Testing brick mux e2e failures

e2e/brickmux_test.go

@@ -21,6 +21,7 @@ func TestBrickMux(t *testing.T) {
 
 	tc, err := setupCluster(t, "./config/1.toml")
 	r.Nil(err)
+	t.Logf("BrickMux Log: Setting up cluster")
 	defer teardownCluster(tc)
 
 	client, err = initRestclient(tc.gds[0])
@@ -245,12 +246,14 @@ func TestBrickMux(t *testing.T) {
 	// Stop glusterd2 instance and kill the glusterfsd into
 	// which all bricks were multiplexed
 	r.Nil(tc.gds[0].Stop())
+	t.Logf("BrickMux Log:Stopping Gluster D2 0")
 	process, err = os.FindProcess(pid)
 	r.Nil(err, fmt.Sprintf("failed to find brick pid: %s", err))
 	err = process.Signal(syscall.Signal(15))
 	r.Nil(err, fmt.Sprintf("failed to kill brick: %s", err))
 
 	// Spawn glusterd2 instance
+	t.Logf("BrickMux Log:Starting Gluster D2 1")
 	gd, err := spawnGlusterd(t, "./config/1.toml", false)
 	r.Nil(err)
 	r.True(gd.IsRunning())
@@ -372,6 +375,7 @@ func TestBrickMux(t *testing.T) {
 	}
 
 	r.Nil(gd.Stop())
+	t.Logf("BrickMux Log: Stopping Gluster D2 1")
 	for k := range pidMap {
 		process, err := os.FindProcess(k)
 		r.Nil(err, fmt.Sprintf("failed to find brick pid: %s", err))
@@ -380,6 +384,7 @@ func TestBrickMux(t *testing.T) {
 	}
 
 	// Spawn glusterd2 instance
+	t.Logf("BrickMux Log: Starting Gluster D2 2")
 	gd, err = spawnGlusterd(t, "./config/1.toml", false)
 	r.Nil(err)
 	r.True(gd.IsRunning())

e2e/process_utils_test.go

@@ -41,7 +41,11 @@ var errFinished = errors.New("os: process already finished")
 // Stop will terminate the associated process. It will attempt a graceful
 // shutdown before killing the process.
 func (tp *testProcess) Stop() error {
-	tp.Cmd.Process.Signal(os.Interrupt) // try shutting down gracefully
+	//tp.Cmd.Process.Signal(os.Interrupt)
+	err := tp.Cmd.Process.Signal(os.Interrupt) // try shutting down gracefully
+	if err != nil {
+		return err
+	}
 	time.Sleep(2 * time.Second)
 	if tp.IsRunning() {
 		time.Sleep(2 * time.Second)
@@ -179,13 +183,17 @@ func spawnGlusterd(t *testing.T, configFilePath string, cleanStart bool) (*gdPro
 		)
 	}
 	g.Cmd = exec.Command(path.Join(binDir, "glusterd2"), args...)
+	logMsg := fmt.Sprintf("BrickMux Log:g.Cmd : %v, g.PeerID: %s, args:%v", g.Cmd, g.PeerID(), args)
+	t.Logf(logMsg)
 
 	if err := g.Cmd.Start(); err != nil {
 		return nil, err
 	}
 
 	go func() {
-		g.Cmd.Wait()
+		err := g.Cmd.Wait()
+		logMsg := fmt.Sprintf("BrickMux Log: Error in wait %v", err)
+		t.Logf(logMsg)
 	}()
 
 	retries := 4

extras/centos-ci.sh

@@ -47,6 +47,9 @@ make test TESTOPTIONS=-v
 # run functional tests
 make functest
 
+# Printing everything
+tail -2000 /tmp/gd2_func_test/TestBrickMux/w1/log/glusterd2.log
+
 # tarball the (default) functional test directory
 TIMESTAMP=$(date +%s)
 tar -czf "/tmp/gd2_func_test_${TIMESTAMP}.tar.gz" /tmp/gd2_func_test