Skip to content
This repository has been archived by the owner on Mar 26, 2020. It is now read-only.

Testing brick mux e2e failures #1429

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open

Testing brick mux e2e failures #1429

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
171 changes: 89 additions & 82 deletions e2e/brickmux_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ func TestBrickMux(t *testing.T) {

tc, err := setupCluster(t, "./config/1.toml")
r.Nil(err)
t.Logf("BrickMux Log: Setting up cluster")
defer teardownCluster(tc)

client, err = initRestclient(tc.gds[0])
Expand Down Expand Up @@ -245,12 +246,14 @@ func TestBrickMux(t *testing.T) {
// Stop glusterd2 instance and kill the glusterfsd into
// which all bricks were multiplexed
r.Nil(tc.gds[0].Stop())
t.Logf("BrickMux Log:Stopping Gluster D2 0")
process, err = os.FindProcess(pid)
r.Nil(err, fmt.Sprintf("failed to find brick pid: %s", err))
err = process.Signal(syscall.Signal(15))
r.Nil(err, fmt.Sprintf("failed to kill brick: %s", err))

// Spawn glusterd2 instance
t.Logf("BrickMux Log:Starting Gluster D2 1")
gd, err := spawnGlusterd(t, "./config/1.toml", false)
r.Nil(err)
r.True(gd.IsRunning())
Expand Down Expand Up @@ -370,8 +373,9 @@ func TestBrickMux(t *testing.T) {
for _, v := range portMap {
r.Equal(v, 5)
}

t.Logf("BrickMux Log: Stopping Gluster D2 1")
r.Nil(gd.Stop())
t.Logf("BrickMux Log: Stopped Gluster D2 2")
for k := range pidMap {
process, err := os.FindProcess(k)
r.Nil(err, fmt.Sprintf("failed to find brick pid: %s", err))
Expand All @@ -380,7 +384,9 @@ func TestBrickMux(t *testing.T) {
}

// Spawn glusterd2 instance
t.Logf("BrickMux Log: Starting Gluster D2 2")
gd, err = spawnGlusterd(t, "./config/1.toml", false)
t.Logf("BrickMux Log: Started Gluster D2 2")
r.Nil(err)
r.True(gd.IsRunning())

Expand Down Expand Up @@ -421,104 +427,105 @@ func TestBrickMux(t *testing.T) {
r.Nil(client.VolumeStop(volname1 + strconv.Itoa(i)))
r.Nil(client.VolumeDelete(volname1 + strconv.Itoa(i)))
}
/*

// Create two volumes with different options, so that bricks from these
// two volumes are multiplexed into bricks from their own volume. Also,
// check if among three bricks of a volume 2 bricks have same pid and
// port while 1 brick has a different pid and port, since num of bricks
// are 3 and max-bricks-per-process is set as 2.
// Create two volumes with different options, so that bricks from these
// two volumes are multiplexed into bricks from their own volume. Also,
// check if among three bricks of a volume 2 bricks have same pid and
// port while 1 brick has a different pid and port, since num of bricks
// are 3 and max-bricks-per-process is set as 2.

// Turn on brick mux cluster option
optReq = api.ClusterOptionReq{
Options: map[string]string{"cluster.max-bricks-per-process": "2"},
}
err = client.ClusterOptionSet(optReq)
r.Nil(err)
// Turn on brick mux cluster option
optReq = api.ClusterOptionReq{
Options: map[string]string{"cluster.max-bricks-per-process": "2"},
}
err = client.ClusterOptionSet(optReq)
r.Nil(err)

createReq = api.VolCreateReq{
Name: volname1,
Subvols: []api.SubvolReq{
{
ReplicaCount: 3,
Type: "replicate",
Bricks: []api.BrickReq{
{PeerID: tc.gds[0].PeerID(), Path: brickPaths[51]},
{PeerID: tc.gds[0].PeerID(), Path: brickPaths[52]},
{PeerID: tc.gds[0].PeerID(), Path: brickPaths[53]},
createReq = api.VolCreateReq{
Name: volname1,
Subvols: []api.SubvolReq{
{
ReplicaCount: 3,
Type: "replicate",
Bricks: []api.BrickReq{
{PeerID: tc.gds[0].PeerID(), Path: brickPaths[51]},
{PeerID: tc.gds[0].PeerID(), Path: brickPaths[52]},
{PeerID: tc.gds[0].PeerID(), Path: brickPaths[53]},
},
},
},
},
Force: true,
}
_, err = client.VolumeCreate(createReq)
r.Nil(err)
Force: true,
}
_, err = client.VolumeCreate(createReq)
r.Nil(err)

// start the volume
err = client.VolumeStart(volname1, false)
r.Nil(err)
// start the volume
err = client.VolumeStart(volname1, false)
r.Nil(err)

createReq = api.VolCreateReq{
Name: volname2,
Subvols: []api.SubvolReq{
{
Type: "distribute",
Bricks: []api.BrickReq{
{PeerID: tc.gds[0].PeerID(), Path: brickPaths[48]},
{PeerID: tc.gds[0].PeerID(), Path: brickPaths[49]},
{PeerID: tc.gds[0].PeerID(), Path: brickPaths[50]},
createReq = api.VolCreateReq{
Name: volname2,
Subvols: []api.SubvolReq{
{
Type: "distribute",
Bricks: []api.BrickReq{
{PeerID: tc.gds[0].PeerID(), Path: brickPaths[48]},
{PeerID: tc.gds[0].PeerID(), Path: brickPaths[49]},
{PeerID: tc.gds[0].PeerID(), Path: brickPaths[50]},
},
},
},
},
Force: true,
}
_, err = client.VolumeCreate(createReq)
r.Nil(err)

// Setting an option in second volume so that second volume doesn't
// multiplex its brick into first volume
var optionReq api.VolOptionReq
optionReq.Options = map[string]string{"io-stats.count-fop-hits": "on"}
optionReq.AllowAdvanced = true
Force: true,
}
_, err = client.VolumeCreate(createReq)
r.Nil(err)

r.Nil(client.VolumeSet(volname2, optionReq))
// Setting an option in second volume so that second volume doesn't
// multiplex its brick into first volume
var optionReq api.VolOptionReq
optionReq.Options = map[string]string{"io-stats.count-fop-hits": "on"}
optionReq.AllowAdvanced = true

// start the volume
err = client.VolumeStart(volname2, false)
r.Nil(err)
r.Nil(client.VolumeSet(volname2, optionReq))

bstatus, err = client.BricksStatus(volname1)
r.Nil(err)
// start the volume
err = client.VolumeStart(volname2, false)
r.Nil(err)

// Keep track of used unique pids and ports used in multiplexing bricks
// of volname1 and calculate length length of maps, which should be equal to 2
pidMap = make(map[int]int)
portMap = make(map[int]int)
for _, b := range bstatus {
pidMap[b.Pid] = 1
portMap[b.Port] = 1
}
r.Equal(len(pidMap), 2)
r.Equal(len(portMap), 2)
bstatus, err = client.BricksStatus(volname1)
r.Nil(err)

bstatus2, err := client.BricksStatus(volname2)
r.Nil(err)
// Keep track of used unique pids and ports used in multiplexing bricks
// of volname1 and calculate length length of maps, which should be equal to 2
pidMap = make(map[int]int)
portMap = make(map[int]int)
for _, b := range bstatus {
pidMap[b.Pid] = 1
portMap[b.Port] = 1
}
r.Equal(len(pidMap), 2)
r.Equal(len(portMap), 2)

// Keep track of used unique pids and ports used in multiplexing bricks
// of volname1 and calculate length length of maps, which should be equal to 2
pidMap = make(map[int]int)
portMap = make(map[int]int)
for _, b := range bstatus2 {
pidMap[b.Pid] = 1
portMap[b.Port] = 1
}
r.Equal(len(pidMap), 2)
r.Equal(len(portMap), 2)
bstatus2, err := client.BricksStatus(volname2)
r.Nil(err)

r.Nil(client.VolumeStop(volname1))
r.Nil(client.VolumeDelete(volname1))
// Keep track of used unique pids and ports used in multiplexing bricks
// of volname1 and calculate length length of maps, which should be equal to 2
pidMap = make(map[int]int)
portMap = make(map[int]int)
for _, b := range bstatus2 {
pidMap[b.Pid] = 1
portMap[b.Port] = 1
}
r.Equal(len(pidMap), 2)
r.Equal(len(portMap), 2)

r.Nil(client.VolumeStop(volname2))
r.Nil(client.VolumeDelete(volname2))
r.Nil(client.VolumeStop(volname1))
r.Nil(client.VolumeDelete(volname1))

r.Nil(client.VolumeStop(volname2))
r.Nil(client.VolumeDelete(volname2))
*/
r.Nil(gd.Stop())
}
27 changes: 25 additions & 2 deletions e2e/process_utils_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,11 @@ var errFinished = errors.New("os: process already finished")
// Stop will terminate the associated process. It will attempt a graceful
// shutdown before killing the process.
func (tp *testProcess) Stop() error {
tp.Cmd.Process.Signal(os.Interrupt) // try shutting down gracefully
//tp.Cmd.Process.Signal(os.Interrupt)
err := tp.Cmd.Process.Signal(os.Interrupt) // try shutting down gracefully
if err != nil {
return err
}
time.Sleep(2 * time.Second)
if tp.IsRunning() {
time.Sleep(2 * time.Second)
Expand Down Expand Up @@ -179,13 +183,32 @@ func spawnGlusterd(t *testing.T, configFilePath string, cleanStart bool) (*gdPro
)
}
g.Cmd = exec.Command(path.Join(binDir, "glusterd2"), args...)
logMsg := fmt.Sprintf("BrickMux Log:g.Cmd : %v, g.PeerID: %s, args:%v\n", g.Cmd, g.PeerID(), args)
t.Logf(logMsg)

stdout, err := g.Cmd.StdoutPipe()
if err != nil {
return nil, err
}
stderr, err := g.Cmd.StderrPipe()
if err != nil {
return nil, err
}

if err := g.Cmd.Start(); err != nil {
return nil, err
}

go func() {
g.Cmd.Wait()
sout, _ := ioutil.ReadAll(stdout)
logOutput := fmt.Sprintf("BrickMux Log output: %s\n", sout)
t.Logf(logOutput)
serr, _ := ioutil.ReadAll(stderr)
logErr := fmt.Sprintf("BrickMux Log error: %s\n", serr)
t.Logf(logErr)
err := g.Cmd.Wait()
logMsg := fmt.Sprintf("BrickMux Log: Error in wait %v\n", err)
t.Logf(logMsg)
}()

retries := 4
Expand Down
7 changes: 5 additions & 2 deletions extras/centos-ci.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,9 @@ make test TESTOPTIONS=-v
# run functional tests
make functest

# Printing everything
tail -1000 /tmp/gd2_func_test/TestBrickMux/w1/log/glusterd2.log

# tarball the (default) functional test directory
TIMESTAMP=$(date +%s)
tar -czf "/tmp/gd2_func_test_${TIMESTAMP}.tar.gz" /tmp/gd2_func_test
#TIMESTAMP=$(date +%s)
#tar -czf "/tmp/gd2_func_test_${TIMESTAMP}.tar.gz" /tmp/gd2_func_test