Allow for slow standby take-over in MIQM test

In TestMultiInstanceContainerStop, if the standby hasn't taken over by the time the active has stopped, the test fails.  This causes problems on slow machines for the CI/CD pipeline.  This commit adds a 30 second timeout on the take-over.

Author: arthurbarr

test/docker/mq_multi_instance_test.go

@@ -1,5 +1,5 @@
 /*
-© Copyright IBM Corporation 2019, 2020
+© Copyright IBM Corporation 2019, 2022
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -16,6 +16,7 @@ limitations under the License.
 package main
 
 import (
+	"context"
 	"strings"
 	"testing"
 	"time"
@@ -92,15 +93,28 @@ func TestMultiInstanceContainerStop(t *testing.T) {
 	waitForReady(t, cli, qm1aId)
 	waitForReady(t, cli, qm1bId)
 
-	err, active, standby := getActiveStandbyQueueManager(t, cli, qm1aId, qm1bId)
+	err, originalActive, originalStandby := getActiveStandbyQueueManager(t, cli, qm1aId, qm1bId)
 	if err != nil {
 		t.Fatal(err)
 	}
 
-	stopContainer(t, cli, active)
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+	stopContainer(t, cli, originalActive)
 
-	if status := getQueueManagerStatus(t, cli, standby, "QM1"); strings.Compare(status, "Running") != 0 {
-		t.Fatalf("Expected QM1 to be running as active queue manager, dspmq returned status of %v", status)
+	for {
+		status := getQueueManagerStatus(t, cli, originalStandby, "QM1")
+		select {
+		case <-time.After(1 * time.Second):
+			if status == "Running" {
+				t.Logf("Original standby is now the active")
+				return
+			} else if status == "Starting" {
+				t.Logf("Original standby is starting")
+			}
+		case <-ctx.Done():
+			t.Fatalf("%s Timed out waiting for standby to become the active.  Status=%v", time.Now().Format(time.RFC3339), status)
+		}
 	}
 }
 

test/docker/mq_multi_instance_test_util.go

@@ -1,5 +1,5 @@
 /*
-© Copyright IBM Corporation 2019
+© Copyright IBM Corporation 2019, 2022
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -77,6 +77,7 @@ func getActiveStandbyQueueManager(t *testing.T, cli *client.Client, qm1aId strin
 
 func getQueueManagerStatus(t *testing.T, cli *client.Client, containerID string, queueManagerName string) string {
 	_, dspmqOut := execContainer(t, cli, containerID, "", []string{"bash", "-c", "dspmq", "-m", queueManagerName})
+	t.Logf("dspmq for %v (%v) returned: %v", containerID, queueManagerName, dspmqOut)
 	regex := regexp.MustCompile(`STATUS\(.*\)`)
 	status := regex.FindString(dspmqOut)
 	status = strings.TrimSuffix(strings.TrimPrefix(status, "STATUS("), ")")