OAS-9946 Handle DB creation failures in tests. (#616)

Author: jwierzbo

test/backup_test.go

@@ -333,6 +333,11 @@ func waitForServerRestart(ctx context.Context, c driver.Client, t *testing.T) dr
 }
 
 func TestBackupRestore(t *testing.T) {
+	if os.Getenv("TEST_CONNECTION") == "vst" {
+		t.Skip("VST is dropped since 3.12")
+		return
+	}
+
 	c := createClient(t, nil)
 	skipIfNoBackup(c, t)
 	ctx := context.Background()
@@ -721,6 +726,11 @@ func TestBackupCreateManyBackupsFast(t *testing.T) {
 }
 
 func TestBackupRestoreWithViews(t *testing.T) {
+	if os.Getenv("TEST_CONNECTION") == "vst" {
+		t.Skip("VST is dropped since 3.12")
+		return
+	}
+
 	c := createClient(t, nil)
 	skipIfNoBackup(c, t)
 	ctx := context.Background()

v2/arangodb/shared/error.go

@@ -102,15 +102,12 @@ func newArangoError(code, errorNum int, errorMessage string) error {
 	}
 }
 
-// IsArangoError returns true when the given error is an ArangoError.
-func IsArangoError(err error) bool {
-	return checkCause(err, func(err error) bool {
-		if a, ok := err.(ArangoError); ok {
-			return a.HasError
-		}
+// IsArangoError returns true when the given error is an ArangoError
+func IsArangoError(err error) (bool, ArangoError) {
+	var arangoErr ArangoError
+	ok := errors.As(err, &arangoErr)
 
-		return false
-	})
+	return ok, arangoErr
 }
 
 // IsArangoErrorWithCode returns true when the given error is an ArangoError and its Code field is equal to the given code.

v2/tests/backup_test.go

@@ -23,6 +23,7 @@ package tests
 import (
 	"context"
 	"encoding/json"
+	"fmt"
 	"os"
 	"strings"
 	"testing"
@@ -31,6 +32,7 @@ import (
 	"github.com/stretchr/testify/require"
 
 	"github.com/arangodb/go-driver/v2/arangodb"
+	"github.com/arangodb/go-driver/v2/arangodb/shared"
 )
 
 func Test_CreateBackupSimple(t *testing.T) {
@@ -132,6 +134,8 @@ func Test_RestoreBackupSimple(t *testing.T) {
 
 					err = client.BackupDelete(ctx, backup.ID)
 					require.NoError(t, err, "DeleteBackup failed")
+
+					waitForSync(t, ctx, client)
 				})
 			})
 		})
@@ -140,6 +144,40 @@ func Test_RestoreBackupSimple(t *testing.T) {
 	})
 }
 
+/*
+Sometimes after restore, we observe the following error during db creation:
+
+```
+Could not create database: executing createSystemCollectionsAndIndices
+(creates all system collections including their indices) failed.
+```
+
+Looks like not all DB servers have reported ready in Current/DBServers in the agency when the database creation attempt runs.
+This could be the case if DB servers are all restarted (as after a hotbackup) and start reacting to liveliness probes each,
+but the coordinator has not yet fetched the latest agency state.
+There may be a small window of time in which a DB server already responds to `/_admin/server/availablity`,
+but has not reported ready to the agency, or the coordinator has not yet fetched the latest state from the agency and
+does not yet see the server available in Current/DBServers.
+*/
+func waitForSync(t *testing.T, ctx context.Context, client arangodb.Client) {
+	NewTimeout(func() error {
+		name := GenerateUUID("test-backup-DB")
+
+		db, err := client.CreateDatabase(ctx, name, nil)
+		if err != nil {
+			if ok, arangoErr := shared.IsArangoError(err); ok {
+				t.Logf("waitForSync ERROR: errorNum: %d, errCode: %d, msg: %s", arangoErr.ErrorNum, arangoErr.Code, arangoErr.ErrorMessage)
+				if strings.Contains(arangoErr.ErrorMessage, "executing createSystemCollectionsAndIndices (creates all system collections including their indices) failed") {
+					return err
+				}
+			}
+		}
+		require.NoError(t, err, fmt.Sprintf("waitForSync Failed to create DB %s", name))
+		require.NoError(t, db.Remove(ctx))
+		return Interrupt{}
+	}).TimeoutT(t, 2*time.Minute, 125*time.Millisecond)
+}
+
 func Test_BackupFullFlow(t *testing.T) {
 	requireClusterMode(t)
 
@@ -184,6 +222,7 @@ func Test_BackupFullFlow(t *testing.T) {
 				WaitForHealthyCluster(t, client, time.Minute, true)
 			})
 
+			waitForSync(t, ctx, client)
 		})
 	}, WrapOptions{
 		Parallel: newBool(false),

v2/tests/call_test.go

@@ -84,10 +84,9 @@ func Test_CallWithChecks(t *testing.T) {
 					[]int{http.StatusOK, http.StatusNoContent})
 				require.Error(t, err)
 
-				arangoErr, ok := err.(shared.ArangoError)
+				ok, arangoErr := shared.IsArangoError(err)
 				require.True(t, ok)
 				require.True(t, arangoErr.HasError)
-				require.True(t, shared.IsArangoError(arangoErr))
 				require.Equal(t, http.StatusNotFound, resp.Code())
 			})
 		})