[9.1] Fix testRoleMigration race condition (#138742) (#138746)

* Fix testRoleMigration race condition (#138742)

* Remove mute

Author: jfreden

muted-tests.yml

@@ -377,9 +377,6 @@ tests:
 - class: org.elasticsearch.threadpool.ThreadPoolTests
   method: testDetailedUtilizationMetric
   issue: https://github.com/elastic/elasticsearch/issues/138242
-- class: org.elasticsearch.upgrades.SecurityIndexRolesMetadataMigrationIT
-  method: testRoleMigration
-  issue: https://github.com/elastic/elasticsearch/issues/138731
 
 # Examples:
 #

x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/SecurityIndexRolesMetadataMigrationIT.java

@@ -54,8 +54,9 @@ public void testRoleMigration() throws Exception {
             assertMigratedDocInSecurityIndex(mixed1TestRole, "meta", "test");
             assertMigratedDocInSecurityIndex(mixed2TestRole, "meta", "test");
             assertMigratedDocInSecurityIndex(upgradedTestRole, "meta", "test");
-            // queries all roles by metadata
-            assertAllRoles(client(), "mixed1-test-role", "mixed2-test-role", "old-test-role", "upgraded-test-role");
+            // query all roles by metadata - use assertBusy to handle the case where the node handling the query is not yet aware of the
+            // successful migration
+            assertBusy(() -> assertAllRoles(client(), "mixed1-test-role", "mixed2-test-role", "old-test-role", "upgraded-test-role"));
         }
     }
 
@@ -175,7 +176,13 @@ private void assertAllRoles(RestClient client, String... roleNames) throws IOExc
             {"query":{"bool":{"must":[{"exists":{"field":"metadata.meta"}}]}},"sort":["name"]}""";
         Request request = new Request(randomFrom("POST", "GET"), "/_security/_query/role");
         request.setJsonEntity(metadataQuery);
-        Response response = client.performRequest(request);
+        Response response = null;
+        try {
+            response = client.performRequest(request);
+        } catch (ResponseException e) {
+            fail(e);
+        }
+        assertNotNull(response);
         assertOK(response);
         Map<String, Object> responseMap = responseAsMap(response);
         assertThat(responseMap.get("total"), is(roleNames.length));