Fix CCR Tests. (#3734)

* Fix CCR Tests.

- WaitForActiveShards is a new feature in 6.7.0, which defaults to false, this is opposite behaviour of 6.6.0 (which implicitly waits for active shards)
- Unfollowing a closed index in 6.7.0 now throws an exception

* Add IntrusiveOperationSeeded cluster that indexes the default data and change ClusterRerouteApiTests to use this cluster

Author: codebrain

src/Tests/Tests.Core/ManagedElasticsearch/Clusters/IntrusiveOperationSeededCluster.cs

@@ -0,0 +1,24 @@
+using Elastic.Managed.Ephemeral.Plugins;
+using Tests.Core.ManagedElasticsearch.NodeSeeders;
+
+namespace Tests.Core.ManagedElasticsearch.Clusters
+{
+	/// <summary>
+	/// Use this cluster for heavy API's, either on ES's side or the client (intricate setup etc)
+	/// </summary>
+	public class IntrusiveOperationSeededCluster : ClientTestClusterBase
+	{
+		public IntrusiveOperationSeededCluster() : base(new ClientTestClusterConfiguration(
+			ElasticsearchPlugin.IngestGeoIp, ElasticsearchPlugin.IngestAttachment
+		)
+		{
+			MaxConcurrency = 1
+		}) { }
+
+		protected override void SeedCluster()
+		{
+			var seeder = new DefaultSeeder(Client);
+			seeder.SeedNode();
+		}
+	}
+}

src/Tests/Tests/Cluster/ClusterReroute/ClusterRerouteApiTests.cs

@@ -12,10 +12,10 @@
 namespace Tests.Cluster.ClusterReroute
 {
 	public class ClusterRerouteApiTests
-		: ApiIntegrationTestBase<IntrusiveOperationCluster, IClusterRerouteResponse, IClusterRerouteRequest, ClusterRerouteDescriptor,
+		: ApiIntegrationTestBase<IntrusiveOperationSeededCluster, IClusterRerouteResponse, IClusterRerouteRequest, ClusterRerouteDescriptor,
 			ClusterRerouteRequest>
 	{
-		public ClusterRerouteApiTests(IntrusiveOperationCluster cluster, EndpointUsage usage) : base(cluster, usage) { }
+		public ClusterRerouteApiTests(IntrusiveOperationSeededCluster cluster, EndpointUsage usage) : base(cluster, usage) { }
 
 		protected override bool ExpectIsValid => false;
 

src/Tests/Tests/XPack/CrossClusterReplication/CrossClusterReplicationFollowTests.cs

@@ -21,6 +21,7 @@ namespace Tests.XPack.CrossClusterReplication
 	[SkipVersion("<6.5.0", "")]
 	public class CrossClusterReplicationFollowTests : CoordinatedIntegrationTestBase<WritableCluster>
 	{
+		private readonly WritableCluster _cluster;
 		private const string CloseIndexStep = nameof(CloseIndexStep);
 		private const string CountAfterStep = nameof(CountAfterStep);
 		private const string CountBeforeStep = nameof(CountBeforeStep);
@@ -73,14 +74,37 @@ public CrossClusterReplicationFollowTests(WritableCluster cluster, EndpointUsage
 			{
 				FollowIndexStep, u =>
 					u.Calls<CreateFollowIndexDescriptor, CreateFollowIndexRequest, ICreateFollowIndexRequest, ICreateFollowIndexResponse>(
-						v => new CreateFollowIndexRequest(CopyIndex(v))
+						v =>
 						{
-							RemoteCluster = DefaultSeeder.RemoteClusterName,
-							LeaderIndex = v
+							if (cluster.ClusterConfiguration.Version < "6.7.0")
+							{
+								return new CreateFollowIndexRequest(CopyIndex(v))
+								{
+									RemoteCluster = DefaultSeeder.RemoteClusterName,
+									LeaderIndex = v
+								};
+							}
+							return new CreateFollowIndexRequest(CopyIndex(v))
+							{
+								RemoteCluster = DefaultSeeder.RemoteClusterName,
+								LeaderIndex = v,
+								WaitForActiveShards = "1"
+							};
+						},
+						(v, d) =>
+						{
+							if (cluster.ClusterConfiguration.Version < "6.7.0")
+							{
+								return d
+									.RemoteCluster(DefaultSeeder.RemoteClusterName)
+									.LeaderIndex(v);
+							}
+
+							return d
+								.RemoteCluster(DefaultSeeder.RemoteClusterName)
+								.LeaderIndex(v)
+								.WaitForActiveShards("1");
 						},
-						(v, d) => d
-							.RemoteCluster(DefaultSeeder.RemoteClusterName)
-							.LeaderIndex(v),
 						(v, c, f) => c.CreateFollowIndex(CopyIndex(v), f),
 						(v, c, f) => c.CreateFollowIndexAsync(CopyIndex(v), f),
 						(v, c, r) => c.CreateFollowIndex(r),
@@ -197,7 +221,10 @@ public CrossClusterReplicationFollowTests(WritableCluster cluster, EndpointUsage
 					(v, c, r) => c.UnfollowIndexAsync(r)
 				)
 			},
-		}) { }
+		})
+		{
+			_cluster = cluster;
+		}
 
 		protected static string Prefix { get; } = $"f{Guid.NewGuid().ToString("N").Substring(0, 4)}";
 
@@ -318,6 +345,16 @@ [I] public async Task PauseBeforeCloseIsAcked() =>
 		[I] public async Task CloseIsAcked() => await Assert<CloseIndexResponse>(CloseIndexStep, r => r.Acknowledged.Should().BeTrue());
 
 		[I] public async Task UnfollowAfterCloseIsAcked() =>
-			await Assert<UnfollowIndexResponse>(UnfollowAgainStep, r => r.Acknowledged.Should().BeTrue());
+			await Assert<UnfollowIndexResponse>(UnfollowAgainStep, r =>
+			{
+				// Unfollowing an index after closing on 6.7.0 throws an exception.
+				if (_cluster.ClusterConfiguration.Version < "6.7.0")
+				{
+					r.Acknowledged.Should().BeTrue();
+					return;
+				}
+				r.IsValid.Should().BeFalse();
+				r.ServerError.Should().NotBeNull();
+			});
 	}
 }