Add mirror test implementation

    - Implement mirror test structure
    - Add test/common functionality
    - Create ctl_v3_make_mirror_test

Signed-off-by: Ronald Ngounou <ronald.ngounou@yahoo.com>

Author: ronaldngounou

tests/common/e2e_test.go

@@ -120,5 +120,37 @@ func WithUnixClient() config.ClusterOption {
 }
 
 func WithTCPClient() config.ClusterOption {
-	return e2e.WithTCPClient()
+	return func(c *config.ClusterConfig) {
+		ctx := ensureE2EClusterContext(c)
+		ctx.UseUnix = false
+		c.ClusterContext = ctx
+	}
+}
+
+func WithBasePort(port int) config.ClusterOption {
+	return func(c *config.ClusterConfig) {
+		ctx := ensureE2EClusterContext(c)
+		ctx.BasePort = port
+		c.ClusterContext = ctx
+	}
+}
+
+func ensureE2EClusterContext(c *config.ClusterConfig) *e2e.ClusterContext {
+	ctx, _ := c.ClusterContext.(*e2e.ClusterContext)
+	if ctx == nil {
+		ctx = &e2e.ClusterContext{}
+	}
+	return ctx
+}
+
+func configureMirrorDestTLS(mm *config.MakeMirrorOptions, tls config.TLSConfig) {
+	switch tls {
+	case config.ManualTLS:
+		mm.DestCert = e2e.CertPath
+		mm.DestKey = e2e.PrivateKeyPath
+		mm.DestCACert = e2e.CaPath
+		mm.DestInsecureTransport = false
+	default:
+		mm.DestInsecureTransport = true
+	}
 }

tests/common/integration_test.go

@@ -65,5 +65,23 @@ func WithUnixClient() config.ClusterOption {
 }
 
 func WithTCPClient() config.ClusterOption {
-	return integration.WithTCPClient()
+	return func(c *config.ClusterConfig) {
+		ctx := ensureIntegrationClusterContext(c)
+		ctx.UseUnix = false
+		c.ClusterContext = ctx
+	}
+}
+
+func WithBasePort(port int) config.ClusterOption {
+	return func(c *config.ClusterConfig) {}
 }
+
+func ensureIntegrationClusterContext(c *config.ClusterConfig) *integration.ClusterContext {
+	ctx, _ := c.ClusterContext.(*integration.ClusterContext)
+	if ctx == nil {
+		ctx = &integration.ClusterContext{}
+	}
+	return ctx
+}
+
+func configureMirrorDestTLS(mm *config.MakeMirrorOptions, _ config.TLSConfig) {}

tests/common/make_mirror_test.go

@@ -0,0 +1,164 @@
+// Copyright 2016 The etcd Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package common
+
+import (
+	"context"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+	"go.etcd.io/etcd/tests/v3/framework/config"
+	"go.etcd.io/etcd/tests/v3/framework/testutils"
+)
+
+func TestMakeMirrorModifyDestPrefix(t *testing.T) {
+	testRunner.BeforeTest(t)
+	for _, srcTC := range clusterTestCases() {
+		for _, destTC := range clusterTestCases() {
+			t.Run(fmt.Sprintf("Source=%s/Destination=%s", srcTC.name, destTC.name), func(t *testing.T) {
+
+				var (
+					mmOpts     = config.MakeMirrorOptions{Prefix: "o_", DestPrefix: "d_"}
+					sourcekvs  = []testutils.KV{{Key: "o_key1", Val: "val1"}, {Key: "o_key2", Val: "val2"}, {Key: "o_key3", Val: "val3"}}
+					destkvs    = []testutils.KV{{Key: "d_key1", Val: "val1"}, {Key: "d_key2", Val: "val2"}, {Key: "d_key3", Val: "val3"}}
+					srcprefix  = "o_"
+					destprefix = "d_"
+				)
+
+				testMirror(t, srcTC, destTC, mmOpts, sourcekvs, destkvs, srcprefix, destprefix)
+			})
+		}
+	}
+}
+
+func TestMakeMirrorNoDestPrefix(t *testing.T) {
+	testRunner.BeforeTest(t)
+	for _, srcTC := range clusterTestCases() {
+		for _, destTC := range clusterTestCases() {
+
+			t.Run(fmt.Sprintf("Source=%s/Destination=%s", srcTC.name, destTC.name), func(t *testing.T) {
+				var (
+					mmOpts     = config.MakeMirrorOptions{Prefix: "o_", NoDestPrefix: true}
+					sourcekvs  = []testutils.KV{{Key: "o_key1", Val: "val1"}, {Key: "o_key2", Val: "val2"}, {Key: "o_key3", Val: "val3"}}
+					destkvs    = []testutils.KV{{Key: "key1", Val: "val1"}, {Key: "key2", Val: "val2"}, {Key: "key3", Val: "val3"}}
+					srcprefix  = "o_"
+					destprefix = "key"
+				)
+
+				testMirror(t, srcTC, destTC, mmOpts, sourcekvs, destkvs, srcprefix, destprefix)
+			})
+		}
+	}
+}
+
+func TestMakeMirror(t *testing.T) {
+	testRunner.BeforeTest(t)
+	for _, srcTC := range clusterTestCases() {
+		for _, destTC := range clusterTestCases() {
+			t.Run(fmt.Sprintf("Source=%s/Destination=%s", srcTC.name, destTC.name), func(t *testing.T) {
+				var (
+					mmOpts    config.MakeMirrorOptions
+					sourcekvs = []testutils.KV{{Key: "key1", Val: "val1"}, {Key: "key2", Val: "val2"}, {Key: "key3", Val: "val3"}}
+					destkvs   = []testutils.KV{{Key: "key1", Val: "val1"}, {Key: "key2", Val: "val2"}, {Key: "key3", Val: "val3"}}
+					prefix    = "key"
+				)
+
+				testMirror(t, srcTC, destTC, mmOpts, sourcekvs, destkvs, prefix, prefix)
+
+			})
+		}
+	}
+}
+
+func TestMakeMirrorWithWatchRev(t *testing.T) {
+	testRunner.BeforeTest(t)
+	for _, srcTC := range clusterTestCases() {
+		for _, destTC := range clusterTestCases() {
+			t.Run(fmt.Sprintf("Source=%s/Destination=%s", srcTC.name, destTC.name), func(t *testing.T) {
+				var (
+					mmOpts     = config.MakeMirrorOptions{Prefix: "o_", NoDestPrefix: true, Rev: 4}
+					sourcekvs  = []testutils.KV{{Key: "o_key1", Val: "val1"}, {Key: "o_key2", Val: "val2"}, {Key: "o_key3", Val: "val3"}, {Key: "o_key4", Val: "val4"}}
+					destkvs    = []testutils.KV{{Key: "key3", Val: "val3"}, {Key: "key4", Val: "val4"}}
+					srcprefix  = "o_"
+					destprefix = "key"
+				)
+
+				testMirror(t, srcTC, destTC, mmOpts, sourcekvs, destkvs, srcprefix, destprefix)
+
+			})
+		}
+	}
+}
+
+func testMirror(t *testing.T, srcTC, destTC testCase, mmOpts config.MakeMirrorOptions, sourcekvs, destkvs []testutils.KV, srcprefix, destprefix string) {
+	t.Helper()
+
+	if destTC.config.ClientTLS == config.AutoTLS {
+		// AutoTLS destination unsupported here: we cannot pass the destination CA to etcdctl (--dest-cacert)
+		// so tLS verification would fail.
+		t.Skip("Skipping: destingation uses Client AutoTLS, but the test cannot expose the dest CA for etcdctl (--dest-cacert); tLS verification would fail.")
+	}
+
+	ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second)
+	defer cancel()
+	// Source cluster
+	src := testRunner.NewCluster(ctx, t, config.WithClusterConfig(srcTC.config))
+	defer src.Close()
+	srcClient := testutils.MustClient(src.Client())
+
+	// Destination cluster
+	destCfg := destTC.config
+	dest := testRunner.NewCluster(ctx, t, config.WithClusterConfig(destTC.config), WithBasePort(10000)) // destTC.BasePort
+	defer dest.Close()
+	destClient := testutils.MustClient(dest.Client())
+
+	// Configure TLS for destination before starting make-mirror
+	configureMirrorDestTLS(&mmOpts, destCfg.ClientTLS)
+
+	// Start make mirror
+	errCh := make(chan error)
+	go func(opts config.MakeMirrorOptions) {
+		errCh <- srcClient.MakeMirror(ctx, dest.Endpoints()[0], mmOpts)
+	}(mmOpts)
+	defer func() {
+		// Need to cancel the context to ensure the MakeMirror goroutine is cancelled before catching the error.
+		cancel()
+		require.NoError(t, <-errCh)
+	}()
+
+	// Write to source
+	for i := range sourcekvs {
+		require.NoError(t, srcClient.Put(ctx, sourcekvs[i].Key, sourcekvs[i].Val, config.PutOptions{}))
+	}
+
+	// Source assertion
+	srcResp, err := srcClient.Get(ctx, srcprefix, config.GetOptions{Prefix: true})
+	require.NoError(t, err)
+	require.Equal(t, sourcekvs, testutils.KeyValuesFromGetResponse(srcResp))
+
+	// Destination assertion
+	wCtx, wCancel := context.WithCancel(ctx)
+	defer wCancel()
+
+	watchChan := destClient.Watch(wCtx, destprefix, config.WatchOptions{Prefix: true, Revision: 1})
+
+	// Compare the result of what we obtained using the make-mirror command versus what we had using
+	// the Watch command
+	destResp, err := testutils.KeyValuesFromWatchChan(watchChan, len(destkvs), 10*time.Second)
+	require.NoError(t, err)
+	require.Equal(t, destkvs, destResp)
+}

tests/common/unit_test.go

@@ -56,3 +56,7 @@ func WithTCPClient() config.ClusterOption {
 func WithUnixClient() config.ClusterOption {
 	return func(c *config.ClusterConfig) {}
 }
+
+func WithBasePort(port int) config.ClusterOption {
+	return func(c *config.ClusterConfig) {}
+}

tests/framework/config/client.go

@@ -77,3 +77,14 @@ type WatchOptions struct {
 	Revision int64
 	RangeEnd string
 }
+
+type MakeMirrorOptions struct {
+	Prefix                string
+	Rev                   int64
+	DestPrefix            string
+	NoDestPrefix          bool
+	DestCACert            string
+	DestCert              string
+	DestKey               string
+	DestInsecureTransport bool
+}

tests/framework/e2e/config.go

@@ -44,6 +44,7 @@ type ClusterContext struct {
 	Version ClusterVersion
 	EnvVars map[string]string
 	UseUnix bool
+	BasePort int
 }
 
 func WithHTTP2Debug() config.ClusterOption {

tests/framework/e2e/e2e.go

@@ -66,6 +66,9 @@ func (e e2eRunner) NewCluster(ctx context.Context, tb testing.TB, opts ...config
 		if ctx.UseUnix {
 			e2eConfig.BaseClientScheme = "unix"
 		}
+		if ctx.BasePort != 0 {
+			e2eConfig.BasePort = ctx.BasePort
+		}
 	}
 
 	switch cfg.ClientTLS {

tests/framework/e2e/etcdctl.go

@@ -745,3 +745,50 @@ func (ctl *EtcdctlV3) Watch(ctx context.Context, key string, opts config.WatchOp
 
 	return ch
 }
+
+func (ctl *EtcdctlV3) MakeMirror(ctx context.Context, destEndpoints string, opts config.MakeMirrorOptions) error {
+
+	args := ctl.cmdArgs()
+	args = append(args, "make-mirror")
+
+	if opts.Prefix != "" {
+		args = append(args, "--prefix", opts.Prefix)
+	}
+	if opts.Rev != 0 {
+		args = append(args, "--rev", fmt.Sprint(opts.Rev))
+	}
+	if opts.NoDestPrefix {
+		args = append(args, "--no-dest-prefix")
+	}
+	if opts.DestPrefix != "" {
+		args = append(args, "--dest-prefix", opts.DestPrefix)
+	}
+	if opts.DestCACert != "" {
+		args = append(args, "--dest-cacert", opts.DestCACert)
+	}
+	if opts.DestCert != "" {
+		args = append(args, "--dest-cert", opts.DestCert)
+	}
+	if opts.DestKey != "" {
+		args = append(args, "--dest-key", opts.DestKey)
+	}
+	if opts.DestInsecureTransport {
+		// Bool flags in cobra must be provided as --flag=false, not as separate args.
+		args = append(args, "--dest-insecure-transport", "--dest-insecure-transport=false")
+	}
+
+	args = append(args, destEndpoints)
+
+	proc, err := SpawnCmd(args, nil)
+	if err != nil {
+		return err
+	}
+
+	defer proc.Stop()
+
+	// Wait until context is cancelled or its timeout is reached so that the make-mirror command can keep running in the background.
+	<-ctx.Done()
+
+	return nil
+
+}

tests/framework/integration/integration.go

@@ -16,6 +16,7 @@ package integration
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"strings"
 	"testing"
@@ -475,3 +476,8 @@ func (c integrationClient) MemberList(ctx context.Context, serializable bool) (*
 	}
 	return c.Client.MemberList(ctx)
 }
+
+func (c integrationClient) MakeMirror(ctx context.Context, destEndpoints string, opts config.MakeMirrorOptions) error {
+
+	return errors.New("error")
+}

tests/framework/interfaces/interface.go

@@ -84,6 +84,8 @@ type Client interface {
 	MemberRemove(ctx context.Context, id uint64) (*clientv3.MemberRemoveResponse, error)
 
 	Watch(ctx context.Context, key string, opts config.WatchOptions) clientv3.WatchChan
+
+	MakeMirror(ctx context.Context, destEndpoints string, opts config.MakeMirrorOptions) error
 }
 
 type TemplateEndpoints interface {