diff --git a/modules/core/pom.xml b/modules/core/pom.xml index 9f80f623aa498..5f302fb70bdeb 100644 --- a/modules/core/pom.xml +++ b/modules/core/pom.xml @@ -231,6 +231,20 @@ 0.23.0 test + + + org.testcontainers + testcontainers + ${testcontainers.version} + test + + + + org.slf4j + slf4j-simple + 2.0.12 + test + diff --git a/modules/core/src/test/java/org/apache/ignite/testcontainers/IgniteClusterContainer.java b/modules/core/src/test/java/org/apache/ignite/testcontainers/IgniteClusterContainer.java new file mode 100644 index 0000000000000..50c23b9f0995f --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/testcontainers/IgniteClusterContainer.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.testcontainers; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import org.testcontainers.containers.Network; +import org.testcontainers.lifecycle.Startable; +import org.testcontainers.lifecycle.Startables; + +/** Ignite cluster container. */ +public class IgniteClusterContainer implements Startable { + /** Containers. */ + private final List containers; + + /** Network. */ + private final Network net = Network.newNetwork(); + + /** @param nodeIds Node ids. */ + public IgniteClusterContainer(List nodeIds) { + containers = new ArrayList<>(nodeIds.size()); + + for (int i = 0; i < nodeIds.size(); i++) { + String hostname = "node" + (1 + i); + + IgniteContainer ignite = new IgniteContainer(net, hostname, nodeIds.get(i)); + + containers.add(ignite); + } + } + + /** {@inheritDoc} */ + @Override public void start() { + Startables.deepStart(containers).join(); + } + + /** {@inheritDoc} */ + @Override public void stop() { + for (IgniteContainer container : containers) + container.stop(); + } + + /** @return All started nodes. */ + public List nodes() { + return Collections.unmodifiableList(containers); + } + + /** @return First started node in cluster. */ + public IgniteContainer firstNode() { + return containers.get(0); + } +} diff --git a/modules/core/src/test/java/org/apache/ignite/testcontainers/IgniteContainer.java b/modules/core/src/test/java/org/apache/ignite/testcontainers/IgniteContainer.java new file mode 100644 index 0000000000000..c8fb515c92cd4 --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/testcontainers/IgniteContainer.java @@ -0,0 +1,307 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.testcontainers; + +import java.io.File; +import java.io.IOException; +import java.time.Duration; +import java.time.ZoneId; +import java.util.Arrays; +import java.util.UUID; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import com.github.dockerjava.api.command.InspectContainerResponse; +import org.apache.ignite.IgniteException; +import org.apache.ignite.Ignition; +import org.apache.ignite.client.IgniteClient; +import org.apache.ignite.cluster.ClusterState; +import org.apache.ignite.configuration.ClientConfiguration; +import org.apache.ignite.internal.IgniteInterruptedCheckedException; +import org.apache.ignite.internal.util.typedef.internal.U; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.Network; +import org.testcontainers.containers.wait.strategy.Wait; +import org.testcontainers.utility.DockerImageName; +import org.testcontainers.utility.MountableFile; + +import static org.apache.ignite.testframework.GridTestUtils.waitForCondition; +import static org.testcontainers.utility.MountableFile.forClasspathResource; + +/** Ignite container. */ +public class IgniteContainer extends GenericContainer { + /** Logger. */ + private static final Logger LOGGER = LoggerFactory.getLogger(IgniteContainer.class); + + /** Docker image name. */ + private static final DockerImageName DOCKER_IMAGE_NAME = DockerImageName.parse("apacheignite/ignite:2.18.0"); + + /** Work directory. */ + private static final String WORK_DIR = "/opt/ignite/apache-ignite/"; + + /** Libs directory in container. */ + private static final File LIBS_DIR = new File(WORK_DIR + "libs"); + + /** Target libs directory in container. */ + private static final File TARGET_LIBS_DIR = new File("/opt/ignite/target-libs"); + + /** Local target libs directory to copy in container. */ + private static final String LOCAL_TARGET_LIBS_DIR = System.getProperty("local.target.libs", "/tmp/target-libs"); + + /** */ + private static final String CFG_PATH = WORK_DIR + "config/test-config.xml"; + + /** */ + private static final String ENABLE_EXPERIMENTAL_FLAG = "--enable-experimental"; + + /** */ + private static final Pattern CLUSTER_STATE_PATTERN = Pattern.compile("Cluster state: (ACTIVE|INACTIVE)"); + + /** */ + private static final Pattern RU_STATUS_PATTERN = Pattern.compile("Rolling upgrade status: (enabled|disabled)"); + + /** */ + private static final String WRAPPER_SCRIPT = "/opt/ignite/run-wrapper.sh"; + + /** Default thin client port. */ + private static final int THIN_CLIENT_PORT = 10800; + + /** Ignite thin client. */ + private IgniteClient client; + + /** Constructor. */ + public IgniteContainer() { + this(Network.newNetwork(), "node0", UUID.randomUUID().toString()); + } + + /** Constructor. */ + public IgniteContainer(Network net, String hostname, String nodeId) { + super(DOCKER_IMAGE_NAME); + + withEnv("CONFIG_URI", "file://" + CFG_PATH); + withEnv("IGNITE_QUIET", "false"); + withEnv("IGNITE_NODE_NAME", nodeId); + withEnv("TZ", ZoneId.systemDefault().toString()); + //withEnv("JVM_OPTS", String.format("-Xmx%s", heapSize)); + + withCopyFileToContainer(forClasspathResource("docker/test-config.xml"), CFG_PATH); + withCopyFileToContainer(forClasspathResource("docker/run-wrapper.sh"), WRAPPER_SCRIPT); + withCopyToContainer(MountableFile.forHostPath(LOCAL_TARGET_LIBS_DIR), TARGET_LIBS_DIR.getAbsolutePath()); + withNetwork(net); + withNetworkAliases(hostname); + withExposedPorts(THIN_CLIENT_PORT); + + withCommand("sh", "-c", "chmod +x " + WRAPPER_SCRIPT + " && exec " + WRAPPER_SCRIPT); + + waitingFor(Wait.forLogMessage(".*Node started.*", 1) + .withStartupTimeout(Duration.ofSeconds(60))); + } + + /** @return Thin client instance. */ + public IgniteClient client() { + if (client == null) + client = Ignition.startClient(clientConfig()); + + return client; + } + + /** */ + public void activateCluster() { + execControl("--set-state", "ACTIVE", "--yes"); + + try { + waitForCondition(() -> { + String out = execControl("--state"); + + Matcher matcher = CLUSTER_STATE_PATTERN.matcher(out); + + if (matcher.find()) + return ClusterState.valueOf(matcher.group(1)) == ClusterState.ACTIVE; + + return false; + }, 30_000); + } + catch (IgniteInterruptedCheckedException e) { + throw new IgniteException(e); + } + } + + /** + * 1. Stop ignite node. + * 2. Remove current libs dir. + * 3. Upgrade JAR's (copy libs from target dir). + * 4. Start ignite node. + */ + public void upgrade() throws Exception { + LOGGER.info(">>> Upgrade container {}", getContainerName()); + + closeClient(); + + exec("Failed to kill Ignite process", "kill", "-INT", "1"); + + for (int i = 0; i < 30; i++) { + ExecResult res = execInContainer("pgrep", "-f", "org.apache.ignite.startup.cmdline.CommandLineStartup"); + + if (res.getExitCode() == 1) + break; + + U.sleep(1_000); + } + + exec("Failed to remove old libs", "sh", "-c", "rm -rf " + LIBS_DIR + "/*"); + + exec("Failed to copy new libs", "sh", "-c", "cp -r " + TARGET_LIBS_DIR + "/* " + LIBS_DIR + "/"); + + execInContainer("sh", "-c", WORK_DIR + "run.sh &"); + + waitForCondition(() -> { + try { + return client().cluster().nodes().size() == 3; + } + catch (Exception e) { + return false; + } + }, 30_000); + } + + /** */ + public RollingUpgradeStatus rollingUpgradeStatus() { + String out = execControl("--rolling-upgrade", "status"); + + LOGGER.info(">>> Rolling upgrade status: {}", out); + + Matcher matcher = RU_STATUS_PATTERN.matcher(out); + + if (matcher.find()) + return RollingUpgradeStatus.valueOf(matcher.group(1).toUpperCase()); + + throw new IllegalStateException("Failed to parse rolling upgrade status from output:\n" + out); + } + + /** */ + public void rollingUpgradeEnable(String targetVer) { + execControl("--rolling-upgrade", "enable", targetVer, "--yes"); + } + + /** */ + public void rollingUpgradeDisable() { + execControl("--rolling-upgrade", "disable"); + } + + /** */ + public int nodesCountForVersion(String targetVer) { + String out = execControl("--rolling-upgrade", "status"); + + // Match the version block + Pattern verPattern = Pattern.compile( + "Version\\s+" + Pattern.quote(targetVer) + ".*?:\\s*\n" + + "((?:\\s*Node\\[.*?\\]\\s*\n)*)", + Pattern.DOTALL + ); + + Matcher verMatcher = verPattern.matcher(out); + + if (!verMatcher.find()) + return 0; + + String nodesBlock = verMatcher.group(1); + + // Count Node entries + Pattern nodePattern = Pattern.compile("^\\s*Node\\[.*?\\]", Pattern.MULTILINE); + Matcher nodeMatcher = nodePattern.matcher(nodesBlock); + + int cnt = 0; + + while (nodeMatcher.find()) + cnt++; + + return cnt; + } + + /** {@inheritDoc} */ + @Override protected void containerIsStopping(InspectContainerResponse containerInfo) { + closeClient(); + } + + /** */ + private String execControl(String... cmd) { + String[] fullCmd = new String[cmd.length + 2]; + + fullCmd[0] = WORK_DIR + "bin/control.sh"; + fullCmd[1] = ENABLE_EXPERIMENTAL_FLAG; + + System.arraycopy(cmd, 0, fullCmd, 2, cmd.length); + + ExecResult result; + + try { + LOGGER.info("Running command: {}", Arrays.toString(fullCmd).replace(", ", " ")); + + result = execInContainer(fullCmd); + } + catch (IOException | InterruptedException e) { + throw new IgniteException(e); + } + + if (result.getExitCode() != 0) + throw new IllegalStateException(result.toString()); + + return result.getStdout(); + } + + /** */ + private ExecResult exec(String errMsg, String... cmd) { + try { + ExecResult res = execInContainer(cmd); + + if (res.getExitCode() != 0) + throw new IllegalStateException(errMsg + ": " + res.getStderr()); + + return res; + } + catch (IOException | InterruptedException e) { + throw new IgniteException(e); + } + } + + /** */ + private void closeClient() { + if (client != null) { + client.close(); + + client = null; + } + } + + /** */ + private ClientConfiguration clientConfig() { + return new ClientConfiguration() + .setAddresses("127.0.0.1:" + getMappedPort(THIN_CLIENT_PORT)) + .setRequestTimeout(30_000); + } + + /** */ + public enum RollingUpgradeStatus { + /** */ + ENABLED, + + /** */ + DISABLED + } +} diff --git a/modules/core/src/test/java/org/apache/ignite/testcontainers/IgniteRebalanceOnUpgradeTest.java b/modules/core/src/test/java/org/apache/ignite/testcontainers/IgniteRebalanceOnUpgradeTest.java new file mode 100644 index 0000000000000..d531847316369 --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/testcontainers/IgniteRebalanceOnUpgradeTest.java @@ -0,0 +1,90 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.testcontainers; + +import java.util.List; +import org.apache.ignite.cache.CacheAtomicityMode; +import org.apache.ignite.client.ClientCache; +import org.apache.ignite.client.ClientCacheConfiguration; +import org.junit.Test; + +import static org.apache.ignite.testcontainers.IgniteContainer.RollingUpgradeStatus.DISABLED; +import static org.apache.ignite.testcontainers.IgniteContainer.RollingUpgradeStatus.ENABLED; +import static org.junit.Assert.assertEquals; + +/** Smoke test for rolling upgrade with persistence. */ +public class IgniteRebalanceOnUpgradeTest { + /** Node IDs. */ + private static final List NODE_IDS = List.of( + "ad26bff6-5ff5-49f1-9a61-425a827953ed", + "c1099d16-e7d7-49f4-925c-53329286c444", + "7b880b69-8a9e-4b84-b555-250d365e2e67" + ); + + /** Target version for RU. */ + private static final String TARGET_VER = "2.18.1"; + + /** */ + private static final String CACHE_NAME = "ru-test-cache"; + + /** */ + @Test + public void testRollingUpgrade() throws Exception { + try (IgniteClusterContainer cluster = new IgniteClusterContainer(NODE_IDS)) { + cluster.start(); + + IgniteContainer node = cluster.firstNode(); + + node.activateCluster(); + + ClientCacheConfiguration cfg = new ClientCacheConfiguration() + .setName(CACHE_NAME) + .setBackups(1) + .setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL); + + ClientCache cache = node.client().createCache(cfg); + + for (int i = 0; i < 1000; i++) + cache.put(i, i); + + assertEquals(DISABLED, node.rollingUpgradeStatus()); + + node.rollingUpgradeEnable(TARGET_VER); + + assertEquals(ENABLED, node.rollingUpgradeStatus()); + + for (IgniteContainer container : cluster.nodes()) + container.upgrade(); + + assertEquals(NODE_IDS.size(), node.nodesCountForVersion(TARGET_VER)); + + node.rollingUpgradeDisable(); + + assertEquals(DISABLED, node.rollingUpgradeStatus()); + + ClientCache targetCache = node.client().getOrCreateCache(CACHE_NAME); + + for (int i = 0; i < 1000; i++) + assertEquals("Data mismatch after upgrade at key: " + i, i, (int)targetCache.get(i)); + + targetCache.put(1001, 1001); + + assertEquals(1001, (int)targetCache.get(1001)); + } + } +} diff --git a/modules/core/src/test/resources/docker/run-wrapper.sh b/modules/core/src/test/resources/docker/run-wrapper.sh new file mode 100644 index 0000000000000..41bb74cba7d28 --- /dev/null +++ b/modules/core/src/test/resources/docker/run-wrapper.sh @@ -0,0 +1,30 @@ +#!/bin/sh + +# Wrapper script for starting Ignite that doesn't die on SIGINT +IGNITE_SCRIPT="/opt/ignite/apache-ignite/run.sh" + +# Function to stop Java process +stop_ignite() { + echo "Received SIGINT/SIGTERM, stopping Ignite..." + pkill -f "org.apache.ignite.startup.cmdline.CommandLineStartup" + + if [ -n "$IGNITE_PID" ]; then + wait $IGNITE_PID 2>/dev/null + fi + echo "Ignite stopped." +} + +# Catch SIGTERM and SIGINT, but don't exit — only stop the Java process +trap 'stop_ignite' TERM INT + +# Start Ignite in the background +$IGNITE_SCRIPT & +IGNITE_PID=$! + +# Wait for Java process to be alive +wait $IGNITE_PID + +# If Java process died on its own, just wait for the next start +while true; do + sleep 1 +done diff --git a/modules/core/src/test/resources/docker/test-config.xml b/modules/core/src/test/resources/docker/test-config.xml new file mode 100644 index 0000000000000..ce6f034dbdd5d --- /dev/null +++ b/modules/core/src/test/resources/docker/test-config.xml @@ -0,0 +1,42 @@ + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/parent/pom.xml b/parent/pom.xml index ac68955a0d172..c28aea123f4c5 100644 --- a/parent/pom.xml +++ b/parent/pom.xml @@ -110,6 +110,7 @@ 5.3.39 3.5.4 10.0.27 + 2.0.5 0.8.3 3.9.5 1.5.7-8