From 7bb0f8255f99c86d4ba9dac5f1f9ab1161ededdf Mon Sep 17 00:00:00 2001 From: Ngo Quoc Dat Date: Sun, 25 Oct 2026 00:26:32 +0700 Subject: [PATCH 1/2] test(linux): try a fixture's image pull again when it fails --- linux/crates/ssh/tests/openssh_session.rs | 15 ++-- .../test-fixtures/src/cassandra_fixture.rs | 4 +- .../test-fixtures/src/clickhouse_fixture.rs | 4 +- .../test-fixtures/src/container_start.rs | 90 +++++++++++++++++++ linux/crates/test-fixtures/src/lib.rs | 2 + .../crates/test-fixtures/src/mongo_fixture.rs | 4 +- .../crates/test-fixtures/src/mssql_fixture.rs | 4 +- .../crates/test-fixtures/src/mysql_fixture.rs | 7 +- .../test-fixtures/src/openssh_fixture.rs | 4 +- .../test-fixtures/src/pgbouncer_fixture.rs | 4 +- .../test-fixtures/src/postgres_fixture.rs | 4 +- .../crates/test-fixtures/src/redis_fixture.rs | 4 +- linux/docs/testing.md | 2 + 13 files changed, 121 insertions(+), 27 deletions(-) create mode 100644 linux/crates/test-fixtures/src/container_start.rs diff --git a/linux/crates/ssh/tests/openssh_session.rs b/linux/crates/ssh/tests/openssh_session.rs index f59770de08..4dad745aeb 100644 --- a/linux/crates/ssh/tests/openssh_session.rs +++ b/linux/crates/ssh/tests/openssh_session.rs @@ -9,9 +9,9 @@ use secrecy::SecretString; use tablepro_core::credentials::{CredentialInteraction, PromptPurpose, PromptReply}; use tablepro_core::{LivenessPolicy, NetworkEndpoint, SshFailure, TransportError, TransportRoute}; use tablepro_ssh::{SshAuth, SshConfig, SshDestination, SshRuntimeCell, SshServices, SshSession, forget_host_key}; +use tablepro_test_fixtures::start_container; use tablepro_test_support::FakePrompter; use testcontainers::core::{IntoContainerPort, WaitFor}; -use testcontainers::runners::AsyncRunner; use testcontainers::{ContainerAsync, GenericImage, ImageExt}; use tokio::io::AsyncReadExt; use tokio_util::sync::CancellationToken; @@ -45,12 +45,13 @@ fn write_config(config: &Path, known_hosts: &Path, strict_host_key_checking: &st } async fn start() -> TestResult { - let container = GenericImage::new("alpine", "3.22") - .with_exposed_port(2222.tcp()) - .with_wait_for(WaitFor::message_on_stderr("Server listening on")) - .with_cmd(["sh", "-c", SSHD_SCRIPT]) - .start() - .await?; + let container = start_container( + GenericImage::new("alpine", "3.22") + .with_exposed_port(2222.tcp()) + .with_wait_for(WaitFor::message_on_stderr("Server listening on")) + .with_cmd(["sh", "-c", SSHD_SCRIPT]), + ) + .await?; let host = container.get_host().await?.to_string(); let port = container.get_host_port_ipv4(2222).await?; diff --git a/linux/crates/test-fixtures/src/cassandra_fixture.rs b/linux/crates/test-fixtures/src/cassandra_fixture.rs index 83ff5ee6d8..ff8e124611 100644 --- a/linux/crates/test-fixtures/src/cassandra_fixture.rs +++ b/linux/crates/test-fixtures/src/cassandra_fixture.rs @@ -1,12 +1,12 @@ use std::time::Duration; use testcontainers::core::IntoContainerPort; -use testcontainers::runners::AsyncRunner; use testcontainers::{ContainerAsync, ContainerRequest, GenericImage, ImageExt}; use tokio::time::Instant; use crate::container_file::with_files; use crate::container_report::{container_state, log_tail}; +use crate::container_start::start_container; use crate::{ CassandraFlavour, CassandraTls, ContainerFile, ExecOutput, FixtureCredentials, FixtureError, PkiFile, TestNetwork, TestPki, @@ -188,7 +188,7 @@ impl CassandraFixture { request = request.with_network(network.name()).with_container_name(alias.as_str()); } let files = Self::files(flavour, server_version, tls, &pki, &password)?; - let container = with_files(request, files).start().await?; + let container = start_container(with_files(request, files)).await?; let host = container.get_host().await?.to_string(); let port = container.get_host_port_ipv4(Self::CONTAINER_PORT).await?; let fixture = Self { diff --git a/linux/crates/test-fixtures/src/clickhouse_fixture.rs b/linux/crates/test-fixtures/src/clickhouse_fixture.rs index a2d94f7c92..95c7e5bbb1 100644 --- a/linux/crates/test-fixtures/src/clickhouse_fixture.rs +++ b/linux/crates/test-fixtures/src/clickhouse_fixture.rs @@ -1,7 +1,6 @@ use std::time::Duration; use testcontainers::core::IntoContainerPort; -use testcontainers::runners::AsyncRunner; use testcontainers::{ContainerAsync, ContainerRequest, GenericImage, ImageExt, TestcontainersError}; use tokio::io::{AsyncReadExt, AsyncWriteExt}; use tokio::net::TcpStream; @@ -9,6 +8,7 @@ use tokio::time::Instant; use crate::container_file::with_files; use crate::container_report::{container_state, log_tail}; +use crate::container_start::start_container; use crate::{ ClickHouseVersion, ContainerFile, ExecOutput, FixtureCredentials, FixtureError, PkiFile, ServerVersion, TestNetwork, TestPki, @@ -72,7 +72,7 @@ impl ClickHouseFixture { } let mut files = Self::files(&pki, &password)?; files.extend(extra_files); - let container = with_files(request, files).start().await?; + let container = start_container(with_files(request, files)).await?; let (host, http_port, https_port) = match published(&container).await { Ok(published) => published, Err(error) => { diff --git a/linux/crates/test-fixtures/src/container_start.rs b/linux/crates/test-fixtures/src/container_start.rs new file mode 100644 index 0000000000..3fbadaa67c --- /dev/null +++ b/linux/crates/test-fixtures/src/container_start.rs @@ -0,0 +1,90 @@ +use std::time::Duration; + +use testcontainers::bollard::Docker; +use testcontainers::bollard::errors::Error as BollardError; +use testcontainers::runners::AsyncRunner; +use testcontainers::{ContainerAsync, ContainerRequest, GenericImage, Image, ImageExt}; + +use crate::FixtureError; + +const PULL_ATTEMPTS: u32 = 3; +const PULL_RETRY_DELAY: Duration = Duration::from_secs(5); + +/// Starts the container, pulling its image first when Docker does not have +/// it. testcontainers pulls once, so a pull the registry cut short ("bytes +/// remaining on stream") failed every test that needed the image. +pub async fn start_container(request: ContainerRequest) -> Result, FixtureError> { + let descriptor = request.descriptor(); + if image_missing(&descriptor).await { + pull(&descriptor, request.platform().as_deref()).await?; + } + Ok(request.start().await?) +} + +async fn image_missing(descriptor: &str) -> bool { + // Any other answer leaves the pull to testcontainers, as before. + let Ok(docker) = Docker::connect_with_defaults() else { + return false; + }; + matches!( + docker.inspect_image(descriptor).await, + Err(BollardError::DockerResponseServerError { status_code: 404, .. }) + ) +} + +async fn pull(descriptor: &str, platform: Option<&str>) -> Result<(), FixtureError> { + let (name, tag) = descriptor.rsplit_once(':').unwrap_or((descriptor, "latest")); + let mut attempt = 1; + loop { + let image = GenericImage::new(name, tag); + // testcontainers falls back to linux/amd64 itself when no platform is + // named, which the SQL Server images need on arm64. + let pulled = match platform { + Some(platform) => image.with_platform(platform).pull_image().await, + None => image.pull_image().await, + }; + match pulled { + Ok(_) => return Ok(()), + Err(_) if attempt < PULL_ATTEMPTS => { + attempt += 1; + tokio::time::sleep(PULL_RETRY_DELAY).await; + } + Err(error) => return Err(error.into()), + } + } +} + +#[cfg(test)] +mod tests { + use std::time::Instant; + + use testcontainers::TestcontainersError; + use testcontainers::core::error::ClientError; + + use super::*; + + #[tokio::test] + #[ignore = "requires docker"] + async fn a_pull_that_keeps_failing_is_tried_three_times_and_reported() { + let started = Instant::now(); + + let error = start_container(ContainerRequest::from(GenericImage::new( + "alpine", + "tablepro-no-such-tag", + ))) + .await + .err() + .unwrap(); + + let FixtureError::Container(TestcontainersError::Client(ClientError::PullImage { descriptor, .. })) = &error + else { + panic!("unexpected error: {error}"); + }; + assert_eq!(descriptor, "alpine:tablepro-no-such-tag"); + assert!( + started.elapsed() >= PULL_RETRY_DELAY * (PULL_ATTEMPTS - 1), + "took {:?}", + started.elapsed() + ); + } +} diff --git a/linux/crates/test-fixtures/src/lib.rs b/linux/crates/test-fixtures/src/lib.rs index 05e31f58eb..605c99751f 100644 --- a/linux/crates/test-fixtures/src/lib.rs +++ b/linux/crates/test-fixtures/src/lib.rs @@ -6,6 +6,7 @@ mod clickhouse_fixture; mod clickhouse_version; mod container_file; mod container_report; +mod container_start; mod exec_output; mod fixture_credentials; mod fixture_error; @@ -51,6 +52,7 @@ pub use cassandra_version::CassandraVersion; pub use clickhouse_fixture::ClickHouseFixture; pub use clickhouse_version::ClickHouseVersion; pub use container_file::ContainerFile; +pub use container_start::start_container; pub use exec_output::ExecOutput; pub use fixture_credentials::FixtureCredentials; pub use fixture_error::FixtureError; diff --git a/linux/crates/test-fixtures/src/mongo_fixture.rs b/linux/crates/test-fixtures/src/mongo_fixture.rs index 32a5663b62..c5f0910664 100644 --- a/linux/crates/test-fixtures/src/mongo_fixture.rs +++ b/linux/crates/test-fixtures/src/mongo_fixture.rs @@ -1,10 +1,10 @@ use std::time::Duration; use testcontainers::core::{IntoContainerPort, WaitFor}; -use testcontainers::runners::AsyncRunner; use testcontainers::{ContainerAsync, ContainerRequest, GenericImage, ImageExt}; use crate::container_file::with_files; +use crate::container_start::start_container; use crate::{ ContainerFile, ExecOutput, FixtureCredentials, FixtureError, MongoVersion, PkiFile, ServerVersion, TestNetwork, TestPki, @@ -115,7 +115,7 @@ impl MongoFixture { if let (Some(network), Some(alias)) = (network, &network_alias) { request = request.with_network(network.name()).with_container_name(alias); } - let container = with_files(request, Self::files(&pki)?).start().await?; + let container = start_container(with_files(request, Self::files(&pki)?)).await?; let host = container.get_host().await?.to_string(); let port = container.get_host_port_ipv4(PORT).await?; let certificate_port = match listening { diff --git a/linux/crates/test-fixtures/src/mssql_fixture.rs b/linux/crates/test-fixtures/src/mssql_fixture.rs index 8ab89a5f82..7d0cfbcc2e 100644 --- a/linux/crates/test-fixtures/src/mssql_fixture.rs +++ b/linux/crates/test-fixtures/src/mssql_fixture.rs @@ -1,11 +1,11 @@ use std::time::Duration; -use testcontainers::runners::AsyncRunner; use testcontainers::{ContainerAsync, ImageExt}; use testcontainers_modules::mssql_server::MssqlServer; use uuid::Uuid; use crate::container_file::with_files; +use crate::container_start::start_container; use crate::{ ContainerFile, ExecOutput, FixtureCredentials, FixtureError, MssqlVersion, PkiFile, ServerVersion, TestPki, }; @@ -41,7 +41,7 @@ impl MssqlFixture { // SQL Server otherwise takes 80% of the host's memory, and a // container left behind by a killed test keeps all of it. .with_env_var("MSSQL_MEMORY_LIMIT_MB", MEMORY_LIMIT_MB); - let container = with_files(request, Self::files(&pki)?).start().await?; + let container = start_container(with_files(request, Self::files(&pki)?)).await?; let host = container.get_host().await?.to_string(); let port = container.get_host_port_ipv4(CONTAINER_PORT).await?; Ok(Self { diff --git a/linux/crates/test-fixtures/src/mysql_fixture.rs b/linux/crates/test-fixtures/src/mysql_fixture.rs index 9c42023e1a..f67d7d6d13 100644 --- a/linux/crates/test-fixtures/src/mysql_fixture.rs +++ b/linux/crates/test-fixtures/src/mysql_fixture.rs @@ -1,8 +1,8 @@ use testcontainers::core::IntoContainerPort; -use testcontainers::runners::AsyncRunner; use testcontainers::{ContainerAsync, GenericImage, ImageExt}; use crate::container_file::with_files; +use crate::container_start::start_container; use crate::{ ContainerFile, ExecOutput, FixtureCredentials, FixtureError, MysqlFlavour, MysqlOptions, PkiFile, TestNetwork, TestPki, @@ -61,9 +61,8 @@ impl MysqlFixture { if let (Some(network), Some(alias)) = (network, &network_alias) { request = request.with_network(network.name()).with_container_name(alias); } - let container = with_files(request, Self::files(&pki, &options, &certificate_password)?) - .start() - .await?; + let container = + start_container(with_files(request, Self::files(&pki, &options, &certificate_password)?)).await?; let host = container.get_host().await?.to_string(); let port = container.get_host_port_ipv4(Self::CONTAINER_PORT).await?; Ok(Self { diff --git a/linux/crates/test-fixtures/src/openssh_fixture.rs b/linux/crates/test-fixtures/src/openssh_fixture.rs index 79e3804a69..03194d5f0d 100644 --- a/linux/crates/test-fixtures/src/openssh_fixture.rs +++ b/linux/crates/test-fixtures/src/openssh_fixture.rs @@ -1,7 +1,6 @@ use std::time::Duration; use testcontainers::core::{IntoContainerPort, WaitFor}; -use testcontainers::runners::AsyncRunner; use testcontainers::{ContainerAsync, GenericImage, ImageExt}; use tokio::io::AsyncReadExt; use tokio::net::TcpStream; @@ -9,6 +8,7 @@ use tokio::time::Instant; use uuid::Uuid; use crate::container_file::with_files; +use crate::container_start::start_container; use crate::{ ContainerFile, ExecOutput, FixtureCredentials, FixtureError, HostCertificate, OpenSshOptions, OpenSshVersion, ServerVersion, SshAuthVariant, TestNetwork, @@ -95,7 +95,7 @@ impl OpenSshFixture { if let (Some(network), Some(alias)) = (network, &network_alias) { request = request.with_network(network.name()).with_container_name(alias); } - let container = with_files(request, files).start().await?; + let container = start_container(with_files(request, files)).await?; let host = container.get_host().await?.to_string(); let port = container.get_host_port_ipv4(Self::CONTAINER_PORT).await?; wait_for_banner(&host, port).await?; diff --git a/linux/crates/test-fixtures/src/pgbouncer_fixture.rs b/linux/crates/test-fixtures/src/pgbouncer_fixture.rs index d099472e53..7c84959725 100644 --- a/linux/crates/test-fixtures/src/pgbouncer_fixture.rs +++ b/linux/crates/test-fixtures/src/pgbouncer_fixture.rs @@ -1,10 +1,10 @@ use std::num::NonZeroU32; use testcontainers::core::{IntoContainerPort, WaitFor}; -use testcontainers::runners::AsyncRunner; use testcontainers::{ContainerAsync, GenericImage, ImageExt}; use crate::container_file::with_files; +use crate::container_start::start_container; use crate::{ ContainerFile, FixtureCredentials, FixtureError, FixtureImage, HbaMode, PoolMode, PostgresFixture, TestNetwork, }; @@ -50,7 +50,7 @@ impl PgBouncerFixture { .with_wait_for(WaitFor::message_on_either_std("process up")) .with_network(network.name()) .with_container_name(network.container_name(NETWORK_ROLE)); - let container = with_files(request, files).start().await?; + let container = start_container(with_files(request, files)).await?; let host = container.get_host().await?.to_string(); let port = container.get_host_port_ipv4(CONTAINER_PORT).await?; Ok(Self { diff --git a/linux/crates/test-fixtures/src/postgres_fixture.rs b/linux/crates/test-fixtures/src/postgres_fixture.rs index add9120ea2..0140f7b5cc 100644 --- a/linux/crates/test-fixtures/src/postgres_fixture.rs +++ b/linux/crates/test-fixtures/src/postgres_fixture.rs @@ -2,12 +2,12 @@ use std::net::TcpListener; use std::time::Duration; use testcontainers::core::IntoContainerPort; -use testcontainers::runners::AsyncRunner; use testcontainers::{ContainerAsync, ImageExt}; use testcontainers_modules::postgres::Postgres; use tokio::time::Instant; use crate::container_file::with_files; +use crate::container_start::start_container; use crate::{ ContainerFile, ExecOutput, FixtureCredentials, FixtureError, HbaMode, PkiAlgorithm, PkiFile, PostgresVersion, ServerVersion, TestNetwork, TestPki, @@ -106,7 +106,7 @@ impl PostgresFixture { if let Some(host_port) = host_port { request = request.with_mapped_port(host_port, CONTAINER_PORT.tcp()); } - let container = with_files(request, Self::files(hba, &pki)?).start().await?; + let container = start_container(with_files(request, Self::files(hba, &pki)?)).await?; let host = container.get_host().await?.to_string(); let port = container.get_host_port_ipv4(CONTAINER_PORT).await?; Ok(Self { diff --git a/linux/crates/test-fixtures/src/redis_fixture.rs b/linux/crates/test-fixtures/src/redis_fixture.rs index 58513239e7..d25c94dbd6 100644 --- a/linux/crates/test-fixtures/src/redis_fixture.rs +++ b/linux/crates/test-fixtures/src/redis_fixture.rs @@ -1,11 +1,11 @@ use std::time::Duration; use testcontainers::core::IntoContainerPort; -use testcontainers::runners::AsyncRunner; use testcontainers::{ContainerAsync, ContainerRequest, GenericImage, ImageExt}; use tokio::time::Instant; use crate::container_file::with_files; +use crate::container_start::start_container; use crate::{ ContainerFile, ExecOutput, FixtureCredentials, FixtureError, PkiFile, RedisVersion, ServerVersion, TestNetwork, TestPki, @@ -74,7 +74,7 @@ impl RedisFixture { if let (Some(network), Some(alias)) = (network, &network_alias) { request = request.with_network(network.name()).with_container_name(alias); } - let container = with_files(request, Self::files(&pki, &password)?).start().await?; + let container = start_container(with_files(request, Self::files(&pki, &password)?)).await?; let host = container.get_host().await?.to_string(); let port = container.get_host_port_ipv4(PORT).await?; let certificate_port = match listening { diff --git a/linux/docs/testing.md b/linux/docs/testing.md index 21004fbe60..acf39f7cde 100644 --- a/linux/docs/testing.md +++ b/linux/docs/testing.md @@ -351,6 +351,8 @@ cargo nextest run -p tablepro-driver-postgres --run-ignored only Do not bother with `TESTCONTAINERS_RYUK_DISABLED`. That is a testcontainers-java / go setting; the Rust crate has no Ryuk reaper and stops each container when its handle drops. +Every fixture starts its container through `tablepro_test_fixtures::start_container`, which pulls a missing image itself and tries a failed pull twice more, 5 seconds apart. testcontainers pulls once, and Docker Hub sometimes ends a pull early ("bytes remaining on stream"), which failed every test that needed the image. A test that starts a container of its own goes through it too. + `curl --unix-socket "${DOCKER_HOST#unix://}" http://localhost/_ping` should print `OK` before you run the suite. `--unix-socket` takes a filesystem path, so the `unix://` prefix has to come off. A test that panics hard can still leave a container behind. `podman container prune` clears them. From 32be0c5c51a71444403d320979ff15d8c5d45c07 Mon Sep 17 00:00:00 2001 From: Ngo Quoc Dat Date: Sun, 25 Oct 2026 00:41:12 +0700 Subject: [PATCH 2/2] test(linux): retry the start from testcontainers' own pull error --- linux/crates/ssh/tests/openssh_session.rs | 6 +- .../test-fixtures/src/cassandra_fixture.rs | 47 +++++++------ .../test-fixtures/src/clickhouse_fixture.rs | 23 ++++--- .../test-fixtures/src/container_start.rs | 68 ++++++------------- .../crates/test-fixtures/src/mongo_fixture.rs | 34 ++++++---- .../crates/test-fixtures/src/mssql_fixture.rs | 24 ++++--- .../crates/test-fixtures/src/mysql_fixture.rs | 26 ++++--- .../test-fixtures/src/openssh_fixture.rs | 41 +++++------ .../test-fixtures/src/pgbouncer_fixture.rs | 18 +++-- .../test-fixtures/src/postgres_fixture.rs | 36 +++++----- .../crates/test-fixtures/src/redis_fixture.rs | 24 ++++--- linux/docs/testing.md | 2 +- 12 files changed, 177 insertions(+), 172 deletions(-) diff --git a/linux/crates/ssh/tests/openssh_session.rs b/linux/crates/ssh/tests/openssh_session.rs index 4dad745aeb..0347ca5a51 100644 --- a/linux/crates/ssh/tests/openssh_session.rs +++ b/linux/crates/ssh/tests/openssh_session.rs @@ -45,12 +45,12 @@ fn write_config(config: &Path, known_hosts: &Path, strict_host_key_checking: &st } async fn start() -> TestResult { - let container = start_container( + let container = start_container(|| { GenericImage::new("alpine", "3.22") .with_exposed_port(2222.tcp()) .with_wait_for(WaitFor::message_on_stderr("Server listening on")) - .with_cmd(["sh", "-c", SSHD_SCRIPT]), - ) + .with_cmd(["sh", "-c", SSHD_SCRIPT]) + }) .await?; let host = container.get_host().await?.to_string(); let port = container.get_host_port_ipv4(2222).await?; diff --git a/linux/crates/test-fixtures/src/cassandra_fixture.rs b/linux/crates/test-fixtures/src/cassandra_fixture.rs index ff8e124611..60373baab0 100644 --- a/linux/crates/test-fixtures/src/cassandra_fixture.rs +++ b/linux/crates/test-fixtures/src/cassandra_fixture.rs @@ -166,29 +166,34 @@ impl CassandraFixture { // ScyllaDB's image exposes "9042" with no protocol, so publishing every // exposed port gives Docker 28 two 9042/tcp bindings on one host port. // Host port 0 lets Docker pick one. - let mut request: ContainerRequest = - image.with_cmd(command).with_mapped_port(0, Self::CONTAINER_PORT.tcp()); - if flavour == CassandraFlavour::Cassandra { - // A joining node waits out the ring delay to learn who owns - // which tokens, so only the first node skips it. - let options = match &seed { - None => Self::fast_start_options(), - Some(_) => "-Dcassandra.superuser_setup_delay_ms=0", - }; - request = request - .with_env_var("MAX_HEAP_SIZE", CASSANDRA_HEAP) - .with_env_var("HEAP_NEWSIZE", CASSANDRA_NEW_HEAP) - .with_env_var("JVM_EXTRA_OPTS", options); - if let Some(seed) = &seed { - request = request.with_env_var("CASSANDRA_SEEDS", seed.as_str()); - } - } let network_alias = network.as_ref().map(|(_, alias)| alias.clone()); - if let Some((network, alias)) = &network { - request = request.with_network(network.name()).with_container_name(alias.as_str()); - } let files = Self::files(flavour, server_version, tls, &pki, &password)?; - let container = start_container(with_files(request, files)).await?; + let container = start_container(|| { + let mut request: ContainerRequest = image + .clone() + .with_cmd(command.clone()) + .with_mapped_port(0, Self::CONTAINER_PORT.tcp()); + if flavour == CassandraFlavour::Cassandra { + // A joining node waits out the ring delay to learn who owns + // which tokens, so only the first node skips it. + let options = match &seed { + None => Self::fast_start_options(), + Some(_) => "-Dcassandra.superuser_setup_delay_ms=0", + }; + request = request + .with_env_var("MAX_HEAP_SIZE", CASSANDRA_HEAP) + .with_env_var("HEAP_NEWSIZE", CASSANDRA_NEW_HEAP) + .with_env_var("JVM_EXTRA_OPTS", options); + if let Some(seed) = &seed { + request = request.with_env_var("CASSANDRA_SEEDS", seed.as_str()); + } + } + if let Some((network, alias)) = &network { + request = request.with_network(network.name()).with_container_name(alias.as_str()); + } + with_files(request, files.clone()) + }) + .await?; let host = container.get_host().await?.to_string(); let port = container.get_host_port_ipv4(Self::CONTAINER_PORT).await?; let fixture = Self { diff --git a/linux/crates/test-fixtures/src/clickhouse_fixture.rs b/linux/crates/test-fixtures/src/clickhouse_fixture.rs index 95c7e5bbb1..9831c75829 100644 --- a/linux/crates/test-fixtures/src/clickhouse_fixture.rs +++ b/linux/crates/test-fixtures/src/clickhouse_fixture.rs @@ -60,19 +60,22 @@ impl ClickHouseFixture { let pki = TestPki::generate()?; let password = FixtureCredentials::generate_password(); let version = ClickHouseVersion::selected()?; - let mut request: ContainerRequest = version - .image() - .generic() - .with_exposed_port(HTTP_PORT.tcp()) - .with_exposed_port(HTTPS_PORT.tcp()) - .into(); let network_alias = network.map(|network| network.container_name(Self::NETWORK_ROLE)); - if let (Some(network), Some(alias)) = (network, &network_alias) { - request = request.with_network(network.name()).with_container_name(alias); - } let mut files = Self::files(&pki, &password)?; files.extend(extra_files); - let container = start_container(with_files(request, files)).await?; + let container = start_container(|| { + let mut request: ContainerRequest = version + .image() + .generic() + .with_exposed_port(HTTP_PORT.tcp()) + .with_exposed_port(HTTPS_PORT.tcp()) + .into(); + if let (Some(network), Some(alias)) = (network, &network_alias) { + request = request.with_network(network.name()).with_container_name(alias); + } + with_files(request, files.clone()) + }) + .await?; let (host, http_port, https_port) = match published(&container).await { Ok(published) => published, Err(error) => { diff --git a/linux/crates/test-fixtures/src/container_start.rs b/linux/crates/test-fixtures/src/container_start.rs index 3fbadaa67c..8519897fed 100644 --- a/linux/crates/test-fixtures/src/container_start.rs +++ b/linux/crates/test-fixtures/src/container_start.rs @@ -1,55 +1,29 @@ use std::time::Duration; -use testcontainers::bollard::Docker; -use testcontainers::bollard::errors::Error as BollardError; +use testcontainers::core::error::ClientError; use testcontainers::runners::AsyncRunner; -use testcontainers::{ContainerAsync, ContainerRequest, GenericImage, Image, ImageExt}; +use testcontainers::{ContainerAsync, ContainerRequest, Image, TestcontainersError}; use crate::FixtureError; -const PULL_ATTEMPTS: u32 = 3; +const START_ATTEMPTS: u32 = 3; const PULL_RETRY_DELAY: Duration = Duration::from_secs(5); -/// Starts the container, pulling its image first when Docker does not have -/// it. testcontainers pulls once, so a pull the registry cut short ("bytes -/// remaining on stream") failed every test that needed the image. -pub async fn start_container(request: ContainerRequest) -> Result, FixtureError> { - let descriptor = request.descriptor(); - if image_missing(&descriptor).await { - pull(&descriptor, request.platform().as_deref()).await?; - } - Ok(request.start().await?) -} - -async fn image_missing(descriptor: &str) -> bool { - // Any other answer leaves the pull to testcontainers, as before. - let Ok(docker) = Docker::connect_with_defaults() else { - return false; - }; - matches!( - docker.inspect_image(descriptor).await, - Err(BollardError::DockerResponseServerError { status_code: 404, .. }) - ) -} - -async fn pull(descriptor: &str, platform: Option<&str>) -> Result<(), FixtureError> { - let (name, tag) = descriptor.rsplit_once(':').unwrap_or((descriptor, "latest")); +/// Starts the container `request` builds, building and starting it again +/// when pulling its image failed. testcontainers pulls once, so a pull the +/// registry cut short ("bytes remaining on stream") failed every test that +/// needed the image. A request is consumed by its start, hence the builder. +pub async fn start_container( + request: impl Fn() -> ContainerRequest, +) -> Result, FixtureError> { let mut attempt = 1; loop { - let image = GenericImage::new(name, tag); - // testcontainers falls back to linux/amd64 itself when no platform is - // named, which the SQL Server images need on arm64. - let pulled = match platform { - Some(platform) => image.with_platform(platform).pull_image().await, - None => image.pull_image().await, - }; - match pulled { - Ok(_) => return Ok(()), - Err(_) if attempt < PULL_ATTEMPTS => { + match request().start().await { + Err(TestcontainersError::Client(ClientError::PullImage { .. })) if attempt < START_ATTEMPTS => { attempt += 1; tokio::time::sleep(PULL_RETRY_DELAY).await; } - Err(error) => return Err(error.into()), + started => return Ok(started?), } } } @@ -58,8 +32,7 @@ async fn pull(descriptor: &str, platform: Option<&str>) -> Result<(), FixtureErr mod tests { use std::time::Instant; - use testcontainers::TestcontainersError; - use testcontainers::core::error::ClientError; + use testcontainers::GenericImage; use super::*; @@ -68,13 +41,10 @@ mod tests { async fn a_pull_that_keeps_failing_is_tried_three_times_and_reported() { let started = Instant::now(); - let error = start_container(ContainerRequest::from(GenericImage::new( - "alpine", - "tablepro-no-such-tag", - ))) - .await - .err() - .unwrap(); + let error = start_container(|| GenericImage::new("alpine", "tablepro-no-such-tag").into()) + .await + .err() + .unwrap(); let FixtureError::Container(TestcontainersError::Client(ClientError::PullImage { descriptor, .. })) = &error else { @@ -82,7 +52,7 @@ mod tests { }; assert_eq!(descriptor, "alpine:tablepro-no-such-tag"); assert!( - started.elapsed() >= PULL_RETRY_DELAY * (PULL_ATTEMPTS - 1), + started.elapsed() >= PULL_RETRY_DELAY * (START_ATTEMPTS - 1), "took {:?}", started.elapsed() ); diff --git a/linux/crates/test-fixtures/src/mongo_fixture.rs b/linux/crates/test-fixtures/src/mongo_fixture.rs index c5f0910664..61d35febdc 100644 --- a/linux/crates/test-fixtures/src/mongo_fixture.rs +++ b/linux/crates/test-fixtures/src/mongo_fixture.rs @@ -100,22 +100,26 @@ impl MongoFixture { let pki = TestPki::generate()?; let password = FixtureCredentials::generate_password(); let version = MongoVersion::selected()?; - let mut request: ContainerRequest = version - .image() - .generic() - .with_wait_for(WaitFor::message_on_stdout(READY)) - .with_exposed_port(PORT.tcp()) - .with_exposed_port(CERTIFICATE_PORT.tcp()) - .with_entrypoint("sh") - .with_cmd([format!("{CONFIG_DIR}/start.sh")]) - .with_env_var("TABLEPRO_PASSWORD", &password) - .with_env_var("TABLEPRO_TLS", if listening == Listening::Tls { "1" } else { "0" }) - .with_startup_timeout(READY_TIMEOUT); let network_alias = network.map(|network| network.container_name(Self::NETWORK_ROLE)); - if let (Some(network), Some(alias)) = (network, &network_alias) { - request = request.with_network(network.name()).with_container_name(alias); - } - let container = start_container(with_files(request, Self::files(&pki)?)).await?; + let files = Self::files(&pki)?; + let container = start_container(|| { + let mut request: ContainerRequest = version + .image() + .generic() + .with_wait_for(WaitFor::message_on_stdout(READY)) + .with_exposed_port(PORT.tcp()) + .with_exposed_port(CERTIFICATE_PORT.tcp()) + .with_entrypoint("sh") + .with_cmd([format!("{CONFIG_DIR}/start.sh")]) + .with_env_var("TABLEPRO_PASSWORD", &password) + .with_env_var("TABLEPRO_TLS", if listening == Listening::Tls { "1" } else { "0" }) + .with_startup_timeout(READY_TIMEOUT); + if let (Some(network), Some(alias)) = (network, &network_alias) { + request = request.with_network(network.name()).with_container_name(alias); + } + with_files(request, files.clone()) + }) + .await?; let host = container.get_host().await?.to_string(); let port = container.get_host_port_ipv4(PORT).await?; let certificate_port = match listening { diff --git a/linux/crates/test-fixtures/src/mssql_fixture.rs b/linux/crates/test-fixtures/src/mssql_fixture.rs index 7d0cfbcc2e..be2bfd24d0 100644 --- a/linux/crates/test-fixtures/src/mssql_fixture.rs +++ b/linux/crates/test-fixtures/src/mssql_fixture.rs @@ -32,16 +32,20 @@ impl MssqlFixture { let password = format!("Tp1!{}", Uuid::new_v4().simple()); let version = MssqlVersion::selected()?; let image = version.image(); - let request = MssqlServer::default() - .with_accept_eula() - .with_sa_password(&password) - .with_name(image.name) - .with_tag(image.tag) - .with_startup_timeout(STARTUP_TIMEOUT) - // SQL Server otherwise takes 80% of the host's memory, and a - // container left behind by a killed test keeps all of it. - .with_env_var("MSSQL_MEMORY_LIMIT_MB", MEMORY_LIMIT_MB); - let container = start_container(with_files(request, Self::files(&pki)?)).await?; + let files = Self::files(&pki)?; + let container = start_container(|| { + let request = MssqlServer::default() + .with_accept_eula() + .with_sa_password(&password) + .with_name(image.name) + .with_tag(image.tag) + .with_startup_timeout(STARTUP_TIMEOUT) + // SQL Server otherwise takes 80% of the host's memory, and a + // container left behind by a killed test keeps all of it. + .with_env_var("MSSQL_MEMORY_LIMIT_MB", MEMORY_LIMIT_MB); + with_files(request, files.clone()) + }) + .await?; let host = container.get_host().await?.to_string(); let port = container.get_host_port_ipv4(CONTAINER_PORT).await?; Ok(Self { diff --git a/linux/crates/test-fixtures/src/mysql_fixture.rs b/linux/crates/test-fixtures/src/mysql_fixture.rs index f67d7d6d13..66bd61e3e7 100644 --- a/linux/crates/test-fixtures/src/mysql_fixture.rs +++ b/linux/crates/test-fixtures/src/mysql_fixture.rs @@ -51,18 +51,22 @@ impl MysqlFixture { .into_iter() .fold(release.generic(), GenericImage::with_wait_for) .with_exposed_port(Self::CONTAINER_PORT.tcp()); - let mut request = image - .with_env_var(format!("{prefix}_ROOT_PASSWORD"), &root_password) - .with_env_var(format!("{prefix}_DATABASE"), DATABASE) - .with_env_var(format!("{prefix}_USER"), PASSWORD_USER) - .with_env_var(format!("{prefix}_PASSWORD"), &password) - .with_cmd(Self::server_arguments(&options)); let network_alias = network.map(|network| network.container_name(Self::NETWORK_ROLE)); - if let (Some(network), Some(alias)) = (network, &network_alias) { - request = request.with_network(network.name()).with_container_name(alias); - } - let container = - start_container(with_files(request, Self::files(&pki, &options, &certificate_password)?)).await?; + let files = Self::files(&pki, &options, &certificate_password)?; + let container = start_container(|| { + let mut request = image + .clone() + .with_env_var(format!("{prefix}_ROOT_PASSWORD"), &root_password) + .with_env_var(format!("{prefix}_DATABASE"), DATABASE) + .with_env_var(format!("{prefix}_USER"), PASSWORD_USER) + .with_env_var(format!("{prefix}_PASSWORD"), &password) + .with_cmd(Self::server_arguments(&options)); + if let (Some(network), Some(alias)) = (network, &network_alias) { + request = request.with_network(network.name()).with_container_name(alias); + } + with_files(request, files.clone()) + }) + .await?; let host = container.get_host().await?.to_string(); let port = container.get_host_port_ipv4(Self::CONTAINER_PORT).await?; Ok(Self { diff --git a/linux/crates/test-fixtures/src/openssh_fixture.rs b/linux/crates/test-fixtures/src/openssh_fixture.rs index 03194d5f0d..38872336b0 100644 --- a/linux/crates/test-fixtures/src/openssh_fixture.rs +++ b/linux/crates/test-fixtures/src/openssh_fixture.rs @@ -60,21 +60,6 @@ impl OpenSshFixture { | SshAuthVariant::UserCertificateAuthority(_) | SshAuthVariant::KeyboardInteractive => None, }; - let mut request = options - .version - .image() - .generic() - .with_exposed_port(Self::CONTAINER_PORT.tcp()) - .with_wait_for(WaitFor::message_on_stdout("[ls.io-init] done.")) - .with_env_var("USER_NAME", USERNAME) - .with_env_var("USER_PASSWORD", &password) - .with_env_var( - "PASSWORD_ACCESS", - if variant.password_access() { "true" } else { "false" }, - ); - if let SshAuthVariant::PublicKey(key) = &variant { - request = request.with_env_var("PUBLIC_KEY", key.public_key()); - } let mut files = vec![ContainerFile::readable( Self::DROP_IN, Self::drop_in(&variant, &options), @@ -92,10 +77,28 @@ impl OpenSshFixture { let suffix = Uuid::new_v4().simple().to_string(); network.container_name(&format!("sshd-{}", &suffix[..8])) }); - if let (Some(network), Some(alias)) = (network, &network_alias) { - request = request.with_network(network.name()).with_container_name(alias); - } - let container = start_container(with_files(request, files)).await?; + let container = start_container(|| { + let mut request = options + .version + .image() + .generic() + .with_exposed_port(Self::CONTAINER_PORT.tcp()) + .with_wait_for(WaitFor::message_on_stdout("[ls.io-init] done.")) + .with_env_var("USER_NAME", USERNAME) + .with_env_var("USER_PASSWORD", &password) + .with_env_var( + "PASSWORD_ACCESS", + if variant.password_access() { "true" } else { "false" }, + ); + if let SshAuthVariant::PublicKey(key) = &variant { + request = request.with_env_var("PUBLIC_KEY", key.public_key()); + } + if let (Some(network), Some(alias)) = (network, &network_alias) { + request = request.with_network(network.name()).with_container_name(alias); + } + with_files(request, files.clone()) + }) + .await?; let host = container.get_host().await?.to_string(); let port = container.get_host_port_ipv4(Self::CONTAINER_PORT).await?; wait_for_banner(&host, port).await?; diff --git a/linux/crates/test-fixtures/src/pgbouncer_fixture.rs b/linux/crates/test-fixtures/src/pgbouncer_fixture.rs index 7c84959725..9218464ec0 100644 --- a/linux/crates/test-fixtures/src/pgbouncer_fixture.rs +++ b/linux/crates/test-fixtures/src/pgbouncer_fixture.rs @@ -44,13 +44,17 @@ impl PgBouncerFixture { upstream.database(), &upstream.password_credentials(), ); - let request = FixtureImage::PGBOUNCER - .generic() - .with_exposed_port(CONTAINER_PORT.tcp()) - .with_wait_for(WaitFor::message_on_either_std("process up")) - .with_network(network.name()) - .with_container_name(network.container_name(NETWORK_ROLE)); - let container = start_container(with_files(request, files)).await?; + let container_name = network.container_name(NETWORK_ROLE); + let container = start_container(|| { + let request = FixtureImage::PGBOUNCER + .generic() + .with_exposed_port(CONTAINER_PORT.tcp()) + .with_wait_for(WaitFor::message_on_either_std("process up")) + .with_network(network.name()) + .with_container_name(&container_name); + with_files(request, files.clone()) + }) + .await?; let host = container.get_host().await?.to_string(); let port = container.get_host_port_ipv4(CONTAINER_PORT).await?; Ok(Self { diff --git a/linux/crates/test-fixtures/src/postgres_fixture.rs b/linux/crates/test-fixtures/src/postgres_fixture.rs index 0140f7b5cc..74e930ff0a 100644 --- a/linux/crates/test-fixtures/src/postgres_fixture.rs +++ b/linux/crates/test-fixtures/src/postgres_fixture.rs @@ -90,23 +90,27 @@ impl PostgresFixture { let pki = TestPki::generate_with(algorithm)?; let password = FixtureCredentials::generate_password(); let image = version.image(); - let mut request = Postgres::default() - .with_db_name(DATABASE) - .with_user(PASSWORD_USER) - .with_password(&password) - .with_name(image.name) - .with_tag(image.tag); - if let Some(command) = Self::server_command(hba) { - request = request.with_cmd(command); - } let network_alias = network.map(|network| network.container_name(Self::NETWORK_ROLE)); - if let (Some(network), Some(alias)) = (network, &network_alias) { - request = request.with_network(network.name()).with_container_name(alias); - } - if let Some(host_port) = host_port { - request = request.with_mapped_port(host_port, CONTAINER_PORT.tcp()); - } - let container = start_container(with_files(request, Self::files(hba, &pki)?)).await?; + let files = Self::files(hba, &pki)?; + let container = start_container(|| { + let mut request = Postgres::default() + .with_db_name(DATABASE) + .with_user(PASSWORD_USER) + .with_password(&password) + .with_name(image.name) + .with_tag(image.tag); + if let Some(command) = Self::server_command(hba) { + request = request.with_cmd(command); + } + if let (Some(network), Some(alias)) = (network, &network_alias) { + request = request.with_network(network.name()).with_container_name(alias); + } + if let Some(host_port) = host_port { + request = request.with_mapped_port(host_port, CONTAINER_PORT.tcp()); + } + with_files(request, files.clone()) + }) + .await?; let host = container.get_host().await?.to_string(); let port = container.get_host_port_ipv4(CONTAINER_PORT).await?; Ok(Self { diff --git a/linux/crates/test-fixtures/src/redis_fixture.rs b/linux/crates/test-fixtures/src/redis_fixture.rs index d25c94dbd6..abeb30780b 100644 --- a/linux/crates/test-fixtures/src/redis_fixture.rs +++ b/linux/crates/test-fixtures/src/redis_fixture.rs @@ -64,17 +64,21 @@ impl RedisFixture { let pki = TestPki::generate()?; let password = FixtureCredentials::generate_password(); let version = RedisVersion::selected()?; - let mut request: ContainerRequest = version - .image() - .generic() - .with_exposed_port(PORT.tcp()) - .with_exposed_port(CERTIFICATE_PORT.tcp()) - .with_cmd(["sh".to_owned(), "-c".to_owned(), start_command(listening)]); let network_alias = network.map(|network| network.container_name(Self::NETWORK_ROLE)); - if let (Some(network), Some(alias)) = (network, &network_alias) { - request = request.with_network(network.name()).with_container_name(alias); - } - let container = start_container(with_files(request, Self::files(&pki, &password)?)).await?; + let files = Self::files(&pki, &password)?; + let container = start_container(|| { + let mut request: ContainerRequest = version + .image() + .generic() + .with_exposed_port(PORT.tcp()) + .with_exposed_port(CERTIFICATE_PORT.tcp()) + .with_cmd(["sh".to_owned(), "-c".to_owned(), start_command(listening)]); + if let (Some(network), Some(alias)) = (network, &network_alias) { + request = request.with_network(network.name()).with_container_name(alias); + } + with_files(request, files.clone()) + }) + .await?; let host = container.get_host().await?.to_string(); let port = container.get_host_port_ipv4(PORT).await?; let certificate_port = match listening { diff --git a/linux/docs/testing.md b/linux/docs/testing.md index acf39f7cde..99418edfd8 100644 --- a/linux/docs/testing.md +++ b/linux/docs/testing.md @@ -351,7 +351,7 @@ cargo nextest run -p tablepro-driver-postgres --run-ignored only Do not bother with `TESTCONTAINERS_RYUK_DISABLED`. That is a testcontainers-java / go setting; the Rust crate has no Ryuk reaper and stops each container when its handle drops. -Every fixture starts its container through `tablepro_test_fixtures::start_container`, which pulls a missing image itself and tries a failed pull twice more, 5 seconds apart. testcontainers pulls once, and Docker Hub sometimes ends a pull early ("bytes remaining on stream"), which failed every test that needed the image. A test that starts a container of its own goes through it too. +Every fixture starts its container through `tablepro_test_fixtures::start_container`, which takes a closure that builds the request and, when pulling the image failed, builds and starts it again, up to three times 5 seconds apart. testcontainers pulls once, and Docker Hub sometimes ends a pull early ("bytes remaining on stream"), which failed every test that needed the image. A test that starts a container of its own goes through it too. `curl --unix-socket "${DOCKER_HOST#unix://}" http://localhost/_ping` should print `OK` before you run the suite. `--unix-socket` takes a filesystem path, so the `unix://` prefix has to come off.