Testcontainer fixes

tests/integration/baikal.rs

@@ -76,7 +76,8 @@ impl Baikal {
         let request = image
             .with_exposed_port(HTTP_PORT.tcp())
             .with_wait_for(WaitFor::message_on_stderr("start worker process"))
-            .with_startup_timeout(Duration::from_secs(180));
+            .with_startup_timeout(Duration::from_secs(180))
+            .with_labels([(super::OWNER_LABEL, "1")]);
 
         let container = request.start()?;
         let host = container.get_host()?.to_string();

tests/integration/cyrus.rs

@@ -154,7 +154,8 @@ impl Cyrus {
         let request = image
             .with_exposed_port(IMAP_PORT.tcp())
             .with_wait_for(WaitFor::message_on_stdout("cyrus provisioned"))
-            .with_startup_timeout(Duration::from_secs(180));
+            .with_startup_timeout(Duration::from_secs(180))
+            .with_labels([(super::OWNER_LABEL, "1")]);
 
         let container = request.start()?;
         let host = container.get_host()?.to_string();

tests/integration/dav_client.rs

@@ -4,6 +4,7 @@
  * SPDX-License-Identifier: Apache-2.0 OR MIT
  */
 
+use std::io;
 use std::time::Duration;
 
 use base64::Engine;
@@ -56,24 +57,35 @@ impl DavSeed {
     ) -> ContainerResult<(u16, Vec<u8>)> {
         let parsed = Method::from_bytes(method.as_bytes())
             .map_err(|e| ContainerError::Protocol(format!("bad method {method}: {e}")))?;
-        let mut builder = Request::builder()
-            .method(parsed)
-            .uri(url)
-            .header("Authorization", &self.auth);
-        if let Some(d) = depth {
-            builder = builder.header("Depth", d.to_string());
-        }
-        if let Some(ct) = content_type {
-            builder = builder.header("Content-Type", ct);
-        }
         let payload: Vec<u8> = body.map(|b| b.to_vec()).unwrap_or_default();
-        let request = builder
+        let mut attempt = 0u32;
-            .body(payload)
+        loop {
-            .map_err(|e| ContainerError::Protocol(format!("build request: {e}")))?;
+            let mut builder = Request::builder()
-        let mut response = self.agent.run(request)?;
+                .method(parsed.clone())
-        let status = response.status().as_u16();
+                .uri(url)
-        let bytes = response.body_mut().read_to_vec()?;
+                .header("Authorization", &self.auth);
-        Ok((status, bytes))
+            if let Some(d) = depth {
+                builder = builder.header("Depth", d.to_string());
+            }
+            if let Some(ct) = content_type {
+                builder = builder.header("Content-Type", ct);
+            }
+            let request = builder
+                .body(payload.clone())
+                .map_err(|e| ContainerError::Protocol(format!("build request: {e}")))?;
+            match self.agent.run(request) {
+                Ok(mut response) => {
+                    let status = response.status().as_u16();
+                    let bytes = response.body_mut().read_to_vec()?;
+                    return Ok((status, bytes));
+                }
+                Err(e) if attempt < 20 && is_transient(&e) => {
+                    attempt += 1;
+                    std::thread::sleep(Duration::from_millis(250));
+                }
+                Err(e) => return Err(e.into()),
+            }
+        }
     }
 
     pub fn mkcol(&self, path: &str, body: Option<&str>) -> ContainerResult<u16> {
@@ -154,6 +166,21 @@ impl DavSeed {
     }
 }
 
+fn is_transient(e: &ureq::Error) -> bool {
+    match e {
+        ureq::Error::ConnectionFailed => true,
+        ureq::Error::Io(io) => matches!(
+            io.kind(),
+            io::ErrorKind::UnexpectedEof
+                | io::ErrorKind::ConnectionReset
+                | io::ErrorKind::ConnectionAborted
+                | io::ErrorKind::ConnectionRefused
+                | io::ErrorKind::BrokenPipe
+        ),
+        _ => false,
+    }
+}
+
 fn is_collection_created(status: u16) -> bool {
     matches!(status, 200 | 201 | 204)
 }

tests/integration/dovecot.rs

@@ -132,7 +132,8 @@ impl Dovecot {
             .with_exposed_port(IMAP_PORT.tcp())
             .with_exposed_port(SIEVE_PORT.tcp())
             .with_wait_for(WaitFor::message_on_stderr("starting up"))
-            .with_startup_timeout(Duration::from_secs(120));
+            .with_startup_timeout(Duration::from_secs(120))
+            .with_labels([(super::OWNER_LABEL, "1")]);
 
         let container = request.start()?;
         let host = container.get_host()?.to_string();

tests/integration/mod.rs

@@ -22,6 +22,8 @@ pub mod webdav;
 
 pub use error::{ContainerError, ContainerResult};
 
+pub const OWNER_LABEL: &str = "art.stalw.vandelay.itest";
+
 use std::sync::Once;
 
 static CRYPTO_INIT: Once = Once::new();

tests/integration/radicale.rs

@@ -5,7 +5,7 @@
  */
 
 use std::collections::HashSet;
-use std::time::Duration;
+use std::time::{Duration, Instant};
 
 use testcontainers::core::{IntoContainerPort, WaitFor};
 use testcontainers::runners::SyncRunner;
@@ -56,12 +56,15 @@ impl Radicale {
         let request = image
             .with_copy_to("/config/config", CONFIG.as_bytes().to_vec())
             .with_copy_to("/config/users", users.into_bytes())
-            .with_startup_timeout(Duration::from_secs(90));
+            .with_startup_timeout(Duration::from_secs(90))
+            .with_labels([(super::OWNER_LABEL, "1")]);
 
         let container = request.start()?;
         let host = container.get_host()?.to_string();
         let port = container.get_host_port_ipv4(RADICALE_PORT.tcp())?;
 
+        wait_ready(&format!("http://{host}:{port}/"), Duration::from_secs(30))?;
+
         let accounts: Vec<Account> = layouts::accounts()
             .iter()
             .map(|name| Account {
@@ -338,6 +341,26 @@ impl AccountSeed {
     }
 }
 
+fn wait_ready(base: &str, total: Duration) -> ContainerResult<()> {
+    let agent = ureq::Agent::config_builder()
+        .http_status_as_error(false)
+        .timeout_global(Some(Duration::from_secs(5)))
+        .build()
+        .new_agent();
+    let deadline = Instant::now() + total;
+    let mut last_err = String::from("no probe attempted");
+    while Instant::now() < deadline {
+        match agent.get(base).call() {
+            Ok(_) => return Ok(()),
+            Err(e) => last_err = e.to_string(),
+        }
+        std::thread::sleep(Duration::from_millis(250));
+    }
+    Err(ContainerError::Protocol(format!(
+        "radicale did not become ready in {total:?}: {last_err}"
+    )))
+}
+
 fn collection_segment(prefix: &str, idx: usize, name: &str) -> String {
     let mut out = format!("{prefix}-{:02}-", idx);
     for c in name.chars() {

tests/integration/stalwart.rs

@@ -5,6 +5,7 @@
  */
 
 use std::net::TcpListener;
+use std::process::Command;
 use std::sync::{Arc, OnceLock};
 use std::time::{Duration, Instant};
 
@@ -17,6 +18,7 @@ use ureq::Agent;
 use ureq::config::RedirectAuthHeaders;
 use ureq::tls::{TlsConfig, TlsProvider};
 
+use super::OWNER_LABEL;
 use super::error::{ContainerError, ContainerResult};
 
 const IMAGE_NAME: &str = "stalwartlabs/stalwart";
@@ -45,6 +47,7 @@ pub struct Stalwart {
 
 impl Stalwart {
     pub fn start() -> ContainerResult<Self> {
+        sweep_abandoned_containers();
         let https_port = pick_free_port()?;
         let imap_port = pick_free_port()?;
         let imaps_port = pick_free_port()?;
@@ -63,6 +66,7 @@ impl Stalwart {
         let request = image
             .with_env_var("STALWART_PUBLIC_URL", &public_url)
             .with_env_var("STALWART_RECOVERY_ADMIN", "admin:admin")
+            .with_labels([(OWNER_LABEL, "1")])
             .with_copy_to(CONFIG_PATH, CONFIG_JSON.as_bytes().to_vec())
             .with_mapped_port(https_port, HTTPS_PORT.tcp())
             .with_mapped_port(imap_port, IMAP_PORT.tcp())
@@ -132,6 +136,24 @@ pub fn shared() -> &'static Stalwart {
     SHARED.get_or_init(|| Stalwart::start().expect("start shared stalwart container"))
 }
 
+fn sweep_abandoned_containers() {
+    let Ok(listed) = Command::new("docker")
+        .args(["ps", "-aq", "--filter", &format!("label={OWNER_LABEL}")])
+        .output()
+    else {
+        return;
+    };
+    let ids: Vec<String> = String::from_utf8_lossy(&listed.stdout)
+        .split_whitespace()
+        .map(String::from)
+        .collect();
+    if ids.is_empty() {
+        return;
+    }
+    eprintln!("removing {} abandoned test container(s)", ids.len());
+    let _ = Command::new("docker").args(["rm", "-f"]).args(&ids).output();
+}
+
 fn pick_free_port() -> ContainerResult<u16> {
     let listener = TcpListener::bind("127.0.0.1:0")?;
     Ok(listener.local_addr()?.port())

tests/integration/webdav.rs

@@ -148,7 +148,8 @@ impl WebDav {
         let request = image
             .with_exposed_port(HTTP_PORT.tcp())
             .with_wait_for(WaitFor::message_on_stderr("AH00558"))
-            .with_startup_timeout(Duration::from_secs(120));
+            .with_startup_timeout(Duration::from_secs(120))
+            .with_labels([(super::OWNER_LABEL, "1")]);
 
         let container = request.start()?;
         let host = container.get_host()?.to_string();