From 76070a8abe30190671db38e7215e29276a715fa9 Mon Sep 17 00:00:00 2001 From: John Coffey Date: Tue, 3 Nov 2026 20:15:24 -0700 Subject: [PATCH] Give up on a stalled read or write after 5 minutes A transfer that stalled partway waited out the budget for the whole body: 30 minutes for a download, over two hours for the largest uploads. ureq 3.5 can time each read and write separately, so a silent socket now fails after 5 minutes and is retried like any other timeout. --- CHANGELOG.md | 3 ++ docs/usage.md | 3 +- src/dav/client.rs | 2 + src/exchange_ews/client.rs | 2 + src/exchange_graph/client.rs | 2 + src/jmap/http.rs | 2 + src/net.rs | 88 +++++++++++++++++++++++++++++++++++- 7 files changed, 99 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1ffffee..f81f7d0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,9 @@ candidates before a series ships (`27.1-rc.1`). The tag is the version with a - Google Takeout: a contact without a `UID` is stored once rather than once per contact group file. Archives from an earlier version keep the old copies, so re-import into a fresh archive. +- A download or upload that stalls partway now fails after 5 minutes of silence + and is retried, rather than waiting out the limit on the whole body: 30 + minutes for a download, over two hours for the largest uploads. ## [27.1-rc.10] diff --git a/docs/usage.md b/docs/usage.md index e4349fa..677a1fb 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -55,7 +55,8 @@ a server to trust. Connections time out rather than wait forever: 30 seconds to connect, 5 minutes for the server's first byte, and 30 minutes to read a whole -response. A timed-out request is retried like any other transient failure. +response. A transfer that goes silent for 5 minutes in either direction also +times out. A timed-out request is retried like any other transient failure. Secrets come from the `INBUXA_MIGRATE_*` environment variables or a prompt; see [Credentials](../README.md#credentials). The command line takes them too, diff --git a/src/dav/client.rs b/src/dav/client.rs index 83efd75..dceb419 100644 --- a/src/dav/client.rs +++ b/src/dav/client.rs @@ -1023,6 +1023,8 @@ mod tests { ureq::Timeout::SendBody, ureq::Timeout::RecvResponse, ureq::Timeout::RecvBody, + ureq::Timeout::PerRead, + ureq::Timeout::PerWrite, ] { let err = map_ureq_error(ureq::Error::Timeout(t)); assert!(matches!(err, JmapError::Transport(_)), "{t:?} -> {err:?}"); diff --git a/src/exchange_ews/client.rs b/src/exchange_ews/client.rs index 90e2289..0d64d44 100644 --- a/src/exchange_ews/client.rs +++ b/src/exchange_ews/client.rs @@ -559,6 +559,8 @@ mod tests { ureq::Timeout::SendBody, ureq::Timeout::RecvResponse, ureq::Timeout::RecvBody, + ureq::Timeout::PerRead, + ureq::Timeout::PerWrite, ] { let err = map_ureq_error(ureq::Error::Timeout(t)); assert!(matches!(err, EwsError::Transport(_)), "{t:?} -> {err:?}"); diff --git a/src/exchange_graph/client.rs b/src/exchange_graph/client.rs index d540a2e..14c13de 100644 --- a/src/exchange_graph/client.rs +++ b/src/exchange_graph/client.rs @@ -498,6 +498,8 @@ mod tests { ureq::Timeout::SendBody, ureq::Timeout::RecvResponse, ureq::Timeout::RecvBody, + ureq::Timeout::PerRead, + ureq::Timeout::PerWrite, ] { let err = map_ureq_error(ureq::Error::Timeout(t)); assert!(matches!(err, GraphError::Transport(_)), "{t:?} -> {err:?}"); diff --git a/src/jmap/http.rs b/src/jmap/http.rs index c70b234..314f013 100644 --- a/src/jmap/http.rs +++ b/src/jmap/http.rs @@ -824,6 +824,8 @@ mod tests { ureq::Timeout::SendBody, ureq::Timeout::RecvResponse, ureq::Timeout::RecvBody, + ureq::Timeout::PerRead, + ureq::Timeout::PerWrite, ] { let err = map_ureq_error(ureq::Error::Timeout(t)); assert!(matches!(err, JmapError::Transport(_)), "{t:?} -> {err:?}"); diff --git a/src/net.rs b/src/net.rs index 951c735..3d19baa 100644 --- a/src/net.rs +++ b/src/net.rs @@ -25,10 +25,21 @@ pub const RECV_RESPONSE: Duration = Duration::from_secs(5 * 60); /// Reading the whole response body. ureq counts this as one budget for the /// entire body, not per read, so it has to cover the largest body a client -/// accepts (512 MiB) on a slow link: 30 minutes is about 300 KB/s. A stalled -/// transfer is abandoned and retried after at most this long. +/// accepts (512 MiB) on a slow link: 30 minutes is about 300 KB/s. pub const RECV_BODY: Duration = Duration::from_secs(30 * 60); +/// Silence on the socket during any one read, which restarts with every read. +/// A transfer that stalls partway is abandoned and retried after this long, +/// rather than waiting out [`RECV_BODY`]. It matches [`RECV_RESPONSE`] so it +/// never cuts the server's thinking time short. +pub const PER_READ: Duration = Duration::from_secs(5 * 60); + +/// Silence on the socket during any one write, which restarts with every write. +/// An upload that stalls partway is abandoned and retried after this long, +/// rather than waiting out its send budget, which runs past two hours for the +/// largest bodies. +pub const PER_WRITE: Duration = Duration::from_secs(5 * 60); + /// Sending a request body when its size is not known in advance. Uploads know /// their size and get [`send_body_budget`] instead. pub const SEND_BODY: Duration = Duration::from_secs(30 * 60); @@ -57,6 +68,8 @@ macro_rules! with_timeouts { .timeout_send_body(Some($crate::net::SEND_BODY)) .timeout_recv_response(Some($crate::net::RECV_RESPONSE)) .timeout_recv_body(Some($crate::net::RECV_BODY)) + .timeout_per_read(Some($crate::net::PER_READ)) + .timeout_per_write(Some($crate::net::PER_WRITE)) }; } pub(crate) use with_timeouts; @@ -280,5 +293,76 @@ mod tests { assert_eq!(t.send_body, Some(SEND_BODY)); assert_eq!(t.recv_response, Some(RECV_RESPONSE)); assert_eq!(t.recv_body, Some(RECV_BODY)); + assert_eq!(config.timeout_per_read(), Some(PER_READ)); + assert_eq!(config.timeout_per_write(), Some(PER_WRITE)); + } + + #[test] + fn a_stalled_body_times_out_without_waiting_for_the_body_budget() { + use std::io::{Read, Write}; + + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let url = format!("http://{}/", listener.local_addr().unwrap()); + let (done, wait) = std::sync::mpsc::channel::<()>(); + let server = std::thread::spawn(move || { + let (mut stream, _) = listener.accept().unwrap(); + let mut request = [0u8; 1024]; + let _ = stream.read(&mut request); + stream + .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 10\r\n\r\nhalf") + .unwrap(); + let _ = wait.recv(); + }); + + let agent = with_timeouts!(ureq::config::Config::builder()) + .timeout_per_read(Some(Duration::from_millis(200))) + .build() + .new_agent(); + let started = std::time::Instant::now(); + let err = agent + .get(&url) + .call() + .unwrap() + .into_body() + .read_to_vec() + .unwrap_err(); + assert!( + matches!(err, ureq::Error::Timeout(ureq::Timeout::PerRead)), + "got {err:?}" + ); + assert!(started.elapsed() < Duration::from_secs(10)); + + drop(done); + server.join().unwrap(); + } + + #[test] + fn a_stalled_upload_times_out_without_waiting_for_the_send_budget() { + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let url = format!("http://{}/", listener.local_addr().unwrap()); + let (done, wait) = std::sync::mpsc::channel::<()>(); + // Accepts and never reads, so the socket buffers fill and writes stop. + let server = std::thread::spawn(move || { + let (_stream, _) = listener.accept().unwrap(); + let _ = wait.recv(); + }); + + let agent = with_timeouts!(ureq::config::Config::builder()) + .timeout_per_write(Some(Duration::from_millis(200))) + .build() + .new_agent(); + let started = std::time::Instant::now(); + let err = agent + .post(&url) + .send(&vec![0u8; 64 * 1024 * 1024][..]) + .unwrap_err(); + assert!( + matches!(err, ureq::Error::Timeout(ureq::Timeout::PerWrite)), + "got {err:?}" + ); + assert!(started.elapsed() < Duration::from_secs(30)); + + drop(done); + server.join().unwrap(); } } -- 2.54.0