From 7b97efbb7fd2e401cd140b21d059fd4b3a78af96 Mon Sep 17 00:00:00 2001 From: John Coffey Date: Sun, 27 Sep 2026 18:19:00 -0700 Subject: [PATCH] Legal holds, step 3: deleted items in a held account are kept Every way of deleting mail (JMAP, IMAP EXPUNGE, POP3, mailbox removal, Trash emptying) and Sieve scripts, events, contacts and files now asks how the account's deletions are kept: a hold keeps them with no expiry (archivedUntil 9999-12-31), even with undelete off; otherwise undelete's period applies as before (LH-4). A hold's date range decides by the item's own date (LH-3). Mail is noted as held at deletion and settled when it's archived, once its received date is known; outside the range it gets undelete's deadline or isn't kept. Events go by their start, with a day's slack for time zones; recurring events, contacts, files and scripts are held whole. A groupware item's note now stays until its archive succeeds, and a failure retries the task instead of being logged and lost (LH-5). --- crates/common/src/hold.rs | 11 +- crates/email/src/mailbox/destroy.rs | 10 +- crates/email/src/message/delete.rs | 10 +- crates/email/src/sieve/delete.rs | 12 +-- crates/features/src/hold/mod.rs | 100 ++++++++++++++++++ crates/features/src/undelete/data.rs | 8 ++ crates/features/src/undelete/email.rs | 44 ++++++-- crates/features/src/undelete/groupware.rs | 33 ++++++ crates/imap/src/op/expunge.rs | 10 +- crates/services/src/task_manager/index.rs | 35 +++++-- tests/src/system/legal_hold.rs | 119 +++++++++++++++++++++- 11 files changed, 350 insertions(+), 42 deletions(-) diff --git a/crates/common/src/hold.rs b/crates/common/src/hold.rs index 58be497..f92815e 100644 --- a/crates/common/src/hold.rs +++ b/crates/common/src/hold.rs @@ -10,7 +10,7 @@ //! there are few holds. use crate::Server; -use inbuxa_features::hold::{self, Hold, Member}; +use inbuxa_features::hold::{self, Hold, Keeping, Member}; impl Server { /// The active holds covering `account_id`, through its own name, its @@ -36,6 +36,15 @@ impl Server { hold::covering(self.store(), &member).await } + /// How `account_id`'s deleted items are kept: its holds' ranges and the + /// undelete period in force now (LH-4, UD-6a). + pub async fn keeping(&self, account_id: u32) -> trc::Result { + let retention = inbuxa_features::undelete::settings::retention(self.registry()) + .await? + .items; + Ok(Keeping::new(retention, &self.holds_on(account_id).await?)) + } + /// Whether any active hold covers `account_id` at all. pub async fn is_held(&self, account_id: u32) -> trc::Result { Ok(!self.holds_on(account_id).await?.is_empty()) diff --git a/crates/email/src/mailbox/destroy.rs b/crates/email/src/mailbox/destroy.rs index 91e89ef..1e095e8 100644 --- a/crates/email/src/mailbox/destroy.rs +++ b/crates/email/src/mailbox/destroy.rs @@ -92,10 +92,8 @@ impl MailboxDestroy for Server { let mut deleted_ids = RoaringBitmap::new(); let mut thread_ids = RoaringBitmap::new(); - // inbuxa: UD-1, UD-6a: the retention in force now - let retention = inbuxa_features::undelete::settings::retention(self.registry()) - .await? - .items; + // inbuxa: UD-1, UD-6a, LH-4: how this account's deletions are kept + let keeping = self.keeping(account_id).await?; self.archives( account_id, Collection::Email, @@ -125,10 +123,10 @@ impl MailboxDestroy for Server { deleted_ids.insert(message_id); thread_ids.insert(prev_message_data.inner.thread_id.to_native()); // inbuxa: UD-1, UD-4: a deleted message is noted for archiving - if let Some(retention) = retention { + if keeping.keeps_anything() { inbuxa_features::undelete::email::note( &mut batch, - retention, + &keeping, account_id, message_id, prev_message_data.inner.size.to_native() as u64, diff --git a/crates/email/src/message/delete.rs b/crates/email/src/message/delete.rs index cc781f0..256fb37 100644 --- a/crates/email/src/message/delete.rs +++ b/crates/email/src/message/delete.rs @@ -69,10 +69,8 @@ impl EmailDeletion for Server { batch .with_account_id(account_id) .with_collection(Collection::Email); - // inbuxa: UD-1, UD-6a: the retention in force now - let retention = inbuxa_features::undelete::settings::retention(self.registry()) - .await? - .items; + // inbuxa: UD-1, UD-6a, LH-4: how this account's deletions are kept + let keeping = self.keeping(account_id).await?; self.archives( account_id, Collection::Email, @@ -90,10 +88,10 @@ impl EmailDeletion for Server { } thread_ids.insert(metadata.inner.thread_id.to_native()); // inbuxa: UD-1, UD-4: a deleted message is noted for archiving - if let Some(retention) = retention { + if keeping.keeps_anything() { inbuxa_features::undelete::email::note( batch, - retention, + &keeping, account_id, document_id, metadata.inner.size.to_native() as u64, diff --git a/crates/email/src/sieve/delete.rs b/crates/email/src/sieve/delete.rs index c982ada..9ac94b6 100644 --- a/crates/email/src/sieve/delete.rs +++ b/crates/email/src/sieve/delete.rs @@ -44,12 +44,12 @@ impl SieveScriptDelete for Server { )) .await? { - // inbuxa: UD-1: a deleted script is kept, when archiving is on - if let Some(retention) = - inbuxa_features::undelete::settings::retention(self.registry()) - .await? - .items - { + // inbuxa: UD-1, LH-4: a deleted script is kept, when archiving + // is on or a hold covers the account (whole: scripts have no date) + let keeping = self.keeping(account_id).await?; + let now = store::write::now(); + if let Some(until) = keeping.until(now, keeping.is_held()) { + let retention = until.saturating_sub(now); let script = obj_ .deserialize::() .caused_by(trc::location!())?; diff --git a/crates/features/src/hold/mod.rs b/crates/features/src/hold/mod.rs index b940b47..4480dde 100644 --- a/crates/features/src/hold/mod.rs +++ b/crates/features/src/hold/mod.rs @@ -26,6 +26,85 @@ use store::{ }; use trc::AddContext; +/// The deadline a held archived item carries: the last second of 9999. It +/// never passes, so every expiry check keeps the item without knowing about +/// holds (LH-4, LH-5); releasing a hold gives it a real deadline (LH-10). +pub const HELD_UNTIL: u64 = 253_402_300_799; + +/// Whether an archived item's deadline marks it as held. Anything past the +/// year 9000 counts, so a deadline computed from a hold a moment earlier or +/// later still reads as held. +pub fn is_held_until(until: u64) -> bool { + until >= 221_845_392_000 +} + +/// A day, in seconds: the slack either side of a range for an event's start, +/// whose time zone isn't known here. +const DAY: u64 = 86_400; + +/// How an account's deleted items are kept: its holds' ranges, and the +/// undelete period for whatever no hold covers (LH-3, LH-4). +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct Keeping { + /// `archiveDeletedItemsFor`, in seconds, if undelete is on. + pub retention: Option, + /// Each active hold's range on this account; `(None, None)` is a whole + /// account. Empty when nothing holds it. + pub ranges: Vec<(Option, Option)>, +} + +impl Keeping { + pub fn new(retention: Option, holds: &[Hold]) -> Keeping { + Keeping { + retention, + ranges: holds.iter().map(|h| (h.from, h.to)).collect(), + } + } + + /// Whether any hold reaches the account at all. + pub fn is_held(&self) -> bool { + !self.ranges.is_empty() + } + + /// Whether deleted items need noting: something may keep them. + pub fn keeps_anything(&self) -> bool { + self.is_held() || self.retention.is_some() + } + + /// Whether a hold covers an item dated `date`. No date means the item is + /// held whole, whatever the range (LH-3). + pub fn covers(&self, date: Option) -> bool { + self.ranges.iter().any(|(from, to)| match date { + None => true, + Some(at) => { + from.is_none_or(|from| at >= from) && to.is_none_or(|to| at <= to) + } + }) + } + + /// Like `covers`, for an event's start: a day of slack either side, since + /// its time zone isn't known here. + pub fn covers_event(&self, start: Option) -> bool { + self.ranges.iter().any(|(from, to)| match start { + None => true, + Some(at) => { + from.is_none_or(|from| at + DAY >= from) + && to.is_none_or(|to| at <= to.saturating_add(DAY)) + } + }) + } + + /// Until when an item deleted at `now` is kept: held, the undelete + /// period, or not at all. + pub fn until(&self, now: u64, held: bool) -> Option { + if held { + Some(HELD_UNTIL) + } else { + self.retention.map(|retention| now + retention) + } + } +} + const FEATURE: u8 = b'H'; const KIND_HOLD: u8 = b'h'; @@ -511,6 +590,27 @@ mod tests { assert!(held.scope.covers(&member) && !held.scope.covers(&moved)); } + #[test] + fn keeping_deleted_items() { + let whole = Keeping::new(None, &[hold(accounts(&[2]), None, None)]); + assert!(whole.covers(Some(5)) && whole.covers(None)); + assert_eq!(whole.until(100, whole.covers(Some(5))), Some(HELD_UNTIL)); + assert!(is_held_until(whole.until(100, true).unwrap())); + + // LH-3: a range holds only what's inside it; outside, undelete's rules + let ranged = Keeping::new(Some(30), &[hold(accounts(&[2]), Some(1_000), Some(2_000))]); + assert!(ranged.covers(Some(1_500)) && !ranged.covers(Some(2_500))); + assert!(ranged.covers(None), "contacts, files and scripts are held whole"); + assert_eq!(ranged.until(100, ranged.covers(Some(2_500))), Some(130)); + assert!(ranged.covers_event(Some(2_000 + 3_600)), "a day of slack for an event"); + + // Neither held nor undelete: nothing is kept + let none = Keeping::new(None, &[]); + assert!(!none.keeps_anything()); + assert_eq!(none.until(100, false), None); + assert!(!is_held_until(100 + 30 * 365 * 86_400)); + } + #[test] fn stored_as_json() { let current = hold(accounts(&[2]), Some(100), None); diff --git a/crates/features/src/undelete/data.rs b/crates/features/src/undelete/data.rs index c398800..b170c98 100644 --- a/crates/features/src/undelete/data.rs +++ b/crates/features/src/undelete/data.rs @@ -123,6 +123,14 @@ pub struct EmailNote { pub size: u64, pub mailboxes: Vec, pub keywords: Vec, + /// LH-3: the ranges of the holds on the account when it was deleted. + /// Its received date is only known when it's archived, which decides + /// whether a hold keeps it after all. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub held_ranges: Vec<(Option, Option)>, + /// The undelete deadline for when no range covers it. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub otherwise_until: Option, } /// What restore needs beyond the kept copy (UD-4, UD-8). diff --git a/crates/features/src/undelete/email.rs b/crates/features/src/undelete/email.rs index 83cc244..7d67380 100644 --- a/crates/features/src/undelete/email.rs +++ b/crates/features/src/undelete/email.rs @@ -12,9 +12,12 @@ //! is made if archiving is on, fixing the deadline then. When the data is //! finally removed, a noted message becomes an archived item. -use crate::undelete::{ - data::{self, EmailNote, Extra}, - records, +use crate::{ + hold::Keeping, + undelete::{ + data::{self, EmailNote, Extra}, + records, + }, }; use registry::{ schema::structs::{ArchivedEmail, ArchivedItem}, @@ -26,10 +29,12 @@ use store::{ }; use types::{blob::BlobId, blob_hash::BlobHash}; -/// Notes a deleted message, when archiving is on (`retention` seconds). +/// Notes a deleted message, when anything keeps it: undelete, or a legal +/// hold on the account (LH-4). A held note keeps it until it's archived, +/// when its received date says whether the hold's range covers it. pub fn note( batch: &mut BatchBuilder, - retention: u64, + keeping: &Keeping, account_id: u32, document_id: u32, size: u64, @@ -37,16 +42,23 @@ pub fn note( keywords: Vec, ) -> trc::Result<()> { let archived_at = now(); + // Held until the date is known; the undelete deadline otherwise + let otherwise_until = keeping.until(archived_at, false); + let Some(archived_until) = keeping.until(archived_at, keeping.is_held()) else { + return Ok(()); + }; data::note_email( batch, account_id, document_id, &EmailNote { archived_at, - archived_until: archived_at + retention, + archived_until, size, mailboxes, keywords, + held_ranges: keeping.ranges.clone(), + otherwise_until: if keeping.is_held() { otherwise_until } else { None }, }, ) } @@ -78,9 +90,27 @@ pub async fn archive( document_id: u32, summary: Summary<'_>, ) -> trc::Result { - let Some(note) = data::email_note(data, account_id, document_id).await? else { + let Some(mut note) = data::email_note(data, account_id, document_id).await? else { return Ok(false); }; + // LH-3: a held note's range decides now that the date is known; outside + // it, undelete's deadline, or nothing kept at all + if !note.held_ranges.is_empty() { + let keeping = Keeping { + retention: None, + ranges: std::mem::take(&mut note.held_ranges), + }; + if !keeping.covers(Some(summary.received_at)) { + match note.otherwise_until { + Some(until) => note.archived_until = until, + None => { + let mut batch = BatchBuilder::new(); + data::clear_email_note(&mut batch, account_id, document_id); + return data.write(batch.build_all()).await.map(|_| false); + } + } + } + } let item = ArchivedItem::Email(ArchivedEmail { from: summary.from.unwrap_or_default().to_string(), subject: summary.subject.unwrap_or_default().to_string(), diff --git a/crates/features/src/undelete/groupware.rs b/crates/features/src/undelete/groupware.rs index b94e7dc..a77eb0f 100644 --- a/crates/features/src/undelete/groupware.rs +++ b/crates/features/src/undelete/groupware.rs @@ -97,6 +97,39 @@ pub async fn take( Ok(Some(note)) } +/// A note, left in place: for a held account it's cleared only once its item +/// is archived, so a failure leaves it for the retry (LH-5). +pub async fn peek( + data: &Store, + kind: Kind, + account_id: u32, + document_id: u32, +) -> trc::Result> { + Ok(data + .get_value::>(ValueKey::from(note_class(kind, account_id, document_id))) + .await? + .map(|Json(note)| note)) +} + +/// Removes a note once its item is archived or needn't be. +pub async fn clear(data: &Store, kind: Kind, account_id: u32, document_id: u32) -> trc::Result<()> { + let mut batch = BatchBuilder::new(); + batch.clear(note_class(kind, account_id, document_id)); + data.write(batch.build_all()).await.map(|_| ()) +} + +/// An event's start, for a hold's range (LH-3). None for a recurring event, +/// which may have an occurrence anywhere, so a hold keeps it whole. +pub fn event_start(note: &Note) -> Option { + let text = note.content.as_deref()?; + if property(text, "RRULE").is_some() || property(text, "RDATE").is_some() { + return None; + } + property(text, "DTSTART") + .and_then(|v| ical_time(&v)) + .map(|t| t.max(0) as u64) +} + /// The value of the first line starting with `name` (as `NAME:` or /// `NAME;params:`) in iCalendar or vCard text, unfolded. fn property(text: &str, name: &str) -> Option { diff --git a/crates/imap/src/op/expunge.rs b/crates/imap/src/op/expunge.rs index 918f177..0a8aebb 100644 --- a/crates/imap/src/op/expunge.rs +++ b/crates/imap/src/op/expunge.rs @@ -243,10 +243,8 @@ impl SessionData { let mut fully_deleted = RoaringBitmap::new(); let mut thread_ids = RoaringBitmap::new(); - // inbuxa: UD-1, UD-6a: the retention in force now - let retention = inbuxa_features::undelete::settings::retention(self.server.registry()) - .await? - .items; + // inbuxa: UD-1, UD-6a, LH-4: how this account's deletions are kept + let keeping = self.server.keeping(account_id).await?; self.server .archives( account_id, @@ -270,10 +268,10 @@ impl SessionData { fully_deleted.insert(document_id); thread_ids.insert(metadata.inner.thread_id.to_native()); // inbuxa: UD-1, UD-4: a deleted message is noted for archiving - if let Some(retention) = retention { + if keeping.keeps_anything() { inbuxa_features::undelete::email::note( batch, - retention, + &keeping, account_id, document_id, metadata.inner.size.to_native() as u64, diff --git a/crates/services/src/task_manager/index.rs b/crates/services/src/task_manager/index.rs index 07ce970..53161e9 100644 --- a/crates/services/src/task_manager/index.rs +++ b/crates/services/src/task_manager/index.rs @@ -229,11 +229,19 @@ impl SearchIndexTask for Server { IndexDocumentType::Email => None, } && let Err(err) = archive_noted(self, kind, account_id, document_id).await { + // inbuxa: LH-5: the note stays, so the retry archives + // it; nothing a hold keeps is lost to a failure trc::error!( err.account_id(account_id) .document_id(document_id) .details("Failed to archive a deleted item") ); + results.push(IndexTaskResult { + task_type: TaskType::Delete, + index: task.document_type, + result: TaskResult::temporary("Failed to archive a deleted item"), + }); + continue; } document_deletions[idx] @@ -696,8 +704,9 @@ pub fn trace_search_document( document } -// inbuxa: UD-1, UD-4: archives a deleted file, event or contact noted at -// deletion, when archiving is on; otherwise its note is dropped +// inbuxa: UD-1, UD-4, LH-4: archives a deleted file, event or contact noted +// at deletion, when archiving is on or a hold covers it; otherwise its note is +// dropped. The note goes only once the item is archived. async fn archive_noted( server: &Server, kind: undelete::groupware::Kind, @@ -705,12 +714,22 @@ async fn archive_noted( document_id: u32, ) -> trc::Result<()> { let data = &server.core.storage.data; - let Some(note) = undelete::groupware::take(data, kind, account_id, document_id).await? else { + let Some(note) = undelete::groupware::peek(data, kind, account_id, document_id).await? else { return Ok(()); }; - let Some(retention) = undelete::settings::retention(server.registry()).await?.items else { - return Ok(()); + // LH-3: events by their start; contacts and files whole + let keeping = server.keeping(account_id).await?; + let held = match kind { + undelete::groupware::Kind::CalendarEvent => { + keeping.covers_event(undelete::groupware::event_start(¬e)) + } + _ => keeping.covers(None), }; + let now = store::write::now(); + let Some(until) = keeping.until(now, held) else { + return undelete::groupware::clear(data, kind, account_id, document_id).await; + }; + let retention = until.saturating_sub(now); let blob_hash = match (¬e.content, ¬e.blob_hash) { (Some(text), _) => { server @@ -723,11 +742,11 @@ async fn archive_noted( .into_err() .details("Invalid blob hash in undelete note") })?, - (None, None) => return Ok(()), + (None, None) => return undelete::groupware::clear(data, kind, account_id, document_id).await, }; undelete::groupware::archive(data, server.registry(), account_id, note, blob_hash, retention) - .await - .map(|_| ()) + .await?; + undelete::groupware::clear(data, kind, account_id, document_id).await } async fn delete_email_metadata( diff --git a/tests/src/system/legal_hold.rs b/tests/src/system/legal_hold.rs index f6f524e..12b139f 100644 --- a/tests/src/system/legal_hold.rs +++ b/tests/src/system/legal_hold.rs @@ -14,11 +14,22 @@ use crate::utils::{ }; use registry::schema::{ prelude::{ObjectType, Property}, - structs::{CertificateManagement, DkimManagement, DnsManagement, Domain, Tenant, UserRoles}, + structs::{ + CertificateManagement, DataRetention, DkimManagement, DnsManagement, Domain, Tenant, + UserRoles, + }, }; use serde_json::{Value, json}; +use types::id::Id; -const USING: &[&str] = &["urn:ietf:params:jmap:core", "urn:inbuxa:jmap"]; +const INBOX_ID: u32 = 0; + +const USING: &[&str] = &[ + "urn:ietf:params:jmap:core", + "urn:ietf:params:jmap:mail", + "urn:ietf:params:jmap:contacts", + "urn:inbuxa:jmap", +]; impl Account { async fn hold_call(&self, method: &str, mut arguments: Value) -> (String, Value) { @@ -38,6 +49,13 @@ impl Account { response } + async fn archived_items(&self) -> Vec { + let (_, response) = self + .hold_call("x:ArchivedItem/get", json!({"ids": null})) + .await; + response["list"].as_array().cloned().unwrap_or_default() + } + async fn hold_get(&self, id: &str) -> Value { let (name, response) = self .hold_call("inbuxa:LegalHold/get", json!({"ids": [id]})) @@ -266,6 +284,86 @@ pub async fn test(test: &mut TestServer) { "test 7, LH-2: the moved account escaped the hold" ); + // Test 6, LH-4: what a hold keeps, with undelete switched off, so only + // the hold can be keeping anything + admin + .registry_update_setting( + DataRetention { + archive_deleted_items_for: None, + ..Default::default() + }, + &[Property::ArchiveDeletedItemsFor], + ) + .await; + let held = admin + .create_user_account("held@example.com", "held-secret-4419", "Held", &[], vec![]) + .await; + let ranged = admin + .create_user_account("ranged@example.com", "ranged-secret-5530", "Ranged", &[], vec![]) + .await; + let response = admin + .hold_set(json!({"reason": "Preserve everything", "create": { + "w": {"name": "Matter 6001", "scope": {"accounts": [held.id_string()]}}, + "r": {"name": "Matter 6002", "from": "2020-01-01T00:00:00Z", "to": "2020-12-31T23:59:59Z", + "scope": {"accounts": [ranged.id_string()]}}}})) + .await; + assert!(response["created"]["w"]["id"].is_string(), "LH-1: {response}"); + assert!(response["created"]["r"]["id"].is_string(), "LH-1: {response}"); + + let held_client = held.jmap_client().await; + let ranged_client = ranged.jmap_client().await; + let whole = import(&held_client, "Held whole", None).await; + held_client.email_destroy(&whole).await.unwrap(); + // 2020-03-15: inside the range; now: outside it + let inside = import(&ranged_client, "Inside the range", Some(1_584_230_400)).await; + let outside = import(&ranged_client, "Outside the range", None).await; + ranged_client.email_destroy(&inside).await.unwrap(); + ranged_client.email_destroy(&outside).await.unwrap(); + + // LH-3: a contact is held whole, whatever the range + let (_, books) = ranged + .hold_call("AddressBook/get", json!({"ids": null})) + .await; + let book = books["list"][0]["id"] + .as_str() + .unwrap_or_else(|| panic!("no address book: {books}")) + .to_string(); + { + let (_, created) = ranged + .hold_call( + "ContactCard/set", + json!({"create": {"c": {"addressBookIds": {book: true}, + "name": {"full": "Kept Contact"}}}}), + ) + .await; + let card = created["created"]["c"]["id"].as_str().unwrap_or_default().to_string(); + let (_, destroyed) = ranged + .hold_call("ContactCard/set", json!({"destroy": [card]})) + .await; + assert!(destroyed["destroyed"][0].is_string(), "{destroyed}"); + } + test.wait_for_tasks().await; + + let is_held = |item: &Value| item["archivedUntil"].as_str().is_some_and(|u| u.starts_with("9999-")); + let kept = held.archived_items().await; + assert!( + kept.iter().any(|i| i["subject"] == "Held whole" && is_held(i)), + "test 6, LH-4: a held account's mail wasn't kept: {kept:?}" + ); + let kept = ranged.archived_items().await; + assert!( + kept.iter().any(|i| i["subject"] == "Inside the range" && is_held(i)), + "LH-3: mail inside the range wasn't kept: {kept:?}" + ); + assert!( + !kept.iter().any(|i| i["subject"] == "Outside the range"), + "LH-3: mail outside the range was kept, with undelete off: {kept:?}" + ); + assert!( + kept.iter().any(|i| i["name"] == "Kept Contact" && is_held(i)), + "LH-3: a contact wasn't kept whole: {kept:?}" + ); + // LH-10: release needs a reason, and a released hold stays, read-only let response = admin .hold_set(json!({"update": {hold_id.as_str(): {"released": true}}})) @@ -319,6 +417,23 @@ pub async fn test(test: &mut TestServer) { } } +async fn import( + client: &jmap_client::client::Client, + subject: &str, + received_at: Option, +) -> String { + client + .email_import( + format!("From: a@example.org\r\nSubject: {subject}\r\n\r\nBody.\r\n").into_bytes(), + [Id::from(INBOX_ID).to_string()], + None::>, + received_at, + ) + .await + .unwrap() + .take_id() +} + /// Runs these tests alone: `cargo test -p tests legal_hold_tests -- --ignored`. #[ignore] #[tokio::test(flavor = "multi_thread")]