Export what a legal hold keeps as a ZIP (LH-12)
inbuxa:HoldExport/set takes a hold, optionally some of the accounts it covers, and a reason; the collection runs in the background and get says when it's ready. The ZIP has, per account, mail as .eml under its folders, calendars as .ics, contacts as .vcf, files as stored, and the archived items the hold keeps under archived/; a manifest.csv gives each entry's account, kind, folder, date, whether it was archived, size and SHA-256, and manifest.sha256 hashes the manifest. Accounts the hold doesn't cover are left out, and items outside its date range are too: live mail by arrival, events by start, and archived items the same way, so an export doesn't carry deleted items that only another hold keeps. The finished file is a blob of whoever started the export, so only they download it, and it lasts as long as any upload (uploadTtl). Exports are records under the hold (SUBSPACE_INBUXA H/e): never changed or destroyed, each with its status, counts, size and checksum. Starting one needs sysLegalHoldExport, an active hold and a reason, and is recorded in the audit log like the audit log's own export. The build is in memory and capped at 2 GB; bigger holds fail with a message saying so, and are split by picking accounts. Tested: unit tests for safe ZIP names and the manifest and its hash; the legal_hold system test, on RocksDB, PostgreSQL and MySQL, exports a hold end to end (live and archived mail, the manifest's hash, an asked- for account the hold doesn't cover left out) and checks the refusals (no reason, a user without the permission, a released hold) and the audit record; and by hand from the console on a local server. Not covered by a test: the archived-item date range with two holds of different ranges over one account.
This commit is contained in:
@@ -39,6 +39,7 @@ base64 = "0.23"
|
||||
p256 = { version = "0.13", features = ["ecdh"] }
|
||||
sha1 = "0.11"
|
||||
sha2 = "0.11"
|
||||
zip = "8.6" # inbuxa: legal hold exports (LH-12)
|
||||
reqwest = { version = "0.13", default-features = false, features = ["rustls", "http2"]}
|
||||
tokio-tungstenite = "0.30"
|
||||
tungstenite = "0.30"
|
||||
|
||||
@@ -97,6 +97,7 @@ impl JmapAuthorization for AccessToken {
|
||||
// inbuxa: account lock (AL-12)
|
||||
GetRequestMethod::AccountLock(_) => Permission::SysAccountLockGet,
|
||||
GetRequestMethod::LegalHold(_) => Permission::SysLegalHoldGet,
|
||||
GetRequestMethod::HoldExport(_) => Permission::SysLegalHoldExport,
|
||||
// inbuxa: legacy protocols off. It takes listeners away and
|
||||
// puts them back, so it takes the listener's permissions
|
||||
GetRequestMethod::ProtocolPolicy(_) => Permission::SysNetworkListenerGet,
|
||||
@@ -232,6 +233,14 @@ impl JmapAuthorization for AccessToken {
|
||||
Permission::SysLegalHoldUpdate,
|
||||
Permission::SysLegalHoldUpdate,
|
||||
),
|
||||
// inbuxa: LH-12, exporting held data
|
||||
SetRequestMethod::HoldExport(s) => validate_set(
|
||||
s,
|
||||
self,
|
||||
Permission::SysLegalHoldExport,
|
||||
Permission::SysLegalHoldExport,
|
||||
Permission::SysLegalHoldExport,
|
||||
),
|
||||
SetRequestMethod::AuditVerification(s) => validate_set(
|
||||
s,
|
||||
self,
|
||||
@@ -380,6 +389,7 @@ impl JmapAuthorization for AccessToken {
|
||||
| MethodObject::AuditVerification
|
||||
| MethodObject::AccountLock
|
||||
| MethodObject::LegalHold
|
||||
| MethodObject::HoldExport
|
||||
| MethodObject::ProtocolPolicy
|
||||
| MethodObject::TenantProtocolPolicy => Permission::JmapEmailChanges,
|
||||
// inbuxa: x:MaskedEmail/changes reads what /get reads
|
||||
|
||||
@@ -276,6 +276,9 @@ impl RequestHandler for Server {
|
||||
SetResponseMethod::LegalHold(set_response) => {
|
||||
set_response.update_created_ids(&mut response);
|
||||
}
|
||||
SetResponseMethod::HoldExport(set_response) => {
|
||||
set_response.update_created_ids(&mut response);
|
||||
}
|
||||
SetResponseMethod::Explanation(set_response) => {
|
||||
set_response.update_created_ids(&mut response);
|
||||
}
|
||||
@@ -450,6 +453,11 @@ impl RequestHandler for Server {
|
||||
.into()
|
||||
}
|
||||
// inbuxa: legal hold (LH-1)
|
||||
// inbuxa: legal hold exports (LH-12)
|
||||
GetRequestMethod::HoldExport(mut req) => {
|
||||
resolve_account_id(&mut req.account_id, method_name.obj, access_token)?;
|
||||
crate::inbuxa::hold_export_api::get(self, *req).await?.into()
|
||||
}
|
||||
GetRequestMethod::LegalHold(mut req) => {
|
||||
resolve_account_id(&mut req.account_id, method_name.obj, access_token)?;
|
||||
crate::inbuxa::legal_hold::get(self, *req).await?.into()
|
||||
@@ -807,6 +815,34 @@ impl RequestHandler for Server {
|
||||
}
|
||||
// inbuxa: legal hold (LH-1), each change recorded with its
|
||||
// reason (AU-12)
|
||||
// inbuxa: legal hold exports, recorded with their reason
|
||||
// (AU-1.9, AU-12)
|
||||
SetRequestMethod::HoldExport(mut req) => {
|
||||
resolve_account_id(&mut req.account_id, method_name.obj, access_token)?;
|
||||
let reason = req.arguments.reason.clone().or_else(|| {
|
||||
req.create.as_ref().and_then(|create| {
|
||||
create.values().find_map(|value| {
|
||||
serde_json::to_value(value)
|
||||
.ok()?
|
||||
.get("reason")?
|
||||
.as_str()
|
||||
.map(str::to_string)
|
||||
})
|
||||
})
|
||||
});
|
||||
crate::inbuxa::audit::recorded(
|
||||
self,
|
||||
access_token,
|
||||
session,
|
||||
&method_name.obj.to_string(),
|
||||
None,
|
||||
reason,
|
||||
*req,
|
||||
|req| Box::pin(crate::inbuxa::hold_export_api::set(self, access_token, req)),
|
||||
)
|
||||
.await?
|
||||
.into()
|
||||
}
|
||||
SetRequestMethod::LegalHold(mut req) => {
|
||||
resolve_account_id(&mut req.account_id, method_name.obj, access_token)?;
|
||||
let reason = req.arguments.reason.clone().or_else(|| {
|
||||
|
||||
@@ -425,6 +425,7 @@ impl IntermediateChangesResponse {
|
||||
| MethodObject::AuditVerification
|
||||
| MethodObject::AccountLock
|
||||
| MethodObject::LegalHold
|
||||
| MethodObject::HoldExport
|
||||
| MethodObject::ProtocolPolicy
|
||||
| MethodObject::TenantProtocolPolicy
|
||||
| MethodObject::Registry(_) => unreachable!(),
|
||||
|
||||
@@ -0,0 +1,429 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2026 Coffey Labs
|
||||
*
|
||||
* SPDX-License-Identifier: AGPL-3.0-only
|
||||
*/
|
||||
|
||||
//! Collecting what a legal hold keeps, as a ZIP (audit-hold-lock spec,
|
||||
//! LH-12). For each account the hold covers (or those asked for): its mail,
|
||||
//! calendars, contacts and files, and the deleted items the hold keeps, each
|
||||
//! with its SHA-256 in `manifest.csv`, and the manifest's own hash beside
|
||||
//! it. The hold's date range applies as it does to what's kept (LH-3).
|
||||
|
||||
use common::{Server, hold::kept_member};
|
||||
use email::{
|
||||
cache::MessageCacheFetch,
|
||||
message::metadata::{MESSAGE_RECEIVED_MASK, MessageMetadata},
|
||||
};
|
||||
use groupware::{cache::GroupwareCache, calendar::CalendarEvent, contact::ContactCard, file::FileNode};
|
||||
use inbuxa_features::{
|
||||
hold::{Hold, Keeping, is_held_until},
|
||||
undelete::records,
|
||||
};
|
||||
use registry::schema::{prelude::ObjectType, structs::ArchivedItem};
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::io::{Cursor, Write};
|
||||
use store::{
|
||||
ValueKey,
|
||||
registry::RegistryQuery,
|
||||
write::{AlignedBytes, Archive},
|
||||
};
|
||||
use trc::AddContext;
|
||||
use types::{
|
||||
collection::{Collection, SyncCollection},
|
||||
field::EmailField,
|
||||
id::Id,
|
||||
};
|
||||
use zip::{CompressionMethod, ZipWriter, write::SimpleFileOptions};
|
||||
|
||||
/// The largest ZIP built in memory. A bigger collection is refused with a
|
||||
/// clear error rather than taking the node down; export fewer accounts.
|
||||
pub const MAX_EXPORT: u64 = 2 * 1024 * 1024 * 1024;
|
||||
|
||||
/// One line of `manifest.csv`.
|
||||
struct Entry {
|
||||
path: String,
|
||||
account: String,
|
||||
kind: &'static str,
|
||||
folder: String,
|
||||
date: Option<i64>,
|
||||
archived: bool,
|
||||
size: usize,
|
||||
sha256: String,
|
||||
}
|
||||
|
||||
fn hex(bytes: &[u8]) -> String {
|
||||
bytes.iter().map(|b| format!("{b:02x}")).collect()
|
||||
}
|
||||
|
||||
fn csv(field: &str) -> String {
|
||||
if field.contains([',', '"', '\n', '\r']) {
|
||||
format!("\"{}\"", field.replace('"', "\"\""))
|
||||
} else {
|
||||
field.to_string()
|
||||
}
|
||||
}
|
||||
|
||||
/// A path segment that's safe in a ZIP: no separators, no leading dots.
|
||||
fn segment(name: &str) -> String {
|
||||
let cleaned: String = name
|
||||
.chars()
|
||||
.map(|c| if c == '/' || c == '\\' || c.is_control() { '_' } else { c })
|
||||
.collect();
|
||||
let trimmed = cleaned.trim_start_matches('.').trim();
|
||||
if trimmed.is_empty() { "_".into() } else { trimmed.chars().take(120).collect() }
|
||||
}
|
||||
|
||||
fn date_text(at: Option<i64>) -> String {
|
||||
at.map(|at| jmap_proto::types::date::UTCDate::from_timestamp(at).to_string())
|
||||
.unwrap_or_default()
|
||||
}
|
||||
|
||||
struct Builder {
|
||||
zip: ZipWriter<Cursor<Vec<u8>>>,
|
||||
entries: Vec<Entry>,
|
||||
written: u64,
|
||||
}
|
||||
|
||||
impl Builder {
|
||||
fn new() -> Self {
|
||||
Builder {
|
||||
zip: ZipWriter::new(Cursor::new(Vec::new())),
|
||||
entries: Vec::new(),
|
||||
written: 0,
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
fn add(
|
||||
&mut self,
|
||||
path: String,
|
||||
bytes: &[u8],
|
||||
account: &str,
|
||||
kind: &'static str,
|
||||
folder: &str,
|
||||
date: Option<i64>,
|
||||
archived: bool,
|
||||
) -> trc::Result<()> {
|
||||
self.written += bytes.len() as u64;
|
||||
if self.written > MAX_EXPORT {
|
||||
return Err(trc::StoreEvent::UnexpectedError
|
||||
.into_err()
|
||||
.details("The collection is larger than one export can hold (2 GB). Export fewer accounts at a time."));
|
||||
}
|
||||
// Unique within the ZIP, however names collide
|
||||
let mut name = path.clone();
|
||||
let mut n = 1;
|
||||
while self.entries.iter().any(|e| e.path == name) {
|
||||
n += 1;
|
||||
name = match path.rsplit_once('.') {
|
||||
Some((stem, ext)) if !stem.ends_with('/') => format!("{stem} ({n}).{ext}"),
|
||||
_ => format!("{path} ({n})"),
|
||||
};
|
||||
}
|
||||
let options = SimpleFileOptions::default().compression_method(CompressionMethod::Deflated);
|
||||
self.zip
|
||||
.start_file(name.as_str(), options)
|
||||
.and_then(|_| self.zip.write_all(bytes).map_err(Into::into))
|
||||
.map_err(|err| {
|
||||
trc::StoreEvent::UnexpectedError
|
||||
.into_err()
|
||||
.details("Failed to write the export")
|
||||
.reason(err)
|
||||
})?;
|
||||
self.entries.push(Entry {
|
||||
path: name,
|
||||
account: account.to_string(),
|
||||
kind,
|
||||
folder: folder.to_string(),
|
||||
date,
|
||||
archived,
|
||||
size: bytes.len(),
|
||||
sha256: hex(&Sha256::digest(bytes)),
|
||||
});
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Closes the ZIP with its manifest and the manifest's hash. Returns the
|
||||
/// bytes and how many items went in.
|
||||
fn finish(mut self) -> trc::Result<(Vec<u8>, usize)> {
|
||||
let mut manifest = String::from("path,account,kind,folder,date,archived,size,sha256\n");
|
||||
for e in &self.entries {
|
||||
manifest.push_str(&format!(
|
||||
"{},{},{},{},{},{},{},{}\n",
|
||||
csv(&e.path),
|
||||
csv(&e.account),
|
||||
e.kind,
|
||||
csv(&e.folder),
|
||||
date_text(e.date),
|
||||
e.archived,
|
||||
e.size,
|
||||
e.sha256
|
||||
));
|
||||
}
|
||||
let manifest_hash = hex(&Sha256::digest(manifest.as_bytes()));
|
||||
let options = SimpleFileOptions::default().compression_method(CompressionMethod::Deflated);
|
||||
let fail = |err: zip::result::ZipError| {
|
||||
trc::StoreEvent::UnexpectedError
|
||||
.into_err()
|
||||
.details("Failed to write the export")
|
||||
.reason(err)
|
||||
};
|
||||
self.zip.start_file("manifest.csv", options).map_err(fail)?;
|
||||
self.zip.write_all(manifest.as_bytes()).map_err(|e| fail(e.into()))?;
|
||||
self.zip.start_file("manifest.sha256", options).map_err(fail)?;
|
||||
self.zip
|
||||
.write_all(format!("{manifest_hash} manifest.csv\n").as_bytes())
|
||||
.map_err(|e| fail(e.into()))?;
|
||||
let items = self.entries.len();
|
||||
let bytes = self.zip.finish().map_err(fail)?.into_inner();
|
||||
Ok((bytes, items))
|
||||
}
|
||||
}
|
||||
|
||||
/// The accounts to collect: those asked for that the hold covers, or every
|
||||
/// account it covers, deleted ones it keeps included.
|
||||
async fn accounts(server: &Server, hold: &Hold, asked: &[u32]) -> trc::Result<Vec<u32>> {
|
||||
let mut covered = Vec::new();
|
||||
for id in server
|
||||
.registry()
|
||||
.query::<Vec<Id>>(RegistryQuery::new(ObjectType::Account))
|
||||
.await
|
||||
.caused_by(trc::location!())?
|
||||
{
|
||||
let account_id = id.document_id();
|
||||
if let Some(member) = server.member_of(account_id).await
|
||||
&& hold.scope.covers(&member)
|
||||
{
|
||||
covered.push(account_id);
|
||||
}
|
||||
}
|
||||
for (account_id, kept) in inbuxa_features::undelete::data::kept_accounts(server.store()).await? {
|
||||
if hold.scope.covers(&kept_member(account_id, &kept)) {
|
||||
covered.push(account_id);
|
||||
}
|
||||
}
|
||||
covered.sort_unstable();
|
||||
covered.dedup();
|
||||
if !asked.is_empty() {
|
||||
covered.retain(|id| asked.contains(id));
|
||||
}
|
||||
Ok(covered)
|
||||
}
|
||||
|
||||
async fn blob(server: &Server, hash: &[u8]) -> trc::Result<Option<Vec<u8>>> {
|
||||
server.blob_store().get_blob(hash, 0..usize::MAX).await
|
||||
}
|
||||
|
||||
/// Collects `accounts` under `hold` into a ZIP. Returns its bytes and item
|
||||
/// count.
|
||||
pub async fn build(server: &Server, hold: &Hold, asked: &[u32]) -> trc::Result<(Vec<u8>, usize)> {
|
||||
let keeping = Keeping::new(None, std::slice::from_ref(hold));
|
||||
let data = server.store();
|
||||
let mut out = Builder::new();
|
||||
|
||||
for account_id in accounts(server, hold, asked).await? {
|
||||
let address = server.audit_account_name(account_id).await;
|
||||
let base = format!("{}/", segment(&address));
|
||||
let live = server.account(account_id).await.is_ok();
|
||||
|
||||
if live {
|
||||
// Mail, by the folder it's in
|
||||
let cache = server
|
||||
.get_cached_messages(account_id)
|
||||
.await
|
||||
.caused_by(trc::location!())?;
|
||||
for message in cache.emails.items.iter() {
|
||||
let Some(metadata_) = data
|
||||
.get_value::<Archive<AlignedBytes>>(ValueKey::property(
|
||||
account_id,
|
||||
Collection::Email,
|
||||
message.document_id,
|
||||
EmailField::Metadata,
|
||||
))
|
||||
.await?
|
||||
else {
|
||||
continue;
|
||||
};
|
||||
let metadata = metadata_
|
||||
.unarchive::<MessageMetadata>()
|
||||
.caused_by(trc::location!())?;
|
||||
let received = metadata.rcvd_attach.to_native() & MESSAGE_RECEIVED_MASK;
|
||||
if !keeping.covers(Some(received)) {
|
||||
continue;
|
||||
}
|
||||
let folder = message
|
||||
.mailboxes
|
||||
.first()
|
||||
.and_then(|m| cache.mailboxes.items.iter().find(|b| b.document_id == m.mailbox_id))
|
||||
.map(|b| b.path.clone())
|
||||
.unwrap_or_default();
|
||||
let hash = types::blob_hash::BlobHash::from(&metadata.blob_hash);
|
||||
if let Some(bytes) = blob(server, hash.as_slice()).await? {
|
||||
let path = format!(
|
||||
"{base}mail/{}/{}.eml",
|
||||
folder.split('/').map(segment).collect::<Vec<_>>().join("/"),
|
||||
Id::from(message.document_id)
|
||||
);
|
||||
out.add(path, &bytes, &address, "email", &folder, Some(received as i64), false)?;
|
||||
}
|
||||
}
|
||||
|
||||
// Calendars, contacts and files, by their DAV paths
|
||||
for (sync, kind) in [
|
||||
(SyncCollection::Calendar, "event"),
|
||||
(SyncCollection::AddressBook, "contact"),
|
||||
(SyncCollection::FileNode, "file"),
|
||||
] {
|
||||
let resources = server
|
||||
.fetch_dav_resources(account_id, account_id, sync)
|
||||
.await
|
||||
.caused_by(trc::location!())?;
|
||||
for path in resources.paths.iter() {
|
||||
let Some(resource) = resources.resources.get(path.resource_idx) else {
|
||||
continue;
|
||||
};
|
||||
let folder = path.path.rsplit_once('/').map(|(f, _)| f).unwrap_or_default();
|
||||
let zip_path = |ext: Option<&str>| {
|
||||
let mut p = format!(
|
||||
"{base}{}/{}",
|
||||
match kind {
|
||||
"event" => "calendar",
|
||||
"contact" => "contacts",
|
||||
_ => "files",
|
||||
},
|
||||
path.path.split('/').map(segment).collect::<Vec<_>>().join("/")
|
||||
);
|
||||
if let Some(ext) = ext
|
||||
&& !p.ends_with(ext)
|
||||
{
|
||||
p.push_str(ext);
|
||||
}
|
||||
p
|
||||
};
|
||||
use common::DavResourceMetadata as M;
|
||||
match &resource.data {
|
||||
M::CalendarEvent { start, .. } => {
|
||||
if !keeping.covers_event(Some((*start).max(0) as u64)) {
|
||||
continue;
|
||||
}
|
||||
let Some(event_) = data
|
||||
.get_value::<Archive<AlignedBytes>>(ValueKey::archive(
|
||||
account_id,
|
||||
Collection::CalendarEvent,
|
||||
resource.document_id,
|
||||
))
|
||||
.await?
|
||||
else {
|
||||
continue;
|
||||
};
|
||||
let event = event_.unarchive::<CalendarEvent>().caused_by(trc::location!())?;
|
||||
let text = event.data.event.to_string();
|
||||
out.add(zip_path(Some(".ics")), text.as_bytes(), &address, kind, folder, Some(*start), false)?;
|
||||
}
|
||||
M::ContactCard { .. } => {
|
||||
let Some(card_) = data
|
||||
.get_value::<Archive<AlignedBytes>>(ValueKey::archive(
|
||||
account_id,
|
||||
Collection::ContactCard,
|
||||
resource.document_id,
|
||||
))
|
||||
.await?
|
||||
else {
|
||||
continue;
|
||||
};
|
||||
let card = card_.unarchive::<ContactCard>().caused_by(trc::location!())?;
|
||||
let mut text = String::with_capacity(256);
|
||||
let _ = card.card.write_to(&mut text, server.core.groupware.vcard_version);
|
||||
out.add(zip_path(Some(".vcf")), text.as_bytes(), &address, kind, folder, None, false)?;
|
||||
}
|
||||
M::File { size: Some(_), .. } => {
|
||||
let Some(file_) = data
|
||||
.get_value::<Archive<AlignedBytes>>(ValueKey::archive(
|
||||
account_id,
|
||||
Collection::FileNode,
|
||||
resource.document_id,
|
||||
))
|
||||
.await?
|
||||
else {
|
||||
continue;
|
||||
};
|
||||
let file = file_.unarchive::<FileNode>().caused_by(trc::location!())?;
|
||||
let Some(props) = file.file.as_ref() else {
|
||||
continue;
|
||||
};
|
||||
let hash = types::blob_hash::BlobHash::from(&props.blob_hash);
|
||||
if let Some(bytes) = blob(server, hash.as_slice()).await? {
|
||||
out.add(zip_path(None), &bytes, &address, kind, folder, None, false)?;
|
||||
}
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// What the hold keeps of what was deleted
|
||||
for (id, item) in records::of_account(data, server.registry(), account_id).await? {
|
||||
if !is_held_until(item.archived_until().timestamp().max(0) as u64) {
|
||||
continue;
|
||||
}
|
||||
let (kind, ext, date) = match &item {
|
||||
ArchivedItem::Email(e) => ("email", ".eml", Some(e.received_at.timestamp())),
|
||||
ArchivedItem::CalendarEvent(e) => ("event", ".ics", e.start_time.map(|t| t.timestamp())),
|
||||
ArchivedItem::ContactCard(_) => ("contact", ".vcf", None),
|
||||
ArchivedItem::FileNode(_) => ("file", "", None),
|
||||
ArchivedItem::SieveScript(_) => ("sieve", ".sieve", None),
|
||||
};
|
||||
// Kept by this hold, not only by another one over the same account
|
||||
let in_range = match kind {
|
||||
"event" => keeping.covers_event(date.map(|d| d.max(0) as u64)),
|
||||
_ => keeping.covers(date.map(|d| d.max(0) as u64)),
|
||||
};
|
||||
if !in_range {
|
||||
continue;
|
||||
}
|
||||
let name = match &item {
|
||||
ArchivedItem::FileNode(f) => segment(&f.name),
|
||||
ArchivedItem::SieveScript(s) => format!("{}{ext}", segment(&s.name)),
|
||||
_ => format!("{id}{ext}"),
|
||||
};
|
||||
if let Some(bytes) = blob(server, item.blob_id().hash.as_slice()).await? {
|
||||
out.add(format!("{base}archived/{kind}/{name}"), &bytes, &address, kind, "", date, true)?;
|
||||
}
|
||||
}
|
||||
}
|
||||
out.finish()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn names_are_safe_in_a_zip() {
|
||||
assert_eq!(segment("../etc/passwd"), "_etc_passwd");
|
||||
assert_eq!(segment(" "), "_");
|
||||
assert_eq!(segment("Q3 report.pdf"), "Q3 report.pdf");
|
||||
assert_eq!(csv("a,b"), "\"a,b\"");
|
||||
assert_eq!(csv("say \"hi\""), "\"say \"\"hi\"\"\"");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_zip_carries_its_manifest_and_its_hash() {
|
||||
let mut b = Builder::new();
|
||||
b.add("[email protected]/mail/INBOX/b.eml".into(), b"Subject: x\r\n\r\ny", "[email protected]", "email", "INBOX", Some(0), false)
|
||||
.unwrap();
|
||||
b.add("[email protected]/mail/INBOX/b.eml".into(), b"other", "[email protected]", "email", "INBOX", None, true)
|
||||
.unwrap();
|
||||
let (bytes, items) = b.finish().unwrap();
|
||||
assert_eq!(items, 2);
|
||||
let mut zip = zip::ZipArchive::new(Cursor::new(bytes)).unwrap();
|
||||
let mut manifest = String::new();
|
||||
std::io::Read::read_to_string(&mut zip.by_name("manifest.csv").unwrap(), &mut manifest).unwrap();
|
||||
assert!(manifest.contains("[email protected]/mail/INBOX/b (2).eml"), "{manifest}");
|
||||
let mut hash = String::new();
|
||||
std::io::Read::read_to_string(&mut zip.by_name("manifest.sha256").unwrap(), &mut hash).unwrap();
|
||||
assert!(hash.starts_with(&hex(&Sha256::digest(manifest.as_bytes()))));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,294 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2026 Coffey Labs
|
||||
*
|
||||
* SPDX-License-Identifier: AGPL-3.0-only
|
||||
*/
|
||||
|
||||
//! `inbuxa:HoldExport` (audit-hold-lock spec, LH-12): starting a collection
|
||||
//! of what a hold keeps, and seeing how it went. The ZIP is the creator's
|
||||
//! blob, to download once it's ready. Creating one is audited, with its
|
||||
//! reason (AU-1.9, AU-12).
|
||||
|
||||
use common::{Server, auth::AccessToken};
|
||||
use inbuxa_features::hold::{self, Export, ExportStatus};
|
||||
use jmap_proto::{
|
||||
error::set::SetError,
|
||||
method::{
|
||||
get::{GetRequest, GetResponse},
|
||||
set::{SetRequest, SetResponse},
|
||||
},
|
||||
object::inbuxa_hold_export::{
|
||||
HoldExport, HoldExportProperty as P, HoldExportSetArguments, HoldExportValue,
|
||||
},
|
||||
types::date::UTCDate,
|
||||
};
|
||||
use jmap_tools::{Key, Map, Value};
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::str::FromStr;
|
||||
use store::write::now;
|
||||
use types::id::Id;
|
||||
|
||||
type LValue = Value<'static, P, HoldExportValue>;
|
||||
|
||||
const ALL: &[P] = &[
|
||||
P::Id,
|
||||
P::HoldId,
|
||||
P::AccountIds,
|
||||
P::Reason,
|
||||
P::Status,
|
||||
P::CreatedAt,
|
||||
P::CreatedBy,
|
||||
P::FinishedAt,
|
||||
P::BlobId,
|
||||
P::Size,
|
||||
P::Items,
|
||||
P::Sha256,
|
||||
P::Error,
|
||||
];
|
||||
|
||||
fn date(seconds: u64) -> LValue {
|
||||
Value::Str(UTCDate::from_timestamp(seconds as i64).to_string().into())
|
||||
}
|
||||
|
||||
fn opt_text(value: &Option<String>) -> LValue {
|
||||
value.as_ref().map_or(Value::Null, |v| Value::Str(v.clone().into()))
|
||||
}
|
||||
|
||||
fn to_value(export: &Export, properties: &[P]) -> LValue {
|
||||
let mut out = Map::with_capacity(properties.len());
|
||||
for property in properties {
|
||||
let value = match property {
|
||||
P::Id => Value::Element(HoldExportValue::Id(Id::from(export.id))),
|
||||
P::HoldId => Value::Str(Id::from(export.hold_id).to_string().into()),
|
||||
P::AccountIds => Value::Array(
|
||||
export
|
||||
.accounts
|
||||
.iter()
|
||||
.map(|id| Value::Str(Id::from(*id).to_string().into()))
|
||||
.collect(),
|
||||
),
|
||||
P::Reason => Value::Str(export.reason.clone().into()),
|
||||
P::Status => Value::Str(
|
||||
match export.status {
|
||||
ExportStatus::Running => "running",
|
||||
ExportStatus::Ready => "ready",
|
||||
ExportStatus::Failed => "failed",
|
||||
}
|
||||
.into(),
|
||||
),
|
||||
P::CreatedAt => date(export.created_at),
|
||||
P::CreatedBy => Value::Str(export.created_by.clone().into()),
|
||||
P::FinishedAt => export.finished_at.map_or(Value::Null, date),
|
||||
P::BlobId => opt_text(&export.blob_id),
|
||||
P::Size => Value::Number(export.size.into()),
|
||||
P::Items => Value::Number(export.items.into()),
|
||||
P::Sha256 => opt_text(&export.sha256),
|
||||
P::Error => opt_text(&export.error),
|
||||
};
|
||||
out.insert_unchecked(Key::Property(property.clone()), value);
|
||||
}
|
||||
Value::Object(out)
|
||||
}
|
||||
|
||||
/// `inbuxa:HoldExport/get`: every export, newest first.
|
||||
pub async fn get(
|
||||
server: &Server,
|
||||
mut request: GetRequest<HoldExport>,
|
||||
) -> trc::Result<GetResponse<HoldExport>> {
|
||||
let properties = request.unwrap_properties(ALL);
|
||||
let (ids, not_found) = request.unwrap_ids(server.core.jmap.get_max_objects)?;
|
||||
let mut response = GetResponse {
|
||||
account_id: request.account_id.into(),
|
||||
state: None,
|
||||
list: Vec::new(),
|
||||
not_found,
|
||||
};
|
||||
let mut exports = hold::exports(server.store()).await?;
|
||||
exports.reverse();
|
||||
match ids {
|
||||
None => response
|
||||
.list
|
||||
.extend(exports.iter().map(|e| to_value(e, &properties))),
|
||||
Some(ids) => {
|
||||
for id in ids {
|
||||
match exports.iter().find(|e| u64::from(e.id) == id.id()) {
|
||||
Some(export) => response.list.push(to_value(export, &properties)),
|
||||
None => response.push_not_found(id),
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(response)
|
||||
}
|
||||
|
||||
fn invalid(property: P, why: &str) -> SetError<P> {
|
||||
SetError::invalid_properties()
|
||||
.with_property(property)
|
||||
.with_description(why.to_string())
|
||||
}
|
||||
|
||||
/// `inbuxa:HoldExport/set`: create starts an export; nothing else is
|
||||
/// allowed. The request layer records it with its reason.
|
||||
pub async fn set(
|
||||
server: &Server,
|
||||
access_token: &AccessToken,
|
||||
mut request: SetRequest<'_, HoldExport>,
|
||||
) -> trc::Result<SetResponse<HoldExport>> {
|
||||
let mut response = SetResponse::from_request(&request, server.core.jmap.set_max_objects)?;
|
||||
let arguments: HoldExportSetArguments = std::mem::take(&mut request.arguments);
|
||||
let data = server.store();
|
||||
let actor = server.audit_actor(access_token).await;
|
||||
|
||||
'create: for (client_id, value) in request.unwrap_create() {
|
||||
let mut hold_id = None;
|
||||
let mut accounts = Vec::new();
|
||||
let mut reason = arguments.reason.clone();
|
||||
for (key, value) in value.into_expanded_object() {
|
||||
match (&key, &value) {
|
||||
(Key::Property(P::HoldId), Value::Str(id)) => {
|
||||
hold_id = Id::from_str(id).ok().and_then(|id| u32::try_from(id.id()).ok())
|
||||
}
|
||||
(Key::Property(P::AccountIds), Value::Array(items)) => {
|
||||
for item in items {
|
||||
match item {
|
||||
Value::Str(id) => match Id::from_str(id) {
|
||||
Ok(id) => accounts.push(id.document_id()),
|
||||
Err(_) => {
|
||||
response.not_created.append(
|
||||
client_id,
|
||||
invalid(P::AccountIds, "accountIds must be account ids."),
|
||||
);
|
||||
continue 'create;
|
||||
}
|
||||
},
|
||||
_ => {
|
||||
response.not_created.append(
|
||||
client_id,
|
||||
invalid(P::AccountIds, "accountIds must be account ids."),
|
||||
);
|
||||
continue 'create;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
(Key::Property(P::Reason), Value::Str(r)) => reason = Some(r.to_string()),
|
||||
_ => {
|
||||
response.not_created.append(
|
||||
client_id,
|
||||
SetError::invalid_properties().with_property(key.clone().into_owned()),
|
||||
);
|
||||
continue 'create;
|
||||
}
|
||||
}
|
||||
}
|
||||
let Some(reason) = reason
|
||||
.map(|r| r.trim().chars().take(500).collect::<String>())
|
||||
.filter(|r| !r.is_empty())
|
||||
else {
|
||||
response.not_created.append(
|
||||
client_id,
|
||||
invalid(P::Reason, "Say why: a reason is required and is kept in the audit log."),
|
||||
);
|
||||
continue;
|
||||
};
|
||||
let hold = match hold_id {
|
||||
Some(id) => hold::get(data, id).await?,
|
||||
None => None,
|
||||
};
|
||||
let Some(hold) = hold else {
|
||||
response
|
||||
.not_created
|
||||
.append(client_id, invalid(P::HoldId, "No such legal hold."));
|
||||
continue;
|
||||
};
|
||||
if !hold.is_active() {
|
||||
response.not_created.append(
|
||||
client_id,
|
||||
invalid(P::HoldId, "That hold was released; export while a hold is in place."),
|
||||
);
|
||||
continue;
|
||||
}
|
||||
let Some(created_by_id) = actor.account_id else {
|
||||
response
|
||||
.not_created
|
||||
.append(client_id, SetError::forbidden().with_description("Sign in as a person to export."));
|
||||
continue;
|
||||
};
|
||||
accounts.sort_unstable();
|
||||
accounts.dedup();
|
||||
let export = Export {
|
||||
id: 0,
|
||||
hold_id: hold.id,
|
||||
accounts,
|
||||
reason,
|
||||
created_at: now(),
|
||||
created_by: actor.name.clone(),
|
||||
created_by_id,
|
||||
status: ExportStatus::Running,
|
||||
finished_at: None,
|
||||
blob_id: None,
|
||||
size: 0,
|
||||
items: 0,
|
||||
sha256: None,
|
||||
error: None,
|
||||
};
|
||||
let id = hold::create_export(data, &export).await?;
|
||||
let export = Export { id, ..export };
|
||||
|
||||
// The collection runs on its own; get says when it's ready
|
||||
let server = server.clone();
|
||||
tokio::spawn(async move {
|
||||
let mut done = export.clone();
|
||||
match crate::inbuxa::hold_export::build(&server, &hold, &export.accounts).await {
|
||||
Ok((bytes, items)) => match server.put_jmap_blob(export.created_by_id, &bytes).await {
|
||||
Ok(blob) => {
|
||||
done.status = ExportStatus::Ready;
|
||||
done.blob_id = Some(blob.to_string());
|
||||
done.size = bytes.len() as u64;
|
||||
done.items = items as u64;
|
||||
done.sha256 = Some(
|
||||
Sha256::digest(&bytes).iter().map(|b| format!("{b:02x}")).collect(),
|
||||
);
|
||||
}
|
||||
Err(err) => {
|
||||
done.status = ExportStatus::Failed;
|
||||
done.error = Some(err.to_string());
|
||||
}
|
||||
},
|
||||
Err(err) => {
|
||||
done.status = ExportStatus::Failed;
|
||||
done.error = Some(
|
||||
err.value_as_str(trc::Key::Details)
|
||||
.map(str::to_string)
|
||||
.unwrap_or_else(|| err.to_string()),
|
||||
);
|
||||
}
|
||||
}
|
||||
done.finished_at = Some(now());
|
||||
if let Err(err) = hold::update_export(server.store(), &done).await {
|
||||
trc::error!(err.details("Failed to save a legal hold export's result"));
|
||||
}
|
||||
});
|
||||
|
||||
let mut out = Map::with_capacity(1);
|
||||
out.insert_unchecked(
|
||||
Key::Property(P::Id),
|
||||
Value::Element(HoldExportValue::Id(Id::from(id))),
|
||||
);
|
||||
response.created.insert(client_id, Value::Object(out));
|
||||
}
|
||||
|
||||
for (id, _) in request.unwrap_update() {
|
||||
response.not_updated.append(
|
||||
id,
|
||||
SetError::forbidden().with_description("An export can't be changed; start a new one."),
|
||||
);
|
||||
}
|
||||
for id in request.unwrap_destroy() {
|
||||
response.not_destroyed.append(
|
||||
id,
|
||||
SetError::forbidden().with_description("Exports stay listed; the file expires on its own."),
|
||||
);
|
||||
}
|
||||
Ok(response)
|
||||
}
|
||||
@@ -10,6 +10,8 @@
|
||||
pub mod access;
|
||||
pub mod account_lock;
|
||||
pub mod legal_hold;
|
||||
pub mod hold_export;
|
||||
pub mod hold_export_api;
|
||||
pub mod audit;
|
||||
pub mod audit_log;
|
||||
pub mod ai_limits;
|
||||
|
||||
Reference in New Issue
Block a user