inbuxa/inbuxa-migrate
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

v1.0.9

Browse Source
This commit is contained in:
Maurus Decimus
2026-08-22 18:46:54 +02:00
parent 1cc633f39c
commit 423b761617
34 changed files with 1936 additions and 123 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/sync/export/flat.rs
+22 -8
View File
@@ -48,6 +48,9 @@ pub fn reconcile(
};
let targets = target_get_all(net, ty).map_err(Error::from)?;
let mut default_claimed = targets
.iter()
.any(|t| t.get("isDefault").and_then(Value::as_bool).unwrap_or(false));
let mut tmatched = std::collections::HashSet::new();
let mut to_create: Vec<(i64, bool)> = Vec::new();
@@ -99,14 +102,25 @@ pub fn reconcile(
maps.insert(ty, local, JmapId(id.clone()));
counts.created += 1;
if to_create.iter().any(|(l, d)| *l == local && *d) {
let mut req = crate::jmap::request::Request::new();
req.call(
format!("{}/set", ty.jmap_name()),
json!({ "accountId": net.account, "onSuccessSetIsDefault": id }),
"d",
);
if let Err(e) = req.send(&net.client, &net.api) {
logger.warn(&format!("{} isDefault not set: {e}", ty.jmap_name()));
if default_claimed {
logger.warn(&format!(
"{} {}: the target account already has a default, leaving it alone",
ty.jmap_name(),
id
));
} else {
let mut req = crate::jmap::request::Request::new();
req.call(
format!("{}/set", ty.jmap_name()),
json!({ "accountId": net.account, "onSuccessSetIsDefault": id }),
"d",
);
match req.send(&net.client, &net.api) {
Ok(_) => default_claimed = true,
Err(e) => {
logger.warn(&format!("{} isDefault not set: {e}", ty.jmap_name()));
}
}
}
}
}
src/sync/export/tree.rs
+1 -1
View File
@@ -442,7 +442,7 @@ fn build_create(
.map_err(Error::from)?;
let mut wire = fnrow.wire;
if let (crate::jmap::wire::file_node::NodeType::File, Some(blob_local)) =
(&wire.node_type, fnrow.blob_local_id)
(wire.effective_node_type(), fnrow.blob_local_id)
{
let ct = wire
.media_type
src/sync/import_imap/coordinator.rs
+30 -5
View File
@@ -20,7 +20,7 @@ use crate::error::Error;
use crate::imap::client::{ConnectMode, ImapClient};
use crate::imap::command;
use crate::imap::error::ImapError;
use crate::imap::name::encode_mailbox_name_with;
use crate::imap::name::{alternate_mailbox_name, encode_mailbox_name_with};
use crate::imap::response::{NamespaceEntry, Untagged};
use crate::imap::retry::{
BackoffState, Disposition, RetryPolicy, classify, is_negotiation_failure,
@@ -111,6 +111,33 @@ pub(super) fn control_run_collect(
}
}
pub(super) fn select_folder(
client: &mut ImapClient,
ctx: &ControlCtx,
folder: &str,
) -> Result<crate::imap::CollectedResponse, Error> {
let utf8_accept = client.utf8_accept();
let wire_name = encode_mailbox_name_with(folder, utf8_accept);
let first = control_run_collect(client, ctx, &command::select(&wire_name));
let Err(Error::Partial(_)) = &first else {
return first;
};
let Some(alternate) = alternate_mailbox_name(folder, utf8_accept) else {
return first;
};
match control_run_collect(client, ctx, &command::select(&alternate)) {
Ok(r) => {
log_at(
ctx.logger,
LEVEL_DEFAULT,
&format!("folder {folder:?}: selected with the alternate mailbox-name encoding"),
);
Ok(r)
}
Err(_) => first,
}
}
pub(super) fn call_with_retry<F, T>(
client: &mut ImapClient,
ctx: &ControlCtx,
@@ -713,8 +740,7 @@ fn reconcile_folder(
}
}
let wire_name = encode_mailbox_name_with(&folder.name, client.utf8_accept());
let resp = control_run_collect(client, control_ctx, &command::select(&wire_name))?;
let resp = select_folder(client, control_ctx, &folder.name)?;
let mut uidvalidity: u32 = 0;
let mut uidnext: u32 = 0;
for u in &resp.untagged {
@@ -1123,8 +1149,7 @@ fn dry_run_summary(
}
for folder in folders {
let wire = encode_mailbox_name_with(&folder.name, client.utf8_accept());
let select_resp = match control_run_collect(client, control_ctx, &command::select(&wire)) {
let select_resp = match select_folder(client, control_ctx, &folder.name) {
Ok(r) => r,
Err(e) => {
log_at(
src/sync/import_imap/pool.rs
+11 -3
View File
@@ -13,7 +13,7 @@ use crossbeam_channel::{Receiver, Sender, unbounded};
use crate::imap::client::{ConnectMode, ImapClient};
use crate::imap::command;
use crate::imap::error::ImapError;
use crate::imap::name::encode_mailbox_name_with;
use crate::imap::name::{alternate_mailbox_name, encode_mailbox_name_with};
use crate::imap::response::Untagged;
use crate::imap::retry::{BackoffState, Disposition, RetryPolicy, classify};
use crate::imap::transport::Connector;
@@ -231,8 +231,16 @@ fn run_one_job(
event_tx: &Sender<FetchEvent>,
) -> Result<(), ImapError> {
if current_folder.as_deref() != Some(job.folder.as_str()) {
let wire = encode_mailbox_name_with(&job.folder, client.utf8_accept());
client.run_collect(&command::select(&wire))?;
let utf8_accept = client.utf8_accept();
let wire = encode_mailbox_name_with(&job.folder, utf8_accept);
if let Err(e) = client.run_collect(&command::select(&wire)) {
match alternate_mailbox_name(&job.folder, utf8_accept) {
Some(alternate) if matches!(e, ImapError::No(_)) => {
client.run_collect(&command::select(&alternate))?;
}
_ => return Err(e),
}
}
*current_folder = Some(job.folder.clone());
}
let set = command::format_uid_set(&job.uids, true);
src/sync/import_jmap.rs
+23 -7
View File
@@ -29,6 +29,7 @@ use crate::jmap::http::{Auth, HttpClient};
use crate::jmap::request::{get_all, get_changes, get_objects, get_state, query_all_ids};
use crate::jmap::session::{Limits, Session};
use crate::jmap::wire::JmapId;
use crate::jmap::wire::common::parse_utc_date;
use crate::jmap::wire::email::Email;
use crate::jmap::wire::file_node::{FileNode, NodeType};
use crate::jmap::wire::sieve_script::SieveScript;
@@ -478,7 +479,7 @@ fn insert_objects(
continue;
}
};
match insert_one(&tx, ty, source_id, obj, &blobs) {
match insert_one(&tx, ty, source_id, obj, &blobs, logger) {
Ok(local_id) => {
db::ids::insert(&tx, source_id, ty, &jmap_id, local_id)
.map_err(|e| Error::Partial(e.to_string()))?;
@@ -649,6 +650,7 @@ fn insert_one(
source_id: i64,
obj: &Value,
blobs: &HashMap<String, Vec<u8>>,
logger: &Logger,
) -> Result<i64, JmapError> {
let resolver = DbResolver { conn, source_id };
match ty {
@@ -673,14 +675,28 @@ fn insert_one(
mapping::insert_participant_identity(conn, &w)
}
ObjectType::Email => {
let w: Email = serde_json::from_value(obj.clone())?;
let mut w: Email = serde_json::from_value(obj.clone())?;
let data = blobs
.get(&w.blob_id.0)
.ok_or_else(|| JmapError::malformed("email blob missing"))?;
let blob_local = db::blobs::intern_blob(conn, data)?;
let mm = crate::sync::keys::index_to_json(
&crate::sync::emailmeta::email_index_from_blob(data),
);
let (index, date_header) = crate::sync::emailmeta::email_meta_from_blob(data);
let mm = crate::sync::keys::index_to_json(&index);
if w.received_at.is_none() {
w.received_at = date_header.as_deref().and_then(parse_utc_date);
let id = obj.get("id").and_then(Value::as_str).unwrap_or("?");
let raw = obj
.get("receivedAt")
.and_then(Value::as_str)
.unwrap_or("<missing>");
logger.warn(&format!(
"Email {id}: unusable receivedAt {raw:?}; falling back to {}",
match &w.received_at {
Some(_) => "the Date header",
None => "1970-01-01T00:00:00Z",
}
));
}
mapping::insert_email(conn, &w, blob_local, &mm, &resolver)
}
ObjectType::SieveScript => {
@@ -693,7 +709,7 @@ fn insert_one(
}
ObjectType::FileNode => {
let w: FileNode = serde_json::from_value(obj.clone())?;
let blob_local = match (&w.node_type, &w.blob_id) {
let blob_local = match (w.effective_node_type(), &w.blob_id) {
(NodeType::File, Some(b)) => {
let data = blobs
.get(&b.0)
@@ -934,7 +950,7 @@ fn update_one(
}
ObjectType::FileNode => {
let w: FileNode = serde_json::from_value(obj.clone())?;
let blob_local = match (&w.node_type, &w.blob_id) {
let blob_local = match (w.effective_node_type(), &w.blob_id) {
(NodeType::File, Some(b)) => {
let data = blobs
.get(&b.0)
src/sync/import_jmap/mapping.rs
+29 -10
View File
@@ -8,6 +8,7 @@ use indexmap::IndexMap;
use rusqlite::{Connection, Row, params};
use serde_json::{Map, Value};
use crate::db::defaults::unique_default;
use crate::jmap::blob::{BlobWalkError, InlineShape, import_blob_ids, inline_blob_data_uris};
use crate::jmap::error::JmapError;
use crate::jmap::wire::JmapId;
@@ -168,7 +169,7 @@ pub fn insert_email(
VALUES (?1, ?2, ?3, ?4, ?5)",
params![
blob_local_id,
format_utc(&wire.received_at)?,
format_utc(&wire.received_at.unwrap_or(time::OffsetDateTime::UNIX_EPOCH))?,
id_array_json(&mailbox_locals),
Value::Array(keywords.iter().map(|k| Value::from(k.as_str())).collect()).to_string(),
message_match
@@ -248,6 +249,7 @@ pub fn insert_sieve_script(
pub const SIEVE_SELECT: &str = "SELECT id, name, is_active, blob_id FROM sieve_scripts";
pub fn insert_address_book(conn: &Connection, wire: &AddressBook) -> Result<i64, JmapError> {
let is_default = unique_default(conn, ObjectType::AddressBook, wire.is_default, None)?;
conn.execute(
"INSERT INTO address_books (name, description, sort_order, is_default, is_subscribed)
VALUES (?1, ?2, ?3, ?4, ?5)",
@@ -255,7 +257,7 @@ pub fn insert_address_book(conn: &Connection, wire: &AddressBook) -> Result<i64,
wire.name,
wire.description,
wire.sort_order,
wire.is_default as i64,
is_default as i64,
wire.is_subscribed as i64
],
)?;
@@ -266,6 +268,7 @@ pub const ADDRESS_BOOK_SELECT: &str =
"SELECT id, name, description, sort_order, is_default, is_subscribed FROM address_books";
pub fn insert_calendar(conn: &Connection, wire: &Calendar) -> Result<i64, JmapError> {
let is_default = unique_default(conn, ObjectType::Calendar, wire.is_default, None)?;
conn.execute(
"INSERT INTO calendars (name, description, color, sort_order, is_subscribed, is_visible,
is_default, include_in_availability, default_alerts_with_time,
@@ -278,7 +281,7 @@ pub fn insert_calendar(conn: &Connection, wire: &Calendar) -> Result<i64, JmapEr
wire.sort_order,
wire.is_subscribed as i64,
wire.is_visible as i64,
wire.is_default as i64,
is_default as i64,
wire.include_in_availability,
opt_json(&wire.default_alerts_with_time)?,
opt_json(&wire.default_alerts_without_time)?,
@@ -296,10 +299,11 @@ pub fn insert_participant_identity(
conn: &Connection,
wire: &ParticipantIdentity,
) -> Result<i64, JmapError> {
let is_default = unique_default(conn, ObjectType::ParticipantIdentity, wire.is_default, None)?;
conn.execute(
"INSERT INTO participant_identities (name, calendar_address, is_default)
VALUES (?1, ?2, ?3)",
params![wire.name, wire.calendar_address, wire.is_default as i64],
params![wire.name, wire.calendar_address, is_default as i64],
)?;
Ok(conn.last_insert_rowid())
}
@@ -314,7 +318,7 @@ pub fn insert_file_node(
resolver: &impl LocalResolver,
) -> Result<i64, JmapError> {
let parent = opt_parent(resolver, ObjectType::FileNode, &wire.parent_id);
let node_type = serde_json::to_value(wire.node_type)?
let node_type = serde_json::to_value(wire.effective_node_type())?
.as_str()
.unwrap_or("file")
.to_owned();
@@ -495,6 +499,12 @@ pub fn update_address_book(
local_id: i64,
wire: &AddressBook,
) -> Result<bool, JmapError> {
let is_default = unique_default(
conn,
ObjectType::AddressBook,
wire.is_default,
Some(local_id),
)?;
let n = conn.execute(
"UPDATE address_books SET name = ?1, description = ?2, sort_order = ?3, is_default = ?4,
is_subscribed = ?5
@@ -504,7 +514,7 @@ pub fn update_address_book(
wire.name,
wire.description,
wire.sort_order,
wire.is_default as i64,
is_default as i64,
wire.is_subscribed as i64,
local_id
],
@@ -517,6 +527,7 @@ pub fn update_calendar(
local_id: i64,
wire: &Calendar,
) -> Result<bool, JmapError> {
let is_default = unique_default(conn, ObjectType::Calendar, wire.is_default, Some(local_id))?;
let n = conn.execute(
"UPDATE calendars SET name = ?1, description = ?2, color = ?3, sort_order = ?4,
is_subscribed = ?5, is_visible = ?6, is_default = ?7, include_in_availability = ?8,
@@ -533,7 +544,7 @@ pub fn update_calendar(
wire.sort_order,
wire.is_subscribed as i64,
wire.is_visible as i64,
wire.is_default as i64,
is_default as i64,
wire.include_in_availability,
opt_json(&wire.default_alerts_with_time)?,
opt_json(&wire.default_alerts_without_time)?,
@@ -549,13 +560,19 @@ pub fn update_participant_identity(
local_id: i64,
wire: &ParticipantIdentity,
) -> Result<bool, JmapError> {
let is_default = unique_default(
conn,
ObjectType::ParticipantIdentity,
wire.is_default,
Some(local_id),
)?;
let n = conn.execute(
"UPDATE participant_identities SET name = ?1, calendar_address = ?2, is_default = ?3
WHERE id = ?4 AND (name IS NOT ?1 OR calendar_address IS NOT ?2 OR is_default IS NOT ?3)",
params![
wire.name,
wire.calendar_address,
wire.is_default as i64,
is_default as i64,
local_id
],
)?;
@@ -570,7 +587,7 @@ pub fn update_file_node(
resolver: &impl LocalResolver,
) -> Result<bool, JmapError> {
let parent = opt_parent(resolver, ObjectType::FileNode, &wire.parent_id);
let node_type = serde_json::to_value(wire.node_type)?
let node_type = serde_json::to_value(wire.effective_node_type())?
.as_str()
.unwrap_or("file")
.to_owned();
@@ -828,7 +845,9 @@ pub fn row_to_file_node(
),
None => None,
};
let node_type = serde_json::from_value(Value::from(row.get::<_, String>(2)?))?;
let node_type = Some(serde_json::from_value(Value::from(
row.get::<_, String>(2)?,
))?);
let target: Option<Vec<String>> = from_opt_json(row.get::<_, Option<String>>(4)?)?;
let blob_local_id: Option<i64> = row.get(3)?;
Ok(FileNodeRow {
src/sync/import_maildir/messages.rs
+80 -3
View File
@@ -16,7 +16,7 @@ use time::format_description::well_known::Rfc3339;
use crate::db;
use crate::sync::TypeCounts;
use crate::sync::emailmeta::email_index_from_blob;
use crate::sync::emailmeta::email_meta_from_blob;
use crate::sync::keys::index_to_json;
use super::keywords::{Translation, flags_from_filename, translate_flags, unique_id_from_filename};
@@ -149,9 +149,10 @@ pub fn insert_new(
if translation.has_trashed_flag && !ctx.include_deleted {
return Ok(None);
}
let received_at = format_unix_rfc3339(entry.mtime_unix);
let (index, date_header) = email_meta_from_blob(&bytes);
let received_at = pick_received_at(&entry.filename, entry.mtime_unix, date_header.as_deref());
let blob_id = db::blobs::intern_blob(tx, &bytes)?;
let message_match = index_to_json(&email_index_from_blob(&bytes));
let message_match = index_to_json(&index);
let mailbox_ids = Value::Array(vec![Value::from(ctx.mailbox_local)]);
let keywords = keywords_json(&translation);
tx.execute(
@@ -170,6 +171,37 @@ pub fn insert_new(
Ok(Some(local_id))
}
pub fn delivery_time_from_filename(filename: &str) -> Option<u64> {
let digits: String = filename.chars().take_while(char::is_ascii_digit).collect();
if digits.is_empty() || digits.len() > 12 {
return None;
}
let secs: u64 = digits.parse().ok()?;
if secs == 0 {
return None;
}
let now = OffsetDateTime::now_utc().unix_timestamp().max(0) as u64;
if secs > now.saturating_add(86_400) {
return None;
}
Some(secs)
}
pub fn pick_received_at(filename: &str, mtime_unix: u64, date_header: Option<&str>) -> String {
if let Some(secs) = delivery_time_from_filename(filename) {
return format_unix_rfc3339(secs);
}
if let Some(d) = date_header {
return d.to_owned();
}
if mtime_unix > 0 {
return format_unix_rfc3339(mtime_unix);
}
OffsetDateTime::now_utc()
.format(&Rfc3339)
.unwrap_or_else(|_| "1970-01-01T00:00:00Z".to_owned())
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum PresentOutcome {
Unchanged,
@@ -721,4 +753,49 @@ mod tests {
);
tx.commit().unwrap();
}
#[test]
fn filename_delivery_time_wins_over_mtime() {
let got = pick_received_at(
"1763065233.M12345P678.host,S=1234:2,S",
99,
Some("2025-05-12T10:00:00+02:00"),
);
assert_eq!(got, "2025-11-13T20:20:33Z");
}
#[test]
fn date_header_used_when_the_filename_carries_no_timestamp() {
let got = pick_received_at(
"M12345P678.host:2,S",
1763065233,
Some("2025-05-12T10:00:00+02:00"),
);
assert_eq!(got, "2025-05-12T10:00:00+02:00");
}
#[test]
fn mtime_is_the_last_resort_before_now() {
let got = pick_received_at("M12345P678.host:2,S", 1763065233, None);
assert_eq!(got, "2025-11-13T20:20:33Z");
}
#[test]
fn implausible_filename_timestamps_are_rejected() {
assert_eq!(delivery_time_from_filename("0.M1P1.host"), None);
assert_eq!(delivery_time_from_filename("M1P1.host"), None);
assert_eq!(
delivery_time_from_filename("99999999999999.M1P1.host"),
None
);
let far_future = (OffsetDateTime::now_utc().unix_timestamp() as u64) + 400_000;
assert_eq!(
delivery_time_from_filename(&format!("{far_future}.M1P1.host")),
None
);
assert_eq!(
delivery_time_from_filename("1763065233.M12345P678.host"),
Some(1763065233)
);
}
}