inbuxa/inbuxa-migrate
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

EWS fixes

Browse Source
This commit is contained in:
Maurus Decimus
2026-06-10 20:26:41 +02:00
parent f7371cc2ae
commit adbd0276a6
11 changed files with 631 additions and 280 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/sync/import_exchange_ews/calendar.rs
+23 -14
View File
@@ -19,7 +19,7 @@ use crate::sync::TypeCounts;
use super::attachments::{fetch_attachments, intern_attachment};
use super::folders::FolderPlan;
use super::items::{
EnumerationMode, ItemRunCtx, delete_vanished, enumerate_folder, get_items, plan_for,
EnumerationMode, ItemRunCtx, delete_vanished, enumerate_folder, for_each_fetched_item, plan_for,
};
pub fn reconcile_all(
@@ -41,9 +41,18 @@ pub fn reconcile_all(
Some(id) => id,
None => continue,
};
if let Err(e) = reconcile_one(conn, ctx, folder_id, local_folder_id, counts) {
ctx.logger
.warn(&format!("calendar folder {} failed: {}", folder_id.id, e));
if let Err(e) = reconcile_one(
conn,
ctx,
folder_id,
&folder.folder.display_name,
local_folder_id,
counts,
) {
ctx.logger.warn(&format!(
"calendar folder {:?} failed: {}",
folder.folder.display_name, e
));
counts.failed += 1;
}
}
@@ -54,6 +63,7 @@ fn reconcile_one(
conn: &mut Connection,
ctx: &ItemRunCtx<'_>,
folder: &crate::exchange_ews::types::FolderId,
folder_name: &str,
local_folder_id: i64,
counts: &mut TypeCounts,
) -> Result<(), Error> {
@@ -81,8 +91,8 @@ fn reconcile_one(
let plan = plan_for(&outcome, &local);
if ctx.logger.enabled(LEVEL_PROGRESS) {
eprintln!(
"EWS calendar folder {}: new={} changed={} vanished={}",
folder.id,
"EWS calendar folder {:?}: new={} changed={} vanished={}",
folder_name,
plan.new.len(),
plan.present_changed.len(),
plan.vanished.len()
@@ -93,9 +103,7 @@ fn reconcile_one(
to_fetch.push(id.clone());
}
if !to_fetch.is_empty() {
let outcome = get_items(ctx, ItemShape::CalendarItem, &to_fetch).map_err(Error::from)?;
counts.failed += outcome.failed_items;
for msg in outcome.messages {
let failed_items = for_each_fetched_item(ctx, ItemShape::CalendarItem, &to_fetch, |msg| {
if !msg.success {
if matches!(
msg.response_code,
@@ -107,19 +115,19 @@ fn reconcile_one(
ctx.logger
.warn(&format!("GetItem (calendar) error: {}", msg.response_code));
}
continue;
return Ok(());
}
let parsed = parse_calendar_item(&msg.inner_xml).map_err(Error::from)?;
if parsed.id.id.is_empty() {
counts.failed += 1;
continue;
return Ok(());
}
if matches!(
parsed.calendar_item_type,
Some(CalendarItemType::Occurrence) | Some(CalendarItemType::Exception)
) {
counts.skipped += 1;
continue;
return Ok(());
}
let existing = plan
.present_changed
@@ -134,8 +142,9 @@ fn reconcile_one(
&folder.id,
existing,
counts,
)?;
}
)
})?;
counts.failed += failed_items;
}
delete_vanished(
conn,
src/sync/import_exchange_ews/contacts.rs
+22 -13
View File
@@ -18,7 +18,7 @@ use crate::sync::TypeCounts;
use super::attachments::fetch_contact_photo;
use super::folders::FolderPlan;
use super::items::{
EnumerationMode, ItemRunCtx, delete_vanished, enumerate_folder, get_items, plan_for,
EnumerationMode, ItemRunCtx, delete_vanished, enumerate_folder, for_each_fetched_item, plan_for,
};
pub fn reconcile_all(
@@ -40,9 +40,18 @@ pub fn reconcile_all(
Some(id) => id,
None => continue,
};
if let Err(e) = reconcile_one(conn, ctx, folder_id, local_folder_id, counts) {
ctx.logger
.warn(&format!("contact folder {} failed: {}", folder_id.id, e));
if let Err(e) = reconcile_one(
conn,
ctx,
folder_id,
&folder.folder.display_name,
local_folder_id,
counts,
) {
ctx.logger.warn(&format!(
"contact folder {:?} failed: {}",
folder.folder.display_name, e
));
counts.failed += 1;
}
}
@@ -53,6 +62,7 @@ fn reconcile_one(
conn: &mut Connection,
ctx: &ItemRunCtx<'_>,
folder: &crate::exchange_ews::types::FolderId,
folder_name: &str,
local_folder_id: i64,
counts: &mut TypeCounts,
) -> Result<(), Error> {
@@ -74,8 +84,8 @@ fn reconcile_one(
let plan = plan_for(&outcome, &local);
if ctx.logger.enabled(LEVEL_PROGRESS) {
eprintln!(
"EWS contacts folder {}: new={} changed={} vanished={}",
folder.id,
"EWS contacts folder {:?}: new={} changed={} vanished={}",
folder_name,
plan.new.len(),
plan.present_changed.len(),
plan.vanished.len()
@@ -86,9 +96,7 @@ fn reconcile_one(
to_fetch.push(id.clone());
}
if !to_fetch.is_empty() {
let outcome = get_items(ctx, ItemShape::Contact, &to_fetch).map_err(Error::from)?;
counts.failed += outcome.failed_items;
for msg in outcome.messages {
let failed_items = for_each_fetched_item(ctx, ItemShape::Contact, &to_fetch, |msg| {
if !msg.success {
if matches!(
msg.response_code,
@@ -100,12 +108,12 @@ fn reconcile_one(
ctx.logger
.warn(&format!("GetItem (contact) error: {}", msg.response_code));
}
continue;
return Ok(());
}
let parsed = parse_contact_item(&msg.inner_xml).map_err(Error::from)?;
if parsed.id.id.is_empty() {
counts.failed += 1;
continue;
return Ok(());
}
let existing = plan
.present_changed
@@ -120,8 +128,9 @@ fn reconcile_one(
&folder.id,
existing,
counts,
)?;
}
)
})?;
counts.failed += failed_items;
}
delete_vanished(
conn,
src/sync/import_exchange_ews/items.rs
+23
View File
@@ -368,6 +368,29 @@ pub fn get_items(
})
}
pub fn for_each_fetched_item<F>(
ctx: &ItemRunCtx<'_>,
shape: ItemShape,
ids: &[ItemId],
mut on_message: F,
) -> Result<u64, Error>
where
F: FnMut(crate::exchange_ews::parse::ResponseMessage) -> Result<(), Error>,
{
let batch = ctx.batch_size.max(1);
let workers = ctx.connections.clamp(1, 8);
let window = batch.saturating_mul(workers).max(batch);
let mut failed_items = 0u64;
for win in ids.chunks(window) {
let outcome = get_items(ctx, shape, win).map_err(Error::from)?;
failed_items = failed_items.saturating_add(outcome.failed_items);
for msg in outcome.messages {
on_message(msg)?;
}
}
Ok(failed_items)
}
pub fn delete_vanished(
conn: &mut rusqlite::Connection,
source_id: i64,
src/sync/import_exchange_ews/messages.rs
+22 -13
View File
@@ -21,7 +21,7 @@ use crate::sync::keys::index_to_json;
use super::folders::FolderPlan;
use super::items::{
EnumerationMode, ItemRunCtx, delete_vanished, enumerate_folder, get_items, plan_for,
EnumerationMode, ItemRunCtx, delete_vanished, enumerate_folder, for_each_fetched_item, plan_for,
};
pub fn reconcile_all(
@@ -43,9 +43,18 @@ pub fn reconcile_all(
Some(id) => id,
None => continue,
};
if let Err(e) = reconcile_one_folder(conn, ctx, folder_id, local_folder_id, counts) {
ctx.logger
.warn(&format!("email folder {} failed: {}", folder_id.id, e));
if let Err(e) = reconcile_one_folder(
conn,
ctx,
folder_id,
&folder.folder.display_name,
local_folder_id,
counts,
) {
ctx.logger.warn(&format!(
"email folder {:?} failed: {}",
folder.folder.display_name, e
));
counts.failed += 1;
}
}
@@ -56,6 +65,7 @@ fn reconcile_one_folder(
conn: &mut Connection,
ctx: &ItemRunCtx<'_>,
folder: &crate::exchange_ews::types::FolderId,
folder_name: &str,
local_folder_id: i64,
counts: &mut TypeCounts,
) -> Result<(), Error> {
@@ -73,8 +83,8 @@ fn reconcile_one_folder(
let plan = plan_for(&outcome, &local);
if ctx.logger.enabled(LEVEL_PROGRESS) {
eprintln!(
"EWS folder {}: new={} changed={} vanished={} unchanged={}",
folder.id,
"EWS folder {:?}: new={} changed={} vanished={} unchanged={}",
folder_name,
plan.new.len(),
plan.present_changed.len(),
plan.vanished.len(),
@@ -86,9 +96,7 @@ fn reconcile_one_folder(
to_fetch.push(id.clone());
}
if !to_fetch.is_empty() {
let outcome = get_items(ctx, ItemShape::Message, &to_fetch).map_err(Error::from)?;
counts.failed += outcome.failed_items;
for msg in outcome.messages {
let failed_items = for_each_fetched_item(ctx, ItemShape::Message, &to_fetch, |msg| {
if !msg.success {
if matches!(
msg.response_code,
@@ -102,12 +110,12 @@ fn reconcile_one_folder(
msg.response_code, msg.message_text
));
}
continue;
return Ok(());
}
let parsed = parse_message_item(&msg.inner_xml).map_err(Error::from)?;
if parsed.id.id.is_empty() {
counts.failed += 1;
continue;
return Ok(());
}
let existing = plan
.present_changed
@@ -122,8 +130,9 @@ fn reconcile_one_folder(
&folder.id,
existing,
counts,
)?;
}
)
})?;
counts.failed += failed_items;
}
delete_vanished(
conn,
src/sync/import_jmap.rs
+13 -15
View File
@@ -430,13 +430,6 @@ fn insert_objects(
logger: &Logger,
counts: &mut TypeCounts,
) -> Result<(), Error> {
let blob_refs = blob_references(ty, &objects);
let blobs = if blob_refs.is_empty() {
HashMap::new()
} else {
download_blobs(net, blob_refs, threads, logger, counts)
};
let existing_parents: HashSet<String> =
if matches!(ty, ObjectType::Mailbox | ObjectType::FileNode) {
db::ids::jmap_to_local(&ctx.conn, source_id, ty)
@@ -451,6 +444,12 @@ fn insert_objects(
}
for batch in ordered.chunks(200) {
let blob_refs = blob_references(ty, batch.iter().map(|&i| &objects[i]));
let blobs = if blob_refs.is_empty() {
HashMap::new()
} else {
download_blobs(net, blob_refs, threads, logger, counts)
};
let tx = ctx
.conn
.unchecked_transaction()
@@ -481,7 +480,7 @@ fn insert_objects(
Ok(())
}
fn blob_references(ty: ObjectType, objects: &[Value]) -> Vec<BlobRef> {
fn blob_references<'a>(ty: ObjectType, objects: impl Iterator<Item = &'a Value>) -> Vec<BlobRef> {
let mut seen: HashSet<String> = HashSet::new();
let mut refs = Vec::new();
match ty {
@@ -845,14 +844,13 @@ fn update_objects(
logger: &Logger,
counts: &mut TypeCounts,
) -> Result<(), Error> {
let blob_refs = blob_references(ty, &objects);
let blobs = if blob_refs.is_empty() {
HashMap::new()
} else {
download_blobs(net, blob_refs, threads, logger, counts)
};
for batch in objects.chunks(200) {
let blob_refs = blob_references(ty, batch.iter());
let blobs = if blob_refs.is_empty() {
HashMap::new()
} else {
download_blobs(net, blob_refs, threads, logger, counts)
};
let tx = ctx
.conn
.unchecked_transaction()