Import upstream v0.16.22, stripped
Upstream commit: 474dd0229cb20cf513036619781ed97bd8073c3f Enterprise-only files removed or emptied: 63 Enterprise-only snippets removed: 117 in 50 files Dangling module declarations removed: 5 Cargo edits turning enterprise off: 14 Verification: clean Enterprise feature gates left for rebuilt features: 19 in 18 files Produced by tools/fork/strip.py. The full report is in docs/fork/strip-reports/ on main.
This commit is contained in:
@@ -0,0 +1,299 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
|
||||
*
|
||||
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
||||
*/
|
||||
|
||||
use crate::destroy::destroy_subspace;
|
||||
use common::{Server, manager::SPAM_TRAINER_KEY};
|
||||
use registry::{
|
||||
schema::{
|
||||
prelude::{ObjectType, Property},
|
||||
structs::{ArchivedEmail, ArchivedItem},
|
||||
},
|
||||
types::{EnumImpl, ObjectImpl, datetime::UTCDateTime, id::ObjectId},
|
||||
};
|
||||
use spam_filter::modules::classifier::SpamTrainer;
|
||||
use store::{
|
||||
Deserialize, IterateParams, SUBSPACE_BLOB_LINK, SUBSPACE_DIRECTORY, SUBSPACE_QUOTA,
|
||||
SUBSPACE_REPORT_IN, SUBSPACE_REPORT_OUT, SUBSPACE_TASK_QUEUE, SUBSPACE_TELEMETRY_METRIC,
|
||||
SUBSPACE_TELEMETRY_SPAN, Serialize, SerializeInfallible, U32_LEN, U64_LEN,
|
||||
search::{SearchField, SearchFilter, SearchQuery},
|
||||
write::{
|
||||
AlignedBytes, AnyClass, AnyKey, Archive, Archiver, BatchBuilder, RegistryClass,
|
||||
SearchIndex, ValueClass, key::DeserializeBigEndian, now,
|
||||
},
|
||||
};
|
||||
use trc::AddContext;
|
||||
use types::{blob::BlobId, blob_hash::BLOB_HASH_LEN};
|
||||
|
||||
const LEGACY_SUBSPACE_BLOB_EXTRA: u8 = b'j'; // Now SUBSPACE_DELETED_ITEMS
|
||||
const LEGACY_SUBSPACE_BITMAP_ID: u8 = b'b'; // Now SUBSPACE_REGISTRY_IDX
|
||||
const LEGACY_SUBSPACE_SETTINGS: u8 = b's'; // Now SUBSPACE_REGISTRY
|
||||
const LEGACY_SUBSPACE_FTS_INDEX: u8 = b'g'; // Now SUBSPACE_REGISTRY_PK
|
||||
const LEGACY_SUBSPACE_TELEMETRY_INDEX: u8 = b'w'; // Now SUBSPACE_SPAM_SAMPLES
|
||||
|
||||
pub async fn migrate_v0_16(server: &Server) -> trc::Result<()> {
|
||||
// Delete tracing index
|
||||
server
|
||||
.search_store()
|
||||
.unindex(
|
||||
SearchQuery::new(SearchIndex::Tracing)
|
||||
.with_filter(SearchFilter::lt(SearchField::Id, u64::MAX)),
|
||||
)
|
||||
.await
|
||||
.caused_by(trc::location!())?;
|
||||
|
||||
// Delete old quotas
|
||||
server
|
||||
.store()
|
||||
.delete_range(
|
||||
AnyKey {
|
||||
subspace: SUBSPACE_QUOTA,
|
||||
key: vec![0x04],
|
||||
},
|
||||
AnyKey {
|
||||
subspace: SUBSPACE_QUOTA,
|
||||
key: vec![0x05],
|
||||
},
|
||||
)
|
||||
.await
|
||||
.caused_by(trc::location!())?;
|
||||
|
||||
// Destroy old and incompatible subspaces
|
||||
for namespace in [
|
||||
LEGACY_SUBSPACE_BLOB_EXTRA,
|
||||
LEGACY_SUBSPACE_TELEMETRY_INDEX,
|
||||
LEGACY_SUBSPACE_SETTINGS,
|
||||
LEGACY_SUBSPACE_BITMAP_ID,
|
||||
LEGACY_SUBSPACE_FTS_INDEX,
|
||||
SUBSPACE_REPORT_IN,
|
||||
SUBSPACE_REPORT_OUT,
|
||||
SUBSPACE_DIRECTORY,
|
||||
SUBSPACE_TELEMETRY_METRIC,
|
||||
SUBSPACE_TELEMETRY_SPAN,
|
||||
SUBSPACE_TASK_QUEUE,
|
||||
] {
|
||||
destroy_subspace(server.store(), namespace)
|
||||
.await
|
||||
.caused_by(trc::location!())?;
|
||||
}
|
||||
|
||||
// Migrate blob links
|
||||
migrate_blob_links(server).await?;
|
||||
|
||||
// Migrate spam model
|
||||
migrate_spam_model(server).await?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn migrate_spam_model(server: &Server) -> trc::Result<()> {
|
||||
let Some(mut trainer) = server
|
||||
.blob_store()
|
||||
.get_blob(SPAM_TRAINER_KEY, 0..usize::MAX)
|
||||
.await
|
||||
.and_then(|archive| match archive {
|
||||
Some(archive) => <Archive<AlignedBytes> as Deserialize>::deserialize(&archive)
|
||||
.and_then(|archive| archive.deserialize_untrusted::<SpamTrainer>())
|
||||
.map(Some),
|
||||
None => Ok(None),
|
||||
})
|
||||
.caused_by(trc::location!())?
|
||||
else {
|
||||
return Ok(());
|
||||
};
|
||||
|
||||
if trainer.last_id == 0 {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
if let Some(config) = &server.core.spam.classifier {
|
||||
trainer.reservoir.ham.total_seen = std::cmp::min(
|
||||
config.min_ham_samples + config.reservoir_capacity as u64,
|
||||
trainer.reservoir.ham.total_seen,
|
||||
);
|
||||
trainer.reservoir.spam.total_seen = std::cmp::min(
|
||||
config.min_spam_samples + config.reservoir_capacity as u64,
|
||||
trainer.reservoir.spam.total_seen,
|
||||
);
|
||||
} else {
|
||||
trainer.reservoir.ham.total_seen = std::cmp::min(
|
||||
trainer.reservoir.ham.buffer.len() as u64,
|
||||
trainer.reservoir.ham.total_seen,
|
||||
);
|
||||
trainer.reservoir.spam.total_seen = std::cmp::min(
|
||||
trainer.reservoir.spam.buffer.len() as u64,
|
||||
trainer.reservoir.spam.total_seen,
|
||||
);
|
||||
}
|
||||
|
||||
trainer.reservoir.ham.buffer.clear();
|
||||
trainer.reservoir.spam.buffer.clear();
|
||||
trainer.last_id = 0;
|
||||
|
||||
server
|
||||
.blob_store()
|
||||
.put_blob(
|
||||
SPAM_TRAINER_KEY,
|
||||
&Archiver::new(trainer)
|
||||
.serialize()
|
||||
.caused_by(trc::location!())?,
|
||||
server.core.email.compression,
|
||||
)
|
||||
.await
|
||||
.caused_by(trc::location!())?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn migrate_blob_links(server: &Server) -> trc::Result<()> {
|
||||
let mut delete_keys = Vec::new();
|
||||
let mut archived_items = Vec::new();
|
||||
let now = now();
|
||||
|
||||
server
|
||||
.store()
|
||||
.iterate(
|
||||
IterateParams::new(
|
||||
AnyKey {
|
||||
subspace: SUBSPACE_BLOB_LINK,
|
||||
key: vec![0u8],
|
||||
},
|
||||
AnyKey {
|
||||
subspace: SUBSPACE_BLOB_LINK,
|
||||
key: vec![u8::MAX; 32],
|
||||
},
|
||||
),
|
||||
|key, value| {
|
||||
const TEMP_LINK: usize = BLOB_HASH_LEN + U32_LEN + U64_LEN;
|
||||
|
||||
const QUOTA_LINK: u8 = 0;
|
||||
const UNDELETE_LINK: u8 = 1;
|
||||
const SPAM_SAMPLE_LINK: u8 = 2;
|
||||
|
||||
if key.len() == TEMP_LINK && value.len() == 1 {
|
||||
let until = key.deserialize_be_u64(BLOB_HASH_LEN + U32_LEN)?;
|
||||
if until > now {
|
||||
let account_id = key.deserialize_be_u32(BLOB_HASH_LEN)?;
|
||||
let hash = types::blob_hash::BlobHash::try_from_hash_slice(
|
||||
key.get(0..BLOB_HASH_LEN).ok_or_else(|| {
|
||||
trc::Error::corrupted_key(key, None, trc::location!())
|
||||
})?,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
match value.first().copied() {
|
||||
Some(UNDELETE_LINK) => {
|
||||
archived_items.push((key.to_vec(), account_id, hash, until));
|
||||
}
|
||||
Some(SPAM_SAMPLE_LINK | QUOTA_LINK) => {
|
||||
delete_keys.push(key.to_vec());
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(true)
|
||||
},
|
||||
)
|
||||
.await
|
||||
.caused_by(trc::location!())?;
|
||||
|
||||
// Delete spam samples and quota links
|
||||
let mut batch = BatchBuilder::new();
|
||||
for key in delete_keys {
|
||||
batch.clear(ValueClass::Any(AnyClass {
|
||||
subspace: SUBSPACE_BLOB_LINK,
|
||||
key,
|
||||
}));
|
||||
|
||||
if batch.is_large_batch() {
|
||||
server
|
||||
.store()
|
||||
.write(batch.build_all())
|
||||
.await
|
||||
.caused_by(trc::location!())?;
|
||||
batch = BatchBuilder::new();
|
||||
}
|
||||
}
|
||||
if !batch.is_empty() {
|
||||
server
|
||||
.store()
|
||||
.write(batch.build_all())
|
||||
.await
|
||||
.caused_by(trc::location!())?;
|
||||
}
|
||||
|
||||
// Migrate spam samples
|
||||
let mut batch = BatchBuilder::new();
|
||||
let id_gen = &server.inner.data.registry_id_gen;
|
||||
let mut last_id = 0;
|
||||
for (key, account_id, blob_hash, until) in archived_items {
|
||||
let item = ArchivedItem::Email(ArchivedEmail {
|
||||
account_id: account_id.into(),
|
||||
blob_id: BlobId::new(blob_hash.clone(), Default::default()),
|
||||
archived_until: UTCDateTime::from_timestamp(until as i64),
|
||||
archived_at: UTCDateTime::now(),
|
||||
from: "Unavailable".to_string(),
|
||||
received_at: UTCDateTime::now(),
|
||||
subject: "...".to_string(),
|
||||
size: 0,
|
||||
})
|
||||
.to_pickled_vec();
|
||||
let object_id = ObjectType::ArchivedItem.to_id();
|
||||
|
||||
loop {
|
||||
let new_id = id_gen.generate();
|
||||
if new_id != last_id {
|
||||
last_id = new_id;
|
||||
break;
|
||||
} else {
|
||||
tokio::time::sleep(std::time::Duration::from_millis(10)).await;
|
||||
}
|
||||
}
|
||||
let item_id = last_id;
|
||||
|
||||
batch
|
||||
.set(
|
||||
ValueClass::Any(AnyClass {
|
||||
subspace: SUBSPACE_BLOB_LINK,
|
||||
key,
|
||||
}),
|
||||
ObjectId::new(ObjectType::ArchivedItem, item_id.into()).serialize(),
|
||||
)
|
||||
.set(
|
||||
ValueClass::Registry(RegistryClass::Index {
|
||||
index_id: Property::AccountId.to_id(),
|
||||
object_id,
|
||||
item_id,
|
||||
key: (account_id as u64).serialize(),
|
||||
}),
|
||||
vec![],
|
||||
)
|
||||
.set(
|
||||
ValueClass::Registry(RegistryClass::Item { object_id, item_id }),
|
||||
item,
|
||||
);
|
||||
|
||||
if batch.is_large_batch() {
|
||||
server
|
||||
.store()
|
||||
.write(batch.build_all())
|
||||
.await
|
||||
.caused_by(trc::location!())?;
|
||||
batch = BatchBuilder::new();
|
||||
}
|
||||
}
|
||||
|
||||
if !batch.is_empty() {
|
||||
server
|
||||
.store()
|
||||
.write(batch.build_all())
|
||||
.await
|
||||
.caused_by(trc::location!())?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
Reference in New Issue
Block a user