Import upstream v0.16.22, stripped
Upstream commit: 474dd0229cb20cf513036619781ed97bd8073c3f Enterprise-only files removed or emptied: 63 Enterprise-only snippets removed: 117 in 50 files Dangling module declarations removed: 5 Cargo edits turning enterprise off: 14 Verification: clean Enterprise feature gates left for rebuilt features: 19 in 18 files Produced by tools/fork/strip.py. The full report is in docs/fork/strip-reports/ on main.
This commit is contained in:
@@ -0,0 +1,395 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
|
||||
*
|
||||
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
||||
*/
|
||||
|
||||
use crate::message::metadata::{
|
||||
ArchivedMessageMetadataContents, ArchivedMetadataHeaderValue, ArchivedMetadataPartType,
|
||||
PART_ENCODING_BASE64, PART_ENCODING_QP, PART_SIZE_MASK,
|
||||
};
|
||||
use jmap_proto::object::email::{EmailProperty, EmailValue};
|
||||
use jmap_tools::{Map, Value};
|
||||
use mail_parser::{HeaderValue, MessagePart, MimeHeaders, PartType};
|
||||
use types::blob::BlobId;
|
||||
use utils::chained_bytes::ChainedBytes;
|
||||
|
||||
use super::headers::HeaderToValue;
|
||||
|
||||
pub trait ToBodyPart {
|
||||
fn to_body_part(
|
||||
&self,
|
||||
part_id: u32,
|
||||
properties: &[EmailProperty],
|
||||
raw_message: &ChainedBytes<'_>,
|
||||
blob_id: &BlobId,
|
||||
blob_body_offset: isize,
|
||||
) -> Value<'static, EmailProperty, EmailValue>;
|
||||
}
|
||||
|
||||
impl ToBodyPart for Vec<MessagePart<'_>> {
|
||||
fn to_body_part(
|
||||
&self,
|
||||
part_id: u32,
|
||||
properties: &[EmailProperty],
|
||||
raw_message: &ChainedBytes<'_>,
|
||||
blob_id: &BlobId,
|
||||
blob_body_offset: isize,
|
||||
) -> Value<'static, EmailProperty, EmailValue> {
|
||||
let mut parts = vec![part_id].into_iter();
|
||||
let mut parts_stack = Vec::new();
|
||||
let mut subparts = Vec::with_capacity(1);
|
||||
|
||||
loop {
|
||||
if let Some((part_id, part)) = parts
|
||||
.next()
|
||||
.map(|part_id| (part_id, &self[part_id as usize]))
|
||||
{
|
||||
let mut values = Map::with_capacity(properties.len());
|
||||
let multipart = if let PartType::Multipart(parts) = &part.body {
|
||||
parts.into()
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
for property in properties {
|
||||
let value = match property {
|
||||
EmailProperty::PartId if multipart.is_none() => part_id.to_string().into(),
|
||||
EmailProperty::BlobId if multipart.is_none() => {
|
||||
let base_offset = blob_id.start_offset() as isize + blob_body_offset;
|
||||
BlobId::new_section(
|
||||
blob_id.hash.clone(),
|
||||
blob_id.class.clone(),
|
||||
(part.offset_body as isize + base_offset) as usize,
|
||||
(part.offset_end as isize + base_offset) as usize,
|
||||
part.encoding as u8,
|
||||
)
|
||||
.into()
|
||||
}
|
||||
EmailProperty::Size if multipart.is_none() => match &part.body {
|
||||
PartType::Text(text) | PartType::Html(text) => text.len(),
|
||||
PartType::Binary(bin) | PartType::InlineBinary(bin) => bin.len(),
|
||||
PartType::Message(message) => message.root_part().raw_len() as usize,
|
||||
PartType::Multipart(_) => 0,
|
||||
}
|
||||
.into(),
|
||||
EmailProperty::Name => part.attachment_name().map(|v| v.to_string()).into(),
|
||||
EmailProperty::Type => part
|
||||
.content_type()
|
||||
.map(|ct| {
|
||||
ct.subtype()
|
||||
.map(|st| format!("{}/{}", ct.ctype(), st))
|
||||
.unwrap_or_else(|| ct.ctype().to_string())
|
||||
})
|
||||
.or_else(|| match &part.body {
|
||||
PartType::Text(_) => Some("text/plain".to_string()),
|
||||
PartType::Html(_) => Some("text/html".to_string()),
|
||||
PartType::Message(_) => Some("message/rfc822".to_string()),
|
||||
_ => None,
|
||||
})
|
||||
.into(),
|
||||
EmailProperty::Charset => part
|
||||
.content_type()
|
||||
.and_then(|ct| ct.attribute("charset"))
|
||||
.or(match &part.body {
|
||||
PartType::Text(_) | PartType::Html(_) => Some("us-ascii"),
|
||||
_ => None,
|
||||
})
|
||||
.map(|v| v.to_string())
|
||||
.into(),
|
||||
EmailProperty::Disposition => part
|
||||
.content_disposition()
|
||||
.map(|cd| cd.ctype())
|
||||
.map(|v| v.to_string())
|
||||
.into(),
|
||||
EmailProperty::Cid => part.content_id().map(|v| v.to_string()).into(),
|
||||
EmailProperty::Language => match part.content_language() {
|
||||
HeaderValue::Text(text) => vec![text.to_string()].into(),
|
||||
HeaderValue::TextList(list) => list
|
||||
.iter()
|
||||
.map(|text| text.to_string().into())
|
||||
.collect::<Vec<Value<'static, EmailProperty, EmailValue>>>()
|
||||
.into(),
|
||||
_ => Value::Null,
|
||||
},
|
||||
EmailProperty::Location => {
|
||||
part.content_location().map(|v| v.to_string()).into()
|
||||
}
|
||||
EmailProperty::Header(_) => {
|
||||
part.headers.header_to_value(property, raw_message)
|
||||
}
|
||||
EmailProperty::Headers => part.headers.headers_to_value(raw_message),
|
||||
EmailProperty::SubParts => continue,
|
||||
_ => Value::Null,
|
||||
};
|
||||
values.insert_unchecked(property.clone(), value);
|
||||
}
|
||||
|
||||
subparts.push(values);
|
||||
|
||||
if let Some(multipart) = multipart {
|
||||
if parts_stack.len() == 10_000 {
|
||||
debug_assert!(false, "Too much nesting in message metadata");
|
||||
return Value::Null;
|
||||
}
|
||||
let multipart = multipart.clone();
|
||||
parts_stack.push((
|
||||
parts,
|
||||
std::mem::replace(&mut subparts, Vec::with_capacity(multipart.len())),
|
||||
));
|
||||
parts = multipart.into_iter();
|
||||
}
|
||||
} else if let Some((prev_parts, mut prev_subparts)) = parts_stack.pop() {
|
||||
prev_subparts
|
||||
.last_mut()
|
||||
.unwrap()
|
||||
.insert_unchecked(EmailProperty::SubParts, subparts);
|
||||
parts = prev_parts;
|
||||
subparts = prev_subparts;
|
||||
} else {
|
||||
return subparts.pop().map(Into::into).unwrap_or_default();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl ToBodyPart for ArchivedMessageMetadataContents {
|
||||
fn to_body_part(
|
||||
&self,
|
||||
part_id: u32,
|
||||
properties: &[EmailProperty],
|
||||
raw_message: &ChainedBytes<'_>,
|
||||
blob_id: &BlobId,
|
||||
blob_body_offset: isize,
|
||||
) -> Value<'static, EmailProperty, EmailValue> {
|
||||
let mut parts = vec![part_id].into_iter();
|
||||
let mut parts_stack = Vec::new();
|
||||
let mut subparts = Vec::with_capacity(1);
|
||||
|
||||
loop {
|
||||
if let Some((part_id, part)) = parts
|
||||
.next()
|
||||
.map(|part_id| (part_id, &self.parts[part_id as usize]))
|
||||
{
|
||||
let mut values = Map::with_capacity(properties.len());
|
||||
let multipart = if let ArchivedMetadataPartType::Multipart(parts) = &part.body {
|
||||
parts.into()
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
for property in properties {
|
||||
let value = match property {
|
||||
EmailProperty::PartId if multipart.is_none() => part_id.to_string().into(),
|
||||
EmailProperty::BlobId if multipart.is_none() => {
|
||||
let base_offset = blob_id.start_offset() as isize + blob_body_offset;
|
||||
let flags = part.flags.to_native();
|
||||
let encoding = if flags & PART_ENCODING_BASE64 != 0 {
|
||||
2
|
||||
} else if flags & PART_ENCODING_QP != 0 {
|
||||
1
|
||||
} else {
|
||||
0
|
||||
};
|
||||
BlobId::new_section(
|
||||
blob_id.hash.clone(),
|
||||
blob_id.class.clone(),
|
||||
(u32::from(part.offset_body) as isize + base_offset) as usize,
|
||||
(u32::from(part.offset_end) as isize + base_offset) as usize,
|
||||
encoding,
|
||||
)
|
||||
.into()
|
||||
}
|
||||
EmailProperty::Size if multipart.is_none() => {
|
||||
(part.flags.to_native() & PART_SIZE_MASK).into()
|
||||
}
|
||||
EmailProperty::Name => part.attachment_name().map(|v| v.to_string()).into(),
|
||||
EmailProperty::Type => part
|
||||
.content_type()
|
||||
.map(|ct| {
|
||||
ct.subtype()
|
||||
.map(|st| format!("{}/{}", ct.ctype(), st))
|
||||
.unwrap_or_else(|| ct.ctype().to_string())
|
||||
})
|
||||
.or_else(|| match &part.body {
|
||||
ArchivedMetadataPartType::Text => Some("text/plain".to_string()),
|
||||
ArchivedMetadataPartType::Html => Some("text/html".to_string()),
|
||||
ArchivedMetadataPartType::Message(_) => {
|
||||
Some("message/rfc822".to_string())
|
||||
}
|
||||
_ => None,
|
||||
})
|
||||
.into(),
|
||||
EmailProperty::Charset => {
|
||||
part.content_type()
|
||||
.and_then(|ct| ct.attribute("charset"))
|
||||
.or(match &part.body {
|
||||
ArchivedMetadataPartType::Text
|
||||
| ArchivedMetadataPartType::Html => Some("us-ascii"),
|
||||
_ => None,
|
||||
})
|
||||
.map(|v| v.to_string())
|
||||
.into()
|
||||
}
|
||||
EmailProperty::Disposition => part
|
||||
.content_disposition()
|
||||
.map(|cd| cd.ctype())
|
||||
.map(|v| v.to_string())
|
||||
.into(),
|
||||
EmailProperty::Cid => part.content_id().map(|v| v.to_string()).into(),
|
||||
EmailProperty::Language => match part.content_language() {
|
||||
ArchivedMetadataHeaderValue::Text(text) => {
|
||||
vec![text.to_string()].into()
|
||||
}
|
||||
ArchivedMetadataHeaderValue::TextList(list) => list
|
||||
.iter()
|
||||
.map(|text| text.to_string().into())
|
||||
.collect::<Vec<Value<'static, EmailProperty, EmailValue>>>()
|
||||
.into(),
|
||||
_ => Value::Null,
|
||||
},
|
||||
EmailProperty::Location => {
|
||||
part.content_location().map(|v| v.to_string()).into()
|
||||
}
|
||||
EmailProperty::Header(_) => part.header_to_value(property, raw_message),
|
||||
EmailProperty::Headers => part.headers_to_value(raw_message),
|
||||
EmailProperty::SubParts => continue,
|
||||
_ => Value::Null,
|
||||
};
|
||||
values.insert_unchecked(property.clone(), value);
|
||||
}
|
||||
|
||||
subparts.push(values);
|
||||
|
||||
if let Some(multipart) = multipart {
|
||||
if parts_stack.len() == 10_000 {
|
||||
debug_assert!(false, "Too much nesting in message metadata");
|
||||
return Value::Null;
|
||||
}
|
||||
let multipart = multipart
|
||||
.iter()
|
||||
.map(|id| u16::from(id) as u32)
|
||||
.collect::<Vec<_>>();
|
||||
parts_stack.push((
|
||||
parts,
|
||||
std::mem::replace(&mut subparts, Vec::with_capacity(multipart.len())),
|
||||
));
|
||||
parts = multipart.into_iter();
|
||||
}
|
||||
} else if let Some((prev_parts, mut prev_subparts)) = parts_stack.pop() {
|
||||
prev_subparts
|
||||
.last_mut()
|
||||
.unwrap()
|
||||
.insert_unchecked(EmailProperty::SubParts, subparts);
|
||||
parts = prev_parts;
|
||||
subparts = prev_subparts;
|
||||
} else {
|
||||
return subparts.pop().map(Into::into).unwrap_or_default();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub trait TruncateBody {
|
||||
fn truncate(&self, max_len: usize) -> (bool, String);
|
||||
}
|
||||
|
||||
impl TruncateBody for PartType<'_> {
|
||||
fn truncate(&self, max_len: usize) -> (bool, String) {
|
||||
match self {
|
||||
PartType::Text(text) => truncate_plain(text, max_len),
|
||||
PartType::Html(html) => truncate_html(html, max_len),
|
||||
PartType::Binary(bytes) | PartType::InlineBinary(bytes) => {
|
||||
PartType::Text(String::from_utf8_lossy(bytes)).truncate(max_len)
|
||||
}
|
||||
_ => (false, "".into()),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn truncate_plain(text: &str, mut max_len: usize) -> (bool, String) {
|
||||
if max_len != 0 && text.len() > max_len {
|
||||
let add_dots = max_len > 6;
|
||||
if add_dots {
|
||||
max_len -= 3;
|
||||
}
|
||||
let mut result = String::with_capacity(max_len);
|
||||
for ch in text.chars() {
|
||||
if ch != '\r' {
|
||||
if ch.len_utf8() + result.len() > max_len {
|
||||
break;
|
||||
}
|
||||
result.push(ch);
|
||||
}
|
||||
}
|
||||
if add_dots {
|
||||
result.push_str("...");
|
||||
}
|
||||
(true, result)
|
||||
} else {
|
||||
(false, text.replace('\r', ""))
|
||||
}
|
||||
}
|
||||
|
||||
pub fn truncate_html(html: &str, mut max_len: usize) -> (bool, String) {
|
||||
if max_len != 0 && html.len() > max_len {
|
||||
let add_dots = max_len > 6;
|
||||
if add_dots {
|
||||
max_len -= 3;
|
||||
}
|
||||
|
||||
let mut result = String::with_capacity(max_len);
|
||||
let mut in_tag = false;
|
||||
let mut in_comment = false;
|
||||
let mut last_tag_end_pos = 0;
|
||||
let mut cr_count = 0;
|
||||
for (pos, ch) in html.char_indices() {
|
||||
let mut set_last_tag = 0;
|
||||
match ch {
|
||||
'<' if !in_tag => {
|
||||
in_tag = true;
|
||||
if let Some("!--") = html.get(pos + 1..pos + 4) {
|
||||
in_comment = true;
|
||||
}
|
||||
set_last_tag = pos;
|
||||
}
|
||||
'>' if in_tag => {
|
||||
if in_comment {
|
||||
if let Some("--") = html.get(pos - 2..pos) {
|
||||
in_comment = false;
|
||||
in_tag = false;
|
||||
set_last_tag = pos + 1;
|
||||
}
|
||||
} else {
|
||||
in_tag = false;
|
||||
set_last_tag = pos + 1;
|
||||
}
|
||||
}
|
||||
'\r' => {
|
||||
cr_count += 1;
|
||||
continue;
|
||||
}
|
||||
_ => (),
|
||||
}
|
||||
if ch.len_utf8() + pos - cr_count > max_len {
|
||||
result.push_str(
|
||||
&html[0..if (in_tag || set_last_tag > 0) && last_tag_end_pos > 0 {
|
||||
last_tag_end_pos
|
||||
} else {
|
||||
pos
|
||||
}]
|
||||
.replace('\r', ""),
|
||||
);
|
||||
if add_dots {
|
||||
result.push_str("...");
|
||||
}
|
||||
break;
|
||||
} else if set_last_tag > 0 {
|
||||
last_tag_end_pos = set_last_tag;
|
||||
}
|
||||
}
|
||||
(true, result)
|
||||
} else {
|
||||
(false, html.replace('\r', ""))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,267 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
|
||||
*
|
||||
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
||||
*/
|
||||
|
||||
use super::{
|
||||
ingest::{EmailIngest, IngestedEmail},
|
||||
metadata::{MessageData, MessageMetadata},
|
||||
};
|
||||
use crate::{
|
||||
mailbox::UidMailbox,
|
||||
message::{
|
||||
index::extractors::VisitTextArchived,
|
||||
ingest::ThreadInfo,
|
||||
metadata::{
|
||||
MESSAGE_HAS_ATTACHMENT, MESSAGE_RECEIVED_MASK, MetadataHeaderName, MetadataHeaderValue,
|
||||
},
|
||||
},
|
||||
};
|
||||
use common::{Server, storage::index::ObjectIndexBuilder};
|
||||
use mail_parser::parsers::fields::thread::thread_name;
|
||||
use registry::{
|
||||
schema::{
|
||||
enums::IndexDocumentType,
|
||||
structs::{Task, TaskIndexDocument, TaskMergeThreads, TaskStatus},
|
||||
},
|
||||
types::map::Map,
|
||||
};
|
||||
use store::write::{BatchBuilder, IndexPropertyClass, ValueClass};
|
||||
use store::{
|
||||
ValueKey,
|
||||
write::{AlignedBytes, Archive},
|
||||
};
|
||||
use trc::AddContext;
|
||||
use types::{
|
||||
blob::{BlobClass, BlobId},
|
||||
collection::{Collection, SyncCollection},
|
||||
field::EmailField,
|
||||
keyword::Keyword,
|
||||
};
|
||||
use utils::cheeky_hash::CheekyHash;
|
||||
|
||||
pub enum CopyMessageError {
|
||||
NotFound,
|
||||
OverQuota,
|
||||
AlreadyExists(u32),
|
||||
}
|
||||
|
||||
pub trait EmailCopy: Sync + Send {
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
fn copy_message(
|
||||
&self,
|
||||
from_account_id: u32,
|
||||
from_message_id: u32,
|
||||
to_account_id: u32,
|
||||
mailboxes: Vec<u32>,
|
||||
keywords: Vec<Keyword>,
|
||||
received_at: Option<u64>,
|
||||
session_id: u64,
|
||||
) -> impl Future<Output = trc::Result<Result<IngestedEmail, CopyMessageError>>> + Send;
|
||||
}
|
||||
|
||||
impl EmailCopy for Server {
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
async fn copy_message(
|
||||
&self,
|
||||
from_account_id: u32,
|
||||
from_message_id: u32,
|
||||
to_account_id: u32,
|
||||
mailboxes: Vec<u32>,
|
||||
keywords: Vec<Keyword>,
|
||||
received_at: Option<u64>,
|
||||
session_id: u64,
|
||||
) -> trc::Result<Result<IngestedEmail, CopyMessageError>> {
|
||||
// Obtain metadata
|
||||
let mut metadata = if let Some(metadata) = self
|
||||
.store()
|
||||
.get_value::<Archive<AlignedBytes>>(ValueKey::property(
|
||||
from_account_id,
|
||||
Collection::Email,
|
||||
from_message_id,
|
||||
EmailField::Metadata,
|
||||
))
|
||||
.await?
|
||||
{
|
||||
metadata
|
||||
.deserialize::<MessageMetadata>()
|
||||
.caused_by(trc::location!())?
|
||||
} else {
|
||||
return Ok(Err(CopyMessageError::NotFound));
|
||||
};
|
||||
|
||||
// Check quota
|
||||
let size = metadata.root_part().offset_end;
|
||||
let to_account = self.account(to_account_id).await?;
|
||||
match self.has_available_quota(&to_account, size as u64).await {
|
||||
Ok(_) => (),
|
||||
Err(err) => {
|
||||
if err.matches(trc::EventType::Limit(trc::LimitEvent::Quota))
|
||||
|| err.matches(trc::EventType::Limit(trc::LimitEvent::TenantQuota))
|
||||
{
|
||||
trc::error!(err.account_id(to_account_id).span_id(session_id));
|
||||
return Ok(Err(CopyMessageError::OverQuota));
|
||||
} else {
|
||||
return Err(err);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Set receivedAt
|
||||
if let Some(received_at) = received_at {
|
||||
metadata.rcvd_attach = (metadata.rcvd_attach & MESSAGE_HAS_ATTACHMENT)
|
||||
| (received_at & MESSAGE_RECEIVED_MASK);
|
||||
}
|
||||
|
||||
// Obtain threadId
|
||||
let mut message_ids = Vec::new();
|
||||
let mut subject = "";
|
||||
for header in &metadata.contents[0].parts[0].headers {
|
||||
match &header.name {
|
||||
MetadataHeaderName::MessageId => {
|
||||
header.value.visit_text(|id| {
|
||||
if !id.is_empty() {
|
||||
message_ids.push(CheekyHash::new(id.as_bytes()));
|
||||
}
|
||||
});
|
||||
}
|
||||
MetadataHeaderName::InReplyTo
|
||||
| MetadataHeaderName::References
|
||||
| MetadataHeaderName::ResentMessageId => {
|
||||
header.value.visit_text(|id| {
|
||||
if !id.is_empty() {
|
||||
message_ids.push(CheekyHash::new(id.as_bytes()));
|
||||
}
|
||||
});
|
||||
}
|
||||
MetadataHeaderName::Subject if subject.is_empty() => {
|
||||
subject = thread_name(match &header.value {
|
||||
MetadataHeaderValue::Text(text) => text.as_ref(),
|
||||
MetadataHeaderValue::TextList(list) if !list.is_empty() => {
|
||||
list.first().unwrap().as_ref()
|
||||
}
|
||||
_ => "",
|
||||
});
|
||||
}
|
||||
_ => (),
|
||||
}
|
||||
}
|
||||
|
||||
message_ids.sort_unstable();
|
||||
message_ids.dedup();
|
||||
|
||||
// Obtain threadId
|
||||
let thread_result = self
|
||||
.find_thread_id(to_account_id, subject, &message_ids)
|
||||
.await
|
||||
.caused_by(trc::location!())?;
|
||||
|
||||
if let Some(&existing) = thread_result.duplicate_ids.first() {
|
||||
return Ok(Err(CopyMessageError::AlreadyExists(existing)));
|
||||
}
|
||||
|
||||
// Assign id
|
||||
let mut email = IngestedEmail {
|
||||
size: size as usize,
|
||||
..Default::default()
|
||||
};
|
||||
let blob_hash = metadata.blob_hash.clone();
|
||||
|
||||
// Assign IMAP UIDs
|
||||
let mut mailbox_ids = Vec::with_capacity(mailboxes.len());
|
||||
email.imap_uids = Vec::with_capacity(mailboxes.len());
|
||||
let mut ids = self
|
||||
.assign_email_ids(to_account_id, mailboxes.iter().copied(), true)
|
||||
.await
|
||||
.caused_by(trc::location!())?;
|
||||
let document_id = ids.next().unwrap();
|
||||
for (uid, mailbox_id) in ids.zip(mailboxes.iter().copied()) {
|
||||
mailbox_ids.push(UidMailbox::new(mailbox_id, uid));
|
||||
email.imap_uids.push(uid);
|
||||
}
|
||||
|
||||
// Prepare batch
|
||||
let mut batch = BatchBuilder::new();
|
||||
batch.with_account_id(to_account_id);
|
||||
|
||||
// Determine thread id
|
||||
let tenant_id = to_account.tenant_id();
|
||||
let thread_id = if let Some(thread_id) = thread_result.thread_id {
|
||||
thread_id
|
||||
} else {
|
||||
batch
|
||||
.with_collection(Collection::Thread)
|
||||
.with_document(document_id)
|
||||
.log_container_insert(SyncCollection::Thread);
|
||||
document_id
|
||||
};
|
||||
batch
|
||||
.with_collection(Collection::Email)
|
||||
.with_document(document_id)
|
||||
.custom(
|
||||
ObjectIndexBuilder::<(), _>::new()
|
||||
.with_tenant_id(tenant_id)
|
||||
.with_changes(MessageData {
|
||||
mailboxes: mailbox_ids.into_boxed_slice(),
|
||||
keywords: keywords.into_boxed_slice(),
|
||||
thread_id,
|
||||
size,
|
||||
}),
|
||||
)
|
||||
.caused_by(trc::location!())?
|
||||
.set(
|
||||
ValueClass::IndexProperty(IndexPropertyClass::Hash {
|
||||
property: EmailField::Threading.into(),
|
||||
hash: thread_result.thread_hash,
|
||||
}),
|
||||
ThreadInfo::serialize(thread_id, &message_ids),
|
||||
)
|
||||
.schedule_task(Task::IndexDocument(TaskIndexDocument {
|
||||
account_id: to_account_id.into(),
|
||||
document_id: document_id.into(),
|
||||
document_type: IndexDocumentType::Email,
|
||||
status: TaskStatus::now(),
|
||||
}));
|
||||
|
||||
// Merge threads if necessary
|
||||
if !thread_result.merge_ids.is_empty() {
|
||||
batch.schedule_task(Task::MergeThreads(TaskMergeThreads {
|
||||
account_id: to_account_id.into(),
|
||||
status: TaskStatus::now(),
|
||||
thread_name: thread_result.thread_hash.to_string(),
|
||||
message_ids: Map::new(message_ids.into_iter().map(|id| id.to_string()).collect()),
|
||||
}));
|
||||
}
|
||||
|
||||
metadata
|
||||
.index(&mut batch, true)
|
||||
.caused_by(trc::location!())?;
|
||||
|
||||
// Insert and obtain ids
|
||||
let change_id = self
|
||||
.store()
|
||||
.write(batch.build_all())
|
||||
.await
|
||||
.caused_by(trc::location!())?
|
||||
.last_change_id(to_account_id)?;
|
||||
|
||||
// Request indexing
|
||||
self.notify_task_queue();
|
||||
|
||||
// Update response
|
||||
email.document_id = document_id;
|
||||
email.thread_id = thread_id;
|
||||
email.change_id = change_id;
|
||||
email.blob_id = BlobId::new(
|
||||
blob_hash,
|
||||
BlobClass::Linked {
|
||||
account_id: to_account_id,
|
||||
collection: Collection::Email.into(),
|
||||
document_id,
|
||||
},
|
||||
);
|
||||
|
||||
Ok(Ok(email))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,598 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
|
||||
*
|
||||
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
||||
*/
|
||||
|
||||
use aes::cipher::{BlockModeEncrypt, KeyIvInit, block_padding::Pkcs7};
|
||||
use aes_gcm::{
|
||||
Aes256Gcm,
|
||||
aead::{AeadInOut, KeyInit},
|
||||
};
|
||||
use chacha20poly1305::ChaCha20Poly1305;
|
||||
use common::auth::{
|
||||
ACCOUNT_FLAG_ENCRYPT_ALGO_AES256, ACCOUNT_FLAG_ENCRYPT_ALGO_AES256_GCM,
|
||||
ACCOUNT_FLAG_ENCRYPT_ALGO_CHACHA20_POLY1305, ACCOUNT_FLAG_ENCRYPT_APPEND,
|
||||
ACCOUNT_FLAG_ENCRYPT_METHOD_PGP, ACCOUNT_FLAG_ENCRYPT_TRAIN_SPAM_FILTER, EncryptionKeys,
|
||||
};
|
||||
use mail_builder::{encoders::Base64Encoder, mime::make_boundary};
|
||||
use mail_parser::{Message, MimeHeaders, PartType};
|
||||
use openpgp::{
|
||||
parse::Parse,
|
||||
serialize::stream,
|
||||
types::{KeyFlags, SymmetricAlgorithm},
|
||||
};
|
||||
use rand::{RngCore, SeedableRng, rngs::StdRng};
|
||||
use rasn::Encoder;
|
||||
use rasn::types::{OctetString, Oid, SetOf};
|
||||
use rasn_cms::{
|
||||
AlgorithmIdentifier, AuthEnvelopedData, CONTENT_DATA, CONTENT_ENVELOPED_DATA, EncryptedContent,
|
||||
EncryptedContentInfo, EncryptedKey, EnvelopedData, IssuerAndSerialNumber,
|
||||
KeyTransRecipientInfo, RecipientIdentifier, RecipientInfo,
|
||||
algorithms::{AES128_CBC, AES256_CBC, RSA},
|
||||
pkcs7_compat::EncapsulatedContentInfo,
|
||||
};
|
||||
use rsa::{Oaep, Pkcs1v15Encrypt, RsaPublicKey, pkcs1::DecodeRsaPublicKey, sha2::Sha256};
|
||||
use sequoia_openpgp as openpgp;
|
||||
use std::io::Cursor;
|
||||
|
||||
const AES256_GCM: &Oid =
|
||||
Oid::JOINT_ISO_ITU_T_COUNTRY_US_ORGANIZATION_GOV_CSOR_NIST_ALGORITHMS_AES256_GCM;
|
||||
const CHACHA20_POLY1305: &Oid = Oid::const_new(&[1, 2, 840, 113549, 1, 9, 16, 3, 18]);
|
||||
const CONTENT_AUTH_ENVELOPED_DATA: &Oid =
|
||||
Oid::ISO_MEMBER_BODY_US_RSADSI_PKCS9_SMIME_CT_AUTH_ENVELOPED_DATA;
|
||||
const SHA256: &Oid =
|
||||
Oid::JOINT_ISO_ITU_T_COUNTRY_US_ORGANIZATION_GOV_CSOR_NIST_ALGORITHMS_HASH_SHA256;
|
||||
const MGF1: &Oid = Oid::ISO_MEMBER_BODY_US_RSADSI_PKCS1_MGF1;
|
||||
const RSAES_OAEP: &Oid = Oid::ISO_MEMBER_BODY_US_RSADSI_PKCS1_RSAES_OAEP;
|
||||
|
||||
#[derive(Debug)]
|
||||
pub enum EncryptMessageError {
|
||||
AlreadyEncrypted,
|
||||
Error(String),
|
||||
}
|
||||
|
||||
#[allow(async_fn_in_trait)]
|
||||
pub trait EncryptMessage {
|
||||
async fn encrypt(
|
||||
&self,
|
||||
keys: &EncryptionKeys,
|
||||
flags: u64,
|
||||
) -> Result<Vec<u8>, EncryptMessageError>;
|
||||
fn is_encrypted(&self) -> bool;
|
||||
}
|
||||
|
||||
impl EncryptMessage for Message<'_> {
|
||||
async fn encrypt(
|
||||
&self,
|
||||
keys: &EncryptionKeys,
|
||||
flags: u64,
|
||||
) -> Result<Vec<u8>, EncryptMessageError> {
|
||||
if flags & ACCOUNT_FLAG_ENCRYPT_METHOD_PGP != 0 && flags.cipher().is_aead() {
|
||||
return Err(EncryptMessageError::Error(
|
||||
"AES-256-GCM and ChaCha20-Poly1305 are only supported for S/MIME encryption."
|
||||
.into(),
|
||||
));
|
||||
}
|
||||
|
||||
let root = self.root_part();
|
||||
let raw_message = self.raw_message();
|
||||
let mut outer_message = Vec::with_capacity((raw_message.len() as f64 * 1.5) as usize);
|
||||
let mut inner_message = Vec::with_capacity(raw_message.len());
|
||||
|
||||
// Move MIME headers and body to inner message
|
||||
for header in root.headers() {
|
||||
(if header.name.is_mime_header() {
|
||||
&mut inner_message
|
||||
} else {
|
||||
&mut outer_message
|
||||
})
|
||||
.extend_from_slice(
|
||||
&raw_message[header.offset_field() as usize..header.offset_end() as usize],
|
||||
);
|
||||
}
|
||||
inner_message.extend_from_slice(b"\r\n");
|
||||
inner_message.extend_from_slice(&raw_message[root.raw_body_offset() as usize..]);
|
||||
|
||||
// Encrypt inner message
|
||||
if flags & ACCOUNT_FLAG_ENCRYPT_METHOD_PGP != 0 {
|
||||
// Prepare encrypted message
|
||||
let boundary = make_boundary("_");
|
||||
outer_message.extend_from_slice(
|
||||
concat!(
|
||||
"Content-Type: multipart/encrypted;\r\n\t",
|
||||
"protocol=\"application/pgp-encrypted\";\r\n\t",
|
||||
"boundary=\""
|
||||
)
|
||||
.as_bytes(),
|
||||
);
|
||||
outer_message.extend_from_slice(boundary.as_bytes());
|
||||
outer_message.extend_from_slice(
|
||||
concat!(
|
||||
"\"\r\n\r\n",
|
||||
"OpenPGP/MIME message (Automatically encrypted by Stalwart)\r\n\r\n",
|
||||
"--"
|
||||
)
|
||||
.as_bytes(),
|
||||
);
|
||||
outer_message.extend_from_slice(boundary.as_bytes());
|
||||
outer_message.extend_from_slice(
|
||||
concat!(
|
||||
"\r\nContent-Type: application/pgp-encrypted\r\n\r\n",
|
||||
"Version: 1\r\n\r\n--"
|
||||
)
|
||||
.as_bytes(),
|
||||
);
|
||||
outer_message.extend_from_slice(boundary.as_bytes());
|
||||
outer_message.extend_from_slice(
|
||||
concat!(
|
||||
"\r\nContent-Type: application/octet-stream; name=\"encrypted.asc\"\r\n",
|
||||
"Content-Disposition: inline; filename=\"encrypted.asc\"\r\n\r\n"
|
||||
)
|
||||
.as_bytes(),
|
||||
);
|
||||
|
||||
let certs = keys
|
||||
.iter()
|
||||
.map(openpgp::Cert::from_bytes)
|
||||
.collect::<Result<Vec<_>, _>>()
|
||||
.map_err(|err| {
|
||||
EncryptMessageError::Error(format!(
|
||||
"Failed to parse OpenPGP public key: {}",
|
||||
err
|
||||
))
|
||||
})?;
|
||||
|
||||
// Encrypt contents (TODO: use rayon)
|
||||
let encrypted_contents = tokio::task::spawn_blocking(move || {
|
||||
// Parse public key
|
||||
let mut keys = Vec::with_capacity(certs.len());
|
||||
let policy = openpgp::policy::StandardPolicy::new();
|
||||
|
||||
for cert in &certs {
|
||||
for key in cert
|
||||
.keys()
|
||||
.with_policy(&policy, None)
|
||||
.supported()
|
||||
.alive()
|
||||
.revoked(false)
|
||||
.key_flags(KeyFlags::empty().set_transport_encryption())
|
||||
{
|
||||
keys.push(key);
|
||||
}
|
||||
}
|
||||
|
||||
// Compose a writer stack corresponding to the output format and
|
||||
// packet structure we want.
|
||||
let mut sink = Vec::with_capacity(inner_message.len());
|
||||
|
||||
// Stream an OpenPGP message.
|
||||
let message = stream::Armorer::new(stream::Message::new(&mut sink))
|
||||
.build()
|
||||
.map_err(|err| {
|
||||
EncryptMessageError::Error(format!("Failed to create armorer: {}", err))
|
||||
})?;
|
||||
let message = stream::Encryptor::for_recipients(message, keys)
|
||||
.symmetric_algo(flags.algo())
|
||||
.build()
|
||||
.map_err(|err| {
|
||||
EncryptMessageError::Error(format!("Failed to build encryptor: {}", err))
|
||||
})?;
|
||||
let mut message = stream::LiteralWriter::new(message).build().map_err(|err| {
|
||||
EncryptMessageError::Error(format!("Failed to create literal writer: {}", err))
|
||||
})?;
|
||||
std::io::copy(&mut Cursor::new(inner_message), &mut message).map_err(|err| {
|
||||
EncryptMessageError::Error(format!("Failed to encrypt message: {}", err))
|
||||
})?;
|
||||
message.finalize().map_err(|err| {
|
||||
EncryptMessageError::Error(format!("Failed to finalize message: {}", err))
|
||||
})?;
|
||||
|
||||
String::from_utf8(sink).map_err(|err| {
|
||||
EncryptMessageError::Error(format!(
|
||||
"Failed to convert encrypted message to UTF-8: {}",
|
||||
err
|
||||
))
|
||||
})
|
||||
})
|
||||
.await
|
||||
.map_err(|err| {
|
||||
EncryptMessageError::Error(format!("Failed to encrypt message: {}", err))
|
||||
})??;
|
||||
outer_message.extend_from_slice(encrypted_contents.as_bytes());
|
||||
outer_message.extend_from_slice(b"\r\n--");
|
||||
outer_message.extend_from_slice(boundary.as_bytes());
|
||||
outer_message.extend_from_slice(b"--\r\n");
|
||||
} else {
|
||||
let cipher = flags.cipher();
|
||||
|
||||
// Generate random nonce
|
||||
let mut rng = StdRng::from_entropy();
|
||||
let mut nonce = vec![0u8; cipher.nonce_size()];
|
||||
rng.fill_bytes(&mut nonce);
|
||||
|
||||
// Generate random key
|
||||
let mut key = vec![0u8; cipher.key_size()];
|
||||
rng.fill_bytes(&mut key);
|
||||
|
||||
// Encrypt contents (TODO: use rayon)
|
||||
let (encrypted_contents, mac, key, nonce) = tokio::task::spawn_blocking(move || {
|
||||
let (encrypted_contents, mac) = cipher.encrypt(&key, &nonce, &inner_message);
|
||||
(encrypted_contents, mac, key, nonce)
|
||||
})
|
||||
.await
|
||||
.map_err(|err| {
|
||||
EncryptMessageError::Error(format!("Failed to encrypt message: {}", err))
|
||||
})?;
|
||||
|
||||
// Encrypt key using public keys
|
||||
let key_encryption_algorithm = cipher.key_encryption_algorithm()?;
|
||||
let mut recipient_infos = SetOf::new();
|
||||
for cert in keys.iter() {
|
||||
let cert = rasn::der::decode::<rasn_pkix::Certificate>(cert).map_err(|err| {
|
||||
EncryptMessageError::Error(format!("Failed to parse certificate: {}", err))
|
||||
})?;
|
||||
|
||||
let public_key = RsaPublicKey::from_pkcs1_der(
|
||||
cert.tbs_certificate
|
||||
.subject_public_key_info
|
||||
.subject_public_key
|
||||
.as_raw_slice(),
|
||||
)
|
||||
.map_err(|err| {
|
||||
EncryptMessageError::Error(format!("Failed to parse public key: {}", err))
|
||||
})?;
|
||||
let encrypted_key = if cipher.is_aead() {
|
||||
public_key.encrypt(&mut rng, Oaep::new::<Sha256>(), &key[..])
|
||||
} else {
|
||||
public_key.encrypt(&mut rng, Pkcs1v15Encrypt, &key[..])
|
||||
}
|
||||
.map_err(|err| {
|
||||
EncryptMessageError::Error(format!("Failed to encrypt key: {}", err))
|
||||
})?;
|
||||
|
||||
recipient_infos.insert(RecipientInfo::KeyTransRecipientInfo(
|
||||
KeyTransRecipientInfo {
|
||||
version: 0.into(),
|
||||
rid: RecipientIdentifier::IssuerAndSerialNumber(IssuerAndSerialNumber {
|
||||
issuer: cert.tbs_certificate.issuer,
|
||||
serial_number: cert.tbs_certificate.serial_number,
|
||||
}),
|
||||
key_encryption_algorithm: key_encryption_algorithm.clone(),
|
||||
encrypted_key: EncryptedKey::from(encrypted_key),
|
||||
},
|
||||
));
|
||||
}
|
||||
|
||||
let encrypted_content_info = EncryptedContentInfo {
|
||||
content_type: CONTENT_DATA.into(),
|
||||
content_encryption_algorithm: cipher.content_encryption_algorithm(&nonce)?,
|
||||
encrypted_content: Some(EncryptedContent::from(encrypted_contents)),
|
||||
};
|
||||
|
||||
let (content_type, content) = if let Some(mac) = mac {
|
||||
(
|
||||
CONTENT_AUTH_ENVELOPED_DATA,
|
||||
rasn::der::encode(&AuthEnvelopedData {
|
||||
version: 0.into(),
|
||||
originator_info: None,
|
||||
recipient_infos,
|
||||
auth_encrypted_content_info: encrypted_content_info,
|
||||
auth_attrs: None,
|
||||
mac: OctetString::from(mac),
|
||||
unauth_attrs: None,
|
||||
})
|
||||
.map_err(|err| {
|
||||
EncryptMessageError::Error(format!(
|
||||
"Failed to encode AuthEnvelopedData: {}",
|
||||
err
|
||||
))
|
||||
})?,
|
||||
)
|
||||
} else {
|
||||
(
|
||||
CONTENT_ENVELOPED_DATA,
|
||||
rasn::der::encode(&EnvelopedData {
|
||||
version: 0.into(),
|
||||
originator_info: None,
|
||||
recipient_infos,
|
||||
encrypted_content_info,
|
||||
unprotected_attrs: None,
|
||||
})
|
||||
.map_err(|err| {
|
||||
EncryptMessageError::Error(format!(
|
||||
"Failed to encode EnvelopedData: {}",
|
||||
err
|
||||
))
|
||||
})?,
|
||||
)
|
||||
};
|
||||
|
||||
let pkcs7 = rasn::der::encode(&EncapsulatedContentInfo {
|
||||
content_type: content_type.into(),
|
||||
content: Some(content.into()),
|
||||
})
|
||||
.map_err(|err| {
|
||||
EncryptMessageError::Error(format!("Failed to encode ContentInfo: {}", err))
|
||||
})?;
|
||||
|
||||
// Generate message
|
||||
outer_message.extend_from_slice(b"Content-Type: application/pkcs7-mime;\r\n");
|
||||
outer_message.extend_from_slice(b"\tname=\"smime.p7m\";\r\n\tsmime-type=");
|
||||
outer_message.extend_from_slice(if cipher.is_aead() {
|
||||
b"authenticated-enveloped-data\r\n"
|
||||
} else {
|
||||
b"enveloped-data\r\n"
|
||||
});
|
||||
outer_message.extend_from_slice(
|
||||
concat!(
|
||||
"Content-Disposition: attachment;\r\n",
|
||||
"\tfilename=\"smime.p7m\"\r\n",
|
||||
"Content-Transfer-Encoding: base64\r\n\r\n"
|
||||
)
|
||||
.as_bytes(),
|
||||
);
|
||||
Base64Encoder::new()
|
||||
.wrap_lines()
|
||||
.encode_to_writer(&pkcs7, &mut outer_message)
|
||||
.map_err(|err| {
|
||||
EncryptMessageError::Error(format!("Failed to base64 encode PKCS7: {}", err))
|
||||
})?;
|
||||
}
|
||||
|
||||
Ok(outer_message)
|
||||
}
|
||||
|
||||
fn is_encrypted(&self) -> bool {
|
||||
if self.content_type().is_some_and(|ct| {
|
||||
let main_type = ct.c_type.as_ref();
|
||||
let sub_type = ct
|
||||
.c_subtype
|
||||
.as_ref()
|
||||
.map(|s| s.as_ref())
|
||||
.unwrap_or_default();
|
||||
|
||||
(main_type.eq_ignore_ascii_case("application")
|
||||
&& (sub_type.eq_ignore_ascii_case("pkcs7-mime")
|
||||
|| sub_type.eq_ignore_ascii_case("pkcs7-signature")
|
||||
|| (sub_type.eq_ignore_ascii_case("octet-stream")
|
||||
&& self.attachment_name().is_some_and(|name| {
|
||||
name.rsplit_once('.')
|
||||
.is_some_and(|(_, ext)| ["p7m", "p7s", "p7c", "p7z"].contains(&ext))
|
||||
}))))
|
||||
|| (main_type.eq_ignore_ascii_case("multipart")
|
||||
&& sub_type.eq_ignore_ascii_case("encrypted"))
|
||||
}) {
|
||||
return true;
|
||||
}
|
||||
|
||||
if self.parts.len() <= 2 {
|
||||
let mut text_part = None;
|
||||
let mut is_multipart = false;
|
||||
|
||||
for part in &self.parts {
|
||||
match &part.body {
|
||||
PartType::Text(text) => {
|
||||
text_part = Some(text.as_ref());
|
||||
}
|
||||
PartType::Multipart(_) => {
|
||||
is_multipart = true;
|
||||
}
|
||||
_ => (),
|
||||
}
|
||||
}
|
||||
|
||||
match text_part {
|
||||
Some(text)
|
||||
if (self.parts.len() == 1 || is_multipart)
|
||||
&& text.trim_start().starts_with("-----BEGIN PGP MESSAGE-----") =>
|
||||
{
|
||||
return true;
|
||||
}
|
||||
_ => (),
|
||||
}
|
||||
}
|
||||
|
||||
false
|
||||
}
|
||||
}
|
||||
|
||||
pub trait EncryptionFlags {
|
||||
fn cipher(&self) -> SymmetricCipher;
|
||||
fn can_train_spam_filter(&self) -> bool;
|
||||
fn encrypt_on_append(&self) -> bool;
|
||||
fn algo(&self) -> SymmetricAlgorithm;
|
||||
}
|
||||
|
||||
impl EncryptionFlags for u64 {
|
||||
fn cipher(&self) -> SymmetricCipher {
|
||||
if *self & ACCOUNT_FLAG_ENCRYPT_ALGO_AES256_GCM != 0 {
|
||||
SymmetricCipher::Aes256Gcm
|
||||
} else if *self & ACCOUNT_FLAG_ENCRYPT_ALGO_CHACHA20_POLY1305 != 0 {
|
||||
SymmetricCipher::ChaCha20Poly1305
|
||||
} else if *self & ACCOUNT_FLAG_ENCRYPT_ALGO_AES256 != 0 {
|
||||
SymmetricCipher::Aes256Cbc
|
||||
} else {
|
||||
SymmetricCipher::Aes128Cbc
|
||||
}
|
||||
}
|
||||
|
||||
fn can_train_spam_filter(&self) -> bool {
|
||||
*self & ACCOUNT_FLAG_ENCRYPT_TRAIN_SPAM_FILTER != 0
|
||||
}
|
||||
|
||||
fn encrypt_on_append(&self) -> bool {
|
||||
*self & ACCOUNT_FLAG_ENCRYPT_APPEND != 0
|
||||
}
|
||||
|
||||
fn algo(&self) -> SymmetricAlgorithm {
|
||||
if *self & ACCOUNT_FLAG_ENCRYPT_ALGO_AES256 != 0 {
|
||||
SymmetricAlgorithm::AES256
|
||||
} else {
|
||||
SymmetricAlgorithm::AES128
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, PartialEq, Eq)]
|
||||
pub enum SymmetricCipher {
|
||||
Aes128Cbc,
|
||||
Aes256Cbc,
|
||||
Aes256Gcm,
|
||||
ChaCha20Poly1305,
|
||||
}
|
||||
|
||||
impl SymmetricCipher {
|
||||
fn key_size(self) -> usize {
|
||||
match self {
|
||||
SymmetricCipher::Aes128Cbc => 16,
|
||||
SymmetricCipher::Aes256Cbc
|
||||
| SymmetricCipher::Aes256Gcm
|
||||
| SymmetricCipher::ChaCha20Poly1305 => 32,
|
||||
}
|
||||
}
|
||||
|
||||
fn nonce_size(self) -> usize {
|
||||
match self {
|
||||
SymmetricCipher::Aes128Cbc | SymmetricCipher::Aes256Cbc => 16,
|
||||
SymmetricCipher::Aes256Gcm | SymmetricCipher::ChaCha20Poly1305 => 12,
|
||||
}
|
||||
}
|
||||
|
||||
fn is_aead(self) -> bool {
|
||||
matches!(
|
||||
self,
|
||||
SymmetricCipher::Aes256Gcm | SymmetricCipher::ChaCha20Poly1305
|
||||
)
|
||||
}
|
||||
|
||||
fn encrypt(self, key: &[u8], nonce: &[u8], contents: &[u8]) -> (Vec<u8>, Option<Vec<u8>>) {
|
||||
match self {
|
||||
SymmetricCipher::Aes128Cbc => (
|
||||
cbc::Encryptor::<aes::Aes128>::new_from_slices(key, nonce)
|
||||
.expect("invalid key or iv length")
|
||||
.encrypt_padded_vec::<Pkcs7>(contents),
|
||||
None,
|
||||
),
|
||||
SymmetricCipher::Aes256Cbc => (
|
||||
cbc::Encryptor::<aes::Aes256>::new_from_slices(key, nonce)
|
||||
.expect("invalid key or iv length")
|
||||
.encrypt_padded_vec::<Pkcs7>(contents),
|
||||
None,
|
||||
),
|
||||
SymmetricCipher::Aes256Gcm => {
|
||||
let cipher = Aes256Gcm::new_from_slice(key).expect("invalid key length");
|
||||
let mut buffer = contents.to_vec();
|
||||
let tag = cipher
|
||||
.encrypt_inout_detached(
|
||||
nonce.try_into().expect("invalid nonce length"),
|
||||
b"",
|
||||
buffer.as_mut_slice().into(),
|
||||
)
|
||||
.expect("AES-GCM encryption failed");
|
||||
(buffer, Some(tag.to_vec()))
|
||||
}
|
||||
SymmetricCipher::ChaCha20Poly1305 => {
|
||||
let cipher = ChaCha20Poly1305::new_from_slice(key).expect("invalid key length");
|
||||
let mut buffer = contents.to_vec();
|
||||
let tag = cipher
|
||||
.encrypt_inout_detached(
|
||||
nonce.try_into().expect("invalid nonce length"),
|
||||
b"",
|
||||
buffer.as_mut_slice().into(),
|
||||
)
|
||||
.expect("ChaCha20-Poly1305 encryption failed");
|
||||
(buffer, Some(tag.to_vec()))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn content_encryption_algorithm(
|
||||
self,
|
||||
nonce: &[u8],
|
||||
) -> Result<AlgorithmIdentifier, EncryptMessageError> {
|
||||
let (algorithm, parameters) = match self {
|
||||
SymmetricCipher::Aes128Cbc => (AES128_CBC, encode_octet_string(nonce)?),
|
||||
SymmetricCipher::Aes256Cbc => (AES256_CBC, encode_octet_string(nonce)?),
|
||||
SymmetricCipher::ChaCha20Poly1305 => (CHACHA20_POLY1305, encode_octet_string(nonce)?),
|
||||
SymmetricCipher::Aes256Gcm => (
|
||||
AES256_GCM,
|
||||
rasn::der::encode(&GcmParameters {
|
||||
nonce: OctetString::from_slice(nonce),
|
||||
icv_len: 16,
|
||||
})
|
||||
.map_err(|err| {
|
||||
EncryptMessageError::Error(format!("Failed to encode GCM parameters: {}", err))
|
||||
})?,
|
||||
),
|
||||
};
|
||||
|
||||
Ok(AlgorithmIdentifier {
|
||||
algorithm: algorithm.into(),
|
||||
parameters: Some(parameters.into()),
|
||||
})
|
||||
}
|
||||
|
||||
fn key_encryption_algorithm(self) -> Result<AlgorithmIdentifier, EncryptMessageError> {
|
||||
if self.is_aead() {
|
||||
let sha256 = AlgorithmIdentifier {
|
||||
algorithm: SHA256.into(),
|
||||
parameters: Some(encode_null()?.into()),
|
||||
};
|
||||
let parameters = rasn::der::encode(&OaepParameters {
|
||||
hash_algorithm: sha256.clone(),
|
||||
mask_gen_algorithm: AlgorithmIdentifier {
|
||||
algorithm: MGF1.into(),
|
||||
parameters: Some(
|
||||
rasn::der::encode(&sha256)
|
||||
.map_err(|err| {
|
||||
EncryptMessageError::Error(format!(
|
||||
"Failed to encode MGF1 parameters: {}",
|
||||
err
|
||||
))
|
||||
})?
|
||||
.into(),
|
||||
),
|
||||
},
|
||||
})
|
||||
.map_err(|err| {
|
||||
EncryptMessageError::Error(format!("Failed to encode OAEP parameters: {}", err))
|
||||
})?;
|
||||
|
||||
Ok(AlgorithmIdentifier {
|
||||
algorithm: RSAES_OAEP.into(),
|
||||
parameters: Some(parameters.into()),
|
||||
})
|
||||
} else {
|
||||
Ok(AlgorithmIdentifier {
|
||||
algorithm: RSA.into(),
|
||||
parameters: Some(encode_null()?.into()),
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(rasn::AsnType, rasn::Encode)]
|
||||
struct GcmParameters {
|
||||
nonce: OctetString,
|
||||
icv_len: u8,
|
||||
}
|
||||
|
||||
#[derive(rasn::AsnType, rasn::Encode)]
|
||||
struct OaepParameters {
|
||||
#[rasn(tag(explicit(0)))]
|
||||
hash_algorithm: AlgorithmIdentifier,
|
||||
#[rasn(tag(explicit(1)))]
|
||||
mask_gen_algorithm: AlgorithmIdentifier,
|
||||
}
|
||||
|
||||
fn encode_octet_string(value: &[u8]) -> Result<Vec<u8>, EncryptMessageError> {
|
||||
rasn::der::encode(&OctetString::from_slice(value))
|
||||
.map_err(|err| EncryptMessageError::Error(format!("Failed to encode nonce: {}", err)))
|
||||
}
|
||||
|
||||
fn encode_null() -> Result<Vec<u8>, EncryptMessageError> {
|
||||
rasn::der::encode(&()).map_err(|err| {
|
||||
EncryptMessageError::Error(format!("Failed to encode NULL parameters: {}", err))
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,315 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
|
||||
*
|
||||
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
||||
*/
|
||||
|
||||
use super::metadata::MessageData;
|
||||
use crate::cache::{MessageCacheFetch, email::MessageCacheAccess};
|
||||
use common::{Server, storage::index::ObjectIndexBuilder};
|
||||
use groupware::calendar::storage::ItipAutoExpunge;
|
||||
use registry::schema::enums::IndexDocumentType;
|
||||
use registry::schema::structs::{Task, TaskIndexDocument, TaskStatus};
|
||||
use std::future::Future;
|
||||
use store::write::key::DeserializeBigEndian;
|
||||
use store::write::{IndexPropertyClass, now};
|
||||
use store::{IterateParams, U32_LEN, U64_LEN, ValueKey};
|
||||
use store::{
|
||||
roaring::RoaringBitmap,
|
||||
write::{BatchBuilder, ValueClass},
|
||||
};
|
||||
use trc::AddContext;
|
||||
use types::collection::{Collection, SyncCollection, VanishedCollection};
|
||||
use types::field::{EmailField, EmailSubmissionField};
|
||||
|
||||
pub trait EmailDeletion: Sync + Send {
|
||||
fn emails_delete(
|
||||
&self,
|
||||
account_id: u32,
|
||||
tenant_id: Option<u32>,
|
||||
batch: &mut BatchBuilder,
|
||||
document_ids: RoaringBitmap,
|
||||
) -> impl Future<Output = trc::Result<RoaringBitmap>> + Send;
|
||||
|
||||
fn purge_account(&self, account_id: u32) -> impl Future<Output = trc::Result<()>> + Send;
|
||||
|
||||
fn purge_email_submissions(
|
||||
&self,
|
||||
account_id: u32,
|
||||
hold_period: u64,
|
||||
) -> impl Future<Output = trc::Result<()>> + Send;
|
||||
|
||||
fn emails_auto_expunge(
|
||||
&self,
|
||||
account_id: u32,
|
||||
hold_period: u64,
|
||||
) -> impl Future<Output = trc::Result<()>> + Send;
|
||||
|
||||
fn log_emptied_threads(
|
||||
&self,
|
||||
account_id: u32,
|
||||
batch: &mut BatchBuilder,
|
||||
thread_ids: RoaringBitmap,
|
||||
deleted_ids: &RoaringBitmap,
|
||||
) -> impl Future<Output = trc::Result<()>> + Send;
|
||||
}
|
||||
|
||||
impl EmailDeletion for Server {
|
||||
async fn emails_delete(
|
||||
&self,
|
||||
account_id: u32,
|
||||
tenant_id: Option<u32>,
|
||||
batch: &mut BatchBuilder,
|
||||
document_ids: RoaringBitmap,
|
||||
) -> trc::Result<RoaringBitmap> {
|
||||
let mut deleted_ids = RoaringBitmap::new();
|
||||
let mut thread_ids = RoaringBitmap::new();
|
||||
batch
|
||||
.with_account_id(account_id)
|
||||
.with_collection(Collection::Email);
|
||||
self.archives(
|
||||
account_id,
|
||||
Collection::Email,
|
||||
&document_ids,
|
||||
|document_id, data_| {
|
||||
// Add changes to batch
|
||||
let metadata = data_
|
||||
.to_unarchived::<MessageData>()
|
||||
.caused_by(trc::location!())?;
|
||||
for mailbox in metadata.inner.mailboxes.iter() {
|
||||
batch.log_vanished_item(
|
||||
VanishedCollection::Email,
|
||||
(mailbox.mailbox_id.to_native(), mailbox.uid.to_native()),
|
||||
);
|
||||
}
|
||||
thread_ids.insert(metadata.inner.thread_id.to_native());
|
||||
batch
|
||||
.with_document(document_id)
|
||||
.custom(
|
||||
ObjectIndexBuilder::<_, ()>::new()
|
||||
.with_tenant_id(tenant_id)
|
||||
.with_current(metadata),
|
||||
)
|
||||
.caused_by(trc::location!())?
|
||||
.schedule_task(Task::UnindexDocument(TaskIndexDocument {
|
||||
account_id: account_id.into(),
|
||||
document_id: document_id.into(),
|
||||
document_type: IndexDocumentType::Email,
|
||||
status: TaskStatus::now(),
|
||||
}))
|
||||
.commit_point();
|
||||
|
||||
deleted_ids.insert(document_id);
|
||||
|
||||
Ok(true)
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
|
||||
self.log_emptied_threads(account_id, batch, thread_ids, &deleted_ids)
|
||||
.await?;
|
||||
|
||||
let not_destroyed = if document_ids.len() == deleted_ids.len() {
|
||||
RoaringBitmap::new()
|
||||
} else {
|
||||
deleted_ids ^= document_ids;
|
||||
deleted_ids
|
||||
};
|
||||
|
||||
Ok(not_destroyed)
|
||||
}
|
||||
|
||||
async fn log_emptied_threads(
|
||||
&self,
|
||||
account_id: u32,
|
||||
batch: &mut BatchBuilder,
|
||||
thread_ids: RoaringBitmap,
|
||||
deleted_ids: &RoaringBitmap,
|
||||
) -> trc::Result<()> {
|
||||
if !thread_ids.is_empty() {
|
||||
let cache = self
|
||||
.get_cached_messages(account_id)
|
||||
.await
|
||||
.caused_by(trc::location!())?;
|
||||
for thread_id in &thread_ids {
|
||||
if cache
|
||||
.in_thread(thread_id)
|
||||
.all(|message| deleted_ids.contains(message.document_id))
|
||||
{
|
||||
batch
|
||||
.with_account_id(account_id)
|
||||
.with_collection(Collection::Thread)
|
||||
.with_document(thread_id)
|
||||
.log_container_delete(SyncCollection::Thread);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn purge_account(&self, account_id: u32) -> trc::Result<()> {
|
||||
// Auto-expunge deleted and junk messages
|
||||
if let Some(hold_period) = self.core.email.mail_autoexpunge_after {
|
||||
self.emails_auto_expunge(account_id, hold_period)
|
||||
.await
|
||||
.caused_by(trc::location!())?;
|
||||
}
|
||||
|
||||
// Auto-expunge iMIP messages
|
||||
if let Some(hold_period) = self.core.groupware.itip_inbox_auto_expunge {
|
||||
self.itip_auto_expunge(account_id, hold_period)
|
||||
.await
|
||||
.caused_by(trc::location!())?;
|
||||
}
|
||||
|
||||
// Delete old e-mail submissions
|
||||
if let Some(hold_period) = self.core.email.email_submission_autoexpunge_after {
|
||||
self.purge_email_submissions(account_id, hold_period)
|
||||
.await
|
||||
.caused_by(trc::location!())?;
|
||||
}
|
||||
|
||||
// Purge changelogs
|
||||
self.delete_changes(
|
||||
account_id,
|
||||
self.core.email.changes_max_history,
|
||||
self.core.email.share_notification_max_history,
|
||||
)
|
||||
.await
|
||||
.caused_by(trc::location!())?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn emails_auto_expunge(&self, account_id: u32, hold_period: u64) -> trc::Result<()> {
|
||||
// Filter messages by received date
|
||||
let mut destroy_ids = RoaringBitmap::new();
|
||||
let cutoff = now().saturating_sub(hold_period);
|
||||
self.store()
|
||||
.iterate(
|
||||
IterateParams::new(
|
||||
ValueKey {
|
||||
account_id,
|
||||
collection: Collection::Email.into(),
|
||||
document_id: 0,
|
||||
class: ValueClass::Property(EmailField::DeletedAt.into()),
|
||||
},
|
||||
ValueKey {
|
||||
account_id,
|
||||
collection: Collection::Email.into(),
|
||||
document_id: u32::MAX,
|
||||
class: ValueClass::Property(EmailField::DeletedAt.into()),
|
||||
},
|
||||
)
|
||||
.ascending(),
|
||||
|key, value| {
|
||||
let deleted_at = value.deserialize_be_u64(0)?;
|
||||
if deleted_at <= cutoff {
|
||||
destroy_ids.insert(key.deserialize_be_u32(key.len() - U32_LEN)?);
|
||||
}
|
||||
|
||||
Ok(true)
|
||||
},
|
||||
)
|
||||
.await
|
||||
.caused_by(trc::location!())?;
|
||||
|
||||
if destroy_ids.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
trc::event!(
|
||||
Store(trc::StoreEvent::AutoExpunge),
|
||||
Collection = Collection::Email.as_str(),
|
||||
AccountId = account_id,
|
||||
Total = destroy_ids.len(),
|
||||
);
|
||||
|
||||
// Delete messages
|
||||
let mut batch = BatchBuilder::new();
|
||||
let tenant_id = self
|
||||
.account(account_id)
|
||||
.await
|
||||
.caused_by(trc::location!())?
|
||||
.tenant_id();
|
||||
self.emails_delete(account_id, tenant_id, &mut batch, destroy_ids)
|
||||
.await?;
|
||||
self.commit_batch(batch).await?;
|
||||
self.notify_task_queue();
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn purge_email_submissions(&self, account_id: u32, hold_period: u64) -> trc::Result<()> {
|
||||
// Filter messages by received date
|
||||
let mut destroy_ids = Vec::new();
|
||||
self.store()
|
||||
.iterate(
|
||||
IterateParams::new(
|
||||
ValueKey {
|
||||
account_id,
|
||||
collection: Collection::EmailSubmission.into(),
|
||||
document_id: 0,
|
||||
class: ValueClass::IndexProperty(IndexPropertyClass::Integer {
|
||||
property: EmailSubmissionField::Metadata.into(),
|
||||
value: 0,
|
||||
}),
|
||||
},
|
||||
ValueKey {
|
||||
account_id,
|
||||
collection: Collection::Email.into(),
|
||||
document_id: u32::MAX,
|
||||
class: ValueClass::IndexProperty(IndexPropertyClass::Integer {
|
||||
property: EmailSubmissionField::Metadata.into(),
|
||||
value: now().saturating_sub(hold_period),
|
||||
}),
|
||||
},
|
||||
)
|
||||
.ascending()
|
||||
.no_values(),
|
||||
|key, _| {
|
||||
destroy_ids.push((
|
||||
key.deserialize_be_u32(key.len() - U32_LEN)?,
|
||||
key.deserialize_be_u64(key.len() - U32_LEN - U64_LEN)?,
|
||||
));
|
||||
|
||||
Ok(true)
|
||||
},
|
||||
)
|
||||
.await
|
||||
.caused_by(trc::location!())?;
|
||||
|
||||
if destroy_ids.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
trc::event!(
|
||||
Store(trc::StoreEvent::AutoExpunge),
|
||||
Collection = Collection::EmailSubmission.as_str(),
|
||||
AccountId = account_id,
|
||||
Total = destroy_ids.len(),
|
||||
);
|
||||
|
||||
// Delete messages
|
||||
let mut batch = BatchBuilder::new();
|
||||
batch
|
||||
.with_account_id(account_id)
|
||||
.with_collection(Collection::EmailSubmission);
|
||||
|
||||
for (document_id, send_at) in destroy_ids {
|
||||
batch
|
||||
.with_document(document_id)
|
||||
.clear(EmailSubmissionField::Metadata)
|
||||
.clear(ValueClass::IndexProperty(IndexPropertyClass::Integer {
|
||||
property: EmailSubmissionField::Metadata.into(),
|
||||
value: send_at,
|
||||
}))
|
||||
.commit_point();
|
||||
}
|
||||
|
||||
self.commit_batch(batch).await?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,279 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
|
||||
*
|
||||
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
||||
*/
|
||||
|
||||
use super::ingest::{EmailIngest, IngestEmail, IngestSource};
|
||||
use crate::{mailbox::INBOX_ID, sieve::ingest::SieveScriptIngest};
|
||||
use common::{
|
||||
Server,
|
||||
auth::BuildAccessToken,
|
||||
ipc::{EmailPush, PushNotification},
|
||||
};
|
||||
use mail_parser::MessageParser;
|
||||
use registry::schema::enums::Permission;
|
||||
use std::{borrow::Cow, future::Future};
|
||||
use store::ahash::AHashMap;
|
||||
use types::blob_hash::BlobHash;
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct IngestMessage {
|
||||
pub sender_address: String,
|
||||
pub sender_authenticated: bool,
|
||||
pub recipients: Vec<IngestRecipient>,
|
||||
pub message_blob: BlobHash,
|
||||
pub message_size: u64,
|
||||
pub session_id: u64,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct IngestRecipient {
|
||||
pub address: String,
|
||||
pub orcpt: Option<String>,
|
||||
pub spam_percentage: Option<u8>,
|
||||
}
|
||||
|
||||
impl IngestRecipient {
|
||||
pub fn is_spam(&self) -> bool {
|
||||
self.spam_percentage
|
||||
.is_some_and(|percentage| percentage >= 50)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum LocalDeliveryStatus {
|
||||
Success,
|
||||
TemporaryFailure {
|
||||
reason: Cow<'static, str>,
|
||||
},
|
||||
PermanentFailure {
|
||||
code: [u8; 3],
|
||||
reason: Cow<'static, str>,
|
||||
},
|
||||
}
|
||||
|
||||
pub struct LocalDeliveryResult {
|
||||
pub status: Vec<LocalDeliveryStatus>,
|
||||
pub autogenerated: Vec<AutogeneratedMessage>,
|
||||
}
|
||||
|
||||
pub struct AutogeneratedMessage {
|
||||
pub sender_address: String,
|
||||
pub recipients: Vec<String>,
|
||||
pub message: Vec<u8>,
|
||||
}
|
||||
|
||||
pub trait MailDelivery: Sync + Send {
|
||||
fn deliver_message(
|
||||
&self,
|
||||
message: IngestMessage,
|
||||
) -> impl Future<Output = LocalDeliveryResult> + Send;
|
||||
}
|
||||
|
||||
impl MailDelivery for Server {
|
||||
async fn deliver_message(&self, message: IngestMessage) -> LocalDeliveryResult {
|
||||
// Read message
|
||||
let raw_message = match self
|
||||
.core
|
||||
.storage
|
||||
.blob
|
||||
.get_blob(message.message_blob.as_slice(), 0..usize::MAX)
|
||||
.await
|
||||
{
|
||||
Ok(Some(raw_message)) => raw_message,
|
||||
Ok(None) => {
|
||||
trc::event!(
|
||||
MessageIngest(trc::MessageIngestEvent::Error),
|
||||
Reason = "Blob not found.",
|
||||
SpanId = message.session_id,
|
||||
CausedBy = trc::location!()
|
||||
);
|
||||
|
||||
return LocalDeliveryResult {
|
||||
status: (0..message.recipients.len())
|
||||
.map(|_| LocalDeliveryStatus::TemporaryFailure {
|
||||
reason: "Blob not found.".into(),
|
||||
})
|
||||
.collect::<Vec<_>>(),
|
||||
autogenerated: vec![],
|
||||
};
|
||||
}
|
||||
Err(err) => {
|
||||
trc::error!(
|
||||
err.details("Failed to fetch message blob.")
|
||||
.span_id(message.session_id)
|
||||
.caused_by(trc::location!())
|
||||
);
|
||||
|
||||
return LocalDeliveryResult {
|
||||
status: (0..message.recipients.len())
|
||||
.map(|_| LocalDeliveryStatus::TemporaryFailure {
|
||||
reason: "Temporary I/O error.".into(),
|
||||
})
|
||||
.collect::<Vec<_>>(),
|
||||
autogenerated: vec![],
|
||||
};
|
||||
}
|
||||
};
|
||||
|
||||
// Obtain the account IDs for each recipient
|
||||
let mut account_ids: AHashMap<u32, usize> =
|
||||
AHashMap::with_capacity(message.recipients.len());
|
||||
let mut result = LocalDeliveryResult {
|
||||
status: Vec::with_capacity(message.recipients.len()),
|
||||
autogenerated: Vec::new(),
|
||||
};
|
||||
|
||||
for rcpt in message.recipients {
|
||||
let account_id = match self.account_id_from_email(&rcpt.address, false).await {
|
||||
Ok(Some(account_id)) => account_id,
|
||||
Ok(None) => {
|
||||
// Something went wrong
|
||||
result.status.push(LocalDeliveryStatus::PermanentFailure {
|
||||
code: [5, 5, 0],
|
||||
reason: "Mailbox not found.".into(),
|
||||
});
|
||||
continue;
|
||||
}
|
||||
Err(err) => {
|
||||
trc::error!(
|
||||
err.details("Failed to lookup recipient.")
|
||||
.ctx(trc::Key::To, rcpt.address.to_string())
|
||||
.span_id(message.session_id)
|
||||
.caused_by(trc::location!())
|
||||
);
|
||||
result.status.push(LocalDeliveryStatus::TemporaryFailure {
|
||||
reason: "Address lookup failed.".into(),
|
||||
});
|
||||
continue;
|
||||
}
|
||||
};
|
||||
if let Some(status) = account_ids
|
||||
.get(&account_id)
|
||||
.and_then(|pos| result.status.get(*pos))
|
||||
{
|
||||
result.status.push(status.clone());
|
||||
continue;
|
||||
}
|
||||
|
||||
// Obtain access token
|
||||
let status = match self.access_token(account_id).await.and_then(|token| {
|
||||
token
|
||||
.build()
|
||||
.assert_has_permission(Permission::EmailReceive)
|
||||
}) {
|
||||
Ok(access_token) => {
|
||||
// Check if there is an active sieve script
|
||||
match self.sieve_script_get_active(account_id).await {
|
||||
Ok(None) => {
|
||||
// Ingest message
|
||||
self.email_ingest(IngestEmail {
|
||||
raw_message: &raw_message,
|
||||
blob_hash: Some(&message.message_blob),
|
||||
message: MessageParser::new().parse(&raw_message),
|
||||
access_token: &access_token,
|
||||
mailbox_ids: vec![INBOX_ID],
|
||||
keywords: vec![],
|
||||
received_at: None,
|
||||
source: IngestSource::Smtp {
|
||||
deliver_to: &rcpt.address,
|
||||
is_sender_authenticated: message.sender_authenticated,
|
||||
is_spam: rcpt.is_spam(),
|
||||
},
|
||||
session_id: message.session_id,
|
||||
})
|
||||
.await
|
||||
}
|
||||
Ok(Some(active_script)) => {
|
||||
self.sieve_script_ingest(
|
||||
&access_token,
|
||||
&message.message_blob,
|
||||
&raw_message,
|
||||
&message.sender_address,
|
||||
message.sender_authenticated,
|
||||
&rcpt,
|
||||
message.session_id,
|
||||
active_script,
|
||||
&mut result.autogenerated,
|
||||
)
|
||||
.await
|
||||
}
|
||||
Err(err) => Err(err),
|
||||
}
|
||||
}
|
||||
|
||||
Err(err) => Err(err),
|
||||
};
|
||||
|
||||
let status = match status {
|
||||
Ok(ingested_message) => {
|
||||
// Notify state change
|
||||
if ingested_message.change_id != u64::MAX {
|
||||
self.broadcast_push_notification(PushNotification::EmailPush(EmailPush {
|
||||
account_id,
|
||||
email_id: ingested_message.document_id,
|
||||
change_id: ingested_message.change_id,
|
||||
}))
|
||||
.await;
|
||||
}
|
||||
|
||||
LocalDeliveryStatus::Success
|
||||
}
|
||||
Err(err) => {
|
||||
let status = match err.as_ref() {
|
||||
trc::EventType::Limit(trc::LimitEvent::Quota) => {
|
||||
LocalDeliveryStatus::TemporaryFailure {
|
||||
reason: "Mailbox over quota.".into(),
|
||||
}
|
||||
}
|
||||
trc::EventType::Limit(trc::LimitEvent::TenantQuota) => {
|
||||
LocalDeliveryStatus::TemporaryFailure {
|
||||
reason: "Organization over quota.".into(),
|
||||
}
|
||||
}
|
||||
trc::EventType::Security(trc::SecurityEvent::Unauthorized) => {
|
||||
LocalDeliveryStatus::PermanentFailure {
|
||||
code: [5, 5, 0],
|
||||
reason: "This account is not authorized to receive email.".into(),
|
||||
}
|
||||
}
|
||||
trc::EventType::MessageIngest(trc::MessageIngestEvent::Error) => {
|
||||
LocalDeliveryStatus::PermanentFailure {
|
||||
code: err
|
||||
.value(trc::Key::Code)
|
||||
.and_then(|v| v.to_uint())
|
||||
.map(|n| {
|
||||
[(n / 100) as u8, ((n % 100) / 10) as u8, (n % 10) as u8]
|
||||
})
|
||||
.unwrap_or([5, 5, 0]),
|
||||
reason: err
|
||||
.value_as_str(trc::Key::Reason)
|
||||
.unwrap_or_default()
|
||||
.to_string()
|
||||
.into(),
|
||||
}
|
||||
}
|
||||
_ => LocalDeliveryStatus::TemporaryFailure {
|
||||
reason: "Transient server failure.".into(),
|
||||
},
|
||||
};
|
||||
|
||||
trc::error!(
|
||||
err.ctx(trc::Key::To, rcpt.address.to_string())
|
||||
.span_id(message.session_id)
|
||||
);
|
||||
|
||||
status
|
||||
}
|
||||
};
|
||||
|
||||
// Cache response for UID to avoid duplicate deliveries
|
||||
account_ids.insert(account_id, result.status.len());
|
||||
|
||||
result.status.push(status);
|
||||
}
|
||||
|
||||
result
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,542 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
|
||||
*
|
||||
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
||||
*/
|
||||
|
||||
use crate::message::metadata::{ArchivedMessageMetadataPart, ArchivedMetadataHeaderValue};
|
||||
use jmap_proto::{
|
||||
object::email::{EmailProperty, EmailValue, HeaderForm, HeaderProperty},
|
||||
types::date::UTCDate,
|
||||
};
|
||||
use jmap_tools::{Key, Map, Value};
|
||||
use mail_builder::{
|
||||
MessageBuilder,
|
||||
headers::{
|
||||
address::{Address, EmailAddress, GroupedAddresses},
|
||||
date::Date,
|
||||
message_id::MessageId,
|
||||
raw::Raw,
|
||||
text::Text,
|
||||
url::URL,
|
||||
},
|
||||
};
|
||||
use mail_parser::{Addr, DateTime, Group, Header, HeaderName, HeaderValue, parsers::MessageStream};
|
||||
use utils::chained_bytes::ChainedBytes;
|
||||
|
||||
pub trait HeaderToValue {
|
||||
fn header_to_value(
|
||||
&self,
|
||||
property: &EmailProperty,
|
||||
raw_message: &ChainedBytes<'_>,
|
||||
) -> Value<'static, EmailProperty, EmailValue>;
|
||||
fn headers_to_value(
|
||||
&self,
|
||||
raw_message: &ChainedBytes<'_>,
|
||||
) -> Value<'static, EmailProperty, EmailValue>;
|
||||
}
|
||||
|
||||
pub trait ValueToHeader<'x> {
|
||||
fn try_into_grouped_addresses(self) -> Option<GroupedAddresses<'x>>;
|
||||
fn try_into_address_list(self) -> Option<Vec<Address<'x>>>;
|
||||
fn try_into_address(self) -> Option<EmailAddress<'x>>;
|
||||
}
|
||||
|
||||
pub trait BuildHeader<'x>: Sized {
|
||||
fn build_header(
|
||||
self,
|
||||
header: HeaderProperty,
|
||||
value: Value<'x, EmailProperty, EmailValue>,
|
||||
) -> Result<Self, HeaderProperty>;
|
||||
}
|
||||
|
||||
impl HeaderToValue for Vec<Header<'_>> {
|
||||
fn header_to_value(
|
||||
&self,
|
||||
property: &EmailProperty,
|
||||
raw_message: &ChainedBytes<'_>,
|
||||
) -> Value<'static, EmailProperty, EmailValue> {
|
||||
let (header_name, form, all) = match property {
|
||||
EmailProperty::Header(header) => (
|
||||
HeaderName::parse(header.header.as_str())
|
||||
.unwrap_or_else(|| HeaderName::Other(header.header.as_str().into())),
|
||||
header.form,
|
||||
header.all,
|
||||
),
|
||||
EmailProperty::Sender => (HeaderName::Sender, HeaderForm::Addresses, false),
|
||||
EmailProperty::From => (HeaderName::From, HeaderForm::Addresses, false),
|
||||
EmailProperty::To => (HeaderName::To, HeaderForm::Addresses, false),
|
||||
EmailProperty::Cc => (HeaderName::Cc, HeaderForm::Addresses, false),
|
||||
EmailProperty::Bcc => (HeaderName::Bcc, HeaderForm::Addresses, false),
|
||||
EmailProperty::ReplyTo => (HeaderName::ReplyTo, HeaderForm::Addresses, false),
|
||||
EmailProperty::Subject => (HeaderName::Subject, HeaderForm::Text, false),
|
||||
EmailProperty::MessageId => (HeaderName::MessageId, HeaderForm::MessageIds, false),
|
||||
EmailProperty::InReplyTo => (HeaderName::InReplyTo, HeaderForm::MessageIds, false),
|
||||
EmailProperty::References => (HeaderName::References, HeaderForm::MessageIds, false),
|
||||
EmailProperty::SentAt => (HeaderName::Date, HeaderForm::Date, false),
|
||||
_ => return Value::Null,
|
||||
};
|
||||
|
||||
let is_raw = matches!(form, HeaderForm::Raw) || !header_name.is_structured();
|
||||
let mut headers = Vec::new();
|
||||
let header_name = header_name.as_str();
|
||||
for header in self.iter().rev() {
|
||||
if header.name.as_str().eq_ignore_ascii_case(header_name) {
|
||||
let raw_header;
|
||||
let header_value = if is_raw || matches!(header.value, HeaderValue::Empty) {
|
||||
raw_header =
|
||||
raw_message.get(header.offset_start as usize..header.offset_end as usize);
|
||||
|
||||
if let Some(bytes) = &raw_header {
|
||||
let bytes = bytes.as_ref();
|
||||
match form {
|
||||
HeaderForm::Raw => {
|
||||
HeaderValue::Text(String::from_utf8_lossy(bytes.trim_end()))
|
||||
}
|
||||
HeaderForm::Text => MessageStream::new(bytes).parse_unstructured(),
|
||||
HeaderForm::Addresses
|
||||
| HeaderForm::GroupedAddresses
|
||||
| HeaderForm::URLs => MessageStream::new(bytes).parse_address(),
|
||||
HeaderForm::MessageIds => MessageStream::new(bytes).parse_id(),
|
||||
HeaderForm::Date => MessageStream::new(bytes).parse_date(),
|
||||
}
|
||||
} else {
|
||||
HeaderValue::Empty
|
||||
}
|
||||
} else {
|
||||
header.value.clone()
|
||||
};
|
||||
headers.push(header_value.into_form(&form));
|
||||
if !all {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if !all {
|
||||
headers.pop().unwrap_or_default()
|
||||
} else {
|
||||
if headers.len() > 1 {
|
||||
headers.reverse();
|
||||
}
|
||||
Value::Array(headers)
|
||||
}
|
||||
}
|
||||
|
||||
fn headers_to_value(
|
||||
&self,
|
||||
raw_message: &ChainedBytes<'_>,
|
||||
) -> Value<'static, EmailProperty, EmailValue> {
|
||||
let mut headers = Vec::with_capacity(self.len());
|
||||
for header in self.iter() {
|
||||
headers.push(Value::Object(
|
||||
Map::with_capacity(2)
|
||||
.with_key_value(EmailProperty::Name, header.name().to_string())
|
||||
.with_key_value(
|
||||
EmailProperty::Value,
|
||||
String::from_utf8_lossy(
|
||||
raw_message
|
||||
.get(header.offset_start as usize..header.offset_end as usize)
|
||||
.unwrap_or_default()
|
||||
.as_ref()
|
||||
.trim_end(),
|
||||
)
|
||||
.into_owned(),
|
||||
),
|
||||
));
|
||||
}
|
||||
headers.into()
|
||||
}
|
||||
}
|
||||
|
||||
impl<'x> ValueToHeader<'x> for Value<'x, EmailProperty, EmailValue> {
|
||||
fn try_into_grouped_addresses(self) -> Option<GroupedAddresses<'x>> {
|
||||
let mut obj = self.into_object()?;
|
||||
Some(GroupedAddresses {
|
||||
name: obj
|
||||
.remove(&Key::Property(EmailProperty::Name))
|
||||
.and_then(|n| n.into_string()),
|
||||
addresses: obj
|
||||
.remove(&Key::Property(EmailProperty::Addresses))?
|
||||
.try_into_address_list()?,
|
||||
})
|
||||
}
|
||||
|
||||
fn try_into_address_list(self) -> Option<Vec<Address<'x>>> {
|
||||
let list = self.into_array()?;
|
||||
let mut addresses = Vec::with_capacity(list.len());
|
||||
for value in list {
|
||||
addresses.push(Address::Address(value.try_into_address()?));
|
||||
}
|
||||
Some(addresses)
|
||||
}
|
||||
|
||||
fn try_into_address(self) -> Option<EmailAddress<'x>> {
|
||||
let mut obj = self.into_object()?;
|
||||
Some(EmailAddress {
|
||||
name: obj
|
||||
.remove(&Key::Property(EmailProperty::Name))
|
||||
.and_then(|n| n.into_string()),
|
||||
email: obj
|
||||
.remove(&Key::Property(EmailProperty::Email))?
|
||||
.into_string()?,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl<'x> BuildHeader<'x> for MessageBuilder<'x> {
|
||||
fn build_header(
|
||||
self,
|
||||
header: HeaderProperty,
|
||||
value: Value<'x, EmailProperty, EmailValue>,
|
||||
) -> Result<Self, HeaderProperty> {
|
||||
Ok(match (&header.form, header.all, value) {
|
||||
(HeaderForm::Raw, false, Value::Str(value)) => {
|
||||
self.header(header.header, Raw::from(value))
|
||||
}
|
||||
(HeaderForm::Raw, true, Value::Array(value)) => self.headers(
|
||||
header.header,
|
||||
value
|
||||
.into_iter()
|
||||
.filter_map(|v| Raw::from(v.into_string()?).into()),
|
||||
),
|
||||
(HeaderForm::Date, false, Value::Element(EmailValue::Date(value))) => {
|
||||
self.header(header.header, Date::new(value.timestamp()))
|
||||
}
|
||||
(HeaderForm::Date, true, Value::Array(value)) => self.headers(
|
||||
header.header,
|
||||
value
|
||||
.into_iter()
|
||||
.filter_map(|v| Date::new(unwrap_date(v)?.timestamp()).into()),
|
||||
),
|
||||
(HeaderForm::Text, false, Value::Str(value)) => {
|
||||
self.header(header.header, Text::from(value))
|
||||
}
|
||||
(HeaderForm::Text, true, Value::Array(value)) => self.headers(
|
||||
header.header,
|
||||
value
|
||||
.into_iter()
|
||||
.filter_map(|v| Text::from(v.into_string()?).into()),
|
||||
),
|
||||
(HeaderForm::URLs, false, Value::Array(value)) => self.header(
|
||||
header.header,
|
||||
URL {
|
||||
url: value
|
||||
.into_iter()
|
||||
.filter_map(|v| v.into_string()?.into())
|
||||
.collect(),
|
||||
},
|
||||
),
|
||||
(HeaderForm::URLs, true, Value::Array(value)) => self.headers(
|
||||
header.header,
|
||||
value.into_iter().filter_map(|value| {
|
||||
URL {
|
||||
url: value
|
||||
.into_array()?
|
||||
.into_iter()
|
||||
.filter_map(|v| v.into_string()?.into())
|
||||
.collect(),
|
||||
}
|
||||
.into()
|
||||
}),
|
||||
),
|
||||
(HeaderForm::MessageIds, false, Value::Array(value)) => self.header(
|
||||
header.header,
|
||||
MessageId {
|
||||
id: value
|
||||
.into_iter()
|
||||
.filter_map(|v| v.into_string()?.into())
|
||||
.collect(),
|
||||
},
|
||||
),
|
||||
(HeaderForm::MessageIds, true, Value::Array(value)) => self.headers(
|
||||
header.header,
|
||||
value.into_iter().filter_map(|value| {
|
||||
MessageId {
|
||||
id: value
|
||||
.into_array()?
|
||||
.into_iter()
|
||||
.filter_map(|v| v.into_string()?.into())
|
||||
.collect(),
|
||||
}
|
||||
.into()
|
||||
}),
|
||||
),
|
||||
(HeaderForm::Addresses, false, Value::Array(value)) => self.header(
|
||||
header.header,
|
||||
Address::new_list(
|
||||
value
|
||||
.into_iter()
|
||||
.filter_map(|v| Address::Address(v.try_into_address()?).into())
|
||||
.collect(),
|
||||
),
|
||||
),
|
||||
(HeaderForm::Addresses, true, Value::Array(value)) => self.headers(
|
||||
header.header,
|
||||
value
|
||||
.into_iter()
|
||||
.filter_map(|v| Address::new_list(v.try_into_address_list()?).into()),
|
||||
),
|
||||
(HeaderForm::GroupedAddresses, false, Value::Array(value)) => self.header(
|
||||
header.header,
|
||||
Address::new_list(
|
||||
value
|
||||
.into_iter()
|
||||
.filter_map(|v| Address::Group(v.try_into_grouped_addresses()?).into())
|
||||
.collect(),
|
||||
),
|
||||
),
|
||||
(HeaderForm::GroupedAddresses, true, Value::Array(value)) => self.headers(
|
||||
header.header,
|
||||
value.into_iter().filter_map(|v| {
|
||||
Address::new_list(
|
||||
v.into_array()?
|
||||
.into_iter()
|
||||
.filter_map(|v| Address::Group(v.try_into_grouped_addresses()?).into())
|
||||
.collect::<Vec<_>>(),
|
||||
)
|
||||
.into()
|
||||
}),
|
||||
),
|
||||
_ => {
|
||||
return Err(header);
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl HeaderToValue for ArchivedMessageMetadataPart {
|
||||
fn header_to_value(
|
||||
&self,
|
||||
property: &EmailProperty,
|
||||
raw_message: &ChainedBytes<'_>,
|
||||
) -> Value<'static, EmailProperty, EmailValue> {
|
||||
let (header_name, form, all) = match property {
|
||||
EmailProperty::Header(header) => (
|
||||
HeaderName::parse(header.header.as_str())
|
||||
.unwrap_or_else(|| HeaderName::Other(header.header.as_str().into())),
|
||||
header.form,
|
||||
header.all,
|
||||
),
|
||||
EmailProperty::Sender => (HeaderName::Sender, HeaderForm::Addresses, false),
|
||||
EmailProperty::From => (HeaderName::From, HeaderForm::Addresses, false),
|
||||
EmailProperty::To => (HeaderName::To, HeaderForm::Addresses, false),
|
||||
EmailProperty::Cc => (HeaderName::Cc, HeaderForm::Addresses, false),
|
||||
EmailProperty::Bcc => (HeaderName::Bcc, HeaderForm::Addresses, false),
|
||||
EmailProperty::ReplyTo => (HeaderName::ReplyTo, HeaderForm::Addresses, false),
|
||||
EmailProperty::Subject => (HeaderName::Subject, HeaderForm::Text, false),
|
||||
EmailProperty::MessageId => (HeaderName::MessageId, HeaderForm::MessageIds, false),
|
||||
EmailProperty::InReplyTo => (HeaderName::InReplyTo, HeaderForm::MessageIds, false),
|
||||
EmailProperty::References => (HeaderName::References, HeaderForm::MessageIds, false),
|
||||
EmailProperty::SentAt => (HeaderName::Date, HeaderForm::Date, false),
|
||||
_ => return Value::Null,
|
||||
};
|
||||
|
||||
let is_raw = matches!(form, HeaderForm::Raw) || !header_name.is_structured();
|
||||
let mut headers = Vec::new();
|
||||
let header_name = header_name.as_str();
|
||||
for header in self.headers.iter().rev() {
|
||||
if header.name.as_str().eq_ignore_ascii_case(header_name) {
|
||||
let raw_header;
|
||||
let header_value =
|
||||
if is_raw || matches!(header.value, ArchivedMetadataHeaderValue::Empty) {
|
||||
raw_header = raw_message.get(header.value_range());
|
||||
|
||||
if let Some(bytes) = &raw_header {
|
||||
let bytes = bytes.as_ref();
|
||||
match form {
|
||||
HeaderForm::Raw => {
|
||||
HeaderValue::Text(String::from_utf8_lossy(bytes.trim_end()))
|
||||
}
|
||||
HeaderForm::Text => MessageStream::new(bytes).parse_unstructured(),
|
||||
HeaderForm::Addresses
|
||||
| HeaderForm::GroupedAddresses
|
||||
| HeaderForm::URLs => MessageStream::new(bytes).parse_address(),
|
||||
HeaderForm::MessageIds => MessageStream::new(bytes).parse_id(),
|
||||
HeaderForm::Date => MessageStream::new(bytes).parse_date(),
|
||||
}
|
||||
} else {
|
||||
HeaderValue::Empty
|
||||
}
|
||||
} else {
|
||||
HeaderValue::from(&header.value)
|
||||
};
|
||||
headers.push(header_value.into_form(&form));
|
||||
if !all {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if !all {
|
||||
headers.pop().unwrap_or_default()
|
||||
} else {
|
||||
if headers.len() > 1 {
|
||||
headers.reverse();
|
||||
}
|
||||
Value::Array(headers)
|
||||
}
|
||||
}
|
||||
|
||||
fn headers_to_value(
|
||||
&self,
|
||||
raw_message: &ChainedBytes<'_>,
|
||||
) -> Value<'static, EmailProperty, EmailValue> {
|
||||
let mut headers = Vec::with_capacity(self.headers.len());
|
||||
for header in self.headers.iter() {
|
||||
headers.push(Value::Object(
|
||||
Map::with_capacity(2)
|
||||
.with_key_value(EmailProperty::Name, header.name.as_str().to_string())
|
||||
.with_key_value(
|
||||
EmailProperty::Value,
|
||||
String::from_utf8_lossy(
|
||||
raw_message
|
||||
.get(header.value_range())
|
||||
.unwrap_or_default()
|
||||
.as_ref()
|
||||
.trim_end(),
|
||||
)
|
||||
.into_owned(),
|
||||
),
|
||||
));
|
||||
}
|
||||
headers.into()
|
||||
}
|
||||
}
|
||||
|
||||
trait ByteTrim {
|
||||
fn trim_end(&self) -> Self;
|
||||
}
|
||||
|
||||
impl ByteTrim for &[u8] {
|
||||
fn trim_end(&self) -> Self {
|
||||
let mut end = self.len();
|
||||
while end > 0 && self[end - 1].is_ascii_whitespace() {
|
||||
end -= 1;
|
||||
}
|
||||
&self[..end]
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub(crate) fn unwrap_date(value: Value<'_, EmailProperty, EmailValue>) -> Option<UTCDate> {
|
||||
match value {
|
||||
Value::Element(EmailValue::Date(date)) => Some(date),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
pub trait IntoForm {
|
||||
fn into_form(self, form: &HeaderForm) -> Value<'static, EmailProperty, EmailValue>;
|
||||
}
|
||||
|
||||
impl IntoForm for HeaderValue<'_> {
|
||||
fn into_form(self, form: &HeaderForm) -> Value<'static, EmailProperty, EmailValue> {
|
||||
match (self, form) {
|
||||
(HeaderValue::Text(text), HeaderForm::Raw | HeaderForm::Text) => {
|
||||
text.into_owned().into()
|
||||
}
|
||||
(HeaderValue::TextList(texts), HeaderForm::Raw | HeaderForm::Text) => {
|
||||
texts.join(", ").into()
|
||||
}
|
||||
(HeaderValue::Text(text), HeaderForm::MessageIds) => {
|
||||
Value::Array(vec![text.into_owned().into()])
|
||||
}
|
||||
(HeaderValue::TextList(texts), HeaderForm::MessageIds) => {
|
||||
Value::Array(texts.into_iter().map(|t| t.into_owned().into()).collect())
|
||||
}
|
||||
(HeaderValue::DateTime(datetime), HeaderForm::Date) => from_mail_datetime(datetime),
|
||||
(HeaderValue::Address(mail_parser::Address::List(addrlist)), HeaderForm::URLs) => {
|
||||
Value::Array(
|
||||
addrlist
|
||||
.into_iter()
|
||||
.filter_map(|addr| match addr {
|
||||
Addr {
|
||||
address: Some(addr),
|
||||
..
|
||||
} if addr.contains(':') => Some(addr.into_owned().into()),
|
||||
_ => None,
|
||||
})
|
||||
.collect(),
|
||||
)
|
||||
}
|
||||
(HeaderValue::Address(mail_parser::Address::List(addrlist)), HeaderForm::Addresses) => {
|
||||
from_mail_addrlist(addrlist)
|
||||
}
|
||||
(
|
||||
HeaderValue::Address(mail_parser::Address::Group(grouplist)),
|
||||
HeaderForm::Addresses,
|
||||
) => Value::Array(
|
||||
grouplist
|
||||
.into_iter()
|
||||
.flat_map(|group| group.addresses.into_iter().map(from_mail_addr))
|
||||
.collect(),
|
||||
),
|
||||
(
|
||||
HeaderValue::Address(mail_parser::Address::List(addrlist)),
|
||||
HeaderForm::GroupedAddresses,
|
||||
) => Value::Array(vec![
|
||||
Map::with_capacity(2)
|
||||
.with_key_value(EmailProperty::Name, Value::Null)
|
||||
.with_key_value(EmailProperty::Addresses, from_mail_addrlist(addrlist))
|
||||
.into(),
|
||||
]),
|
||||
(
|
||||
HeaderValue::Address(mail_parser::Address::Group(grouplist)),
|
||||
HeaderForm::GroupedAddresses,
|
||||
) => Value::Array(
|
||||
grouplist
|
||||
.into_iter()
|
||||
.map(from_mail_group)
|
||||
.collect::<Vec<Value<'static, EmailProperty, EmailValue>>>(),
|
||||
),
|
||||
|
||||
_ => Value::Null,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn from_mail_datetime(date: DateTime) -> Value<'static, EmailProperty, EmailValue> {
|
||||
Value::Element(EmailValue::Date(UTCDate {
|
||||
year: date.year,
|
||||
month: date.month,
|
||||
day: date.day,
|
||||
hour: date.hour,
|
||||
minute: date.minute,
|
||||
second: date.second,
|
||||
tz_before_gmt: date.tz_before_gmt,
|
||||
tz_hour: date.tz_hour,
|
||||
tz_minute: date.tz_minute,
|
||||
}))
|
||||
}
|
||||
|
||||
fn from_mail_addr(value: Addr<'_>) -> Value<'static, EmailProperty, EmailValue> {
|
||||
Value::Object(
|
||||
Map::with_capacity(2)
|
||||
.with_key_value(EmailProperty::Name, value.name.map(|v| v.into_owned()))
|
||||
.with_key_value(
|
||||
EmailProperty::Email,
|
||||
value.address.unwrap_or_default().into_owned(),
|
||||
),
|
||||
)
|
||||
}
|
||||
|
||||
fn from_mail_group(group: Group<'_>) -> Value<'static, EmailProperty, EmailValue> {
|
||||
Value::Object(
|
||||
Map::with_capacity(2)
|
||||
.with_key_value(EmailProperty::Name, group.name.map(|v| v.into_owned()))
|
||||
.with_key_value(
|
||||
EmailProperty::Addresses,
|
||||
from_mail_addrlist(group.addresses),
|
||||
),
|
||||
)
|
||||
}
|
||||
|
||||
fn from_mail_addrlist(addrlist: Vec<Addr<'_>>) -> Value<'static, EmailProperty, EmailValue> {
|
||||
Value::Array(
|
||||
addrlist
|
||||
.into_iter()
|
||||
.map(from_mail_addr)
|
||||
.collect::<Vec<Value<'static, EmailProperty, EmailValue>>>(),
|
||||
)
|
||||
}
|
||||
@@ -0,0 +1,291 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
|
||||
*
|
||||
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
||||
*/
|
||||
|
||||
use crate::message::metadata::{
|
||||
ArchivedMessageMetadataContents, ArchivedMessageMetadataPart, ArchivedMetadataHeaderValue,
|
||||
MetadataHeaderName, MetadataHeaderValue,
|
||||
};
|
||||
use mail_parser::{Addr, Address, Group, HeaderValue};
|
||||
use nlp::language::Language;
|
||||
use rkyv::option::ArchivedOption;
|
||||
use std::borrow::Cow;
|
||||
|
||||
impl ArchivedMessageMetadataContents {
|
||||
pub fn is_html_part(&self, part_id: u16) -> bool {
|
||||
self.html_body.iter().any(|&id| id == part_id)
|
||||
}
|
||||
|
||||
pub fn is_text_part(&self, part_id: u16) -> bool {
|
||||
self.text_body.iter().any(|&id| id == part_id)
|
||||
}
|
||||
}
|
||||
|
||||
impl ArchivedMessageMetadataPart {
|
||||
pub fn language(&self) -> Option<Language> {
|
||||
self.header_value(&MetadataHeaderName::ContentLanguage)
|
||||
.and_then(|v| {
|
||||
Language::from_iso_639(v.as_text()?)
|
||||
.unwrap_or(Language::Unknown)
|
||||
.into()
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, PartialEq, Eq)]
|
||||
pub enum AddressElement {
|
||||
Name,
|
||||
Address,
|
||||
GroupName,
|
||||
}
|
||||
|
||||
pub trait VisitText {
|
||||
fn visit_addresses(&self, visitor: impl FnMut(AddressElement, &str));
|
||||
fn visit_text<'x>(&'x self, visitor: impl FnMut(&'x str));
|
||||
fn into_visit_text(self, visitor: impl FnMut(String));
|
||||
}
|
||||
|
||||
impl VisitText for HeaderValue<'_> {
|
||||
fn visit_addresses(&self, mut visitor: impl FnMut(AddressElement, &str)) {
|
||||
match self {
|
||||
HeaderValue::Address(Address::List(addr_list)) => {
|
||||
for addr in addr_list {
|
||||
if let Some(name) = &addr.name {
|
||||
visitor(AddressElement::Name, name);
|
||||
}
|
||||
if let Some(addr) = &addr.address {
|
||||
visitor(AddressElement::Address, addr);
|
||||
}
|
||||
}
|
||||
}
|
||||
HeaderValue::Address(Address::Group(groups)) => {
|
||||
for group in groups {
|
||||
if let Some(name) = &group.name {
|
||||
visitor(AddressElement::GroupName, name);
|
||||
}
|
||||
|
||||
for addr in &group.addresses {
|
||||
if let Some(name) = &addr.name {
|
||||
visitor(AddressElement::Name, name);
|
||||
}
|
||||
if let Some(addr) = &addr.address {
|
||||
visitor(AddressElement::Address, addr);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
_ => (),
|
||||
}
|
||||
}
|
||||
|
||||
fn visit_text<'x>(&'x self, mut visitor: impl FnMut(&'x str)) {
|
||||
match &self {
|
||||
HeaderValue::Text(text) => {
|
||||
visitor(text.as_ref());
|
||||
}
|
||||
HeaderValue::TextList(texts) => {
|
||||
for text in texts {
|
||||
visitor(text.as_ref());
|
||||
}
|
||||
}
|
||||
_ => (),
|
||||
}
|
||||
}
|
||||
|
||||
fn into_visit_text(self, mut visitor: impl FnMut(String)) {
|
||||
match self {
|
||||
HeaderValue::Text(text) => {
|
||||
visitor(text.into_owned());
|
||||
}
|
||||
HeaderValue::TextList(texts) => {
|
||||
for text in texts {
|
||||
visitor(text.into_owned());
|
||||
}
|
||||
}
|
||||
_ => (),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub trait VisitTextArchived {
|
||||
fn visit_addresses(&self, visitor: impl FnMut(AddressElement, &str));
|
||||
fn visit_text(&self, visitor: impl FnMut(&str));
|
||||
}
|
||||
|
||||
impl VisitTextArchived for MetadataHeaderValue {
|
||||
fn visit_addresses(&self, mut visitor: impl FnMut(AddressElement, &str)) {
|
||||
match self {
|
||||
MetadataHeaderValue::AddressList(addr_list) => {
|
||||
for addr in addr_list.iter() {
|
||||
if let Some(name) = &addr.name {
|
||||
visitor(AddressElement::Name, name);
|
||||
}
|
||||
if let Some(addr) = &addr.address {
|
||||
visitor(AddressElement::Address, addr);
|
||||
}
|
||||
}
|
||||
}
|
||||
MetadataHeaderValue::AddressGroup(groups) => {
|
||||
for group in groups.iter() {
|
||||
if let Some(name) = &group.name {
|
||||
visitor(AddressElement::GroupName, name);
|
||||
}
|
||||
|
||||
for addr in group.addresses.iter() {
|
||||
if let Some(name) = &addr.name {
|
||||
visitor(AddressElement::Name, name);
|
||||
}
|
||||
if let Some(addr) = &addr.address {
|
||||
visitor(AddressElement::Address, addr);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
_ => (),
|
||||
}
|
||||
}
|
||||
|
||||
fn visit_text(&self, mut visitor: impl FnMut(&str)) {
|
||||
match &self {
|
||||
MetadataHeaderValue::Text(text) => {
|
||||
visitor(text.as_ref());
|
||||
}
|
||||
MetadataHeaderValue::TextList(texts) => {
|
||||
for text in texts.iter() {
|
||||
visitor(text.as_ref());
|
||||
}
|
||||
}
|
||||
_ => (),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl VisitTextArchived for ArchivedMetadataHeaderValue {
|
||||
fn visit_addresses(&self, mut visitor: impl FnMut(AddressElement, &str)) {
|
||||
match self {
|
||||
ArchivedMetadataHeaderValue::AddressList(addr_list) => {
|
||||
for addr in addr_list.iter() {
|
||||
if let ArchivedOption::Some(name) = &addr.name {
|
||||
visitor(AddressElement::Name, name);
|
||||
}
|
||||
if let ArchivedOption::Some(addr) = &addr.address {
|
||||
visitor(AddressElement::Address, addr);
|
||||
}
|
||||
}
|
||||
}
|
||||
ArchivedMetadataHeaderValue::AddressGroup(groups) => {
|
||||
for group in groups.iter() {
|
||||
if let ArchivedOption::Some(name) = &group.name {
|
||||
visitor(AddressElement::GroupName, name);
|
||||
}
|
||||
|
||||
for addr in group.addresses.iter() {
|
||||
if let ArchivedOption::Some(name) = &addr.name {
|
||||
visitor(AddressElement::Name, name);
|
||||
}
|
||||
if let ArchivedOption::Some(addr) = &addr.address {
|
||||
visitor(AddressElement::Address, addr);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
_ => (),
|
||||
}
|
||||
}
|
||||
|
||||
fn visit_text(&self, mut visitor: impl FnMut(&str)) {
|
||||
match &self {
|
||||
ArchivedMetadataHeaderValue::Text(text) => {
|
||||
visitor(text.as_ref());
|
||||
}
|
||||
ArchivedMetadataHeaderValue::TextList(texts) => {
|
||||
for text in texts.iter() {
|
||||
visitor(text.as_ref());
|
||||
}
|
||||
}
|
||||
_ => (),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub trait TrimTextValue {
|
||||
fn trim_text(self, length: usize) -> Self;
|
||||
}
|
||||
|
||||
impl TrimTextValue for HeaderValue<'_> {
|
||||
fn trim_text(self, length: usize) -> Self {
|
||||
match self {
|
||||
HeaderValue::Address(Address::List(v)) => {
|
||||
HeaderValue::Address(Address::List(v.trim_text(length)))
|
||||
}
|
||||
HeaderValue::Address(Address::Group(v)) => {
|
||||
HeaderValue::Address(Address::Group(v.trim_text(length)))
|
||||
}
|
||||
HeaderValue::Text(v) => HeaderValue::Text(v.trim_text(length)),
|
||||
HeaderValue::TextList(v) => HeaderValue::TextList(v.trim_text(length)),
|
||||
v => v,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl TrimTextValue for Addr<'_> {
|
||||
fn trim_text(self, length: usize) -> Self {
|
||||
Self {
|
||||
name: self.name.map(|v| v.trim_text(length)),
|
||||
address: self.address.map(|v| v.trim_text(length)),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl TrimTextValue for Group<'_> {
|
||||
fn trim_text(self, length: usize) -> Self {
|
||||
Self {
|
||||
name: self.name.map(|v| v.trim_text(length)),
|
||||
addresses: self.addresses.trim_text(length),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl TrimTextValue for &str {
|
||||
fn trim_text(self, length: usize) -> Self {
|
||||
if self.len() < length {
|
||||
self
|
||||
} else {
|
||||
let mut index = 0;
|
||||
|
||||
for (i, _) in self.char_indices() {
|
||||
if i > length {
|
||||
break;
|
||||
}
|
||||
index = i;
|
||||
}
|
||||
|
||||
&self[..index]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl TrimTextValue for Cow<'_, str> {
|
||||
fn trim_text(self, length: usize) -> Self {
|
||||
if self.len() < length {
|
||||
self
|
||||
} else {
|
||||
let mut result = String::with_capacity(length);
|
||||
for (i, c) in self.char_indices() {
|
||||
if i > length {
|
||||
break;
|
||||
}
|
||||
result.push(c);
|
||||
}
|
||||
result.into()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<T: TrimTextValue> TrimTextValue for Vec<T> {
|
||||
fn trim_text(self, length: usize) -> Self {
|
||||
self.into_iter().map(|v| v.trim_text(length)).collect()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,251 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
|
||||
*
|
||||
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
||||
*/
|
||||
|
||||
use crate::message::{
|
||||
index::{IndexMessage, MAX_MESSAGE_PARTS, PREVIEW_LENGTH},
|
||||
metadata::{
|
||||
ArchivedMessageMetadata, ArchivedMessageMetadataPart, ArchivedMetadataHeaderName,
|
||||
MESSAGE_HAS_ATTACHMENT, MESSAGE_RECEIVED_MASK, MessageData, MessageMetadata,
|
||||
MessageMetadataPart, build_metadata_contents,
|
||||
},
|
||||
};
|
||||
use common::storage::index::ObjectIndexBuilder;
|
||||
use mail_parser::{
|
||||
PartType,
|
||||
decoders::html::html_to_text,
|
||||
parsers::{fields::thread::thread_name, preview::preview_text},
|
||||
};
|
||||
use store::{
|
||||
Serialize,
|
||||
write::{Archiver, BatchBuilder, BlobLink, BlobOp, IndexPropertyClass, ValueClass},
|
||||
};
|
||||
use trc::AddContext;
|
||||
use types::{blob_hash::BlobHash, field::EmailField};
|
||||
use utils::cheeky_hash::CheekyHash;
|
||||
|
||||
impl MessageMetadata {
|
||||
#[inline(always)]
|
||||
pub fn root_part(&self) -> &MessageMetadataPart {
|
||||
&self.contents[0].parts[0]
|
||||
}
|
||||
|
||||
pub fn index(self, batch: &mut BatchBuilder, set: bool) -> trc::Result<()> {
|
||||
if set {
|
||||
batch
|
||||
.set(
|
||||
BlobOp::Link {
|
||||
hash: self.blob_hash.clone(),
|
||||
to: BlobLink::Document,
|
||||
},
|
||||
Vec::new(),
|
||||
)
|
||||
.set(EmailField::Metadata, Archiver::new(self).serialize()?);
|
||||
} else {
|
||||
batch
|
||||
.clear(BlobOp::Link {
|
||||
hash: self.blob_hash.clone(),
|
||||
to: BlobLink::Document,
|
||||
})
|
||||
.clear(EmailField::Metadata);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl ArchivedMessageMetadata {
|
||||
#[inline(always)]
|
||||
pub fn root_part(&self) -> &ArchivedMessageMetadataPart {
|
||||
&self.contents[0].parts[0]
|
||||
}
|
||||
|
||||
pub fn unindex(&self, batch: &mut BatchBuilder) {
|
||||
// Delete metadata
|
||||
let thread_name = self
|
||||
.contents
|
||||
.first()
|
||||
.and_then(|c| c.parts.first())
|
||||
.and_then(|p| {
|
||||
p.headers.iter().rev().find_map(|h| {
|
||||
if let ArchivedMetadataHeaderName::Subject = &h.name {
|
||||
h.value.as_text()
|
||||
} else {
|
||||
None
|
||||
}
|
||||
})
|
||||
})
|
||||
.map(thread_name)
|
||||
.unwrap_or_default();
|
||||
|
||||
batch
|
||||
.clear(EmailField::Metadata)
|
||||
.clear(ValueClass::IndexProperty(IndexPropertyClass::Hash {
|
||||
property: EmailField::Threading.into(),
|
||||
hash: CheekyHash::new(if !thread_name.is_empty() {
|
||||
thread_name
|
||||
} else {
|
||||
"!"
|
||||
}),
|
||||
}))
|
||||
.clear(BlobOp::Link {
|
||||
hash: BlobHash::from(&self.blob_hash),
|
||||
to: BlobLink::Document,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
impl IndexMessage for BatchBuilder {
|
||||
fn index_message<'x>(
|
||||
&mut self,
|
||||
tenant_id: Option<u32>,
|
||||
mut message: mail_parser::Message<'x>,
|
||||
extra_headers: Vec<u8>,
|
||||
mut extra_headers_parsed: Vec<mail_parser::Header<'x>>,
|
||||
blob_hash: BlobHash,
|
||||
data: MessageData,
|
||||
received_at: u64,
|
||||
) -> trc::Result<&mut Self> {
|
||||
let mut has_attachments = false;
|
||||
let mut preview = None;
|
||||
let preview_part_id = message
|
||||
.text_body
|
||||
.first()
|
||||
.or_else(|| message.html_body.first())
|
||||
.copied()
|
||||
.unwrap_or(u32::MAX);
|
||||
|
||||
for (part_id, part) in message.parts.iter().take(MAX_MESSAGE_PARTS).enumerate() {
|
||||
let part_id = part_id as u32;
|
||||
match &part.body {
|
||||
mail_parser::PartType::Text(text) => {
|
||||
if part_id == preview_part_id {
|
||||
preview =
|
||||
preview_text(text.replace('\r', "").into(), PREVIEW_LENGTH).into();
|
||||
}
|
||||
|
||||
if !message.text_body.contains(&part_id)
|
||||
&& !message.html_body.contains(&part_id)
|
||||
{
|
||||
has_attachments = true;
|
||||
}
|
||||
}
|
||||
mail_parser::PartType::Html(html) => {
|
||||
let text = html_to_text(html);
|
||||
if part_id == preview_part_id {
|
||||
preview =
|
||||
preview_text(text.replace('\r', "").into(), PREVIEW_LENGTH).into();
|
||||
}
|
||||
|
||||
if !message.text_body.contains(&part_id)
|
||||
&& !message.html_body.contains(&part_id)
|
||||
{
|
||||
has_attachments = true;
|
||||
}
|
||||
}
|
||||
mail_parser::PartType::Binary(_) | mail_parser::PartType::Message(_)
|
||||
if !has_attachments =>
|
||||
{
|
||||
has_attachments = true;
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
|
||||
// Build raw headers
|
||||
let root_part = message.root_part();
|
||||
let mut raw_headers = Vec::with_capacity(
|
||||
(root_part.offset_body - root_part.offset_header) as usize + extra_headers.len(),
|
||||
);
|
||||
raw_headers.extend_from_slice(&extra_headers);
|
||||
raw_headers.extend_from_slice(
|
||||
message
|
||||
.raw_message
|
||||
.as_ref()
|
||||
.get(root_part.offset_header as usize..root_part.offset_body as usize)
|
||||
.unwrap_or_default(),
|
||||
);
|
||||
|
||||
// Add additional headers to message
|
||||
let blob_body_offset = if !extra_headers.is_empty() {
|
||||
// Add extra headers to root part
|
||||
let offset_start = extra_headers.len() as u32;
|
||||
let mut part_iter_stack = Vec::new();
|
||||
let mut part_iter = message.parts.iter_mut();
|
||||
|
||||
loop {
|
||||
if let Some(part) = part_iter.next() {
|
||||
// Increment header offsets
|
||||
for header in part.headers.iter_mut() {
|
||||
header.offset_field += offset_start;
|
||||
header.offset_start += offset_start;
|
||||
header.offset_end += offset_start;
|
||||
}
|
||||
|
||||
// Adjust part offsets
|
||||
part.offset_body += offset_start;
|
||||
part.offset_end += offset_start;
|
||||
part.offset_header += offset_start;
|
||||
|
||||
if let PartType::Message(sub_message) = &mut part.body
|
||||
&& sub_message.root_part().offset_header != 0
|
||||
{
|
||||
part_iter_stack.push(part_iter);
|
||||
part_iter = sub_message.parts.iter_mut();
|
||||
}
|
||||
} else if let Some(iter) = part_iter_stack.pop() {
|
||||
part_iter = iter;
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// Add extra headers to root part
|
||||
let root_part = &mut message.parts[0];
|
||||
extra_headers_parsed.append(&mut root_part.headers);
|
||||
root_part.offset_header = 0;
|
||||
root_part.headers = extra_headers_parsed;
|
||||
root_part.offset_body - offset_start
|
||||
} else {
|
||||
message.root_part().offset_body
|
||||
};
|
||||
|
||||
// Build metadata
|
||||
let metadata = MessageMetadata {
|
||||
preview: preview.unwrap_or_default().into_owned().into_boxed_str(),
|
||||
raw_headers: raw_headers.into_boxed_slice(),
|
||||
contents: build_metadata_contents(message),
|
||||
blob_hash,
|
||||
blob_body_offset,
|
||||
rcvd_attach: (if has_attachments {
|
||||
MESSAGE_HAS_ATTACHMENT
|
||||
} else {
|
||||
0
|
||||
}) | (received_at & MESSAGE_RECEIVED_MASK),
|
||||
};
|
||||
|
||||
self.set(
|
||||
BlobOp::Link {
|
||||
hash: metadata.blob_hash.clone(),
|
||||
to: BlobLink::Document,
|
||||
},
|
||||
Vec::new(),
|
||||
)
|
||||
.custom(
|
||||
ObjectIndexBuilder::<(), _>::new()
|
||||
.with_tenant_id(tenant_id)
|
||||
.with_changes(data),
|
||||
)
|
||||
.caused_by(trc::location!())?
|
||||
.set(
|
||||
EmailField::Metadata,
|
||||
Archiver::new(metadata)
|
||||
.serialize()
|
||||
.caused_by(trc::location!())?,
|
||||
);
|
||||
|
||||
Ok(self)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,111 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
|
||||
*
|
||||
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
||||
*/
|
||||
|
||||
use crate::{
|
||||
mailbox::{JUNK_ID, TRASH_ID},
|
||||
message::metadata::{ArchivedMessageData, MessageData},
|
||||
};
|
||||
use common::storage::index::{IndexItem, IndexValue, IndexableObject};
|
||||
use store::write::now;
|
||||
use types::{blob_hash::BlobHash, collection::SyncCollection, field::EmailField};
|
||||
|
||||
pub mod extractors;
|
||||
pub mod metadata;
|
||||
pub mod search;
|
||||
|
||||
pub(super) const MAX_MESSAGE_PARTS: usize = 1000;
|
||||
pub const PREVIEW_LENGTH: usize = 256;
|
||||
|
||||
impl IndexableObject for MessageData {
|
||||
fn index_values(&self) -> impl Iterator<Item = IndexValue<'_>> {
|
||||
let mut mailboxes = Vec::with_capacity(self.mailboxes.len());
|
||||
let mut is_in_trash = false;
|
||||
|
||||
for mailbox in &self.mailboxes {
|
||||
mailboxes.push(mailbox.mailbox_id);
|
||||
is_in_trash |= mailbox.mailbox_id == TRASH_ID || mailbox.mailbox_id == JUNK_ID;
|
||||
}
|
||||
|
||||
[
|
||||
IndexValue::Property {
|
||||
field: EmailField::DeletedAt.into(),
|
||||
value: if is_in_trash {
|
||||
IndexItem::from(now())
|
||||
} else {
|
||||
IndexItem::None
|
||||
},
|
||||
},
|
||||
IndexValue::Quota { used: self.size },
|
||||
IndexValue::LogItem {
|
||||
sync_collection: SyncCollection::Email,
|
||||
prefix: self.thread_id.into(),
|
||||
},
|
||||
IndexValue::LogContainerProperty {
|
||||
sync_collection: SyncCollection::Thread,
|
||||
ids: vec![self.thread_id],
|
||||
},
|
||||
IndexValue::LogContainerProperty {
|
||||
sync_collection: SyncCollection::Email,
|
||||
ids: mailboxes,
|
||||
},
|
||||
]
|
||||
.into_iter()
|
||||
}
|
||||
}
|
||||
|
||||
impl IndexableObject for &ArchivedMessageData {
|
||||
fn index_values(&self) -> impl Iterator<Item = IndexValue<'_>> {
|
||||
let mut mailboxes = Vec::with_capacity(self.mailboxes.len());
|
||||
let mut is_in_trash = false;
|
||||
|
||||
for mailbox in self.mailboxes.iter() {
|
||||
let mailbox_id = mailbox.mailbox_id.to_native();
|
||||
mailboxes.push(mailbox_id);
|
||||
is_in_trash |= mailbox_id == TRASH_ID || mailbox_id == JUNK_ID;
|
||||
}
|
||||
|
||||
[
|
||||
IndexValue::Property {
|
||||
field: EmailField::DeletedAt.into(),
|
||||
value: if is_in_trash {
|
||||
IndexItem::from(now())
|
||||
} else {
|
||||
IndexItem::None
|
||||
},
|
||||
},
|
||||
IndexValue::Quota {
|
||||
used: self.size.to_native(),
|
||||
},
|
||||
IndexValue::LogItem {
|
||||
sync_collection: SyncCollection::Email,
|
||||
prefix: self.thread_id.to_native().into(),
|
||||
},
|
||||
IndexValue::LogContainerProperty {
|
||||
sync_collection: SyncCollection::Thread,
|
||||
ids: vec![self.thread_id.to_native()],
|
||||
},
|
||||
IndexValue::LogContainerProperty {
|
||||
sync_collection: SyncCollection::Email,
|
||||
ids: mailboxes,
|
||||
},
|
||||
]
|
||||
.into_iter()
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) trait IndexMessage {
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
fn index_message<'x>(
|
||||
&mut self,
|
||||
tenant_id: Option<u32>,
|
||||
message: mail_parser::Message<'x>,
|
||||
extra_headers: Vec<u8>,
|
||||
extra_headers_parsed: Vec<mail_parser::Header<'x>>,
|
||||
blob_hash: BlobHash,
|
||||
data: MessageData,
|
||||
received_at: u64,
|
||||
) -> trc::Result<&mut Self>;
|
||||
}
|
||||
@@ -0,0 +1,344 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
|
||||
*
|
||||
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
||||
*/
|
||||
|
||||
use crate::message::{
|
||||
index::{MAX_MESSAGE_PARTS, extractors::VisitTextArchived},
|
||||
metadata::{
|
||||
ArchivedMessageMetadata, ArchivedMetadataHeaderName, ArchivedMetadataHeaderValue,
|
||||
ArchivedMetadataPartType, DecodedPartContent, MESSAGE_HAS_ATTACHMENT,
|
||||
MESSAGE_RECEIVED_MASK, MetadataHeaderName,
|
||||
},
|
||||
};
|
||||
use mail_parser::{DateTime, decoders::html::html_to_text, parsers::fields::thread::thread_name};
|
||||
use nlp::{
|
||||
language::{
|
||||
Language,
|
||||
detect::{LanguageDetector, MIN_LANGUAGE_SCORE},
|
||||
},
|
||||
tokenizers::word::WordTokenizer,
|
||||
};
|
||||
use store::{
|
||||
ahash::AHashSet,
|
||||
backend::MAX_TOKEN_LENGTH,
|
||||
search::{EmailSearchField, IndexDocument, SearchField},
|
||||
write::SearchIndex,
|
||||
};
|
||||
use utils::chained_bytes::ChainedBytes;
|
||||
|
||||
impl ArchivedMessageMetadata {
|
||||
pub fn index_document(
|
||||
&self,
|
||||
account_id: u32,
|
||||
document_id: u32,
|
||||
raw_message: &[u8],
|
||||
index_fields: &AHashSet<SearchField>,
|
||||
default_language: Language,
|
||||
) -> IndexDocument {
|
||||
let mut detector = LanguageDetector::new();
|
||||
let mut language = Language::Unknown;
|
||||
let message_contents = &self.contents[0];
|
||||
let mut document = IndexDocument::new(SearchIndex::Email)
|
||||
.with_account_id(account_id)
|
||||
.with_document_id(document_id);
|
||||
|
||||
let raw_message = ChainedBytes::new(self.raw_headers.as_ref()).with_last(
|
||||
raw_message
|
||||
.get(self.blob_body_offset.to_native() as usize..)
|
||||
.unwrap_or_default(),
|
||||
);
|
||||
|
||||
if index_fields.is_empty()
|
||||
|| index_fields.contains(&SearchField::Email(EmailSearchField::ReceivedAt))
|
||||
{
|
||||
document.index_unsigned(
|
||||
SearchField::Email(EmailSearchField::ReceivedAt),
|
||||
self.rcvd_attach.to_native() & MESSAGE_RECEIVED_MASK,
|
||||
);
|
||||
}
|
||||
if index_fields.is_empty()
|
||||
|| index_fields.contains(&SearchField::Email(EmailSearchField::Size))
|
||||
{
|
||||
document.index_unsigned(
|
||||
SearchField::Email(EmailSearchField::Size),
|
||||
raw_message.len() as u32,
|
||||
);
|
||||
}
|
||||
|
||||
for (part_id, part) in message_contents
|
||||
.parts
|
||||
.iter()
|
||||
.take(MAX_MESSAGE_PARTS)
|
||||
.enumerate()
|
||||
{
|
||||
let part_language = part.language().unwrap_or(language);
|
||||
if part_id == 0 {
|
||||
language = part_language;
|
||||
|
||||
for header in part.headers.iter().rev() {
|
||||
match &header.name {
|
||||
ArchivedMetadataHeaderName::From => {
|
||||
if index_fields.is_empty()
|
||||
|| index_fields
|
||||
.contains(&SearchField::Email(EmailSearchField::From))
|
||||
{
|
||||
header.value.visit_addresses(|_, value| {
|
||||
document.index_text(
|
||||
SearchField::Email(EmailSearchField::From),
|
||||
value,
|
||||
Language::None,
|
||||
);
|
||||
});
|
||||
}
|
||||
}
|
||||
ArchivedMetadataHeaderName::To => {
|
||||
if index_fields.is_empty()
|
||||
|| index_fields.contains(&SearchField::Email(EmailSearchField::To))
|
||||
{
|
||||
header.value.visit_addresses(|_, value| {
|
||||
document.index_text(
|
||||
SearchField::Email(EmailSearchField::To),
|
||||
value,
|
||||
Language::None,
|
||||
);
|
||||
});
|
||||
}
|
||||
}
|
||||
ArchivedMetadataHeaderName::Cc => {
|
||||
if index_fields.is_empty()
|
||||
|| index_fields.contains(&SearchField::Email(EmailSearchField::Cc))
|
||||
{
|
||||
header.value.visit_addresses(|_, value| {
|
||||
document.index_text(
|
||||
SearchField::Email(EmailSearchField::Cc),
|
||||
value,
|
||||
Language::None,
|
||||
);
|
||||
});
|
||||
}
|
||||
}
|
||||
ArchivedMetadataHeaderName::Bcc => {
|
||||
if index_fields.is_empty()
|
||||
|| index_fields.contains(&SearchField::Email(EmailSearchField::Bcc))
|
||||
{
|
||||
header.value.visit_addresses(|_, value| {
|
||||
document.index_text(
|
||||
SearchField::Email(EmailSearchField::Bcc),
|
||||
value,
|
||||
Language::None,
|
||||
);
|
||||
});
|
||||
}
|
||||
}
|
||||
ArchivedMetadataHeaderName::Subject => {
|
||||
if (index_fields.is_empty()
|
||||
|| index_fields
|
||||
.contains(&SearchField::Email(EmailSearchField::Subject)))
|
||||
&& let Some(subject) = header.value.as_text()
|
||||
{
|
||||
let subject = thread_name(subject);
|
||||
|
||||
if part_language.is_unknown() {
|
||||
detector.detect(subject, MIN_LANGUAGE_SCORE);
|
||||
}
|
||||
|
||||
document.index_text(
|
||||
SearchField::Email(EmailSearchField::Subject),
|
||||
subject,
|
||||
part_language,
|
||||
);
|
||||
}
|
||||
}
|
||||
ArchivedMetadataHeaderName::Date => {
|
||||
if (index_fields.is_empty()
|
||||
|| index_fields
|
||||
.contains(&SearchField::Email(EmailSearchField::SentAt)))
|
||||
&& let Some(date) = header.value.as_datetime()
|
||||
{
|
||||
document.index_integer(
|
||||
SearchField::Email(EmailSearchField::SentAt),
|
||||
DateTime::from(date).to_timestamp(),
|
||||
);
|
||||
}
|
||||
}
|
||||
_ => {
|
||||
#[cfg(not(feature = "test_mode"))]
|
||||
let index_headers = index_fields
|
||||
.contains(&SearchField::Email(EmailSearchField::Headers));
|
||||
|
||||
#[cfg(feature = "test_mode")]
|
||||
let index_headers = true;
|
||||
|
||||
if index_headers {
|
||||
let mut value = String::new();
|
||||
match &header.value {
|
||||
ArchivedMetadataHeaderValue::AddressList(_)
|
||||
| ArchivedMetadataHeaderValue::AddressGroup(_) => {
|
||||
header.value.visit_addresses(|_, addr| {
|
||||
if !value.is_empty() {
|
||||
value.push(' ');
|
||||
}
|
||||
value.push_str(addr);
|
||||
});
|
||||
}
|
||||
ArchivedMetadataHeaderValue::Text(_)
|
||||
| ArchivedMetadataHeaderValue::TextList(_) => {
|
||||
header.value.visit_text(|text| {
|
||||
if !value.is_empty() {
|
||||
value.push(' ');
|
||||
}
|
||||
value.push_str(text);
|
||||
});
|
||||
}
|
||||
_ => {
|
||||
if let Some(raw_value) =
|
||||
raw_message.get(header.value_range())
|
||||
{
|
||||
let raw_value = std::str::from_utf8(raw_value.as_ref())
|
||||
.unwrap_or_default();
|
||||
|
||||
for word in
|
||||
WordTokenizer::new(raw_value, MAX_TOKEN_LENGTH)
|
||||
{
|
||||
if !value.is_empty() {
|
||||
value.push(' ');
|
||||
}
|
||||
value.push_str(word.word.as_ref());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
document.insert_key_value(
|
||||
EmailSearchField::Headers,
|
||||
header.name.as_str(),
|
||||
value,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let part_id = part_id as u16;
|
||||
match &part.body {
|
||||
ArchivedMetadataPartType::Text | ArchivedMetadataPartType::Html => {
|
||||
let text = match (part.decode_contents(&raw_message), &part.body) {
|
||||
(DecodedPartContent::Text(text), ArchivedMetadataPartType::Text) => text,
|
||||
(DecodedPartContent::Text(html), ArchivedMetadataPartType::Html) => {
|
||||
html_to_text(html.as_ref()).into()
|
||||
}
|
||||
_ => unreachable!(),
|
||||
};
|
||||
|
||||
if message_contents.is_html_part(part_id)
|
||||
|| message_contents.is_text_part(part_id)
|
||||
{
|
||||
if index_fields.is_empty()
|
||||
|| index_fields.contains(&SearchField::Email(EmailSearchField::Body))
|
||||
{
|
||||
if part_language.is_unknown() {
|
||||
detector.detect(text.as_ref(), MIN_LANGUAGE_SCORE);
|
||||
}
|
||||
|
||||
document.index_text(
|
||||
SearchField::Email(EmailSearchField::Body),
|
||||
text.as_ref(),
|
||||
part_language,
|
||||
);
|
||||
}
|
||||
} else if index_fields.is_empty()
|
||||
|| index_fields.contains(&SearchField::Email(EmailSearchField::Attachment))
|
||||
{
|
||||
if part_language.is_unknown() {
|
||||
detector.detect(text.as_ref(), MIN_LANGUAGE_SCORE);
|
||||
}
|
||||
|
||||
document.index_text(
|
||||
SearchField::Email(EmailSearchField::Attachment),
|
||||
text.as_ref(),
|
||||
part_language,
|
||||
);
|
||||
}
|
||||
}
|
||||
ArchivedMetadataPartType::Message(nested_message_id)
|
||||
if index_fields.is_empty()
|
||||
|| index_fields
|
||||
.contains(&SearchField::Email(EmailSearchField::Attachment)) =>
|
||||
{
|
||||
let nested_message = self.message_id(*nested_message_id);
|
||||
let nested_message_language = nested_message
|
||||
.root_part()
|
||||
.language()
|
||||
.unwrap_or(Language::Unknown);
|
||||
if let Some(ArchivedMetadataHeaderValue::Text(subject)) = nested_message
|
||||
.root_part()
|
||||
.header_value(&MetadataHeaderName::Subject)
|
||||
{
|
||||
if nested_message_language.is_unknown() {
|
||||
detector.detect(subject.as_ref(), MIN_LANGUAGE_SCORE);
|
||||
}
|
||||
|
||||
document.index_text(
|
||||
SearchField::Email(EmailSearchField::Attachment),
|
||||
subject.as_ref(),
|
||||
nested_message_language,
|
||||
);
|
||||
}
|
||||
|
||||
for sub_part in nested_message.parts.iter().take(MAX_MESSAGE_PARTS) {
|
||||
let language = sub_part.language().unwrap_or(nested_message_language);
|
||||
match &sub_part.body {
|
||||
ArchivedMetadataPartType::Text | ArchivedMetadataPartType::Html => {
|
||||
let text = match (
|
||||
sub_part.decode_contents(&raw_message),
|
||||
&sub_part.body,
|
||||
) {
|
||||
(
|
||||
DecodedPartContent::Text(text),
|
||||
ArchivedMetadataPartType::Text,
|
||||
) => text,
|
||||
(
|
||||
DecodedPartContent::Text(html),
|
||||
ArchivedMetadataPartType::Html,
|
||||
) => html_to_text(html.as_ref()).into(),
|
||||
_ => unreachable!(),
|
||||
};
|
||||
|
||||
if language.is_unknown() {
|
||||
detector.detect(text.as_ref(), MIN_LANGUAGE_SCORE);
|
||||
}
|
||||
|
||||
document.index_text(
|
||||
SearchField::Email(EmailSearchField::Attachment),
|
||||
text.as_ref(),
|
||||
language,
|
||||
);
|
||||
}
|
||||
_ => (),
|
||||
}
|
||||
}
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(not(feature = "test_mode"))]
|
||||
document.set_unknown_language(
|
||||
detector
|
||||
.most_frequent_language()
|
||||
.unwrap_or(default_language),
|
||||
);
|
||||
|
||||
#[cfg(feature = "test_mode")]
|
||||
document.set_unknown_language(default_language);
|
||||
|
||||
document.index_bool(
|
||||
EmailSearchField::HasAttachment,
|
||||
self.rcvd_attach.to_native() & MESSAGE_HAS_ATTACHMENT != 0,
|
||||
);
|
||||
document
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,15 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
|
||||
*
|
||||
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
||||
*/
|
||||
|
||||
pub mod body;
|
||||
pub mod copy;
|
||||
pub mod crypto;
|
||||
pub mod delete;
|
||||
pub mod delivery;
|
||||
pub mod headers;
|
||||
pub mod index;
|
||||
pub mod ingest;
|
||||
pub mod metadata;
|
||||
Reference in New Issue
Block a user