Import upstream v0.16.22, stripped

Upstream commit: 474dd0229cb20cf513036619781ed97bd8073c3f
Enterprise-only files removed or emptied: 63
Enterprise-only snippets removed: 117 in 50 files
Dangling module declarations removed: 5
Cargo edits turning enterprise off: 14
Verification: clean
Enterprise feature gates left for rebuilt features: 19 in 18 files

Produced by tools/fork/strip.py. The full report is in docs/fork/strip-reports/ on main.

crates/spam-filter/Cargo.toml

@@ -0,0 +1,40 @@
+[package]
+name = "spam-filter"
+version = "0.16.22"
+edition = "2024"
+
+[dependencies]
+utils = { path = "../utils" }
+types = { path = "../types" }
+nlp = { path = "../nlp" }
+store = { path = "../store" }
+trc = { path = "../trc" }
+common = { path =  "../common" }
+registry = { path =  "../registry" }
+smtp-proto = { version = "0.2", features = ["rkyv"] }
+mail-parser = { version = "0.11", features = ["full_encoding"] } 
+mail-auth = { version = "0.13" }
+tokio = { version = "1.53", features = ["net", "macros"] }
+psl = "2"
+hyper = { version = "1.11.1", features = ["server", "http1", "http2"] }
+idna = "1.1"
+decancer = "3.3.3"
+unicode-security = "0.1.2"
+infer = "0.22"
+hashify = "0.2"
+sha1 = "0.11"
+compact_str = "0.10.0"
+rkyv = { version = "0.8.18", features = ["little_endian"] }
+serde = { version = "1.0", features = ["derive"]}
+unicode-general-category = "1.1.0"
+unicode-normalization = "0.1.25"
+
+[features]
+test_mode = []
+enterprise = []
+
+[dev-dependencies]
+tokio = { version = "1.53", features = ["full"] }
+
+[lints]
+workspace = true

crates/spam-filter/src/analysis/classifier.rs

@@ -0,0 +1,51 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
+ */
+
+use crate::{SpamFilterContext, modules::classifier::SpamClassifier};
+use common::Server;
+use std::future::Future;
+
+pub trait SpamFilterAnalyzeClassify: Sync + Send {
+    fn spam_filter_analyze_classify(
+        &self,
+        ctx: &mut SpamFilterContext<'_>,
+    ) -> impl Future<Output = ()> + Send;
+
+    fn spam_filter_analyze_spam_trap(
+        &self,
+        ctx: &mut SpamFilterContext<'_>,
+    ) -> impl Future<Output = bool> + Send;
+}
+
+impl SpamFilterAnalyzeClassify for Server {
+    async fn spam_filter_analyze_classify(&self, ctx: &mut SpamFilterContext<'_>) {
+        if self.core.spam.classifier.is_some()
+            && !ctx.result.has_tag("SPAM_TRAP")
+            && let Err(err) = self.spam_classify(ctx).await
+        {
+            trc::error!(err.span_id(ctx.input.span_id).caused_by(trc::location!()));
+        }
+    }
+
+    async fn spam_filter_analyze_spam_trap(&self, ctx: &mut SpamFilterContext<'_>) -> bool {
+        if let Some(store) = self.get_lookup_store("spam-traps") {
+            for addr in &ctx.output.env_to_orig_addr {
+                match store.key_exists(addr.address.as_str()).await {
+                    Ok(true) => {
+                        ctx.result.add_tag("SPAM_TRAP");
+                        return true;
+                    }
+                    Ok(false) => (),
+                    Err(err) => {
+                        trc::error!(err.span_id(ctx.input.span_id).caused_by(trc::location!()));
+                    }
+                }
+            }
+        }
+
+        false
+    }
+}

crates/spam-filter/src/analysis/date.rs

@@ -0,0 +1,55 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
+ */
+
+use std::future::Future;
+
+use common::Server;
+use mail_parser::HeaderName;
+use store::write::now;
+
+use crate::SpamFilterContext;
+
+pub trait SpamFilterAnalyzeDate: Sync + Send {
+    fn spam_filter_analyze_date(
+        &self,
+        ctx: &mut SpamFilterContext<'_>,
+    ) -> impl Future<Output = ()> + Send;
+}
+
+impl SpamFilterAnalyzeDate for Server {
+    async fn spam_filter_analyze_date(&self, ctx: &mut SpamFilterContext<'_>) {
+        match ctx
+            .input
+            .message
+            .header(HeaderName::Date)
+            .map(|h| h.as_datetime())
+        {
+            Some(Some(date)) => {
+                let date = date.to_timestamp();
+                if date != 0 {
+                    let date_diff = now() as i64 - date;
+
+                    if date_diff > 86400 {
+                        // Older than a day
+                        ctx.result.add_tag("DATE_IN_PAST");
+                    } else if -date_diff > 7200 {
+                        //# More than 2 hours in the future
+                        ctx.result.add_tag("DATE_IN_FUTURE");
+                    }
+                } else {
+                    ctx.result.add_tag("INVALID_DATE");
+                }
+            }
+            Some(None) => {
+                ctx.result.add_tag("INVALID_DATE");
+            }
+
+            None => {
+                ctx.result.add_tag("MISSING_DATE");
+            }
+        }
+    }
+}

crates/spam-filter/src/analysis/dmarc.rs

@@ -0,0 +1,92 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
+ */
+
+use std::future::Future;
+
+use common::Server;
+use mail_auth::{Dkim2Result, DkimResult, DmarcResult, SpfResult, dmarc::Policy};
+
+use crate::SpamFilterContext;
+
+pub trait SpamFilterAnalyzeDmarc: Sync + Send {
+    fn spam_filter_analyze_dmarc(
+        &self,
+        ctx: &mut SpamFilterContext<'_>,
+    ) -> impl Future<Output = ()> + Send;
+}
+
+impl SpamFilterAnalyzeDmarc for Server {
+    async fn spam_filter_analyze_dmarc(&self, ctx: &mut SpamFilterContext<'_>) {
+        ctx.result.add_tag(
+            ctx.input
+                .spf_mail_from_result
+                .map_or("SPF_NA", |r| match r.result() {
+                    SpfResult::Pass => "SPF_ALLOW",
+                    SpfResult::Fail => "SPF_FAIL",
+                    SpfResult::SoftFail => "SPF_SOFTFAIL",
+                    SpfResult::Neutral => "SPF_NEUTRAL",
+                    SpfResult::TempError => "SPF_DNSFAIL",
+                    SpfResult::PermError => "SPF_PERMFAIL",
+                    SpfResult::None => "SPF_NA",
+                }),
+        );
+
+        ctx.result.add_tag(
+            match ctx
+                .input
+                .dkim_result
+                .iter()
+                .find(|r| matches!(r.result(), DkimResult::Pass))
+                .or_else(|| ctx.input.dkim_result.first())
+                .map(|r| r.result())
+                .unwrap_or(&DkimResult::None)
+            {
+                DkimResult::Pass => "DKIM_ALLOW",
+                DkimResult::Fail(_) => "DKIM_REJECT",
+                DkimResult::PermError(_) => "DKIM_PERMFAIL",
+                DkimResult::TempError(_) => "DKIM_TEMPFAIL",
+                DkimResult::Neutral(_) | DkimResult::None => "DKIM_NA",
+            },
+        );
+
+        ctx.result.add_tag(
+            ctx.input
+                .dkim2_result
+                .map_or("DKIM2_NA", |r| match r.result() {
+                    Dkim2Result::Pass => "DKIM2_ALLOW",
+                    Dkim2Result::Fail(_) => "DKIM2_REJECT",
+                    Dkim2Result::PermError(_) => "DKIM2_PERMFAIL",
+                    Dkim2Result::TempError(_) => "DKIM2_TEMPFAIL",
+                    Dkim2Result::None => "DKIM2_NA",
+                }),
+        );
+
+        ctx.result
+            .add_tag(ctx.input.arc_result.map_or("ARC_NA", |r| match r.result() {
+                DkimResult::Pass => "ARC_ALLOW",
+                DkimResult::Fail(_) => "ARC_REJECT",
+                DkimResult::PermError(_) => "ARC_INVALID",
+                DkimResult::TempError(_) => "ARC_DNSFAIL",
+                DkimResult::Neutral(_) | DkimResult::None => "ARC_NA",
+            }));
+
+        ctx.result
+            .add_tag(ctx.input.dmarc_result.map_or("DMARC_NA", |r| match r {
+                DmarcResult::Pass => "DMARC_POLICY_ALLOW",
+                DmarcResult::TempError(_) => "DMARC_DNSFAIL",
+                DmarcResult::PermError(_) => "DMARC_BAD_POLICY",
+                DmarcResult::None => "DMARC_NA",
+                DmarcResult::Fail(_) => ctx.input.dmarc_policy.map_or(
+                    "DMARC_POLICY_SOFTFAIL",
+                    |p| match p {
+                        Policy::Quarantine => "DMARC_POLICY_QUARANTINE",
+                        Policy::Reject => "DMARC_POLICY_REJECT",
+                        Policy::Unspecified | Policy::None => "DMARC_POLICY_SOFTFAIL",
+                    },
+                ),
+            }));
+    }
+}

crates/spam-filter/src/analysis/domain.rs

@@ -0,0 +1,260 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
+ */
+
+use super::{ElementLocation, is_trusted_domain};
+use crate::{
+    Email, Hostname, Recipient, SpamFilterContext, TextPart,
+    modules::{
+        dnsbl::check_dnsbl,
+        expression::StringResolver,
+        html::{A, HREF, HtmlToken},
+    },
+};
+use common::{
+    Server,
+    config::mailstore::spamfilter::{Element, Location},
+};
+use mail_auth::DkimResult;
+use mail_parser::{HeaderName, HeaderValue, Host, parsers::MessageStream};
+use nlp::tokenizers::types::TokenType;
+use std::{collections::HashSet, future::Future};
+
+pub trait SpamFilterAnalyzeDomain: Sync + Send {
+    fn spam_filter_analyze_domain(
+        &self,
+        ctx: &mut SpamFilterContext<'_>,
+    ) -> impl Future<Output = ()> + Send;
+}
+
+impl SpamFilterAnalyzeDomain for Server {
+    async fn spam_filter_analyze_domain(&self, ctx: &mut SpamFilterContext<'_>) {
+        // Obtain email addresses and domains
+        let mut domains: HashSet<ElementLocation<String>> = HashSet::new();
+        let mut emails: HashSet<ElementLocation<Recipient>> = HashSet::new();
+
+        // Add DKIM domains
+        for dkim in ctx.input.dkim_result {
+            if dkim.result() == &DkimResult::Pass
+                && let Some(domain) = dkim.signature().map(|s| &s.d)
+            {
+                domains.insert(ElementLocation::new(
+                    domain.to_lowercase(),
+                    Location::HeaderDkimPass,
+                ));
+            }
+        }
+
+        // Add Received headers
+        for header in ctx.input.message.headers() {
+            match (&header.name, &header.value) {
+                (HeaderName::Received, HeaderValue::Received(received)) => {
+                    for host in [&received.from, &received.helo, &received.by]
+                        .into_iter()
+                        .flatten()
+                    {
+                        if let Host::Name(name) = host {
+                            let host = Hostname::new(name.as_ref());
+
+                            if host.sld.is_some() {
+                                domains.insert(ElementLocation::new(
+                                    host.fqdn,
+                                    Location::HeaderReceived,
+                                ));
+                            }
+                        }
+                    }
+                }
+                (HeaderName::MessageId, value) => {
+                    if let Some(mid_domain) = value
+                        .as_text()
+                        .and_then(|s| s.rsplit_once('@'))
+                        .and_then(|(_, d)| {
+                            let host = Hostname::new(d);
+                            if host.sld.is_some() { Some(host) } else { None }
+                        })
+                    {
+                        domains.insert(ElementLocation::new(mid_domain.fqdn, Location::HeaderMid));
+                    }
+                }
+                (HeaderName::DispositionNotificationTo, _) => {
+                    if let Some(address) = MessageStream::new(
+                        ctx.input
+                            .message
+                            .raw_message
+                            .get(header.offset_start as usize..header.offset_end as usize)
+                            .unwrap_or_default(),
+                    )
+                    .parse_address()
+                    .as_address()
+                    {
+                        for addr in address.iter() {
+                            if let Some(email) = addr.address() {
+                                emails.insert(ElementLocation::new(
+                                    Recipient {
+                                        email: Email::new(email),
+                                        name: None,
+                                    },
+                                    Location::HeaderDnt,
+                                ));
+                            }
+                        }
+                    }
+                }
+                _ => (),
+            }
+        }
+
+        // Add EHLO domain
+        if ctx.output.ehlo_host.sld.is_some() {
+            domains.insert(ElementLocation::new(
+                ctx.output.ehlo_host.fqdn.clone(),
+                Location::Ehlo,
+            ));
+        }
+
+        // Add PTR
+        if let Some(ptr) = &ctx.output.iprev_ptr {
+            domains.insert(ElementLocation::new(ptr.clone(), Location::Tcp));
+        }
+
+        // Add From, Envelope From and Reply-To
+        emails.insert(ElementLocation::new(
+            ctx.output.from.clone(),
+            Location::HeaderFrom,
+        ));
+        if let Some(reply_to) = &ctx.output.reply_to {
+            emails.insert(ElementLocation::new(
+                reply_to.clone(),
+                Location::HeaderReplyTo,
+            ));
+        }
+        emails.insert(ElementLocation::new(
+            Recipient {
+                email: ctx.output.env_from_addr.clone(),
+                name: None,
+            },
+            Location::EnvelopeFrom,
+        ));
+
+        // Add emails found in the message
+        for (part_id, part) in ctx.output.text_parts.iter().enumerate() {
+            let part_id = part_id as u32;
+            let is_body = ctx.input.message.text_body.contains(&part_id)
+                || ctx.input.message.html_body.contains(&part_id);
+            let tokens = match part {
+                TextPart::Plain { tokens, .. } => tokens,
+                TextPart::Html {
+                    tokens,
+                    html_tokens,
+                    ..
+                } => {
+                    emails.extend(html_tokens.iter().filter_map(|token| {
+                        if let HtmlToken::StartTag {
+                            name: A,
+                            attributes,
+                            ..
+                        } = token
+                        {
+                            attributes.iter().find_map(|(attr, value)| {
+                                if *attr == HREF {
+                                    let value = value.as_deref()?.strip_prefix("mailto:")?;
+                                    let email =
+                                        Email::new(value.split_once('?').map_or(value, |(e, _)| e));
+
+                                    if email.is_valid() {
+                                        return Some(ElementLocation::new(
+                                            Recipient { email, name: None },
+                                            if is_body {
+                                                Location::BodyHtml
+                                            } else {
+                                                Location::Attachment
+                                            },
+                                        ));
+                                    }
+                                }
+                                None
+                            })
+                        } else {
+                            None
+                        }
+                    }));
+                    tokens
+                }
+                TextPart::None => continue,
+            };
+
+            for token in tokens {
+                if let TokenType::Email(email) = token {
+                    if !ctx.input.is_train && is_body && !ctx.result.has_tag("RCPT_IN_BODY") {
+                        for rcpt in ctx.output.all_recipients() {
+                            if &rcpt.email == email {
+                                ctx.result.add_tag("RCPT_IN_BODY");
+                                break;
+                            }
+                        }
+                    }
+
+                    if email.is_valid() {
+                        emails.insert(ElementLocation::new(
+                            Recipient {
+                                email: email.clone(),
+                                name: None,
+                            },
+                            if is_body {
+                                Location::BodyText
+                            } else {
+                                Location::Attachment
+                            },
+                        ));
+                    }
+                }
+            }
+        }
+
+        if !ctx.input.is_train {
+            // Validate email
+            for email in &emails {
+                // Skip trusted domains
+                if !email.element.email.is_valid()
+                    || is_trusted_domain(
+                        self,
+                        &email.element.email.domain_part.fqdn,
+                        ctx.input.span_id,
+                    )
+                    .await
+                {
+                    continue;
+                }
+
+                // Check Email DNSBL
+                check_dnsbl(self, ctx, &email.element, Element::Email, email.location).await;
+
+                domains.insert(ElementLocation::new(
+                    email.element.email.domain_part.fqdn.clone(),
+                    email.location,
+                ));
+            }
+
+            // Validate domains
+            for domain in &domains {
+                // Skip trusted domains
+                if !is_trusted_domain(self, &domain.element, ctx.input.span_id).await {
+                    // Check Domain DNSBL
+                    check_dnsbl(
+                        self,
+                        ctx,
+                        &StringResolver(domain.element.as_str()),
+                        Element::Domain,
+                        domain.location,
+                    )
+                    .await;
+                }
+            }
+        }
+        ctx.output.emails = emails;
+        ctx.output.domains = domains;
+    }
+}

crates/spam-filter/src/analysis/ehlo.rs

@@ -0,0 +1,61 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
+ */
+
+use std::future::Future;
+
+use common::Server;
+
+use crate::SpamFilterContext;
+
+pub trait SpamFilterAnalyzeEhlo: Sync + Send {
+    fn spam_filter_analyze_ehlo(
+        &self,
+        ctx: &mut SpamFilterContext<'_>,
+    ) -> impl Future<Output = ()> + Send;
+}
+
+impl SpamFilterAnalyzeEhlo for Server {
+    async fn spam_filter_analyze_ehlo(&self, ctx: &mut SpamFilterContext<'_>) {
+        if let Some(ehlo_ip) = ctx.output.ehlo_host.ip {
+            // Helo host is bare ip
+            ctx.result.add_tag("HELO_BAREIP");
+
+            if ehlo_ip != ctx.input.remote_ip {
+                // Helo A IP != hostname IP
+                ctx.result.add_tag("HELO_IP_A");
+            }
+        } else if ctx.output.ehlo_host.sld.is_some() {
+            if ctx
+                .output
+                .iprev_ptr
+                .as_ref()
+                .is_some_and(|ptr| *ptr != ctx.output.ehlo_host.fqdn)
+            {
+                // Helo does not match reverse IP
+                ctx.result.add_tag("HELO_IPREV_MISMATCH");
+            }
+
+            if matches!(
+                (
+                    self.dns_exists_ip(&ctx.output.ehlo_host.fqdn).await,
+                    self.dns_exists_mx(&ctx.output.ehlo_host.fqdn).await
+                ),
+                (Ok(false), Ok(false))
+            ) {
+                // Helo no resolve to A or MX
+                ctx.result.add_tag("HELO_NORES_A_OR_MX");
+            }
+        } else {
+            if ctx.output.ehlo_host.fqdn.contains("user") {
+                // Helo host contains 'user'
+                ctx.result.add_tag("RCVD_HELO_USER");
+            }
+
+            // Helo not FQDN
+            ctx.result.add_tag("HELO_NOT_FQDN");
+        }
+    }
+}

crates/spam-filter/src/analysis/from.rs

@@ -0,0 +1,227 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
+ */
+
+use crate::{Email, SpamFilterContext};
+use common::Server;
+use mail_parser::HeaderName;
+use nlp::tokenizers::types::{TokenType, TypesTokenizer};
+use smtp_proto::{MAIL_BODY_8BITMIME, MAIL_BODY_BINARYMIME, MAIL_SMTPUTF8};
+use std::future::Future;
+
+pub trait SpamFilterAnalyzeFrom: Sync + Send {
+    fn spam_filter_analyze_from(
+        &self,
+        ctx: &mut SpamFilterContext<'_>,
+    ) -> impl Future<Output = ()> + Send;
+}
+
+impl SpamFilterAnalyzeFrom for Server {
+    async fn spam_filter_analyze_from(&self, ctx: &mut SpamFilterContext<'_>) {
+        let mut from_count = 0;
+        let mut from_raw = b"".as_slice();
+        let mut crt = None;
+        let mut dnt = None;
+
+        for header in ctx.input.message.headers() {
+            match &header.name {
+                HeaderName::From => {
+                    from_count += 1;
+                    from_raw = ctx
+                        .input
+                        .message
+                        .raw_message()
+                        .get(header.offset_start as usize..header.offset_end as usize)
+                        .unwrap_or_default();
+                }
+                HeaderName::DispositionNotificationTo => {
+                    dnt = ctx
+                        .input
+                        .header_as_address(header)
+                        .map(|s| Email::new(s.as_ref()));
+                }
+                HeaderName::Other(name) if name.eq_ignore_ascii_case("X-Confirm-Reading-To") => {
+                    crt = ctx
+                        .input
+                        .header_as_address(header)
+                        .map(|s| Email::new(s.as_ref()));
+                }
+                _ => {}
+            }
+        }
+
+        match from_count {
+            0 => {
+                ctx.result.add_tag("MISSING_FROM");
+            }
+            1 => {}
+            _ => {
+                ctx.result.add_tag("MULTIPLE_FROM");
+            }
+        }
+
+        let env_from_empty = ctx.output.env_from_addr.address.is_empty();
+        let from_addr = &ctx.output.from.email;
+        let from_name = ctx.output.from.name.as_deref().unwrap_or_default();
+        if from_count > 0 {
+            // Validate address
+            let from_addr_is_valid = from_addr.is_valid();
+            if !from_addr_is_valid {
+                ctx.result.add_tag("FROM_INVALID");
+            }
+
+            // Validate from name
+            let from_name_trimmed = from_name.trim();
+            if from_name_trimmed.is_empty() {
+                ctx.result.add_tag("FROM_NO_DN");
+            } else if from_name_trimmed == from_addr.address {
+                ctx.result.add_tag("FROM_DN_EQ_ADDR");
+            } else {
+                if from_addr_is_valid {
+                    ctx.result.add_tag("FROM_HAS_DN");
+                }
+
+                if from_name_trimmed.contains('@')
+                    && let Some(from_name_addr) = TypesTokenizer::new(from_name_trimmed)
+                        .tokenize_numbers(false)
+                        .tokenize_urls(false)
+                        .tokenize_urls_without_scheme(false)
+                        .tokenize_emails(true)
+                        .filter_map(|t| match t.word {
+                            TokenType::Email(email) => {
+                                let email = Email::new(email);
+                                email.is_valid().then_some(email)
+                            }
+                            _ => None,
+                        })
+                        .next()
+                {
+                    if (from_addr_is_valid
+                        && from_name_addr.domain_part.sld != from_addr.domain_part.sld)
+                        || (!env_from_empty
+                            && ctx.output.env_from_addr.domain_part.sld
+                                != from_name_addr.domain_part.sld)
+                        || (env_from_empty
+                            && ctx.output.ehlo_host.sld != from_name_addr.domain_part.sld)
+                    {
+                        ctx.result.add_tag("SPOOF_DISPLAY_NAME");
+                    } else {
+                        ctx.result.add_tag("FROM_NEQ_DISPLAY_NAME");
+                    }
+                }
+            }
+
+            // Check sender
+            if ctx.output.env_from_postmaster {
+                ctx.result.add_tag("FROM_BOUNCE");
+            }
+
+            if !env_from_empty && ctx.output.env_from_addr == *from_addr {
+                ctx.result.add_tag("FROM_EQ_ENV_FROM");
+            } else if from_addr_is_valid {
+                if from_addr.domain_part.sld == ctx.output.ehlo_host.sld {
+                    ctx.result.add_tag("FROMTLD_EQ_ENV_FROMTLD");
+                } else if !ctx.output.env_from_postmaster {
+                    ctx.result.add_tag("FORGED_SENDER");
+                    ctx.result.add_tag("FROM_NEQ_ENV_FROM");
+                }
+            }
+
+            // Validate FROM/TO relationship
+            if ctx.output.recipients_to.len() + ctx.output.recipients_cc.len() == 1 {
+                let rcpt = ctx
+                    .output
+                    .recipients_to
+                    .first()
+                    .or_else(|| ctx.output.recipients_cc.first())
+                    .unwrap();
+                if rcpt.email == *from_addr {
+                    ctx.result.add_tag("TO_EQ_FROM");
+                } else if rcpt.email.domain_part.fqdn == from_addr.domain_part.fqdn {
+                    ctx.result.add_tag("TO_DOM_EQ_FROM_DOM");
+                }
+            }
+
+            // Validate encoding
+            let from_raw_utf8 = std::str::from_utf8(from_raw);
+            if !from_raw.is_ascii() {
+                if (ctx.input.env_from_flags
+                    & (MAIL_SMTPUTF8 | MAIL_BODY_8BITMIME | MAIL_BODY_BINARYMIME))
+                    == 0
+                {
+                    ctx.result.add_tag("FROM_NEEDS_ENCODING");
+                }
+
+                if from_raw_utf8.is_err() {
+                    ctx.result.add_tag("INVALID_FROM_8BIT");
+                }
+            }
+
+            // Validate unnecessary encoding
+            let from_raw_utf8 = from_raw_utf8.unwrap_or_default();
+            if from_name.is_ascii()
+                && from_addr.address.is_ascii()
+                && from_raw_utf8.contains("=?")
+                && from_raw_utf8.contains("?=")
+            {
+                if from_raw_utf8.contains("?q?") || from_raw_utf8.contains("?Q?") {
+                    // From header is unnecessarily encoded in quoted-printable
+                    ctx.result.add_tag("FROM_EXCESS_QP");
+                } else if from_raw_utf8.contains("?b?") || from_raw_utf8.contains("?B?") {
+                    // From header is unnecessarily encoded in base64
+                    ctx.result.add_tag("FROM_EXCESS_BASE64");
+                }
+            }
+
+            // Validate space in FROM
+            if !from_name.is_empty()
+                && !from_addr.address.is_empty()
+                && from_raw_utf8
+                    .as_bytes()
+                    .iter()
+                    .position(|&b| b == b'<')
+                    .and_then(|v| from_raw_utf8.as_bytes().get(v - 1))
+                    .is_none_or(|v| !v.is_ascii_whitespace())
+            {
+                ctx.result.add_tag("NO_SPACE_IN_FROM");
+            }
+
+            // Check whether read confirmation address is different to from address
+            if let Some(crt) = &crt
+                && crt != from_addr
+            {
+                ctx.result.add_tag("HEADER_RCONFIRM_MISMATCH");
+            }
+        }
+
+        if !env_from_empty {
+            // Validate envelope address
+            if ctx.output.env_from_addr.is_valid() {
+                // Mail from no resolve to A or MX
+                if matches!(
+                    (
+                        self.dns_exists_ip(&ctx.output.env_from_addr.domain_part.fqdn)
+                            .await,
+                        self.dns_exists_mx(&ctx.output.env_from_addr.domain_part.fqdn)
+                            .await
+                    ),
+                    (Ok(false), Ok(false))
+                ) {
+                    // Helo no resolve to A or MX
+                    ctx.result.add_tag("FROMHOST_NORES_A_OR_MX");
+                }
+            } else {
+                ctx.result.add_tag("ENV_FROM_INVALID");
+            }
+
+            // Check whether disposition notification address is different to return path
+            if let Some(dnt) = &dnt
+                && *dnt != ctx.output.env_from_addr
+            {
+                ctx.result.add_tag("HEADER_FORGED_MDN");
+            }
+        }
+    }
+}

crates/spam-filter/src/analysis/headers.rs

@@ -0,0 +1,145 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
+ */
+
+use std::future::Future;
+
+use common::Server;
+use mail_parser::HeaderName;
+use store::ahash::AHashSet;
+
+use crate::SpamFilterContext;
+
+pub trait SpamFilterAnalyzeHeaders: Sync + Send {
+    fn spam_filter_analyze_headers(
+        &self,
+        ctx: &mut SpamFilterContext<'_>,
+    ) -> impl Future<Output = ()> + Send;
+}
+
+impl SpamFilterAnalyzeHeaders for Server {
+    async fn spam_filter_analyze_headers(&self, ctx: &mut SpamFilterContext<'_>) {
+        let mut list_score = 0.0;
+        let mut unique_headers = AHashSet::new();
+        let raw_message = ctx.input.message.raw_message();
+
+        for header in ctx.input.message.headers() {
+            // Add header exists tag
+            let hdr_name = header.name();
+            let mut tag: String = String::with_capacity(hdr_name.len() + 5);
+            tag.push_str("X_HDR_");
+            for ch in hdr_name.chars() {
+                if ch.is_ascii_alphanumeric() {
+                    tag.push(ch.to_ascii_uppercase());
+                } else if ch == '-' {
+                    tag.push('_');
+                } else {
+                    tag.push(' ');
+                }
+            }
+            ctx.result.add_tag(tag);
+
+            match &header.name {
+                HeaderName::ContentType
+                | HeaderName::ContentTransferEncoding
+                | HeaderName::Date
+                | HeaderName::From
+                | HeaderName::Sender
+                | HeaderName::To
+                | HeaderName::Cc
+                | HeaderName::Bcc
+                | HeaderName::ReplyTo
+                | HeaderName::Subject
+                | HeaderName::MessageId
+                | HeaderName::References
+                | HeaderName::InReplyTo => {
+                    if !unique_headers.insert(header.name.clone()) {
+                        ctx.result.add_tag("MULTIPLE_UNIQUE_HEADERS");
+                    }
+
+                    let mut value = raw_message
+                        .get(header.offset_start as usize..)
+                        .unwrap_or_default()
+                        .iter();
+                    loop {
+                        match value.next() {
+                            Some(b' ' | b'\t') => {
+                                break;
+                            }
+                            Some(b'\r' | b'\n') => {}
+                            _ => {
+                                ctx.result.add_tag("HEADER_EMPTY_DELIMITER");
+                                break;
+                            }
+                        }
+                    }
+                }
+                HeaderName::ListArchive
+                | HeaderName::ListOwner
+                | HeaderName::ListHelp
+                | HeaderName::ListPost => {
+                    list_score += 0.125;
+                }
+                HeaderName::ListId => {
+                    list_score += 0.5125;
+                }
+                HeaderName::ListSubscribe => {
+                    list_score += 0.25;
+                }
+                HeaderName::ListUnsubscribe => {
+                    list_score += 0.25;
+                    ctx.result.add_tag("HAS_LIST_UNSUB");
+                }
+                HeaderName::Other(name) => {
+                    let value = header
+                        .value()
+                        .as_text()
+                        .unwrap_or_default()
+                        .trim()
+                        .to_lowercase();
+
+                    if name.eq_ignore_ascii_case("Precedence") {
+                        if value == "bulk" {
+                            list_score += 0.25;
+                            ctx.result.add_tag("PRECEDENCE_BULK");
+                        } else if value == "list" {
+                            list_score += 0.25;
+                        }
+                    } else if name.eq_ignore_ascii_case("X-Loop") {
+                        list_score += 0.125;
+                    } else if name.eq_ignore_ascii_case("X-Priority") {
+                        match value.parse::<i32>().unwrap_or(i32::MAX) {
+                            0 => {
+                                ctx.result.add_tag("HAS_X_PRIO_ZERO");
+                            }
+                            1 => {
+                                ctx.result.add_tag("HAS_X_PRIO_ONE");
+                            }
+                            2 => {
+                                ctx.result.add_tag("HAS_X_PRIO_TWO");
+                            }
+                            3 | 4 => {
+                                ctx.result.add_tag("HAS_X_PRIO_THREE");
+                            }
+                            4..=10000 => {
+                                ctx.result.add_tag("HAS_X_PRIO_FIVE");
+                            }
+                            _ => {}
+                        }
+                    }
+                }
+                _ => {}
+            }
+        }
+
+        if list_score >= 1.0 {
+            ctx.result.add_tag("MAILLIST");
+        }
+
+        if unique_headers.is_empty() {
+            ctx.result.add_tag("MISSING_ESSENTIAL_HEADERS");
+        }
+    }
+}

crates/spam-filter/src/analysis/html.rs

@@ -0,0 +1,339 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
+ */
+
+use std::future::Future;
+
+use common::Server;
+use hyper::Uri;
+use mail_parser::MimeHeaders;
+use nlp::tokenizers::types::{TokenType, TypesTokenizer};
+
+use crate::{Hostname, SpamFilterContext, TextPart, modules::html::*};
+
+pub trait SpamFilterAnalyzeHtml: Sync + Send {
+    fn spam_filter_analyze_html(
+        &self,
+        ctx: &mut SpamFilterContext<'_>,
+    ) -> impl Future<Output = ()> + Send;
+}
+
+#[derive(Debug)]
+struct Href {
+    url_parsed: Option<Uri>,
+    host: Option<Hostname>,
+}
+
+impl SpamFilterAnalyzeHtml for Server {
+    async fn spam_filter_analyze_html(&self, ctx: &mut SpamFilterContext<'_>) {
+        // Message only has text/html MIME parts
+        if ctx.input.message.content_type().is_some_and(|ct| {
+            ct.ctype().eq_ignore_ascii_case("text")
+                && ct
+                    .subtype()
+                    .unwrap_or_default()
+                    .eq_ignore_ascii_case("html")
+        }) {
+            ctx.result.add_tag("MIME_HTML_ONLY");
+        }
+
+        for (part_id, part) in ctx.output.text_parts.iter().enumerate() {
+            let part_id = part_id as u32;
+            let is_body_part = ctx.input.message.text_body.contains(&part_id)
+                || ctx.input.message.html_body.contains(&part_id);
+
+            let (html_tokens, tokens) = if let TextPart::Html {
+                html_tokens,
+                tokens,
+                ..
+            } = part
+            {
+                (html_tokens, tokens)
+            } else {
+                continue;
+            };
+
+            let mut has_link_to_img = false;
+            let mut last_href: Option<Href> = None;
+            let mut html_img_words = 0;
+            let mut in_head: i32 = 0;
+            let mut in_body: i32 = 0;
+
+            for token in html_tokens {
+                match token {
+                    HtmlToken::StartTag {
+                        name,
+                        attributes,
+                        is_self_closing,
+                    } => match *name {
+                        A => {
+                            if let Some(attr) = attributes.iter().find_map(|(attr, value)| {
+                                if *attr == HREF {
+                                    value.as_deref()
+                                } else {
+                                    None
+                                }
+                            }) {
+                                let url = attr.trim().to_lowercase();
+                                let url_parsed = url.parse::<Uri>().ok();
+                                let href = Href {
+                                    host: url_parsed
+                                        .as_ref()
+                                        .and_then(|uri| uri.host().map(Hostname::new)),
+                                    url_parsed,
+                                };
+
+                                if is_body_part
+                                    && attr.starts_with("data:")
+                                    && attr.contains(";base64,")
+                                {
+                                    // Has Data URI encoding
+                                    ctx.result.add_tag("HAS_DATA_URI");
+                                    if attr.contains("text/") {
+                                        //  Uses Data URI encoding to obfuscate plain or HTML in base64
+                                        ctx.result.add_tag("DATA_URI_OBFU");
+                                    }
+                                } else if href.host.as_ref().is_some_and(|h| h.ip.is_some()) {
+                                    // HTML anchor points to an IP address
+                                    ctx.result.add_tag("HTTP_TO_IP");
+                                }
+
+                                if !*is_self_closing {
+                                    last_href = Some(href);
+                                }
+                            }
+                        }
+                        IMG if is_body_part => {
+                            let mut img_width = 800;
+                            let mut img_height = 600;
+
+                            for (attr, value) in attributes {
+                                if let Some(value) =
+                                    value.as_deref().map(|v| v.trim()).filter(|v| !v.is_empty())
+                                {
+                                    let dimension = match *attr {
+                                        WIDTH => &mut img_width,
+                                        HEIGHT => &mut img_height,
+                                        SRC => {
+                                            let src = value.to_ascii_lowercase();
+                                            if src.starts_with("data:") && src.contains(";base64,")
+                                            {
+                                                // Has Data URI encoding
+                                                ctx.result.add_tag("HAS_DATA_URI");
+                                            } else if src.starts_with("https://")
+                                                || src.starts_with("http://")
+                                            {
+                                                // Has external image
+                                                ctx.result.add_tag("HAS_EXTERNAL_IMG");
+                                            }
+                                            continue;
+                                        }
+                                        _ => {
+                                            continue;
+                                        }
+                                    };
+                                    if let Some(pct) = value.strip_suffix('%') {
+                                        if let Ok(pct) = pct.trim().parse::<u64>() {
+                                            *dimension = (*dimension * pct) / 100;
+                                        }
+                                    } else if let Ok(value) = value.parse::<u64>() {
+                                        *dimension = value;
+                                    }
+                                }
+                            }
+                            let dimensions = img_width + img_height;
+
+                            if last_href.is_some() {
+                                if dimensions >= 210 {
+                                    ctx.result.add_tag("HAS_LINK_TO_LARGE_IMG");
+                                    has_link_to_img = true;
+                                } else {
+                                    ctx.result.add_tag("HAS_LINK_TO_IMG");
+                                }
+                            }
+
+                            if dimensions > 100 {
+                                // We assume that a single picture 100x200 contains approx 3 words of text
+                                html_img_words += dimensions / 100;
+                            }
+                        }
+                        META => {
+                            let mut has_equiv_refresh = false;
+                            let mut has_content_url = false;
+
+                            for (attr, value) in attributes {
+                                if let Some(value) =
+                                    value.as_deref().map(|v| v.trim()).filter(|v| !v.is_empty())
+                                {
+                                    if *attr == HTTP_EQUIV {
+                                        if value.eq_ignore_ascii_case("refresh") {
+                                            has_equiv_refresh = true;
+                                        }
+                                    } else if *attr == CONTENT
+                                        && value.to_ascii_lowercase().contains("url=")
+                                    {
+                                        has_content_url = true;
+                                    }
+                                }
+                            }
+
+                            if has_equiv_refresh && has_content_url {
+                                // HTML meta refresh tag
+                                ctx.result.add_tag("HTML_META_REFRESH_URL");
+                            }
+                        }
+                        LINK if is_body_part => {
+                            let mut has_rel_style = false;
+                            let mut has_href_css = false;
+
+                            for (attr, value) in attributes {
+                                if let Some(value) =
+                                    value.as_deref().map(|v| v.trim()).filter(|v| !v.is_empty())
+                                {
+                                    if *attr == REL {
+                                        if value.to_ascii_lowercase().contains("stylesheet") {
+                                            has_rel_style = true;
+                                        }
+                                    } else if *attr == HREF
+                                        && value.to_ascii_lowercase().contains(".css")
+                                    {
+                                        has_href_css = true;
+                                    }
+                                }
+                            }
+
+                            if has_rel_style || has_href_css {
+                                // Has external CSS
+                                ctx.result.add_tag("EXT_CSS");
+                            }
+                        }
+                        HEAD if !*is_self_closing => {
+                            in_head += 1;
+                        }
+                        BODY if !*is_self_closing => {
+                            in_body += 1;
+                        }
+                        _ => {}
+                    },
+                    HtmlToken::EndTag { name } => match *name {
+                        A => {
+                            last_href = None;
+                        }
+                        HEAD => {
+                            in_head -= 1;
+                        }
+                        BODY => {
+                            in_body -= 1;
+                        }
+                        _ => (),
+                    },
+                    HtmlToken::Text { text } if in_head == 0 => {
+                        if let Some((href_url, href_host)) = last_href
+                            .as_ref()
+                            .and_then(|href| Some((href.url_parsed.as_ref()?, href.host.as_ref()?)))
+                        {
+                            for token in TypesTokenizer::new(text.as_ref())
+                                .tokenize_numbers(false)
+                                .tokenize_urls(true)
+                                .tokenize_urls_without_scheme(true)
+                                .tokenize_emails(true)
+                            {
+                                let text_url = match token.word {
+                                    TokenType::Url(url) => url.to_lowercase(),
+                                    TokenType::UrlNoScheme(url) => {
+                                        format!("http://{}", url.to_lowercase())
+                                    }
+                                    _ => continue,
+                                };
+                                let text_url_parsed =
+                                    if let Ok(text_url_parsed) = text_url.parse::<Uri>() {
+                                        text_url_parsed
+                                    } else {
+                                        continue;
+                                    };
+
+                                if href_url.scheme().map(|s| s.as_str()).unwrap_or_default()
+                                    == "http"
+                                    && text_url_parsed
+                                        .scheme()
+                                        .map(|s| s.as_str())
+                                        .unwrap_or_default()
+                                        == "https"
+                                {
+                                    // The anchor text contains a distinct scheme compared to the target URL
+                                    ctx.result.add_tag("HTTP_TO_HTTPS");
+                                }
+
+                                if let Some(text_url_host) = text_url_parsed.host() {
+                                    let text_url_host = Hostname::new(text_url_host);
+
+                                    if text_url_host.sld_or_default() != href_host.sld_or_default()
+                                    {
+                                        // The anchor text contains a different domain than the target URL
+                                        ctx.result.add_tag("PHISHING");
+                                    }
+                                }
+                            }
+                        }
+                    }
+                    _ => (),
+                }
+            }
+
+            if is_body_part {
+                if in_head != 0 || in_body != 0 {
+                    // HTML tags are not properly closed
+                    ctx.result.add_tag("HTML_UNBALANCED_TAG");
+                }
+
+                let mut html_words = 0;
+                let mut html_uris = 0;
+                let mut html_text_chars = 0;
+
+                for token in tokens {
+                    match token {
+                        TokenType::Alphabetic(s) | TokenType::Alphanumeric(s) => {
+                            html_words += 1;
+                            html_text_chars += s.len();
+                        }
+                        TokenType::Email(s) => {
+                            html_words += 1;
+                            html_text_chars += s.address.len();
+                        }
+                        TokenType::Url(_) | TokenType::UrlNoScheme(_) => {
+                            html_uris += 1;
+                        }
+                        _ => (),
+                    }
+                }
+
+                match html_text_chars {
+                    0..1024 => {
+                        ctx.result.add_tag("HTML_SHORT_1");
+                    }
+                    1024..1536 => {
+                        ctx.result.add_tag("HTML_SHORT_2");
+                    }
+                    1536..2048 => {
+                        ctx.result.add_tag("HTML_SHORT_3");
+                    }
+                    _ => (),
+                }
+
+                if (!has_link_to_img || html_text_chars >= 2048)
+                    && (html_img_words as f64 / (html_words as f64 + html_img_words as f64) > 0.5)
+                {
+                    // Message contains more images than text
+                    ctx.result.add_tag("HTML_TEXT_IMG_RATIO");
+                }
+
+                if html_uris > 0 && html_words == 0 {
+                    // Message only contains URIs in HTML
+                    ctx.result.add_tag("BODY_URI_ONLY");
+                }
+            }
+        }
+    }
+}

crates/spam-filter/src/analysis/init.rs

@@ -0,0 +1,313 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
+ */
+
+use common::Server;
+
+use mail_auth::DmarcResult;
+use mail_parser::{HeaderName, PartType, parsers::fields::thread::thread_name};
+use nlp::tokenizers::types::{TokenType, TypesTokenizer};
+
+use crate::{
+    Email, Hostname, IpParts, Recipient, SpamFilterContext, SpamFilterInput, SpamFilterOutput,
+    SpamFilterResult, TextPart,
+    modules::html::{HEAD, HtmlToken, html_to_tokens},
+};
+
+use super::url::UrlParts;
+
+pub trait SpamFilterInit {
+    fn spam_filter_init<'x>(&self, input: SpamFilterInput<'x>) -> SpamFilterContext<'x>;
+}
+
+const POSTMASTER_ADDRESSES: [&str; 3] = ["postmaster", "mailer-daemon", "root"];
+
+impl SpamFilterInit for Server {
+    fn spam_filter_init<'x>(&self, mut input: SpamFilterInput<'x>) -> SpamFilterContext<'x> {
+        let mut subject = "";
+        let mut from = None;
+        let mut reply_to = None;
+        let mut recipients_to = Vec::new();
+        let mut recipients_cc = Vec::new();
+        let mut recipients_bcc = Vec::new();
+        let mut found_spam_status = false;
+
+        for header in input.message.headers() {
+            match &header.name {
+                HeaderName::To | HeaderName::Cc | HeaderName::Bcc => {
+                    if let Some(addrs) = header.value().as_address() {
+                        for addr in addrs.iter() {
+                            let rcpt = Recipient {
+                                email: Email::new(addr.address().unwrap_or_default()),
+                                name: addr.name().and_then(|s| {
+                                    let s = s.trim();
+                                    if !s.is_empty() {
+                                        Some(s.to_lowercase())
+                                    } else {
+                                        None
+                                    }
+                                }),
+                            };
+                            if header.name == HeaderName::To {
+                                recipients_to.push(rcpt);
+                            } else if header.name == HeaderName::Cc {
+                                recipients_cc.push(rcpt);
+                            } else {
+                                recipients_bcc.push(rcpt);
+                            }
+                        }
+                    }
+                }
+                HeaderName::ReplyTo => {
+                    reply_to = header
+                        .value()
+                        .as_address()
+                        .and_then(|addrs| addrs.first())
+                        .and_then(|addr| {
+                            Some(Recipient {
+                                email: Email::new(addr.address()?),
+                                name: addr.name().and_then(|s| {
+                                    let s = s.trim();
+                                    if !s.is_empty() {
+                                        Some(s.to_lowercase())
+                                    } else {
+                                        None
+                                    }
+                                }),
+                            })
+                        });
+                }
+                HeaderName::Subject => {
+                    subject = header.value().as_text().unwrap_or_default();
+                }
+                HeaderName::From => {
+                    from = header.value().as_address().and_then(|addrs| addrs.first());
+                }
+                HeaderName::Other(name)
+                    if input.is_train && !found_spam_status && name.eq("X-Spam-Result") =>
+                {
+                    for token in header
+                        .value()
+                        .as_text()
+                        .unwrap_or_default()
+                        .split_ascii_whitespace()
+                    {
+                        if let Some(dmarc) = token.strip_prefix("DMARC_") {
+                            input.dmarc_result = if dmarc == "POLICY_ALLOW" {
+                                Some(&DmarcResult::Pass)
+                            } else {
+                                Some(&DmarcResult::None)
+                            };
+                        } else if let Some(asn) = token
+                            .strip_prefix("SOURCE_ASN_")
+                            .and_then(|v| v.parse().ok())
+                        {
+                            input.asn = Some(asn);
+                        }
+                    }
+
+                    found_spam_status = true;
+                }
+                _ => {}
+            }
+        }
+
+        // Tokenize subject
+        let subject_tokens = TypesTokenizer::new(subject)
+            .tokenize_numbers(false)
+            .tokenize_urls(true)
+            .tokenize_urls_without_scheme(true)
+            .tokenize_emails(true)
+            .map(|t| match t.word {
+                TokenType::Alphabetic(s) => TokenType::Alphabetic(s.into()),
+                TokenType::Alphanumeric(s) => TokenType::Alphanumeric(s.into()),
+                TokenType::Integer(s) => TokenType::Integer(s.into()),
+                TokenType::Other(s) => TokenType::Other(s),
+                TokenType::Punctuation(s) => TokenType::Punctuation(s),
+                TokenType::Space => TokenType::Space,
+                TokenType::Url(url) => TokenType::Url(UrlParts::new(url)),
+                TokenType::UrlNoHost(s) => TokenType::UrlNoHost(s.into()),
+                TokenType::UrlNoScheme(s) => TokenType::UrlNoScheme(UrlParts::no_scheme(s)),
+                TokenType::IpAddr(i) => TokenType::IpAddr(IpParts::new(i)),
+                TokenType::Email(e) => TokenType::Email(Email::new(e)),
+                TokenType::Float(s) => TokenType::Float(s.into()),
+            })
+            .collect::<Vec<_>>();
+
+        // Tokenize and convert text parts
+        let mut text_parts = Vec::new();
+        let mut text_parts_nested = Vec::new();
+        let mut message_stack = Vec::new();
+        let mut message_iter = input.message.parts.iter();
+
+        loop {
+            while let Some(part) = message_iter.next() {
+                let is_main_message = message_stack.is_empty();
+                let text_part = match &part.body {
+                    PartType::Text(text) => TextPart::Plain {
+                        text_body: text.as_ref(),
+                        tokens: TypesTokenizer::new(text.as_ref())
+                            .tokenize_numbers(false)
+                            .tokenize_urls(true)
+                            .tokenize_urls_without_scheme(true)
+                            .tokenize_emails(true)
+                            .map(|t| match t.word {
+                                TokenType::Alphabetic(s) => TokenType::Alphabetic(s.into()),
+                                TokenType::Alphanumeric(s) => TokenType::Alphanumeric(s.into()),
+                                TokenType::Integer(s) => TokenType::Integer(s.into()),
+                                TokenType::Other(s) => TokenType::Other(s),
+                                TokenType::Punctuation(s) => TokenType::Punctuation(s),
+                                TokenType::Space => TokenType::Space,
+                                TokenType::Url(url) => TokenType::Url(UrlParts::new(url)),
+                                TokenType::UrlNoHost(s) => TokenType::UrlNoHost(s.into()),
+                                TokenType::UrlNoScheme(s) => {
+                                    TokenType::UrlNoScheme(UrlParts::no_scheme(s))
+                                }
+                                TokenType::IpAddr(i) => TokenType::IpAddr(IpParts::new(i)),
+                                TokenType::Email(e) => TokenType::Email(Email::new(e)),
+                                TokenType::Float(s) => TokenType::Float(s.into()),
+                            })
+                            .collect::<Vec<_>>(),
+                    },
+                    PartType::Html(html) => {
+                        let html_tokens = html_to_tokens(html);
+                        let text_body_len = html_tokens
+                            .iter()
+                            .filter_map(|t| match t {
+                                HtmlToken::Text { text } => text.len().into(),
+                                _ => None,
+                            })
+                            .sum();
+                        let mut text_body = String::with_capacity(text_body_len);
+                        let mut in_head = false;
+                        for token in &html_tokens {
+                            match token {
+                                HtmlToken::StartTag { name: HEAD, .. } => {
+                                    in_head = true;
+                                }
+                                HtmlToken::EndTag { name: HEAD } => {
+                                    in_head = false;
+                                }
+                                HtmlToken::Text { text } if !in_head => {
+                                    if !text_body.is_empty()
+                                        && !text_body.ends_with(' ')
+                                        && !text.starts_with(' ')
+                                    {
+                                        text_body.push(' ');
+                                    }
+                                    text_body.push_str(text)
+                                }
+                                _ => {}
+                            }
+                        }
+
+                        TextPart::Html {
+                            tokens: TypesTokenizer::new(&text_body)
+                                .tokenize_numbers(false)
+                                .tokenize_urls(true)
+                                .tokenize_urls_without_scheme(true)
+                                .tokenize_emails(true)
+                                .map(|t| match t.word {
+                                    TokenType::Alphabetic(s) => {
+                                        TokenType::Alphabetic(s.to_string().into())
+                                    }
+                                    TokenType::Alphanumeric(s) => {
+                                        TokenType::Alphanumeric(s.to_string().into())
+                                    }
+                                    TokenType::Integer(s) => {
+                                        TokenType::Integer(s.to_string().into())
+                                    }
+                                    TokenType::Other(s) => TokenType::Other(s),
+                                    TokenType::Punctuation(s) => TokenType::Punctuation(s),
+                                    TokenType::Space => TokenType::Space,
+                                    TokenType::Url(url) => {
+                                        TokenType::Url(UrlParts::new(url.to_string()))
+                                    }
+                                    TokenType::UrlNoHost(s) => {
+                                        TokenType::UrlNoHost(s.to_string().into())
+                                    }
+                                    TokenType::UrlNoScheme(s) => {
+                                        TokenType::UrlNoScheme(UrlParts::no_scheme(s.to_string()))
+                                    }
+                                    TokenType::IpAddr(i) => TokenType::IpAddr(IpParts::new(i)),
+                                    TokenType::Email(e) => TokenType::Email(Email::new(e)),
+                                    TokenType::Float(s) => TokenType::Float(s.to_string().into()),
+                                })
+                                .collect::<Vec<_>>(),
+                            html_tokens,
+                            text_body,
+                        }
+                    }
+                    PartType::Message(message) => {
+                        message_stack.push(message_iter);
+                        message_iter = message.parts.iter();
+                        TextPart::None
+                    }
+                    _ => TextPart::None,
+                };
+
+                if is_main_message {
+                    text_parts.push(text_part);
+                } else if !matches!(text_part, TextPart::None) {
+                    text_parts_nested.push(text_part);
+                }
+            }
+
+            if let Some(iter) = message_stack.pop() {
+                message_iter = iter;
+            } else {
+                break;
+            }
+        }
+        text_parts.extend(text_parts_nested);
+
+        let subject_thread = thread_name(subject).to_string();
+        let env_from_addr = Email::new(input.env_from);
+        SpamFilterContext {
+            output: SpamFilterOutput {
+                ehlo_host: Hostname::new(input.ehlo_domain.unwrap_or("unknown")),
+                iprev_ptr: input.iprev_result.and_then(|r| {
+                    r.ptr
+                        .as_ref()
+                        .and_then(|ptr| ptr.first())
+                        .map(|ptr| (ptr.strip_suffix('.').unwrap_or(ptr)).to_lowercase())
+                }),
+                env_from_postmaster: env_from_addr.address.is_empty()
+                    || POSTMASTER_ADDRESSES.contains(&env_from_addr.local_part.as_str()),
+                env_from_addr,
+                env_to_orig_addr: input
+                    .env_rcpt_orig_to
+                    .iter()
+                    .map(|rcpt| Email::new(rcpt))
+                    .collect(),
+                env_to_rewritten_addr: input
+                    .env_rcpt_rewritten_to
+                    .iter()
+                    .map(|rcpt| Email::new(rcpt))
+                    .collect(),
+                from: Recipient {
+                    email: Email::new(from.and_then(|f| f.address()).unwrap_or_default()),
+                    name: from.and_then(|f| f.name()).map(|name| name.to_lowercase()),
+                },
+                reply_to,
+                subject_thread_lc: subject_thread.trim().to_lowercase(),
+                subject_thread,
+                subject_lc: subject.trim().to_lowercase(),
+                subject: subject.to_string(),
+                subject_tokens,
+                recipients_to,
+                recipients_cc,
+                recipients_bcc,
+                text_parts,
+                ips: Default::default(),
+                emails: Default::default(),
+                urls: Default::default(),
+                domains: Default::default(),
+            },
+            input,
+            result: SpamFilterResult::default(),
+        }
+    }
+}

crates/spam-filter/src/analysis/ip.rs

@@ -0,0 +1,133 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
+ */
+
+use super::ElementLocation;
+use crate::{IpParts, SpamFilterContext, TextPart, modules::dnsbl::check_dnsbl};
+use common::{
+    Server,
+    config::mailstore::spamfilter::{Element, IpResolver, Location},
+};
+use mail_auth::IprevResult;
+use mail_parser::{HeaderName, HeaderValue, Host};
+use nlp::tokenizers::types::TokenType;
+use std::future::Future;
+use store::ahash::AHashSet;
+
+pub trait SpamFilterAnalyzeIp: Sync + Send {
+    fn spam_filter_analyze_ip(
+        &self,
+        ctx: &mut SpamFilterContext<'_>,
+    ) -> impl Future<Output = ()> + Send;
+}
+
+impl SpamFilterAnalyzeIp for Server {
+    async fn spam_filter_analyze_ip(&self, ctx: &mut SpamFilterContext<'_>) {
+        // IP Address RBL
+        let mut ips = AHashSet::new();
+
+        ips.insert(ElementLocation::new(ctx.input.remote_ip, Location::Tcp));
+
+        // Obtain IP addresses from Received headers
+        for header in ctx.input.message.headers() {
+            if let (HeaderName::Received, HeaderValue::Received(received)) =
+                (&header.name, &header.value)
+            {
+                if let Some(ip) = received.from_ip()
+                    && !ip.is_loopback()
+                    && !self.is_ip_allowed(ip)
+                {
+                    ips.insert(ElementLocation::new(ip, Location::HeaderReceived));
+                }
+                for host in [&received.from, &received.helo, &received.by]
+                    .into_iter()
+                    .flatten()
+                {
+                    if let Host::IpAddr(ip) = host
+                        && !ip.is_loopback()
+                        && !self.is_ip_allowed(*ip)
+                    {
+                        ips.insert(ElementLocation::new(*ip, Location::HeaderReceived));
+                    }
+                }
+            }
+        }
+
+        // Obtain IP addresses from the message body
+        for (part_id, part) in ctx.output.text_parts.iter().enumerate() {
+            let part_id = part_id as u32;
+            let is_body = ctx.input.message.text_body.contains(&part_id)
+                || ctx.input.message.html_body.contains(&part_id);
+            match part {
+                TextPart::Plain { tokens, .. } | TextPart::Html { tokens, .. } => {
+                    ips.extend(tokens.iter().filter_map(|t| {
+                        if let TokenType::IpAddr(ip) = t {
+                            ip.ip.map(|ip| {
+                                ElementLocation::new(
+                                    ip,
+                                    if is_body {
+                                        Location::BodyText
+                                    } else {
+                                        Location::Attachment
+                                    },
+                                )
+                            })
+                        } else {
+                            None
+                        }
+                    }))
+                }
+
+                TextPart::None => (),
+            }
+        }
+
+        // Validate IP addresses
+        for ip in &ips {
+            if ip.element.is_loopback()
+                || ip.element.is_multicast()
+                || ip.element.is_unspecified()
+                || self.is_ip_allowed(ip.element)
+            {
+                continue;
+            } else if self.is_ip_blocked(ip.element) {
+                ctx.result.add_tag("IP_BLOCKED");
+                continue;
+            }
+
+            check_dnsbl(
+                self,
+                ctx,
+                &IpResolver::new(ip.element),
+                Element::Ip,
+                ip.location,
+            )
+            .await;
+        }
+        ctx.output.ips = ips;
+
+        // Reverse DNS validation
+        if let Some(iprev) = ctx.input.iprev_result {
+            match &iprev.result {
+                IprevResult::TempError(_) => ctx.result.add_tag("RDNS_DNSFAIL"),
+                IprevResult::Fail(_) | IprevResult::PermError(_) => ctx.result.add_tag("RDNS_NONE"),
+                IprevResult::Pass | IprevResult::None => (),
+            }
+        }
+
+        // Add ASN
+        if let Some(asn_id) = &ctx.input.asn {
+            ctx.result.add_tag(format!("SOURCE_ASN_{asn_id}"));
+        }
+    }
+}
+
+impl IpParts {
+    pub fn new(text: &str) -> IpParts {
+        IpParts {
+            ip: text.parse().ok(),
+        }
+    }
+}

crates/spam-filter/src/analysis/messageid.rs

@@ -0,0 +1,98 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
+ */
+
+use std::future::Future;
+
+use common::Server;
+use mail_parser::HeaderName;
+
+use crate::{Hostname, SpamFilterContext};
+
+pub trait SpamFilterAnalyzeMid: Sync + Send {
+    fn spam_filter_analyze_message_id(
+        &self,
+        ctx: &mut SpamFilterContext<'_>,
+    ) -> impl Future<Output = ()> + Send;
+}
+
+impl SpamFilterAnalyzeMid for Server {
+    async fn spam_filter_analyze_message_id(&self, ctx: &mut SpamFilterContext<'_>) {
+        let mut mid = "";
+        let mut mid_raw = "";
+
+        for header in ctx.input.message.headers() {
+            if let (HeaderName::MessageId, value) = (&header.name, &header.value) {
+                mid = value.as_text().unwrap_or_default();
+                mid_raw = std::str::from_utf8(
+                    &ctx.input.message.raw_message()
+                        [header.offset_start as usize..header.offset_end as usize],
+                )
+                .unwrap_or_default()
+                .trim();
+                break;
+            }
+        }
+
+        if !mid.is_empty() {
+            let mid = mid.to_lowercase();
+            if let Some(mid_host) = mid.rsplit_once('@').map(|(_, host)| Hostname::new(host)) {
+                if mid_host.ip.is_some() {
+                    if mid_host.fqdn.starts_with('[') {
+                        ctx.result.add_tag("MID_RHS_IP_LITERAL");
+                    } else {
+                        ctx.result.add_tag("MID_BARE_IP");
+                    }
+                } else if !mid_host.fqdn.contains('.') {
+                    ctx.result.add_tag("MID_RHS_NOT_FQDN");
+                } else if mid_host.fqdn.starts_with("www.") {
+                    ctx.result.add_tag("MID_RHS_WWW");
+                }
+
+                if !mid_raw.is_ascii() || mid_raw.contains('(') || mid.starts_with('@') {
+                    ctx.result.add_tag("INVALID_MSGID");
+                }
+
+                if mid_host.fqdn.len() > 255 {
+                    ctx.result.add_tag("MID_RHS_TOO_LONG");
+                }
+
+                // From address present in Message-ID checks
+                for (part, sender) in [
+                    ("FROM", &ctx.output.from.email),
+                    ("ENV_FROM", &ctx.output.env_from_addr),
+                ] {
+                    if !sender.address.is_empty() {
+                        if mid.contains(sender.address.as_str()) {
+                            ctx.result.add_tag(format!("MID_CONTAINS_{part}"));
+                        } else if mid_host.fqdn == sender.domain_part.fqdn {
+                            ctx.result.add_tag(format!("MID_RHS_MATCH_{part}"));
+                        } else if matches!((&mid_host.sld, &sender.domain_part.sld), (Some(mid_sld), Some(sender_sld)) if mid_sld == sender_sld)
+                        {
+                            ctx.result.add_tag(format!("MID_RHS_MATCH_{part}TLD"));
+                        }
+                    }
+                }
+
+                // To/Cc addresses present in Message-ID checks
+                for rcpt in ctx.output.all_recipients() {
+                    if mid.contains(rcpt.email.address.as_str()) {
+                        ctx.result.add_tag("MID_CONTAINS_TO");
+                    } else if mid_host.fqdn == rcpt.email.domain_part.fqdn {
+                        ctx.result.add_tag("MID_RHS_MATCH_TO");
+                    }
+                }
+            } else {
+                ctx.result.add_tag("INVALID_MSGID");
+            }
+
+            if !mid_raw.starts_with('<') || !mid_raw.contains('>') {
+                ctx.result.add_tag("MID_MISSING_BRACKETS");
+            }
+        } else {
+            ctx.result.add_tag("MISSING_MID");
+        }
+    }
+}

crates/spam-filter/src/analysis/mime.rs

@@ -0,0 +1,442 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
+ */
+
+use std::{collections::HashSet, future::Future, vec};
+
+use common::{
+    Server,
+    scripts::{
+        IsMixedCharset,
+        functions::{array::cosine_similarity, unicode::CharUtils},
+    },
+};
+use mail_parser::{HeaderName, MimeHeaders, PartType};
+use nlp::tokenizers::types::TokenType;
+
+use super::mime_types::{MimeMatch, mime_match};
+use crate::{SpamFilterContext, TextPart};
+
+pub trait SpamFilterAnalyzeMime: Sync + Send {
+    fn spam_filter_analyze_mime(
+        &self,
+        ctx: &mut SpamFilterContext<'_>,
+    ) -> impl Future<Output = ()> + Send;
+}
+
+impl SpamFilterAnalyzeMime for Server {
+    async fn spam_filter_analyze_mime(&self, ctx: &mut SpamFilterContext<'_>) {
+        let mut has_mime_version = false;
+        let mut has_ct = false;
+        let mut has_cte = false;
+        let mut had_cd = false;
+        let mut is_plain_text = false;
+
+        for header in ctx.input.message.headers() {
+            match &header.name {
+                HeaderName::MimeVersion => {
+                    if ctx
+                        .input
+                        .message
+                        .raw_message()
+                        .get(header.offset_field as usize..header.offset_start as usize - 1)
+                        != Some(b"MIME-Version")
+                    {
+                        ctx.result.add_tag("MV_CASE");
+                    }
+                    has_mime_version = true;
+                }
+                HeaderName::ContentType => {
+                    has_ct = true;
+
+                    if let Some(ct) = header.value().as_content_type() {
+                        if ct.ctype().eq_ignore_ascii_case("multipart")
+                            && ct
+                                .subtype()
+                                .is_some_and(|s| s.eq_ignore_ascii_case("report"))
+                            && ct.attribute("report-type").is_some_and(|a| {
+                                a.eq_ignore_ascii_case("delivery-status")
+                                    || a.eq_ignore_ascii_case("disposition-notification")
+                            })
+                        {
+                            // Message is a DSN
+                            ctx.result.add_tag("IS_DSN");
+                        }
+
+                        is_plain_text = ct.ctype().eq_ignore_ascii_case("text")
+                            && ct
+                                .subtype()
+                                .unwrap_or_default()
+                                .eq_ignore_ascii_case("plain");
+                    }
+                }
+                HeaderName::ContentTransferEncoding => {
+                    has_cte = true;
+                }
+                HeaderName::ContentDisposition => {
+                    had_cd = true;
+                }
+                _ => (),
+            }
+        }
+
+        if !has_mime_version && (has_ct || has_cte) {
+            ctx.result.add_tag("MISSING_MIME_VERSION");
+        }
+        if has_ct && !is_plain_text && !has_cte && !had_cd && !has_mime_version {
+            // Only Content-Type header without other MIME headers
+            ctx.result.add_tag("MIME_HEADER_CTYPE_ONLY");
+        }
+        let raw_message = ctx.input.message.raw_message();
+
+        let mut has_text_part = false;
+        let mut is_encrypted = false;
+        let mut is_encrypted_smime = false;
+        let mut is_encrypted_pgp = false;
+
+        let mut num_parts = 0;
+        let mut num_parts_size = 0;
+
+        for (part_id, part) in ctx.input.message.parts.iter().enumerate() {
+            let part_id = part_id as u32;
+            let mut ct = None;
+            let mut cd = None;
+            let mut ct_type = String::new();
+            let mut ct_subtype = String::new();
+            let mut cte = String::new();
+            let mut is_attachment = ctx.input.message.attachments.contains(&part_id);
+            let mut has_content_id = false;
+
+            for header in part.headers() {
+                match &header.name {
+                    HeaderName::ContentType => {
+                        if let Some(ct_) = header.value().as_content_type() {
+                            ct_type = ct_.ctype().to_ascii_lowercase();
+                            ct_subtype = ct_.subtype().unwrap_or_default().to_ascii_lowercase();
+                            ct = Some(ct_);
+                        }
+
+                        if ct_type.is_empty() {
+                            // Content-Type header can't be parsed
+                            ctx.result.add_tag("BROKEN_CONTENT_TYPE");
+                        } else if (ct_type == "message" && ct_subtype == "rfc822")
+                            || (ct_type == "text" && ct_subtype == "rfc822-headers")
+                        {
+                            // Message has parts
+                            ctx.result.add_tag("HAS_MESSAGE_PARTS");
+                        }
+
+                        if raw_message
+                            .get(header.offset_start as usize..header.offset_end as usize)
+                            .and_then(|s| s.trim_ascii_end().last())
+                            == Some(&b';')
+                        {
+                            // Content-Type header ends with a semi-colon
+                            ctx.result.add_tag("CT_EXTRA_SEMI");
+                        }
+                    }
+                    HeaderName::ContentTransferEncoding => {
+                        let cte_ = header.value().as_text().unwrap_or_default();
+                        cte = cte_.to_ascii_lowercase();
+
+                        if cte != cte_ {
+                            ctx.result.add_tag("CTE_CASE");
+                        }
+                    }
+                    HeaderName::ContentDisposition => {
+                        cd = header.value().as_content_type();
+                    }
+                    HeaderName::ContentId => {
+                        has_content_id = true;
+                    }
+                    _ => (),
+                }
+            }
+
+            match ct_type.as_str() {
+                "multipart" => {
+                    let part_ids = match &part.body {
+                        PartType::Multipart(parts) => parts.as_slice(),
+                        _ => &[],
+                    };
+
+                    match ct_subtype.as_str() {
+                        "alternative" => {
+                            let mut has_plain_part = false;
+                            let mut has_html_part = false;
+
+                            let mut text_part_words = vec![];
+                            let mut text_part_uris = 0;
+
+                            let mut html_part_words = vec![];
+                            let mut html_part_uris = 0;
+
+                            for text_part in part_ids
+                                .iter()
+                                .map(|id| &ctx.output.text_parts[*id as usize])
+                            {
+                                let (tokens, words, uri_count) = match text_part {
+                                    TextPart::Plain { tokens, .. } if !has_plain_part => {
+                                        has_plain_part = true;
+                                        (tokens, &mut text_part_words, &mut text_part_uris)
+                                    }
+                                    TextPart::Html { tokens, .. } if !has_html_part => {
+                                        has_html_part = true;
+                                        (tokens, &mut html_part_words, &mut html_part_uris)
+                                    }
+                                    _ => continue,
+                                };
+
+                                let mut uris = HashSet::new();
+                                for token in tokens {
+                                    match token {
+                                        TokenType::Alphabetic(v) | TokenType::Alphanumeric(v) => {
+                                            words.push(v.as_ref());
+                                        }
+                                        TokenType::Url(v) => {
+                                            if let Some(host) =
+                                                v.url_parsed.as_ref().map(|uri| &uri.host)
+                                            {
+                                                uris.insert(host.sld_or_default());
+                                            }
+                                        }
+                                        _ => (),
+                                    }
+                                }
+
+                                *uri_count = uris.len();
+                            }
+
+                            //  Multipart message mostly text/html MIME
+                            if has_html_part {
+                                if !has_plain_part {
+                                    ctx.result.add_tag("MIME_MA_MISSING_TEXT");
+                                }
+                            } else if has_plain_part {
+                                ctx.result.add_tag("MIME_MA_MISSING_HTML");
+                            }
+
+                            // HTML and text parts are different
+                            if has_plain_part
+                                && has_html_part
+                                && (!text_part_words.is_empty() || !html_part_words.is_empty())
+                                && cosine_similarity(&text_part_words, &html_part_words) < 0.95
+                            {
+                                ctx.result.add_tag("PARTS_DIFFER");
+                            }
+
+                            // Odd URI count between parts
+                            if text_part_uris != html_part_uris {
+                                ctx.result.add_tag("URI_COUNT_ODD");
+                            }
+                        }
+                        "mixed" => {
+                            let mut num_text_parts = 0;
+                            let mut has_other_parts = false;
+
+                            for (sub_part_id, sub_part) in part_ids
+                                .iter()
+                                .map(|id| (*id, &ctx.input.message.parts[*id as usize]))
+                            {
+                                let ctype = sub_part
+                                    .content_type()
+                                    .map(|ct| ct.ctype())
+                                    .unwrap_or_default();
+
+                                if ctype.eq_ignore_ascii_case("text")
+                                    && !ctx.input.message.attachments.contains(&sub_part_id)
+                                {
+                                    num_text_parts += 1;
+                                } else if !ctype.eq_ignore_ascii_case("multipart") {
+                                    has_other_parts = true;
+                                }
+                            }
+
+                            // Found multipart/mixed without non-textual part
+                            if !has_other_parts && num_text_parts < 3 {
+                                ctx.result.add_tag("CTYPE_MIXED_BOGUS");
+                            }
+                        }
+                        "encrypted" => {
+                            is_encrypted = true;
+                        }
+                        _ => (),
+                    }
+
+                    continue;
+                }
+                "text" => {
+                    let mut is_7bit = false;
+                    match cte.as_str() {
+                        "" | "7bit" => {
+                            if raw_message
+                                .get(
+                                    part.raw_body_offset() as usize..part.raw_end_offset() as usize,
+                                )
+                                .is_some_and(|bytes| !bytes.is_ascii())
+                            {
+                                // MIME text part claims to be ASCII but isn't
+                                ctx.result.add_tag("BAD_CTE_7BIT");
+                            }
+                            is_7bit = true;
+                        }
+                        "base64" => {
+                            if part.contents().is_ascii() {
+                                // Has text part encoded in base64 that does not contain any 8bit characters
+                                ctx.result.add_tag("MIME_BASE64_TEXT_BOGUS");
+                            } else {
+                                // Has text part encoded in base64
+                                ctx.result.add_tag("MIME_BASE64_TEXT");
+                            }
+                        }
+                        _ => (),
+                    }
+
+                    if !is_7bit
+                        && ct_subtype == "plain"
+                        && ct
+                            .and_then(|ct| ct.attribute("charset"))
+                            .is_none_or(|c| c.is_empty())
+                    {
+                        // Charset header is missing
+                        ctx.result.add_tag("MISSING_CHARSET");
+                    }
+
+                    if ctx
+                        .output
+                        .text_parts
+                        .get(part_id as usize)
+                        .filter(|_| {
+                            ctx.input.message.text_body.contains(&part_id)
+                                || ctx.input.message.html_body.contains(&part_id)
+                        })
+                        .is_some_and(|p| match p {
+                            TextPart::Plain { text_body, .. } => text_body.is_mixed_charset(),
+                            TextPart::Html { text_body, .. } => text_body.is_mixed_charset(),
+                            TextPart::None => false,
+                        })
+                    {
+                        // Text part contains multiple scripts
+                        ctx.result.add_tag("MIXED_CHARSET");
+                    }
+
+                    has_text_part = true;
+                }
+                "application" => match ct_subtype.as_str() {
+                    "pkcs7-mime" => {
+                        ctx.result.add_tag("ENCRYPTED_SMIME");
+                        is_attachment = false;
+                        is_encrypted_smime = true;
+                    }
+                    "pkcs7-signature" => {
+                        ctx.result.add_tag("SIGNED_SMIME");
+                        is_attachment = false;
+                    }
+                    "pgp-encrypted" => {
+                        ctx.result.add_tag("ENCRYPTED_PGP");
+                        is_attachment = false;
+                        is_encrypted_pgp = true;
+                    }
+                    "pgp-signature" => {
+                        ctx.result.add_tag("SIGNED_PGP");
+                        is_attachment = false;
+                    }
+                    "octet-stream"
+                        if !is_encrypted
+                            && !has_content_id
+                            && cd.is_none_or(|cd| {
+                                !cd.c_type.eq_ignore_ascii_case("attachment")
+                                    && !cd.has_attribute("filename")
+                            }) =>
+                    {
+                        ctx.result.add_tag("CTYPE_MISSING_DISPOSITION");
+                    }
+                    _ => (),
+                },
+                _ => (),
+            }
+
+            num_parts += 1;
+            num_parts_size += part.len();
+
+            let ct_full = format!("{ct_type}/{ct_subtype}");
+
+            if is_attachment {
+                // Has a MIME attachment
+                ctx.result.add_tag("HAS_ATTACHMENT");
+                if ct_full != "application/octet-stream"
+                    && let Some(t) = infer::get(part.contents())
+                {
+                    match mime_match(t.mime_type(), &ct_full) {
+                        MimeMatch::Equal => {
+                            // Known content-type
+                            ctx.result.add_tag("MIME_GOOD");
+                        }
+                        MimeMatch::Mismatch => {
+                            // Known bad content-type
+                            ctx.result.add_tag("MIME_BAD");
+                        }
+                        MimeMatch::Compatible => (),
+                    }
+                }
+            }
+
+            // Analyze attachment name
+            if let Some(attach_name) = part.attachment_name() {
+                if attach_name.chars().any(|c| c.is_obscured()) {
+                    // Attachment name contains zero-width space
+                    ctx.result.add_tag("MIME_BAD_UNICODE");
+                }
+                let attach_name = attach_name.trim().to_lowercase();
+                if let Some((name, ext)) = attach_name.rsplit_once('.').and_then(|(name, ext)| {
+                    Some((name, self.core.spam.lists.file_extensions.get(ext)?))
+                }) {
+                    let sub_ext = name
+                        .rsplit_once('.')
+                        .and_then(|(_, ext)| self.core.spam.lists.file_extensions.get(ext));
+
+                    if ext.is_bad {
+                        // Attachment has a bad extension
+                        if sub_ext.is_some_and(|e| e.is_bad) {
+                            ctx.result.add_tag("MIME_DOUBLE_BAD_EXTENSION");
+                        } else {
+                            ctx.result.add_tag("MIME_BAD_EXTENSION");
+                        }
+                    }
+
+                    if ext.is_archive && sub_ext.is_some_and(|e| e.is_archive) {
+                        // Archive in archive
+                        ctx.result.add_tag("MIME_ARCHIVE_IN_ARCHIVE");
+                    }
+
+                    if !ext.known_types.is_empty()
+                        && ct_full != "application/octet-stream"
+                        && !ext.known_types.contains(&ct_full)
+                    {
+                        // Invalid attachment mime type
+                        ctx.result.add_tag("MIME_BAD_ATTACHMENT");
+                    }
+                }
+            }
+        }
+
+        match num_parts_size {
+            0 => {
+                // Message contains no parts
+                ctx.result.add_tag("COMPLETELY_EMPTY");
+            }
+            1..64 if num_parts == 1 => {
+                // Message contains only one short part
+                ctx.result.add_tag("SINGLE_SHORT_PART");
+            }
+            _ => (),
+        }
+
+        if has_text_part && (is_encrypted_pgp || is_encrypted_smime) {
+            // Message contains both text and encrypted parts
+            ctx.result.add_tag("BOGUS_ENCRYPTED_AND_TEXT");
+        }
+    }
+}

crates/spam-filter/src/analysis/mime_types.rs

@@ -0,0 +1,660 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
+ */
+
+pub(super) enum MimeMatch {
+    Equal,
+    Compatible,
+    Mismatch,
+}
+
+struct MimeType {
+    alias_of: u16,
+    container: u64,
+    contained_in: u64,
+}
+
+pub(super) fn mime_match(detected: &str, declared: &str) -> MimeMatch {
+    if detected == declared {
+        return MimeMatch::Equal;
+    }
+
+    match (lookup(detected), lookup(declared)) {
+        (Some(detected), Some(declared)) => {
+            if detected.alias_of != 0 && detected.alias_of == declared.alias_of {
+                MimeMatch::Equal
+            } else if detected.container & declared.contained_in != 0 {
+                MimeMatch::Compatible
+            } else {
+                MimeMatch::Mismatch
+            }
+        }
+        _ => MimeMatch::Mismatch,
+    }
+}
+
+fn lookup(mime: &str) -> Option<&'static MimeType> {
+    hashify::map!(
+        mime.as_bytes(),
+        MimeType,
+        "application/acad" => MimeType { alias_of: 44, container: 0x0, contained_in: 0x0 },
+        "application/acrobat" => MimeType { alias_of: 5, container: 0x0, contained_in: 0x0 },
+        "application/appinstaller" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000000 },
+        "application/appx" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000 },
+        "application/appxbundle" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000 },
+        "application/atom+xml" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000000 },
+        "application/autocad_dwg" => MimeType { alias_of: 44, container: 0x0, contained_in: 0x0 },
+        "application/bizagi-modeler" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000 },
+        "application/bzip2" => MimeType { alias_of: 17, container: 0x0, contained_in: 0x2000 },
+        "application/dash+xml" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000000 },
+        "application/dif+xml" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000000 },
+        "application/dita+xml" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000000 },
+        "application/doc" => MimeType { alias_of: 4, container: 0x0, contained_in: 0x40000 },
+        "application/docbook+xml" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000000 },
+        "application/dwg" => MimeType { alias_of: 44, container: 0x0, contained_in: 0x0 },
+        "application/epub+zip" => MimeType { alias_of: 0, container: 0x1, contained_in: 0x800000 },
+        "application/font-woff" => MimeType { alias_of: 1, container: 0x0, contained_in: 0x0 },
+        "application/futuresplash" => MimeType { alias_of: 19, container: 0x0, contained_in: 0x0 },
+        "application/gml+xml" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000000 },
+        "application/gpx" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000000 },
+        "application/gpx+xml" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000000 },
+        "application/gzip" => MimeType { alias_of: 2, container: 0x2, contained_in: 0x0 },
+        "application/gzip-compressed" => MimeType { alias_of: 2, container: 0x0, contained_in: 0x0 },
+        "application/gzipped" => MimeType { alias_of: 2, container: 0x0, contained_in: 0x0 },
+        "application/hta" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x200000000 },
+        "application/hwp+zip" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000 },
+        "application/ico" => MimeType { alias_of: 45, container: 0x0, contained_in: 0x0 },
+        "application/illustrator" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x8 },
+        "application/illustrator+ps" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x10 },
+        "application/its+xml" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000000 },
+        "application/java" => MimeType { alias_of: 3, container: 0x0, contained_in: 0x0 },
+        "application/java-archive" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000 },
+        "application/java-byte-code" => MimeType { alias_of: 3, container: 0x0, contained_in: 0x0 },
+        "application/java-vm" => MimeType { alias_of: 3, container: 0x0, contained_in: 0x0 },
+        "application/kate" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x8000000 },
+        "application/mathml+xml" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000000 },
+        "application/metalink+xml" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000000 },
+        "application/metalink4+xml" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000000 },
+        "application/microsoftpatch" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x40000 },
+        "application/microsoftupdate" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x40 },
+        "application/msexcel" => MimeType { alias_of: 9, container: 0x0, contained_in: 0x40000 },
+        "application/msix" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000 },
+        "application/msixbundle" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000 },
+        "application/mspowerpoint" => MimeType { alias_of: 10, container: 0x0, contained_in: 0x40000 },
+        "application/msword" => MimeType { alias_of: 4, container: 0x4, contained_in: 0x40000 },
+        "application/msword-template" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x40004 },
+        "application/nappdf" => MimeType { alias_of: 5, container: 0x0, contained_in: 0x0 },
+        "application/ogg" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x8000000 },
+        "application/onix-message+xml" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000000 },
+        "application/onix-message-short+xml" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000000 },
+        "application/ovf" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x100000 },
+        "application/owl+xml" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000000 },
+        "application/oxps" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000 },
+        "application/pdf" => MimeType { alias_of: 5, container: 0x8, contained_in: 0x0 },
+        "application/photoshop" => MimeType { alias_of: 42, container: 0x0, contained_in: 0x0 },
+        "application/postscript" => MimeType { alias_of: 0, container: 0x10, contained_in: 0x0 },
+        "application/powerpoint" => MimeType { alias_of: 10, container: 0x0, contained_in: 0x40000 },
+        "application/rdf+xml" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000000 },
+        "application/rss+xml" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000000 },
+        "application/rtf" => MimeType { alias_of: 6, container: 0x0, contained_in: 0x0 },
+        "application/sldworks" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x40000 },
+        "application/smil" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000000 },
+        "application/smil+xml" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000000 },
+        "application/sparql-results+xml" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000000 },
+        "application/ttml+xml" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000000 },
+        "application/vnd.adobe.flash.movie" => MimeType { alias_of: 19, container: 0x0, contained_in: 0x0 },
+        "application/vnd.adobe.illustrator" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x8 },
+        "application/vnd.adobe.indesign-idml-package" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000 },
+        "application/vnd.adobe.xdp+xml" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000000 },
+        "application/vnd.adobe.xfdf" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000000 },
+        "application/vnd.amazon.mobi8-ebook" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x20000 },
+        "application/vnd.android.app-bundle" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000 },
+        "application/vnd.android.package-archive" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000 },
+        "application/vnd.apple.iwork" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000 },
+        "application/vnd.apple.keynote" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000 },
+        "application/vnd.apple.numbers" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000 },
+        "application/vnd.apple.pages" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000 },
+        "application/vnd.apple.pkpass" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000 },
+        "application/vnd.apple.pkpasses" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000 },
+        "application/vnd.apple.unknown.13" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000 },
+        "application/vnd.bzip3" => MimeType { alias_of: 7, container: 0x20, contained_in: 0x0 },
+        "application/vnd.comicbook+zip" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000 },
+        "application/vnd.comicbook-rar" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x400 },
+        "application/vnd.cyclonedx+xml" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000000 },
+        "application/vnd.debian.binary-package" => MimeType { alias_of: 8, container: 0x0, contained_in: 0x200000 },
+        "application/vnd.etsi.asic-e+zip" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000 },
+        "application/vnd.etsi.asic-s+zip" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000 },
+        "application/vnd.google-earth.kml+xml" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000000 },
+        "application/vnd.google-earth.kmz" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000 },
+        "application/vnd.hancom.hwpx" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000 },
+        "application/vnd.iptc.g2.newsmessage+xml" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000000 },
+        "application/vnd.microsoft.windows.thumbnail-cache" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x40000 },
+        "application/vnd.mindjet.mindmanager" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000 },
+        "application/vnd.mozilla.xul+xml" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000000 },
+        "application/vnd.ms-3mfdocument" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000 },
+        "application/vnd.ms-asf" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x20000000000 },
+        "application/vnd.ms-cab-compressed" => MimeType { alias_of: 0, container: 0x40, contained_in: 0x0 },
+        "application/vnd.ms-excel" => MimeType { alias_of: 9, container: 0x0, contained_in: 0x40000 },
+        "application/vnd.ms-excel.addin.macroenabled.12" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800100 },
+        "application/vnd.ms-excel.sheet.binary.macroenabled.12" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800100 },
+        "application/vnd.ms-excel.sheet.macroenabled.12" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800100 },
+        "application/vnd.ms-excel.template.macroenabled.12" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000 },
+        "application/vnd.ms-officetheme" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000 },
+        "application/vnd.ms-outlook" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x40000 },
+        "application/vnd.ms-powerpoint" => MimeType { alias_of: 10, container: 0x0, contained_in: 0x40000 },
+        "application/vnd.ms-powerpoint.addin.macroenabled.12" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000 },
+        "application/vnd.ms-powerpoint.presentation.macroenabled.12" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800080 },
+        "application/vnd.ms-powerpoint.slide.macroenabled.12" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000 },
+        "application/vnd.ms-powerpoint.slideshow.macroenabled.12" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000 },
+        "application/vnd.ms-powerpoint.template.macroenabled.12" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000 },
+        "application/vnd.ms-project" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x40000 },
+        "application/vnd.ms-publisher" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x40000 },
+        "application/vnd.ms-spreadsheetml" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000000 },
+        "application/vnd.ms-visio" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x40000 },
+        "application/vnd.ms-visio.drawing" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000 },
+        "application/vnd.ms-visio.drawing.macroenabled.12" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000 },
+        "application/vnd.ms-visio.drawing.macroenabled.main+xml" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000 },
+        "application/vnd.ms-visio.drawing.main+xml" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000 },
+        "application/vnd.ms-visio.stencil" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000 },
+        "application/vnd.ms-visio.stencil.macroenabled.12" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000 },
+        "application/vnd.ms-visio.stencil.macroenabled.main+xml" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000 },
+        "application/vnd.ms-visio.stencil.main+xml" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000 },
+        "application/vnd.ms-visio.template" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000 },
+        "application/vnd.ms-visio.template.macroenabled.12" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000 },
+        "application/vnd.ms-visio.template.macroenabled.main+xml" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000 },
+        "application/vnd.ms-visio.template.main+xml" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000 },
+        "application/vnd.ms-word" => MimeType { alias_of: 4, container: 0x0, contained_in: 0x40000 },
+        "application/vnd.ms-word.document.macroenabled.12" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800200 },
+        "application/vnd.ms-word.template.macroenabled.12" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000 },
+        "application/vnd.ms-word2006ml" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000000 },
+        "application/vnd.ms-wordml" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000000 },
+        "application/vnd.ms-works" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x40000 },
+        "application/vnd.ms-xpsdocument" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000 },
+        "application/vnd.oasis.docbook+xml" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000000 },
+        "application/vnd.oasis.opendocument.base" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000 },
+        "application/vnd.oasis.opendocument.chart" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000 },
+        "application/vnd.oasis.opendocument.chart-template" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000 },
+        "application/vnd.oasis.opendocument.database" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000 },
+        "application/vnd.oasis.opendocument.flat.presentation" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000000 },
+        "application/vnd.oasis.opendocument.flat.spreadsheet" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000000 },
+        "application/vnd.oasis.opendocument.flat.text" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000000 },
+        "application/vnd.oasis.opendocument.formula" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000 },
+        "application/vnd.oasis.opendocument.formula-template" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000 },
+        "application/vnd.oasis.opendocument.graphics" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000 },
+        "application/vnd.oasis.opendocument.graphics-flat-xml" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000000 },
+        "application/vnd.oasis.opendocument.graphics-template" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000 },
+        "application/vnd.oasis.opendocument.image" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000 },
+        "application/vnd.oasis.opendocument.image-template" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000 },
+        "application/vnd.oasis.opendocument.presentation" => MimeType { alias_of: 11, container: 0x0, contained_in: 0x800000 },
+        "application/vnd.oasis.opendocument.presentation-flat-xml" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000000 },
+        "application/vnd.oasis.opendocument.presentation-template" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000 },
+        "application/vnd.oasis.opendocument.spreadsheet" => MimeType { alias_of: 12, container: 0x0, contained_in: 0x800000 },
+        "application/vnd.oasis.opendocument.spreadsheet-flat-xml" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000000 },
+        "application/vnd.oasis.opendocument.spreadsheet-template" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000 },
+        "application/vnd.oasis.opendocument.text" => MimeType { alias_of: 13, container: 0x0, contained_in: 0x800000 },
+        "application/vnd.oasis.opendocument.text-flat-xml" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000000 },
+        "application/vnd.oasis.opendocument.text-master" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000 },
+        "application/vnd.oasis.opendocument.text-master-template" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000 },
+        "application/vnd.oasis.opendocument.text-template" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000 },
+        "application/vnd.oasis.opendocument.text-web" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000 },
+        "application/vnd.oasis.opendocument.tika.flat.document" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000000 },
+        "application/vnd.openofficeorg.autotext" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000 },
+        "application/vnd.openofficeorg.extension" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000 },
+        "application/vnd.openxmlformats-officedocument.presentationml.presentation" => MimeType { alias_of: 0, container: 0x80, contained_in: 0x800000 },
+        "application/vnd.openxmlformats-officedocument.presentationml.slide" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000 },
+        "application/vnd.openxmlformats-officedocument.presentationml.slideshow" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000 },
+        "application/vnd.openxmlformats-officedocument.presentationml.template" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000 },
+        "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" => MimeType { alias_of: 0, container: 0x100, contained_in: 0x800000 },
+        "application/vnd.openxmlformats-officedocument.spreadsheetml.template" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000 },
+        "application/vnd.openxmlformats-officedocument.wordprocessingml.document" => MimeType { alias_of: 0, container: 0x200, contained_in: 0x800000 },
+        "application/vnd.openxmlformats-officedocument.wordprocessingml.template" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000 },
+        "application/vnd.rar" => MimeType { alias_of: 14, container: 0x400, contained_in: 0x0 },
+        "application/vnd.sqlite3" => MimeType { alias_of: 15, container: 0x800, contained_in: 0x0 },
+        "application/vnd.stardivision.calc" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x40000 },
+        "application/vnd.stardivision.chart" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x40000 },
+        "application/vnd.stardivision.draw" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x40000 },
+        "application/vnd.stardivision.impress" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x40000 },
+        "application/vnd.stardivision.mail" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x40000 },
+        "application/vnd.stardivision.math" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x40000 },
+        "application/vnd.stardivision.writer" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x40000 },
+        "application/vnd.stardivision.writer-global" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x40000 },
+        "application/vnd.sun.xml.base" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000 },
+        "application/vnd.sun.xml.calc" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000 },
+        "application/vnd.sun.xml.calc.template" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000 },
+        "application/vnd.sun.xml.draw" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000 },
+        "application/vnd.sun.xml.draw.template" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000 },
+        "application/vnd.sun.xml.impress" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000 },
+        "application/vnd.sun.xml.impress.template" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000 },
+        "application/vnd.sun.xml.math" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000 },
+        "application/vnd.sun.xml.writer" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000 },
+        "application/vnd.sun.xml.writer.global" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000 },
+        "application/vnd.sun.xml.writer.template" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000 },
+        "application/vnd.visio" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x40000 },
+        "application/vnd.youtube.yt" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000 },
+        "application/wwf" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x8 },
+        "application/x-7z" => MimeType { alias_of: 16, container: 0x0, contained_in: 0x0 },
+        "application/x-7z-compressed" => MimeType { alias_of: 16, container: 0x1000, contained_in: 0x0 },
+        "application/x-abiword" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000000 },
+        "application/x-acad" => MimeType { alias_of: 44, container: 0x0, contained_in: 0x0 },
+        "application/x-adobe-indesign-interchange" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000000 },
+        "application/x-alpine-package-keeper-package" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x2 },
+        "application/x-amf" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000000 },
+        "application/x-apple-systemprofiler+xml" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000000 },
+        "application/x-archive" => MimeType { alias_of: 21, container: 0x0, contained_in: 0x0 },
+        "application/x-autocad" => MimeType { alias_of: 44, container: 0x0, contained_in: 0x0 },
+        "application/x-bentley-besqlite" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800 },
+        "application/x-bentley-localization" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800 },
+        "application/x-bzdvi" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x2000 },
+        "application/x-bzip" => MimeType { alias_of: 17, container: 0x0, contained_in: 0x2000 },
+        "application/x-bzip-compressed-tar" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x2000 },
+        "application/x-bzip2" => MimeType { alias_of: 17, container: 0x2000, contained_in: 0x0 },
+        "application/x-bzip2-compressed-tar" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x2000 },
+        "application/x-bzip3" => MimeType { alias_of: 7, container: 0x0, contained_in: 0x0 },
+        "application/x-bzip3-compressed-tar" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x20 },
+        "application/x-bzpdf" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x2000 },
+        "application/x-bzpostscript" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x2000 },
+        "application/x-cb7" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x1000 },
+        "application/x-cbr" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x400 },
+        "application/x-cbt" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x100000 },
+        "application/x-cbz" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000 },
+        "application/x-compress" => MimeType { alias_of: 0, container: 0x4000, contained_in: 0x0 },
+        "application/x-compressed" => MimeType { alias_of: 23, container: 0x0, contained_in: 0x0 },
+        "application/x-compressed-tar" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x2 },
+        "application/x-corelpresentations" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x40000 },
+        "application/x-cpio-compressed" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x2 },
+        "application/x-csh" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x400000000 },
+        "application/x-deb" => MimeType { alias_of: 8, container: 0x0, contained_in: 0x200000 },
+        "application/x-debian-package" => MimeType { alias_of: 8, container: 0x0, contained_in: 0x200000 },
+        "application/x-designer" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000000 },
+        "application/x-dia-diagram" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000000 },
+        "application/x-dia-shape" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000000 },
+        "application/x-docbook+xml" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000000 },
+        "application/x-dwg" => MimeType { alias_of: 44, container: 0x0, contained_in: 0x0 },
+        "application/x-esri-layer" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x40000 },
+        "application/x-esri-spatially-enabled-db" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800 },
+        "application/x-fictionbook" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000000 },
+        "application/x-fictionbook+xml" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000000 },
+        "application/x-flash-video" => MimeType { alias_of: 52, container: 0x0, contained_in: 0x0 },
+        "application/x-font-ttx" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000000 },
+        "application/x-font-type1" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x10 },
+        "application/x-fossil-checkout" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800 },
+        "application/x-fossil-global-conf" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800 },
+        "application/x-fossil-repository" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800 },
+        "application/x-freedesktop-appstream-component" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000000 },
+        "application/x-freedesktop-appstream-releases" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000000 },
+        "application/x-geopackage" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800 },
+        "application/x-glade" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000000 },
+        "application/x-gpx" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000000 },
+        "application/x-gpx+xml" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000000 },
+        "application/x-gtar" => MimeType { alias_of: 20, container: 0x0, contained_in: 0x100000 },
+        "application/x-gtk-builder" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000000 },
+        "application/x-gunzip" => MimeType { alias_of: 2, container: 0x0, contained_in: 0x0 },
+        "application/x-gz-font-linux-psf" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x2 },
+        "application/x-gzdvi" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x2 },
+        "application/x-gzip" => MimeType { alias_of: 2, container: 0x0, contained_in: 0x0 },
+        "application/x-gzip-compressed" => MimeType { alias_of: 2, container: 0x0, contained_in: 0x0 },
+        "application/x-gzpdf" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x2 },
+        "application/x-gzpostscript" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x2 },
+        "application/x-hwp+zip" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000 },
+        "application/x-hwp-v5" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x40000 },
+        "application/x-hwpx" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000 },
+        "application/x-ibooks+zip" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800001 },
+        "application/x-itunes-ipa" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000 },
+        "application/x-iwork-keynote-sffkey" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000 },
+        "application/x-iwork-numbers-sffnumbers" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000 },
+        "application/x-iwork-pages-sffpages" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000 },
+        "application/x-jar" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000 },
+        "application/x-java" => MimeType { alias_of: 3, container: 0x0, contained_in: 0x0 },
+        "application/x-java-archive" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000 },
+        "application/x-java-class" => MimeType { alias_of: 3, container: 0x0, contained_in: 0x0 },
+        "application/x-java-jnlp-file" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000000 },
+        "application/x-java-vm" => MimeType { alias_of: 3, container: 0x0, contained_in: 0x0 },
+        "application/x-kexiproject-sqlite" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800 },
+        "application/x-kexiproject-sqlite3" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800 },
+        "application/x-linguist" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000000 },
+        "application/x-lz4" => MimeType { alias_of: 0, container: 0x8000, contained_in: 0x0 },
+        "application/x-lz4-compressed-tar" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x8000 },
+        "application/x-lzip" => MimeType { alias_of: 0, container: 0x10000, contained_in: 0x0 },
+        "application/x-lzip-compressed-tar" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x10000 },
+        "application/x-lzpdf" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x10000 },
+        "application/x-mbtiles" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800 },
+        "application/x-midi" => MimeType { alias_of: 27, container: 0x0, contained_in: 0x0 },
+        "application/x-mobi8-ebook" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x20000 },
+        "application/x-mobipocket-ebook" => MimeType { alias_of: 0, container: 0x20000, contained_in: 0x0 },
+        "application/x-modrinth-modpack+zip" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000 },
+        "application/x-monotone-source-repo" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800 },
+        "application/x-mozilla-bookmarks" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x200000000 },
+        "application/x-ms-asx" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000000 },
+        "application/x-ms-emz" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x2 },
+        "application/x-ms-installer" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x40000 },
+        "application/x-ms-wmz" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x2 },
+        "application/x-msexcel" => MimeType { alias_of: 9, container: 0x0, contained_in: 0x40000 },
+        "application/x-msi" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x40000 },
+        "application/x-mspowerpoint" => MimeType { alias_of: 10, container: 0x0, contained_in: 0x40000 },
+        "application/x-mspublisher" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x40000 },
+        "application/x-msword" => MimeType { alias_of: 4, container: 0x0, contained_in: 0x40000 },
+        "application/x-netscape-bookmarks" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x200000000 },
+        "application/x-netshow-channel" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x20000000000 },
+        "application/x-nzb" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000000 },
+        "application/x-ogg" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x8000000 },
+        "application/x-ole-storage" => MimeType { alias_of: 0, container: 0x40000, contained_in: 0x0 },
+        "application/x-pagemaker" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x40000 },
+        "application/x-pdf" => MimeType { alias_of: 5, container: 0x0, contained_in: 0x0 },
+        "application/x-photoshop" => MimeType { alias_of: 42, container: 0x0, contained_in: 0x0 },
+        "application/x-plist" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000000 },
+        "application/x-pyspread-bz-spreadsheet" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x2000 },
+        "application/x-qbrew" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000000 },
+        "application/x-quattro-pro" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x40000 },
+        "application/x-quicktime-media-link" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x2000000000 },
+        "application/x-quicktimeplayer" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x2000000000 },
+        "application/x-rar" => MimeType { alias_of: 14, container: 0x0, contained_in: 0x0 },
+        "application/x-rar-compressed" => MimeType { alias_of: 14, container: 0x0, contained_in: 0x0 },
+        "application/x-raw-disk-image-xz-compressed" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x400000 },
+        "application/x-redhat-package-manager" => MimeType { alias_of: 18, container: 0x0, contained_in: 0x0 },
+        "application/x-rpm" => MimeType { alias_of: 18, container: 0x80000, contained_in: 0x0 },
+        "application/x-shellscript" => MimeType { alias_of: 47, container: 0x0, contained_in: 0x0 },
+        "application/x-shockwave-flash" => MimeType { alias_of: 19, container: 0x0, contained_in: 0x0 },
+        "application/x-source-rpm" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x80000 },
+        "application/x-speex" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x8000000 },
+        "application/x-sqlite3" => MimeType { alias_of: 15, container: 0x0, contained_in: 0x0 },
+        "application/x-starcalc" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x40000 },
+        "application/x-starchart" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x40000 },
+        "application/x-stardraw" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x40000 },
+        "application/x-starimpress" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x40000 },
+        "application/x-starmath" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x40000 },
+        "application/x-staroffice-template" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x40000 },
+        "application/x-starwriter" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x40000 },
+        "application/x-starwriter-global" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x40000 },
+        "application/x-tar" => MimeType { alias_of: 20, container: 0x100000, contained_in: 0x0 },
+        "application/x-tarz" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x4000 },
+        "application/x-texnicard" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800 },
+        "application/x-tiled-tmx" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000000 },
+        "application/x-tiled-tsx" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000000 },
+        "application/x-tmx" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000000 },
+        "application/x-ufraw" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000000 },
+        "application/x-unix-archive" => MimeType { alias_of: 21, container: 0x200000, contained_in: 0x0 },
+        "application/x-virtualbox-ova" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x100000 },
+        "application/x-vnd.datapackage+gz" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x2 },
+        "application/x-vnd.datapackage+zip" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000 },
+        "application/x-vnd.kde.kexi" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800 },
+        "application/x-vnd.oasis.opendocument.chart" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000 },
+        "application/x-vnd.oasis.opendocument.chart-template" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000 },
+        "application/x-vnd.oasis.opendocument.formula" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000 },
+        "application/x-vnd.oasis.opendocument.formula-template" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000 },
+        "application/x-vnd.oasis.opendocument.graphics" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000 },
+        "application/x-vnd.oasis.opendocument.graphics-template" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000 },
+        "application/x-vnd.oasis.opendocument.image" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000 },
+        "application/x-vnd.oasis.opendocument.image-template" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000 },
+        "application/x-vnd.oasis.opendocument.presentation" => MimeType { alias_of: 11, container: 0x0, contained_in: 0x800000 },
+        "application/x-vnd.oasis.opendocument.presentation-template" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000 },
+        "application/x-vnd.oasis.opendocument.spreadsheet" => MimeType { alias_of: 12, container: 0x0, contained_in: 0x800000 },
+        "application/x-vnd.oasis.opendocument.spreadsheet-template" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000 },
+        "application/x-vnd.oasis.opendocument.text" => MimeType { alias_of: 13, container: 0x0, contained_in: 0x800000 },
+        "application/x-vnd.oasis.opendocument.text-master" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000 },
+        "application/x-vnd.oasis.opendocument.text-template" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000 },
+        "application/x-vnd.oasis.opendocument.text-web" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000 },
+        "application/x-vnd.sun.xml.writer" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000 },
+        "application/x-wacz" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000 },
+        "application/x-windows-installer" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x40000 },
+        "application/x-windows-themepack" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x40 },
+        "application/x-wwf" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x8 },
+        "application/x-x509-ca-cert" => MimeType { alias_of: 22, container: 0x0, contained_in: 0x0 },
+        "application/x-x509-cert" => MimeType { alias_of: 22, container: 0x0, contained_in: 0x0 },
+        "application/x-x509-user-cert" => MimeType { alias_of: 22, container: 0x0, contained_in: 0x0 },
+        "application/x-xbel" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000000 },
+        "application/x-xliff" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000000 },
+        "application/x-xliff+xml" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000000 },
+        "application/x-xliff+zip" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000 },
+        "application/x-xmind" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000 },
+        "application/x-xml" => MimeType { alias_of: 48, container: 0x0, contained_in: 0x0 },
+        "application/x-xpinstall" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000 },
+        "application/x-xspf+xml" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000000 },
+        "application/x-xz" => MimeType { alias_of: 0, container: 0x400000, contained_in: 0x0 },
+        "application/x-xz-compressed-tar" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x400000 },
+        "application/x-xzpdf" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x400000 },
+        "application/x-zip" => MimeType { alias_of: 23, container: 0x0, contained_in: 0x0 },
+        "application/x-zip-compressed" => MimeType { alias_of: 23, container: 0x0, contained_in: 0x0 },
+        "application/x-zip-compressed-fb2" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000 },
+        "application/x-zstd-compressed-tar" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x1000000 },
+        "application/xhtml+xml" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000000 },
+        "application/xliff+xml" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000000 },
+        "application/xml" => MimeType { alias_of: 48, container: 0x0, contained_in: 0x0 },
+        "application/xml-external-parsed-entity" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000000 },
+        "application/xps" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000 },
+        "application/xslfo+xml" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000000 },
+        "application/xslt+xml" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000000 },
+        "application/xspf+xml" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000000 },
+        "application/zip" => MimeType { alias_of: 23, container: 0x800000, contained_in: 0x0 },
+        "application/zstd" => MimeType { alias_of: 0, container: 0x1000000, contained_in: 0x0 },
+        "audio/3gpp" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x13000000000 },
+        "audio/3gpp-encrypted" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x13000000000 },
+        "audio/aac" => MimeType { alias_of: 24, container: 0x0, contained_in: 0x0 },
+        "audio/aiff" => MimeType { alias_of: 30, container: 0x0, contained_in: 0x0 },
+        "audio/amr" => MimeType { alias_of: 25, container: 0x2000000, contained_in: 0x0 },
+        "audio/amr-encrypted" => MimeType { alias_of: 25, container: 0x0, contained_in: 0x0 },
+        "audio/amr-wb" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x2000000 },
+        "audio/amr-wb+" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x2000000 },
+        "audio/amr-wb-encrypted" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x2000000 },
+        "audio/dsd" => MimeType { alias_of: 31, container: 0x0, contained_in: 0x0 },
+        "audio/dsf" => MimeType { alias_of: 31, container: 0x0, contained_in: 0x0 },
+        "audio/flac" => MimeType { alias_of: 32, container: 0x0, contained_in: 0x0 },
+        "audio/m4a" => MimeType { alias_of: 26, container: 0x4000000, contained_in: 0x13000000000 },
+        "audio/mid" => MimeType { alias_of: 27, container: 0x0, contained_in: 0x0 },
+        "audio/midi" => MimeType { alias_of: 27, container: 0x0, contained_in: 0x0 },
+        "audio/mp3" => MimeType { alias_of: 28, container: 0x0, contained_in: 0x0 },
+        "audio/mp4" => MimeType { alias_of: 26, container: 0x0, contained_in: 0x13000000000 },
+        "audio/mpeg" => MimeType { alias_of: 28, container: 0x0, contained_in: 0x0 },
+        "audio/ogg" => MimeType { alias_of: 29, container: 0x8000000, contained_in: 0x0 },
+        "audio/opus" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x8000000 },
+        "audio/speex" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x8000000 },
+        "audio/vnd.wave" => MimeType { alias_of: 33, container: 0x0, contained_in: 0x0 },
+        "audio/vorbis" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x8000000 },
+        "audio/wav" => MimeType { alias_of: 33, container: 0x0, contained_in: 0x0 },
+        "audio/wave" => MimeType { alias_of: 33, container: 0x0, contained_in: 0x0 },
+        "audio/webm" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x4000000000 },
+        "audio/wma" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x20000000000 },
+        "audio/x-aac" => MimeType { alias_of: 24, container: 0x0, contained_in: 0x0 },
+        "audio/x-aiff" => MimeType { alias_of: 30, container: 0x0, contained_in: 0x0 },
+        "audio/x-dsd" => MimeType { alias_of: 31, container: 0x0, contained_in: 0x0 },
+        "audio/x-dsf" => MimeType { alias_of: 31, container: 0x0, contained_in: 0x0 },
+        "audio/x-flac" => MimeType { alias_of: 32, container: 0x0, contained_in: 0x0 },
+        "audio/x-flac+ogg" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x8000000 },
+        "audio/x-hx-aac-adts" => MimeType { alias_of: 24, container: 0x0, contained_in: 0x0 },
+        "audio/x-m4a" => MimeType { alias_of: 26, container: 0x0, contained_in: 0x13000000000 },
+        "audio/x-m4b" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x13004000000 },
+        "audio/x-m4r" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x13000000000 },
+        "audio/x-mid" => MimeType { alias_of: 27, container: 0x0, contained_in: 0x0 },
+        "audio/x-midi" => MimeType { alias_of: 27, container: 0x0, contained_in: 0x0 },
+        "audio/x-mp3" => MimeType { alias_of: 28, container: 0x0, contained_in: 0x0 },
+        "audio/x-mp4a" => MimeType { alias_of: 26, container: 0x0, contained_in: 0x13000000000 },
+        "audio/x-mpeg" => MimeType { alias_of: 28, container: 0x0, contained_in: 0x0 },
+        "audio/x-mpg" => MimeType { alias_of: 28, container: 0x0, contained_in: 0x0 },
+        "audio/x-ms-asx" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000000 },
+        "audio/x-ms-wma" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x20000000000 },
+        "audio/x-ogg" => MimeType { alias_of: 29, container: 0x0, contained_in: 0x8000000 },
+        "audio/x-ogg-flac" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x8000000 },
+        "audio/x-ogg-pcm" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x8000000 },
+        "audio/x-oggflac" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x8000000 },
+        "audio/x-oggpcm" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x8000000 },
+        "audio/x-opus+ogg" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x8000000 },
+        "audio/x-pn-wav" => MimeType { alias_of: 33, container: 0x0, contained_in: 0x0 },
+        "audio/x-rn-3gpp-amr" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x13000000000 },
+        "audio/x-rn-3gpp-amr-encrypted" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x13000000000 },
+        "audio/x-rn-3gpp-amr-wb" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x13000000000 },
+        "audio/x-rn-3gpp-amr-wb-encrypted" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x13000000000 },
+        "audio/x-speex+ogg" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x8000000 },
+        "audio/x-vorbis" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x8000000 },
+        "audio/x-vorbis+ogg" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x8000000 },
+        "audio/x-wav" => MimeType { alias_of: 33, container: 0x0, contained_in: 0x0 },
+        "audio/x-wave" => MimeType { alias_of: 33, container: 0x0, contained_in: 0x0 },
+        "drawing/dwg" => MimeType { alias_of: 44, container: 0x0, contained_in: 0x0 },
+        "flv-application/octet-stream" => MimeType { alias_of: 52, container: 0x0, contained_in: 0x0 },
+        "font/woff" => MimeType { alias_of: 1, container: 0x0, contained_in: 0x0 },
+        "gzip/document" => MimeType { alias_of: 2, container: 0x0, contained_in: 0x0 },
+        "image/apng" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x40000000 },
+        "image/avif" => MimeType { alias_of: 34, container: 0x0, contained_in: 0x0 },
+        "image/avif-sequence" => MimeType { alias_of: 34, container: 0x0, contained_in: 0x0 },
+        "image/bmp" => MimeType { alias_of: 35, container: 0x0, contained_in: 0x0 },
+        "image/gif" => MimeType { alias_of: 36, container: 0x0, contained_in: 0x0 },
+        "image/heic" => MimeType { alias_of: 37, container: 0x0, contained_in: 0x2010000000 },
+        "image/heic-sequence" => MimeType { alias_of: 37, container: 0x0, contained_in: 0x2010000000 },
+        "image/heif" => MimeType { alias_of: 37, container: 0x10000000, contained_in: 0x2000000000 },
+        "image/heif-sequence" => MimeType { alias_of: 37, container: 0x0, contained_in: 0x2010000000 },
+        "image/hevc" => MimeType { alias_of: 37, container: 0x0, contained_in: 0x2010000000 },
+        "image/hevc-sequence" => MimeType { alias_of: 37, container: 0x0, contained_in: 0x2010000000 },
+        "image/ico" => MimeType { alias_of: 45, container: 0x0, contained_in: 0x0 },
+        "image/icon" => MimeType { alias_of: 45, container: 0x0, contained_in: 0x0 },
+        "image/jp2" => MimeType { alias_of: 38, container: 0x0, contained_in: 0x0 },
+        "image/jpe" => MimeType { alias_of: 39, container: 0x0, contained_in: 0x0 },
+        "image/jpeg" => MimeType { alias_of: 39, container: 0x20000000, contained_in: 0x0 },
+        "image/jpeg2000" => MimeType { alias_of: 38, container: 0x0, contained_in: 0x0 },
+        "image/jpeg2000-image" => MimeType { alias_of: 38, container: 0x0, contained_in: 0x0 },
+        "image/jpg" => MimeType { alias_of: 39, container: 0x0, contained_in: 0x0 },
+        "image/jxr" => MimeType { alias_of: 46, container: 0x0, contained_in: 0x0 },
+        "image/openraster" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000 },
+        "image/pdf" => MimeType { alias_of: 5, container: 0x0, contained_in: 0x0 },
+        "image/photoshop" => MimeType { alias_of: 42, container: 0x0, contained_in: 0x0 },
+        "image/pjpeg" => MimeType { alias_of: 39, container: 0x0, contained_in: 0x0 },
+        "image/png" => MimeType { alias_of: 40, container: 0x40000000, contained_in: 0x0 },
+        "image/psd" => MimeType { alias_of: 42, container: 0x0, contained_in: 0x0 },
+        "image/svg+xml" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000000 },
+        "image/svg+xml-compressed" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x2 },
+        "image/tif" => MimeType { alias_of: 41, container: 0x0, contained_in: 0x0 },
+        "image/tiff" => MimeType { alias_of: 41, container: 0x80000000, contained_in: 0x0 },
+        "image/vnd.adobe.photoshop" => MimeType { alias_of: 42, container: 0x0, contained_in: 0x0 },
+        "image/vnd.adobe.premiere" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000000 },
+        "image/vnd.djvu" => MimeType { alias_of: 43, container: 0x100000000, contained_in: 0x0 },
+        "image/vnd.djvu+multipage" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x100000000 },
+        "image/vnd.dwg" => MimeType { alias_of: 44, container: 0x0, contained_in: 0x0 },
+        "image/vnd.fpx" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x40000 },
+        "image/vnd.microsoft.icon" => MimeType { alias_of: 45, container: 0x0, contained_in: 0x0 },
+        "image/vnd.mozilla.apng" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x40000000 },
+        "image/vnd.ms-photo" => MimeType { alias_of: 46, container: 0x0, contained_in: 0x0 },
+        "image/x-adobe-dng" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x80000000 },
+        "image/x-bmp" => MimeType { alias_of: 35, container: 0x0, contained_in: 0x0 },
+        "image/x-bzeps" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x2000 },
+        "image/x-canon-cr2" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x80000000 },
+        "image/x-canon-cr3" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x2000000000 },
+        "image/x-djvu" => MimeType { alias_of: 43, container: 0x0, contained_in: 0x0 },
+        "image/x-dwg" => MimeType { alias_of: 44, container: 0x0, contained_in: 0x0 },
+        "image/x-emf-compressed" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x2 },
+        "image/x-eps" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x10 },
+        "image/x-epson-erf" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x80000000 },
+        "image/x-fpx" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x40000 },
+        "image/x-gif" => MimeType { alias_of: 36, container: 0x0, contained_in: 0x0 },
+        "image/x-gzeps" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x2 },
+        "image/x-hasselblad-3fr" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x80000000 },
+        "image/x-hasselblad-fff" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x80000000 },
+        "image/x-ico" => MimeType { alias_of: 45, container: 0x0, contained_in: 0x0 },
+        "image/x-icon" => MimeType { alias_of: 45, container: 0x0, contained_in: 0x0 },
+        "image/x-jpeg2000-image" => MimeType { alias_of: 38, container: 0x0, contained_in: 0x0 },
+        "image/x-jpg" => MimeType { alias_of: 39, container: 0x0, contained_in: 0x0 },
+        "image/x-kodak-dcr" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x80000000 },
+        "image/x-kodak-k25" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x80000000 },
+        "image/x-kodak-kdc" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x80000000 },
+        "image/x-leaf-mos" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x80000000 },
+        "image/x-mamiya-mef" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x80000000 },
+        "image/x-ms-bmp" => MimeType { alias_of: 35, container: 0x0, contained_in: 0x0 },
+        "image/x-nikon-nef" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x80000000 },
+        "image/x-nikon-nrw" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x80000000 },
+        "image/x-pentax-pef" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x80000000 },
+        "image/x-phaseone-iiq" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x80000000 },
+        "image/x-photoshop" => MimeType { alias_of: 42, container: 0x0, contained_in: 0x0 },
+        "image/x-png" => MimeType { alias_of: 40, container: 0x0, contained_in: 0x0 },
+        "image/x-psd" => MimeType { alias_of: 42, container: 0x0, contained_in: 0x0 },
+        "image/x-raw-adobe" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x80000000 },
+        "image/x-raw-nikon" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x80000000 },
+        "image/x-raw-pentax" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x80000000 },
+        "image/x-raw-sony" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x80000000 },
+        "image/x-samsung-srw" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x80000000 },
+        "image/x-sinar-sti" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x80000000 },
+        "image/x-sony-arw" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x80000000 },
+        "image/x-sony-sr2" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x80000000 },
+        "image/x-sony-srf" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x80000000 },
+        "image/x-tif" => MimeType { alias_of: 41, container: 0x0, contained_in: 0x0 },
+        "image/x-tiff-multipage" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x80000000 },
+        "image/x.djvu" => MimeType { alias_of: 43, container: 0x0, contained_in: 0x0 },
+        "model/3mf" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000 },
+        "model/vnd.dwfx+xps" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000 },
+        "multipart/x-zip" => MimeType { alias_of: 23, container: 0x0, contained_in: 0x0 },
+        "text/html" => MimeType { alias_of: 0, container: 0x200000000, contained_in: 0x0 },
+        "text/ico" => MimeType { alias_of: 45, container: 0x0, contained_in: 0x0 },
+        "text/iso19139+xml" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000000 },
+        "text/mathml" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000000 },
+        "text/rdf" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000000 },
+        "text/rss" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000000 },
+        "text/rtf" => MimeType { alias_of: 6, container: 0x0, contained_in: 0x0 },
+        "text/scriptlet" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000000 },
+        "text/vnd.qt.linguist" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000000 },
+        "text/vnd.trolltech.linguist" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000000 },
+        "text/vnd.wap.wml" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000000 },
+        "text/x-component" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000000 },
+        "text/x-csh" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x400000000 },
+        "text/x-maven+xml" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000000 },
+        "text/x-mrml" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000000 },
+        "text/x-opml" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000000 },
+        "text/x-opml+xml" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000000 },
+        "text/x-sh" => MimeType { alias_of: 47, container: 0x0, contained_in: 0x0 },
+        "text/x-shellscript" => MimeType { alias_of: 47, container: 0x400000000, contained_in: 0x0 },
+        "text/x-xmi" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000000 },
+        "text/x-xslfo" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000000 },
+        "text/xml" => MimeType { alias_of: 48, container: 0x800000000, contained_in: 0x0 },
+        "text/xml-external-parsed-entity" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000000 },
+        "text/xsl" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000000 },
+        "video/3gp" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x13000000000 },
+        "video/3gpp" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x13000000000 },
+        "video/3gpp-encrypted" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x13000000000 },
+        "video/avi" => MimeType { alias_of: 54, container: 0x0, contained_in: 0x0 },
+        "video/daala" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x8000000 },
+        "video/divx" => MimeType { alias_of: 54, container: 0x0, contained_in: 0x0 },
+        "video/flv" => MimeType { alias_of: 52, container: 0x0, contained_in: 0x0 },
+        "video/iso.segment" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x2000000000 },
+        "video/matroska" => MimeType { alias_of: 53, container: 0x0, contained_in: 0x0 },
+        "video/mov" => MimeType { alias_of: 51, container: 0x0, contained_in: 0x0 },
+        "video/mp4" => MimeType { alias_of: 49, container: 0x1000000000, contained_in: 0x12000000000 },
+        "video/mp4v-es" => MimeType { alias_of: 49, container: 0x0, contained_in: 0x13000000000 },
+        "video/mpeg" => MimeType { alias_of: 50, container: 0x0, contained_in: 0x0 },
+        "video/mpeg-system" => MimeType { alias_of: 50, container: 0x0, contained_in: 0x0 },
+        "video/msvideo" => MimeType { alias_of: 54, container: 0x0, contained_in: 0x0 },
+        "video/ogg" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x8000000 },
+        "video/quicktime" => MimeType { alias_of: 51, container: 0x2000000000, contained_in: 0x0 },
+        "video/theora" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x8000000 },
+        "video/vnd.avi" => MimeType { alias_of: 54, container: 0x0, contained_in: 0x0 },
+        "video/vnd.divx" => MimeType { alias_of: 54, container: 0x0, contained_in: 0x0 },
+        "video/vnd.youtube.yt" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000 },
+        "video/webm" => MimeType { alias_of: 0, container: 0x4000000000, contained_in: 0x0 },
+        "video/x-avi" => MimeType { alias_of: 54, container: 0x0, contained_in: 0x0 },
+        "video/x-daala" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x8000000 },
+        "video/x-dirac" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x8000000 },
+        "video/x-flv" => MimeType { alias_of: 52, container: 0x8000000000, contained_in: 0x0 },
+        "video/x-javafx" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x8000000000 },
+        "video/x-m4v" => MimeType { alias_of: 49, container: 0x10000000000, contained_in: 0x3000000000 },
+        "video/x-matroska" => MimeType { alias_of: 53, container: 0x0, contained_in: 0x0 },
+        "video/x-mjpeg" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x20000000 },
+        "video/x-mov" => MimeType { alias_of: 51, container: 0x0, contained_in: 0x0 },
+        "video/x-mpeg" => MimeType { alias_of: 50, container: 0x0, contained_in: 0x0 },
+        "video/x-mpeg-system" => MimeType { alias_of: 50, container: 0x0, contained_in: 0x0 },
+        "video/x-mpeg2" => MimeType { alias_of: 50, container: 0x0, contained_in: 0x0 },
+        "video/x-ms-asf" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x20000000000 },
+        "video/x-ms-asf-plugin" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x20000000000 },
+        "video/x-ms-wax" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000000 },
+        "video/x-ms-wm" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x20000000000 },
+        "video/x-ms-wmv" => MimeType { alias_of: 0, container: 0x20000000000, contained_in: 0x0 },
+        "video/x-ms-wmx" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000000 },
+        "video/x-ms-wvx" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x800000000 },
+        "video/x-msvideo" => MimeType { alias_of: 54, container: 0x0, contained_in: 0x0 },
+        "video/x-ogg" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x8000000 },
+        "video/x-ogg-rgb" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x8000000 },
+        "video/x-ogg-uvs" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x8000000 },
+        "video/x-ogg-yuv" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x8000000 },
+        "video/x-oggrgb" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x8000000 },
+        "video/x-ogguvs" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x8000000 },
+        "video/x-oggyuv" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x8000000 },
+        "video/x-ogm" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x8000000 },
+        "video/x-ogm+ogg" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x8000000 },
+        "video/x-theora" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x8000000 },
+        "video/x-theora+ogg" => MimeType { alias_of: 0, container: 0x0, contained_in: 0x8000000 },
+    )
+}

crates/spam-filter/src/analysis/mod.rs

@@ -0,0 +1,148 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
+ */
+
+use crate::{
+    Recipient, SpamFilterContext, SpamFilterInput, SpamFilterOutput, SpamFilterResult, TextPart,
+};
+use common::{Server, config::mailstore::spamfilter::Location};
+use mail_parser::{Header, parsers::MessageStream};
+use std::{
+    borrow::Cow,
+    hash::{Hash, Hasher},
+};
+
+pub mod classifier;
+pub mod date;
+pub mod dmarc;
+pub mod domain;
+pub mod ehlo;
+pub mod from;
+pub mod headers;
+pub mod html;
+pub mod init;
+pub mod ip;
+pub mod messageid;
+pub mod mime;
+mod mime_types;
+pub mod pyzor;
+pub mod received;
+pub mod recipient;
+pub mod replyto;
+pub mod rules;
+pub mod score;
+pub mod subject;
+pub mod url;
+
+
+impl SpamFilterInput<'_> {
+    pub fn header_as_address(&self, header: &Header<'_>) -> Option<Cow<'_, str>> {
+        self.message
+            .raw_message()
+            .get(header.offset_start as usize..header.offset_end as usize)
+            .map(|bytes| MessageStream::new(bytes).parse_address())
+            .and_then(|addr| addr.into_address())
+            .and_then(|addr| addr.into_list().into_iter().next())
+            .and_then(|addr| addr.address)
+    }
+}
+
+impl SpamFilterOutput<'_> {
+    pub fn all_recipients(&self) -> impl Iterator<Item = &Recipient> {
+        self.recipients_to
+            .iter()
+            .chain(self.recipients_cc.iter())
+            .chain(self.recipients_bcc.iter())
+    }
+}
+
+impl SpamFilterContext<'_> {
+    pub fn text_body(&self) -> Option<&str> {
+        self.input
+            .message
+            .text_body
+            .first()
+            .or_else(|| self.input.message.html_body.first())
+            .and_then(|idx| self.output.text_parts.get(*idx as usize))
+            .and_then(|part| match part {
+                TextPart::Plain { text_body, .. } => Some(*text_body),
+                TextPart::Html { text_body, .. } => Some(text_body.as_str()),
+                TextPart::None => None,
+            })
+    }
+}
+
+impl SpamFilterResult {
+    pub fn add_tag(&mut self, tag: impl Into<String>) {
+        self.tags.insert(tag.into());
+    }
+
+    pub fn has_tag(&self, tag: impl AsRef<str>) -> bool {
+        self.tags.contains(tag.as_ref())
+    }
+}
+
+#[derive(Debug)]
+pub struct ElementLocation<T> {
+    pub element: T,
+    pub location: Location,
+}
+
+impl<T: Hash> Hash for ElementLocation<T> {
+    fn hash<H: Hasher>(&self, state: &mut H) {
+        self.element.hash(state);
+    }
+}
+
+impl<T: PartialEq> PartialEq for ElementLocation<T> {
+    fn eq(&self, other: &Self) -> bool {
+        self.element.eq(&other.element)
+    }
+}
+
+impl<T: Eq> Eq for ElementLocation<T> {}
+
+impl<T> ElementLocation<T> {
+    pub fn new(element: T, location: impl Into<Location>) -> Self {
+        Self {
+            element,
+            location: location.into(),
+        }
+    }
+}
+
+pub(crate) async fn is_trusted_domain(server: &Server, domain: &str, span_id: u64) -> bool {
+    if let Some(store) = server.get_lookup_store("trusted-domains") {
+        match store.key_exists(domain).await {
+            Ok(true) => return true,
+            Ok(false) => (),
+            Err(err) => {
+                trc::error!(err.span_id(span_id).caused_by(trc::location!()));
+            }
+        }
+    }
+
+    match server.domain(domain).await {
+        Ok(result) => result.is_some(),
+        Err(err) => {
+            trc::error!(err.span_id(span_id).caused_by(trc::location!()));
+            false
+        }
+    }
+}
+
+pub(crate) async fn is_url_redirector(server: &Server, url: &str, span_id: u64) -> bool {
+    if let Some(store) = server.get_lookup_store("url-redirectors") {
+        match store.key_exists(url).await {
+            Ok(result) => result,
+            Err(err) => {
+                trc::error!(err.span_id(span_id).caused_by(trc::location!()));
+                false
+            }
+        }
+    } else {
+        false
+    }
+}

crates/spam-filter/src/analysis/pyzor.rs

@@ -0,0 +1,55 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
+ */
+
+use std::{future::Future, time::Instant};
+
+use common::Server;
+
+use crate::{SpamFilterContext, modules::pyzor::pyzor_check};
+
+pub trait SpamFilterAnalyzePyzor: Sync + Send {
+    fn spam_filter_analyze_pyzor(
+        &self,
+        ctx: &mut SpamFilterContext<'_>,
+    ) -> impl Future<Output = ()> + Send;
+}
+
+impl SpamFilterAnalyzePyzor for Server {
+    async fn spam_filter_analyze_pyzor(&self, ctx: &mut SpamFilterContext<'_>) {
+        if let Some(config) = &self.core.spam.pyzor {
+            let time = Instant::now();
+            match pyzor_check(ctx.input.message, config).await {
+                Ok(Some(result)) => {
+                    let is_spam = result.code == 200
+                        && result.count > config.min_count
+                        && (result.wl_count < config.min_wl_count
+                            || (result.wl_count as f64 / result.count as f64) < config.ratio);
+                    if is_spam {
+                        ctx.result.add_tag("PYZOR");
+                    }
+                    trc::event!(
+                        Spam(trc::SpamEvent::Pyzor),
+                        Result = is_spam,
+                        Details = vec![
+                            trc::Value::from(result.code),
+                            trc::Value::from(result.count),
+                            trc::Value::from(result.wl_count)
+                        ],
+                        SpanId = ctx.input.span_id,
+                        Elapsed = time.elapsed()
+                    );
+                }
+                Ok(None) => {}
+                Err(err) => {
+                    trc::error!(
+                        err.span_id(ctx.input.span_id)
+                            .ctx(trc::Key::Elapsed, time.elapsed())
+                    );
+                }
+            }
+        }
+    }
+}

crates/spam-filter/src/analysis/received.rs

@@ -0,0 +1,133 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
+ */
+
+use std::future::Future;
+
+use common::Server;
+use mail_parser::{HeaderName, Host};
+use smtp_proto::MAIL_SMTPUTF8;
+
+use crate::{Email, SpamFilterContext};
+
+pub trait SpamFilterAnalyzeReceived: Sync + Send {
+    fn spam_filter_analyze_received(
+        &self,
+        ctx: &mut SpamFilterContext<'_>,
+    ) -> impl Future<Output = ()> + Send;
+}
+
+impl SpamFilterAnalyzeReceived for Server {
+    async fn spam_filter_analyze_received(&self, ctx: &mut SpamFilterContext<'_>) {
+        let mut rcvd_count = 0;
+        let mut rcvd_from_ip = 0;
+        let mut tls_count = 0;
+
+        let is_smtputf8 = (ctx.input.env_from_flags & MAIL_SMTPUTF8) != 0;
+
+        for header in ctx.input.message.headers() {
+            if let HeaderName::Received = &header.name {
+                if !is_smtputf8
+                    && !ctx
+                        .input
+                        .message
+                        .raw_message()
+                        .get(header.offset_start as usize..header.offset_end as usize)
+                        .unwrap_or_default()
+                        .is_ascii()
+                {
+                    // Received headers have non-ASCII characters
+                    ctx.result.add_tag("RCVD_ILLEGAL_CHARS");
+                }
+
+                if let Some(received) = header.value().as_received() {
+                    let helo_domain = received.from().or_else(|| received.helo());
+                    let ip_rev = received.from_iprev();
+
+                    if matches!(&helo_domain, Some(Host::Name(hostname)) if hostname.eq_ignore_ascii_case("user"))
+                    {
+                        // HELO domain is "user"
+                        ctx.result.add_tag("RCVD_HELO_USER");
+                    } else if let (Some(Host::Name(helo_domain)), Some(ip_rev)) =
+                        (helo_domain, ip_rev)
+                        && helo_domain.to_lowercase() != ip_rev.to_lowercase()
+                    {
+                        // HELO domain does not match PTR record
+                        ctx.result.add_tag("FORGED_RCVD_TRAIL");
+                    }
+
+                    if let Some(delivered_for) = received.for_().map(Email::new)
+                        && ctx
+                            .output
+                            .all_recipients()
+                            .any(|r| r.email == delivered_for)
+                    {
+                        // Recipient appears on Received trail
+                        ctx.result.add_tag("PREVIOUSLY_DELIVERED");
+                    }
+
+                    if matches!(received.from, Some(Host::IpAddr(_))) {
+                        // Received from an IP address rather than a FQDN
+                        rcvd_from_ip += 1;
+                    }
+
+                    if received.tls_version().is_some() {
+                        // Received with TLS
+                        tls_count += 1;
+                    }
+                } else {
+                    // Received header is not RFC 5322 compliant
+                    ctx.result.add_tag("RCVD_UNPARSABLE");
+                }
+
+                rcvd_count += 1;
+            }
+        }
+
+        if rcvd_from_ip >= 2 || (rcvd_from_ip == 1 && ctx.output.ehlo_host.ip.is_some()) {
+            // Has two or more Received headers containing bare IP addresses
+            ctx.result.add_tag("RCVD_DOUBLE_IP_SPAM");
+        }
+
+        // Received from an authenticated user
+        if ctx.input.authenticated_as.is_some() {
+            ctx.result.add_tag("RCVD_VIA_SMTP_AUTH");
+        }
+
+        // Received with TLS checks
+        if rcvd_count > 0 && rcvd_count == tls_count && ctx.input.is_tls {
+            ctx.result.add_tag("RCVD_TLS_ALL");
+        } else if ctx.input.is_tls {
+            ctx.result.add_tag("RCVD_TLS_LAST");
+        } else {
+            ctx.result.add_tag("RCVD_NO_TLS_LAST");
+        }
+
+        match rcvd_count {
+            0 => {
+                ctx.result.add_tag("RCVD_COUNT_ZERO");
+            }
+            1 => {
+                ctx.result.add_tag("RCVD_COUNT_ONE");
+            }
+            2 => {
+                ctx.result.add_tag("RCVD_COUNT_TWO");
+            }
+            3 => {
+                ctx.result.add_tag("RCVD_COUNT_THREE");
+            }
+            4 | 5 => {
+                ctx.result.add_tag("RCVD_COUNT_FIVE");
+            }
+            6 | 7 => {
+                ctx.result.add_tag("RCVD_COUNT_SEVEN");
+            }
+            8..=12 => {
+                ctx.result.add_tag("RCVD_COUNT_TWELVE");
+            }
+            _ => {}
+        }
+    }
+}

crates/spam-filter/src/analysis/recipient.rs

@@ -0,0 +1,268 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
+ */
+
+use std::future::Future;
+
+use common::{Server, scripts::functions::text::levenshtein_distance};
+use mail_parser::HeaderName;
+use smtp_proto::{MAIL_BODY_8BITMIME, MAIL_BODY_BINARYMIME, MAIL_SMTPUTF8};
+use store::ahash::HashSet;
+
+use crate::SpamFilterContext;
+
+pub trait SpamFilterAnalyzeRecipient: Sync + Send {
+    fn spam_filter_analyze_recipient(
+        &self,
+        ctx: &mut SpamFilterContext<'_>,
+    ) -> impl Future<Output = ()> + Send;
+}
+
+impl SpamFilterAnalyzeRecipient for Server {
+    async fn spam_filter_analyze_recipient(&self, ctx: &mut SpamFilterContext<'_>) {
+        let mut to_raw = b"".as_slice();
+        let mut cc_raw = b"".as_slice();
+        let mut bcc_raw = b"".as_slice();
+        let mut has_list_unsubscribe = false;
+        let mut has_list_id = false;
+
+        for header in ctx.input.message.headers() {
+            match &header.name {
+                HeaderName::To | HeaderName::Cc | HeaderName::Bcc => {
+                    let raw = ctx
+                        .input
+                        .message
+                        .raw_message()
+                        .get(header.offset_start as usize..header.offset_end as usize)
+                        .unwrap_or_default();
+                    match header.name {
+                        HeaderName::To => to_raw = raw,
+                        HeaderName::Cc => cc_raw = raw,
+                        HeaderName::Bcc => bcc_raw = raw,
+                        _ => unreachable!(),
+                    }
+                }
+                HeaderName::ListUnsubscribe => {
+                    has_list_unsubscribe = true;
+                }
+                HeaderName::ListId => {
+                    has_list_id = true;
+                }
+                _ => {}
+            }
+        }
+
+        if to_raw.is_empty() {
+            ctx.result.add_tag("MISSING_TO");
+        }
+
+        let to_raw_utf8 = std::str::from_utf8(to_raw);
+        let cc_raw_utf8 = std::str::from_utf8(cc_raw);
+        let bcc_raw_utf8 = std::str::from_utf8(bcc_raw);
+
+        for (raw, raw_utf8, recipients) in [
+            (to_raw, &to_raw_utf8, &ctx.output.recipients_to),
+            (cc_raw, &cc_raw_utf8, &ctx.output.recipients_cc),
+            (bcc_raw, &bcc_raw_utf8, &ctx.output.recipients_bcc),
+        ] {
+            if !raw.is_empty() {
+                // Validate non-ASCII characters in recipient headers
+                if !raw.is_ascii() {
+                    if (ctx.input.env_from_flags
+                        & (MAIL_SMTPUTF8 | MAIL_BODY_8BITMIME | MAIL_BODY_BINARYMIME))
+                        == 0
+                    {
+                        ctx.result.add_tag("TO_NEEDS_ENCODING");
+                    }
+
+                    if raw_utf8.is_err() {
+                        ctx.result.add_tag("INVALID_TO_8BIT");
+                    }
+                }
+
+                // Validate unnecessary encoding in recipient headers
+                let raw_utf8 = raw_utf8.unwrap_or_default();
+                if recipients.iter().all(|rcpt| {
+                    rcpt.name.as_ref().is_none_or(|name| name.is_ascii())
+                        && rcpt.email.address.is_ascii()
+                }) && raw_utf8.contains("=?")
+                    && raw_utf8.contains("?=")
+                {
+                    if raw_utf8.contains("?q?") || raw_utf8.contains("?Q?") {
+                        // To header is unnecessarily encoded in quoted-printable
+                        ctx.result.add_tag("TO_EXCESS_QP");
+                    } else if raw_utf8.contains("?b?") || raw_utf8.contains("?B?") {
+                        // To header is unnecessarily encoded in base64
+                        ctx.result.add_tag("TO_EXCESS_BASE64");
+                    }
+                }
+
+                // Check for spaces in recipient addresses
+                for token in raw_utf8.split('<') {
+                    if let Some((addr, _)) = token.split_once('>')
+                        && (addr.starts_with(' ') || addr.ends_with(' '))
+                    {
+                        ctx.result.add_tag("TO_WRAPPED_IN_SPACES");
+                        break;
+                    }
+                }
+            }
+        }
+
+        let unique_recipients = ctx
+            .output
+            .all_recipients()
+            .filter(|rcpt| !rcpt.email.address.is_empty())
+            .collect::<HashSet<_>>();
+        let rcpt_count = unique_recipients.len();
+
+        match unique_recipients.len() {
+            0 => {
+                ctx.result.add_tag("RCPT_COUNT_ZERO");
+                return;
+            }
+            1 => {
+                ctx.result.add_tag("RCPT_COUNT_ONE");
+            }
+            2 => {
+                ctx.result.add_tag("RCPT_COUNT_TWO");
+            }
+            3 => {
+                ctx.result.add_tag("RCPT_COUNT_THREE");
+            }
+            4 | 5 => {
+                ctx.result.add_tag("RCPT_COUNT_FIVE");
+            }
+            6 | 7 => {
+                ctx.result.add_tag("RCPT_COUNT_SEVEN");
+            }
+            8..=12 => {
+                ctx.result.add_tag("RCPT_COUNT_TWELVE");
+            }
+            13.. => {
+                ctx.result.add_tag("RCPT_COUNT_GT_50");
+            }
+        }
+
+        let mut to_dn_eq_addr_count = 0;
+        let mut to_dn_count = 0;
+        let mut to_match_envrcpt = 0;
+
+        for rcpt in &unique_recipients {
+            // Validate name
+            if let Some(rcpt_name) = &rcpt.name {
+                if *rcpt_name == rcpt.email.address {
+                    to_dn_eq_addr_count += 1;
+                } else {
+                    to_dn_count += 1;
+                }
+            }
+
+            // Recipient is present in envelope
+            if ctx.output.env_to_orig_addr.contains(&rcpt.email) {
+                to_match_envrcpt += 1;
+            }
+
+            // Check if the local part is present in the subject
+            if !rcpt.email.local_part.is_empty() {
+                if ctx.output.subject_lc.contains(rcpt.email.address.as_str()) {
+                    ctx.result.add_tag("RCPT_IN_SUBJECT");
+                } else if rcpt.email.local_part.len() > 3
+                    && ctx
+                        .output
+                        .subject_lc
+                        .contains(rcpt.email.local_part.as_str())
+                {
+                    ctx.result.add_tag("RCPT_LOCAL_IN_SUBJECT");
+                }
+            }
+        }
+
+        if to_dn_count == 0 && to_dn_eq_addr_count == 0 {
+            ctx.result.add_tag("TO_DN_NONE");
+        } else if to_dn_count == rcpt_count {
+            ctx.result.add_tag("TO_DN_ALL");
+        } else if to_dn_count > 0 {
+            ctx.result.add_tag("TO_DN_SOME");
+        }
+
+        if to_dn_eq_addr_count == rcpt_count {
+            ctx.result.add_tag("TO_DN_EQ_ADDR_ALL");
+        } else if to_dn_eq_addr_count > 0 {
+            ctx.result.add_tag("TO_DN_EQ_ADDR_SOME");
+        }
+
+        if to_match_envrcpt == rcpt_count {
+            ctx.result.add_tag("TO_MATCH_ENVRCPT_ALL");
+        } else {
+            if to_match_envrcpt > 0 {
+                ctx.result.add_tag("TO_MATCH_ENVRCPT_SOME");
+            }
+
+            if !has_list_id && !has_list_unsubscribe {
+                for env_rcpt in &ctx.output.env_to_orig_addr {
+                    if !unique_recipients.iter().any(|rcpt| rcpt.email == *env_rcpt)
+                        && env_rcpt != &ctx.output.env_from_addr
+                    {
+                        ctx.result.add_tag("FORGED_RECIPIENTS");
+                        break;
+                    }
+                }
+            }
+        }
+
+        // Message from bounce and over 1 recipient
+        if rcpt_count > 1 && ctx.output.env_from_postmaster {
+            ctx.result.add_tag("RCPT_BOUNCEMOREONE");
+        }
+
+        let rcpts = ctx
+            .output
+            .recipients_to
+            .iter()
+            .chain(ctx.output.recipients_cc.iter())
+            .collect::<Vec<_>>();
+
+        let mut is_sorted = false;
+        if rcpts.len() >= 6 {
+            // Check if the recipients list is sorted
+            let mut sorted = true;
+            for i in 1..rcpts.len() {
+                if rcpts[i - 1].email.address > rcpts[i].email.address {
+                    sorted = false;
+                    break;
+                }
+            }
+            if sorted {
+                ctx.result.add_tag("SORTED_RECIPS");
+                is_sorted = true;
+            }
+        }
+
+        if !is_sorted && rcpt_count >= 5 {
+            // Look for similar recipients
+            let mut hits = 0;
+            let mut combinations = 0;
+            for i in 0..rcpts.len() {
+                for j in i + 1..rcpts.len() {
+                    let a = &rcpts[i].email;
+                    let b = &rcpts[j].email;
+
+                    if levenshtein_distance(&a.local_part, &b.local_part) < 3
+                        || (a.domain_part.fqdn != b.domain_part.fqdn
+                            && levenshtein_distance(&a.domain_part.fqdn, &b.domain_part.fqdn) < 4)
+                    {
+                        hits += 1;
+                    }
+                    combinations += 1;
+                }
+            }
+
+            if hits as f64 / combinations as f64 > 0.65 {
+                ctx.result.add_tag("SUSPICIOUS_RECIPS");
+            }
+        }
+    }
+}

crates/spam-filter/src/analysis/replyto.rs

@@ -0,0 +1,124 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
+ */
+
+use std::future::Future;
+
+use common::Server;
+use mail_parser::HeaderName;
+
+use crate::SpamFilterContext;
+
+pub trait SpamFilterAnalyzeReplyTo: Sync + Send {
+    fn spam_filter_analyze_reply_to(
+        &self,
+        ctx: &mut SpamFilterContext<'_>,
+    ) -> impl Future<Output = ()> + Send;
+}
+
+impl SpamFilterAnalyzeReplyTo for Server {
+    async fn spam_filter_analyze_reply_to(&self, ctx: &mut SpamFilterContext<'_>) {
+        let mut reply_to_raw = b"".as_slice();
+        let mut is_from_list = false;
+
+        for header in ctx.input.message.headers() {
+            match &header.name {
+                HeaderName::ReplyTo => {
+                    reply_to_raw = ctx
+                        .input
+                        .message
+                        .raw_message()
+                        .get(header.offset_start as usize..header.offset_end as usize)
+                        .unwrap_or_default();
+                }
+                HeaderName::ListUnsubscribe | HeaderName::ListId => {
+                    is_from_list = true;
+                }
+
+                HeaderName::AutoSubmitted => {
+                    is_from_list = true;
+                }
+                HeaderName::Other(name) if !is_from_list => {
+                    is_from_list = name.eq_ignore_ascii_case("X-To-Get-Off-This-List")
+                        || name.eq_ignore_ascii_case("X-List");
+                }
+                _ => {}
+            }
+        }
+
+        if reply_to_raw.is_empty() {
+            return;
+        }
+
+        if let Some(reply_to) = &ctx.output.reply_to {
+            let reply_to_name = reply_to.name.as_deref().unwrap_or_default();
+            ctx.result.add_tag("HAS_REPLYTO");
+
+            if reply_to.email == ctx.output.from.email {
+                ctx.result.add_tag("REPLYTO_EQ_FROM");
+            } else {
+                if reply_to.email.domain_part.sld == ctx.output.from.email.domain_part.sld {
+                    ctx.result.add_tag("REPLYTO_DOM_EQ_FROM_DOM");
+                } else {
+                    if !is_from_list
+                        && ctx
+                            .output
+                            .all_recipients()
+                            .any(|r| r.email == reply_to.email)
+                    {
+                        ctx.result.add_tag("REPLYTO_EQ_TO_ADDR");
+                    } else {
+                        ctx.result.add_tag("REPLYTO_DOM_NEQ_FROM_DOM");
+                    }
+
+                    if !(is_from_list
+                        || ctx
+                            .output
+                            .recipients_to
+                            .iter()
+                            .any(|r| r.email == ctx.output.from.email)
+                        || ctx
+                            .output
+                            .env_to_orig_addr
+                            .iter()
+                            .any(|r| r.domain_part.sld == ctx.output.from.email.domain_part.sld)
+                        || ctx.output.env_to_orig_addr.len() == 1
+                            && ctx.output.env_to_orig_addr.contains(&ctx.output.from.email))
+                    {
+                        ctx.result.add_tag("SPOOF_REPLYTO");
+                    }
+                }
+
+                if !reply_to_name.is_empty()
+                    && reply_to_name == ctx.output.from.name.as_deref().unwrap_or_default()
+                {
+                    ctx.result.add_tag("REPLYTO_DN_EQ_FROM_DN");
+                }
+            }
+
+            if reply_to.email == ctx.output.env_from_addr {
+                ctx.result.add_tag("REPLYTO_ADDR_EQ_FROM");
+            }
+
+            // Validate unnecessary encoding
+            let reply_to_raw_utf8 = std::str::from_utf8(reply_to_raw).unwrap_or_default();
+            if reply_to.email.address.is_ascii()
+                && reply_to_name.is_ascii()
+                && reply_to_raw_utf8.contains("=?")
+                && reply_to_raw_utf8.contains("?=")
+            {
+                if reply_to_raw_utf8.contains("?q?") || reply_to_raw_utf8.contains("?Q?") {
+                    // Reply-To header is unnecessarily encoded in quoted-printable
+                    ctx.result.add_tag("REPLYTO_EXCESS_QP");
+                } else if reply_to_raw_utf8.contains("?b?") || reply_to_raw_utf8.contains("?B?") {
+                    // Reply-To header is unnecessarily encoded in base64
+                    ctx.result.add_tag("REPLYTO_EXCESS_BASE64");
+                }
+            }
+        } else {
+            ctx.result.add_tag("REPLYTO_UNPARSABLE");
+        }
+    }
+}

crates/spam-filter/src/analysis/rules.rs

@@ -0,0 +1,198 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
+ */
+
+use std::future::Future;
+
+use common::{
+    Server,
+    config::mailstore::spamfilter::{IpResolver, Location},
+};
+
+use crate::{
+    SpamFilterContext, TextPart,
+    modules::expression::{EmailHeader, SpamFilterResolver, StringResolver},
+};
+
+pub trait SpamFilterAnalyzeRules: Sync + Send {
+    fn spam_filter_analyze_rules(
+        &self,
+        ctx: &mut SpamFilterContext<'_>,
+    ) -> impl Future<Output = ()> + Send;
+}
+
+impl SpamFilterAnalyzeRules for Server {
+    async fn spam_filter_analyze_rules(&self, ctx: &mut SpamFilterContext<'_>) {
+        if !self.core.spam.rules.url.is_empty() {
+            for url in &ctx.output.urls {
+                for rule in &self.core.spam.rules.url {
+                    if let Some(tag) = self
+                        .eval_if::<String, _>(
+                            rule,
+                            &SpamFilterResolver::new(ctx, &url.element, url.location),
+                            ctx.input.span_id,
+                        )
+                        .await
+                    {
+                        ctx.result.tags.insert(tag);
+                    }
+                }
+            }
+        }
+
+        if !self.core.spam.rules.domain.is_empty() {
+            for domain in &ctx.output.domains {
+                let resolver = StringResolver(domain.element.as_str());
+
+                for rule in &self.core.spam.rules.domain {
+                    if let Some(tag) = self
+                        .eval_if::<String, _>(
+                            rule,
+                            &SpamFilterResolver::new(ctx, &resolver, domain.location),
+                            ctx.input.span_id,
+                        )
+                        .await
+                    {
+                        ctx.result.tags.insert(tag);
+                    }
+                }
+            }
+        }
+
+        if !self.core.spam.rules.email.is_empty() {
+            for email in &ctx.output.emails {
+                for rule in &self.core.spam.rules.email {
+                    if let Some(tag) = self
+                        .eval_if::<String, _>(
+                            rule,
+                            &SpamFilterResolver::new(ctx, &email.element, email.location),
+                            ctx.input.span_id,
+                        )
+                        .await
+                    {
+                        ctx.result.tags.insert(tag);
+                    }
+                }
+            }
+
+            for (rcpt, location) in [
+                (&ctx.output.recipients_to, Location::HeaderTo),
+                (&ctx.output.recipients_cc, Location::HeaderCc),
+                (&ctx.output.recipients_bcc, Location::HeaderBcc),
+            ] {
+                for email in rcpt {
+                    for rule in &self.core.spam.rules.email {
+                        if let Some(tag) = self
+                            .eval_if::<String, _>(
+                                rule,
+                                &SpamFilterResolver::new(ctx, email, location),
+                                ctx.input.span_id,
+                            )
+                            .await
+                        {
+                            ctx.result.tags.insert(tag);
+                        }
+                    }
+                }
+            }
+        }
+
+        if !self.core.spam.rules.ip.is_empty() {
+            for ip in &ctx.output.ips {
+                let ip_resolver = IpResolver::new(ip.element);
+
+                for rule in &self.core.spam.rules.ip {
+                    if let Some(tag) = self
+                        .eval_if::<String, _>(
+                            rule,
+                            &SpamFilterResolver::new(ctx, &ip_resolver, ip.location),
+                            ctx.input.span_id,
+                        )
+                        .await
+                    {
+                        ctx.result.tags.insert(tag);
+                    }
+                }
+            }
+        }
+
+        if !self.core.spam.rules.header.is_empty() {
+            for header in ctx.input.message.headers() {
+                let raw = String::from_utf8_lossy(
+                    ctx.input
+                        .message
+                        .raw_message()
+                        .get(header.offset_start as usize..header.offset_end as usize)
+                        .unwrap_or_default(),
+                );
+                let header_resolver = EmailHeader {
+                    header,
+                    raw: raw.as_ref(),
+                };
+
+                for rule in &self.core.spam.rules.header {
+                    if let Some(tag) = self
+                        .eval_if::<String, _>(
+                            rule,
+                            &SpamFilterResolver::new(ctx, &header_resolver, Location::BodyText),
+                            ctx.input.span_id,
+                        )
+                        .await
+                    {
+                        ctx.result.tags.insert(tag);
+                    }
+                }
+            }
+        }
+
+        if !self.core.spam.rules.body.is_empty() {
+            for (idx, part) in ctx.output.text_parts.iter().enumerate() {
+                let text = match part {
+                    TextPart::Plain { text_body, .. } => *text_body,
+                    TextPart::Html { text_body, .. } => text_body.as_str(),
+                    TextPart::None => continue,
+                };
+                let idx = idx as u32;
+                let location = if ctx.input.message.text_body.contains(&idx) {
+                    Location::BodyText
+                } else if ctx.input.message.html_body.contains(&idx) {
+                    Location::BodyHtml
+                } else {
+                    Location::Attachment
+                };
+                let string_resolver = StringResolver(text);
+
+                for rule in &self.core.spam.rules.body {
+                    if let Some(tag) = self
+                        .eval_if::<String, _>(
+                            rule,
+                            &SpamFilterResolver::new(ctx, &string_resolver, location),
+                            ctx.input.span_id,
+                        )
+                        .await
+                    {
+                        ctx.result.tags.insert(tag);
+                    }
+                }
+            }
+        }
+
+        if !self.core.spam.rules.any.is_empty() {
+            let dummy_resolver = StringResolver("");
+            for rule in &self.core.spam.rules.any {
+                if let Some(tag) = self
+                    .eval_if::<String, _>(
+                        rule,
+                        &SpamFilterResolver::new(ctx, &dummy_resolver, Location::BodyText),
+                        ctx.input.span_id,
+                    )
+                    .await
+                {
+                    ctx.result.tags.insert(tag);
+                }
+            }
+        }
+    }
+}

crates/spam-filter/src/analysis/score.rs

@@ -0,0 +1,272 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
+ */
+
+use crate::{
+    SpamFilterContext,
+    analysis::{
+        classifier::SpamFilterAnalyzeClassify, date::SpamFilterAnalyzeDate,
+        dmarc::SpamFilterAnalyzeDmarc, domain::SpamFilterAnalyzeDomain,
+        ehlo::SpamFilterAnalyzeEhlo, from::SpamFilterAnalyzeFrom,
+        headers::SpamFilterAnalyzeHeaders, html::SpamFilterAnalyzeHtml, ip::SpamFilterAnalyzeIp,
+        messageid::SpamFilterAnalyzeMid, mime::SpamFilterAnalyzeMime,
+        pyzor::SpamFilterAnalyzePyzor, received::SpamFilterAnalyzeReceived,
+        recipient::SpamFilterAnalyzeRecipient, replyto::SpamFilterAnalyzeReplyTo,
+        rules::SpamFilterAnalyzeRules, subject::SpamFilterAnalyzeSubject,
+        url::SpamFilterAnalyzeUrl,
+    },
+};
+use common::{Server, config::mailstore::spamfilter::SpamFilterAction};
+use std::{fmt::Write, future::Future, vec};
+
+
+pub trait SpamFilterAnalyzeScore: Sync + Send {
+    fn spam_filter_finalize(
+        &self,
+        ctx: &mut SpamFilterContext<'_>,
+    ) -> impl Future<Output = SpamFilterAction<SpamFilterScore>> + Send;
+
+    fn spam_filter_classify(
+        &self,
+        ctx: &mut SpamFilterContext<'_>,
+    ) -> impl Future<Output = SpamFilterAction<SpamFilterScore>> + Send;
+}
+
+#[derive(Debug, Default)]
+pub struct SpamFilterScore {
+    pub results: Vec<f32>,
+    pub headers: String,
+    pub train_spam: Option<bool>,
+    pub score: f32,
+    pub is_spam: bool,
+}
+
+impl SpamFilterAnalyzeScore for Server {
+    async fn spam_filter_finalize(
+        &self,
+        ctx: &mut SpamFilterContext<'_>,
+    ) -> SpamFilterAction<SpamFilterScore> {
+        // Calculate final score
+        let mut results = vec![];
+        let mut header_len = 60;
+        let mut is_spam_trap = false;
+        let mut rbl_count = 0;
+
+        for tag in &ctx.result.tags {
+            let score = match self.core.spam.lists.scores.get(tag) {
+                Some(SpamFilterAction::Allow(score)) => *score,
+                Some(SpamFilterAction::Discard) => {
+                    return SpamFilterAction::Discard;
+                }
+                Some(SpamFilterAction::Reject) => {
+                    return SpamFilterAction::Reject;
+                }
+                None | Some(SpamFilterAction::Disabled) => 0.0,
+            };
+            if tag == "SPAM_TRAP" {
+                is_spam_trap = true;
+            } else if score > 1.0 && tag.starts_with("RBL_") {
+                rbl_count += 1;
+            }
+            ctx.result.score += score;
+            header_len += tag.len() + 10;
+            if score != 0.0 || !tag.starts_with("X_") {
+                results.push((tag.as_str(), score));
+            }
+        }
+
+        let mut final_score = ctx.result.score;
+        let mut avg_confidence: f32 = 0.0;
+        let mut total_results = 0;
+        let mut user_results = vec![ctx.result.score; ctx.input.env_rcpt_rewritten_to.len()];
+        if !ctx.result.classifier_confidence.is_empty() {
+            for (idx, &confidence) in ctx.result.classifier_confidence.iter().enumerate() {
+                if let Some(confidence) = confidence {
+                    avg_confidence += confidence;
+                    total_results += 1;
+
+                    let user_score = self
+                        .core
+                        .spam
+                        .lists
+                        .scores
+                        .get(confidence.spam_tag())
+                        .and_then(|v| v.as_score())
+                        .copied()
+                        .unwrap_or_default();
+
+                    user_results[idx] = ctx.result.score + user_score;
+                }
+            }
+
+            if total_results > 0 {
+                avg_confidence /= total_results as f32;
+
+                let tag = avg_confidence.spam_tag();
+                let score = self
+                    .core
+                    .spam
+                    .lists
+                    .scores
+                    .get(tag)
+                    .and_then(|v| v.as_score())
+                    .copied()
+                    .unwrap_or_default();
+                results.push((tag, score));
+                final_score += score;
+            }
+        }
+
+        if self.core.spam.scores.reject_threshold > 0.0
+            && final_score >= self.core.spam.scores.reject_threshold
+        {
+            SpamFilterAction::Reject
+        } else if self.core.spam.scores.discard_threshold > 0.0
+            && final_score >= self.core.spam.scores.discard_threshold
+        {
+            SpamFilterAction::Discard
+        } else {
+            let mut headers = String::with_capacity(header_len + 40);
+            results.sort_by(|a, b| a.1.partial_cmp(&b.1).unwrap().then_with(|| a.0.cmp(b.0)));
+            headers.push_str("X-Spam-Result: ");
+            for (idx, (tag, score)) in results.into_iter().enumerate() {
+                if idx > 0 {
+                    headers.push_str(",\r\n\t");
+                }
+                let _ = write!(&mut headers, "{} ({:.2})", tag, score);
+            }
+            headers.push_str("\r\n");
+
+            if let Some((category, explanation)) = &ctx.result.llm_result {
+                let _ = write!(&mut headers, "X-Spam-LLM: {category} ({explanation})\r\n",);
+            }
+
+            let is_spam = final_score >= self.core.spam.scores.spam_threshold;
+            let class = if is_spam { "spam" } else { "ham" };
+
+            if avg_confidence != 0.0 {
+                let _ = write!(
+                    &mut headers,
+                    "X-Spam-Score: {class}, score={final_score:.2}, avg_confidence={avg_confidence:.2}\r\n",
+                );
+            } else {
+                let _ = write!(
+                    &mut headers,
+                    "X-Spam-Score: {class}, score={final_score:.2}\r\n",
+                );
+            }
+
+            // Autolearn SPAM
+            let mut train_spam = None;
+            if is_spam
+                && self.core.spam.classifier.as_ref().is_some_and(|c| {
+                    (c.auto_learn_spam_trap && is_spam_trap)
+                        || (c.auto_learn_spam_rbl_count > 0
+                            && rbl_count >= c.auto_learn_spam_rbl_count)
+                })
+            {
+                train_spam = Some(true);
+            }
+
+            SpamFilterAction::Allow(SpamFilterScore {
+                results: user_results,
+                headers,
+                train_spam,
+                score: final_score,
+                is_spam,
+            })
+        }
+    }
+
+    async fn spam_filter_classify(
+        &self,
+        ctx: &mut SpamFilterContext<'_>,
+    ) -> SpamFilterAction<SpamFilterScore> {
+        // IP address analysis
+        self.spam_filter_analyze_ip(ctx).await;
+
+        // DMARC/SPF/DKIM/ARC analysis
+        self.spam_filter_analyze_dmarc(ctx).await;
+
+        // EHLO hostname analysis
+        self.spam_filter_analyze_ehlo(ctx).await;
+
+        // Generic header analysis
+        self.spam_filter_analyze_headers(ctx).await;
+
+        // Received headers analysis
+        self.spam_filter_analyze_received(ctx).await;
+
+        // Message-ID analysis
+        self.spam_filter_analyze_message_id(ctx).await;
+
+        // Date header analysis
+        self.spam_filter_analyze_date(ctx).await;
+
+        // Subject analysis
+        self.spam_filter_analyze_subject(ctx).await;
+
+        // From and Envelope From analysis
+        self.spam_filter_analyze_from(ctx).await;
+
+        // Reply-To analysis
+        self.spam_filter_analyze_reply_to(ctx).await;
+
+        // Recipient analysis
+        self.spam_filter_analyze_recipient(ctx).await;
+
+        // E-mail and domain analysis
+        self.spam_filter_analyze_domain(ctx).await;
+
+        // URL analysis
+        self.spam_filter_analyze_url(ctx).await;
+
+        // MIME part analysis
+        self.spam_filter_analyze_mime(ctx).await;
+
+        // HTML content analysis
+        self.spam_filter_analyze_html(ctx).await;
+
+
+        // Spam trap
+        self.spam_filter_analyze_spam_trap(ctx).await;
+
+        // Pyzor checks
+        self.spam_filter_analyze_pyzor(ctx).await;
+
+        // Model classification
+        self.spam_filter_analyze_classify(ctx).await;
+
+        // User-defined rules
+        self.spam_filter_analyze_rules(ctx).await;
+
+        // Final score calculation
+        self.spam_filter_finalize(ctx).await
+    }
+}
+
+pub trait ConfidenceStore {
+    fn spam_tag(&self) -> &'static str;
+}
+
+impl ConfidenceStore for f32 {
+    fn spam_tag(&self) -> &'static str {
+        match *self {
+            p if p < 0.15 => "PROB_HAM_HIGH",
+            p if p < 0.25 => "PROB_HAM_MEDIUM",
+            p if p < 0.40 => "PROB_HAM_LOW",
+            p if p < 0.60 => "PROB_SPAM_UNCERTAIN",
+            p if p < 0.75 => "PROB_SPAM_LOW",
+            p if p < 0.85 => "PROB_SPAM_MEDIUM",
+            p => {
+                if p.is_finite() {
+                    "PROB_SPAM_HIGH"
+                } else {
+                    "PROB_SPAM_UNCERTAIN"
+                }
+            }
+        }
+    }
+}

crates/spam-filter/src/analysis/subject.rs

@@ -0,0 +1,179 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
+ */
+
+use std::future::Future;
+
+use common::Server;
+use mail_parser::HeaderName;
+use nlp::tokenizers::types::TokenType;
+use smtp_proto::{MAIL_BODY_8BITMIME, MAIL_BODY_BINARYMIME, MAIL_SMTPUTF8};
+
+use crate::SpamFilterContext;
+
+pub trait SpamFilterAnalyzeSubject: Sync + Send {
+    fn spam_filter_analyze_subject(
+        &self,
+        ctx: &mut SpamFilterContext<'_>,
+    ) -> impl Future<Output = ()> + Send;
+}
+
+impl SpamFilterAnalyzeSubject for Server {
+    async fn spam_filter_analyze_subject(&self, ctx: &mut SpamFilterContext<'_>) {
+        let mut subject_raw = b"".as_slice();
+
+        for header in ctx.input.message.headers() {
+            if header.name == HeaderName::Subject {
+                subject_raw = ctx
+                    .input
+                    .message
+                    .raw_message()
+                    .get(header.offset_start as usize..header.offset_end as usize)
+                    .unwrap_or_default();
+                break;
+            }
+        }
+
+        if subject_raw.is_empty() {
+            // Missing subject header
+            ctx.result.add_tag("MISSING_SUBJECT");
+            return;
+        }
+
+        let mut word_count = 0;
+        let mut upper_count = 0;
+        let mut lower_count = 0;
+
+        let mut last_ch = ' ';
+        let mut is_ascii = true;
+
+        for ch in ctx.output.subject_thread.chars() {
+            if !ch.is_whitespace() {
+                if last_ch.is_whitespace() {
+                    word_count += 1;
+                }
+
+                match ch {
+                    '$' | '€' | '£' | '¥' | '₹' | '₽' | '₿' => {
+                        ctx.result.add_tag("SUBJECT_HAS_CURRENCY");
+                    }
+                    _ => {
+                        if ch.is_alphabetic() {
+                            if ch.is_uppercase() {
+                                upper_count += 1;
+                            } else {
+                                lower_count += 1;
+                            }
+                        }
+                    }
+                }
+            }
+
+            if !ch.is_ascii() {
+                is_ascii = false;
+            }
+
+            last_ch = ch;
+        }
+
+        if ctx.output.subject_lc.is_empty() {
+            // Subject is empty
+            ctx.result.add_tag("EMPTY_SUBJECT");
+        } else if ctx.output.subject.ends_with(' ') {
+            // Subject ends with whitespace
+            ctx.result.add_tag("SUBJECT_ENDS_SPACES");
+        } else if ctx.output.subject
+            == "XJS*C4JDBQADN1.NSBN3*2IDNEN*GTUBE-STANDARD-ANTI-UBE-TEST-EMAIL*C.34X"
+        {
+            ctx.result.add_tag("GTUBE_TEST");
+        }
+
+        if ctx.output.subject_thread.len() >= 10
+            && word_count > 1
+            && upper_count > 2
+            && lower_count == 0
+        {
+            // Subject contains mostly capital letters
+            ctx.result.add_tag("SUBJ_ALL_CAPS");
+        }
+
+        for token in &ctx.output.subject_tokens {
+            match token {
+                TokenType::Url(url) => {
+                    // Subject contains URL
+                    ctx.result.add_tag("URL_IN_SUBJECT");
+
+                    if let Some(url_parsed) = &url.url_parsed {
+                        let host = url_parsed.host.sld_or_default();
+                        for rcpt in ctx.output.all_recipients() {
+                            if rcpt.email.domain_part.sld_or_default() == host {
+                                ctx.result.add_tag("RCPT_DOMAIN_IN_SUBJECT");
+                                break;
+                            }
+                        }
+                    }
+                }
+                TokenType::UrlNoScheme(url) => {
+                    if let Some(url_parsed) = &url.url_parsed {
+                        let host = url_parsed.host.sld_or_default();
+                        for rcpt in ctx.output.all_recipients() {
+                            if rcpt.email.domain_part.sld_or_default() == host {
+                                ctx.result.add_tag("RCPT_DOMAIN_IN_SUBJECT");
+                                break;
+                            }
+                        }
+                    }
+                }
+                TokenType::Email(email) => {
+                    // Subject contains recipient
+                    if ctx.output.env_to_orig_addr.contains(email)
+                        || ctx.output.all_recipients().any(|r| &r.email == email)
+                    {
+                        ctx.result.add_tag("RCPT_IN_SUBJECT");
+                    } else {
+                        let host = email.domain_part.sld_or_default();
+                        for rcpt in ctx.output.all_recipients() {
+                            if &rcpt.email == email {
+                                ctx.result.add_tag("RCPT_IN_SUBJECT");
+                                break;
+                            } else if rcpt.email.domain_part.sld_or_default() == host {
+                                ctx.result.add_tag("RCPT_DOMAIN_IN_SUBJECT");
+                                break;
+                            }
+                        }
+                    }
+                }
+                _ => {}
+            }
+        }
+
+        // Validate encoding
+        let subject_raw_utf8 = std::str::from_utf8(subject_raw);
+        if !subject_raw.is_ascii() {
+            if (ctx.input.env_from_flags
+                & (MAIL_SMTPUTF8 | MAIL_BODY_8BITMIME | MAIL_BODY_BINARYMIME))
+                == 0
+            {
+                ctx.result.add_tag("SUBJECT_NEEDS_ENCODING");
+            }
+
+            if subject_raw_utf8.is_err() {
+                ctx.result.add_tag("INVALID_SUBJECT_8BIT");
+            }
+        }
+
+        // Validate unnecessary encoding
+        let subject_raw_utf8 = subject_raw_utf8.unwrap_or_default();
+        if is_ascii && subject_raw_utf8.contains("=?") && subject_raw_utf8.contains("?=") {
+            if subject_raw_utf8.contains("?q?") || subject_raw_utf8.contains("?Q?") {
+                // Subject header is unnecessarily encoded in quoted-printable
+                ctx.result.add_tag("SUBJ_EXCESS_QP");
+            } else if subject_raw_utf8.contains("?b?") || subject_raw_utf8.contains("?B?") {
+                // Subject header is unnecessarily encoded in base64
+                ctx.result.add_tag("SUBJ_EXCESS_BASE64");
+            }
+        }
+    }
+}

crates/spam-filter/src/analysis/url.rs

@@ -0,0 +1,507 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
+ */
+
+use super::{ElementLocation, is_trusted_domain, is_url_redirector};
+use crate::modules::dnsbl::check_dnsbl;
+use crate::modules::expression::StringResolver;
+use crate::modules::html::SRC;
+use crate::{
+    Hostname, SpamFilterContext, TextPart,
+    modules::html::{A, HREF, HtmlToken},
+};
+use common::Server;
+use common::config::mailstore::spamfilter::{Element, IpResolver, Location};
+use common::scripts::IsMixedCharset;
+use common::scripts::functions::unicode::CharUtils;
+use hyper::{Uri, header::LOCATION};
+use nlp::tokenizers::types::TokenType;
+use std::collections::HashSet;
+use std::hash::{Hash, Hasher};
+use std::{borrow::Cow, future::Future, time::Duration};
+
+const HTTPS_SCHEME: &str = "https://";
+
+pub trait SpamFilterAnalyzeUrl: Sync + Send {
+    fn spam_filter_analyze_url(
+        &self,
+        ctx: &mut SpamFilterContext<'_>,
+    ) -> impl Future<Output = ()> + Send;
+}
+
+#[derive(Clone, Debug)]
+pub struct UrlParts<'x> {
+    pub url: String,
+    pub url_original: Cow<'x, str>,
+    pub url_parsed: Option<UrlParsed>,
+    pub has_scheme: bool,
+}
+
+#[derive(Clone, Debug)]
+pub struct UrlParsed {
+    pub parts: Uri,
+    pub host: Hostname,
+}
+
+impl SpamFilterAnalyzeUrl for Server {
+    async fn spam_filter_analyze_url(&self, ctx: &mut SpamFilterContext<'_>) {
+        // Extract URLs
+        let mut urls: HashSet<ElementLocation<UrlParts<'static>>> = HashSet::new();
+        let mut inferred_urls: HashSet<ElementLocation<UrlParts<'static>>> = HashSet::new();
+        for token in &ctx.output.subject_tokens {
+            if let TokenType::Url(url) | TokenType::UrlNoScheme(url) = token {
+                collect_url(
+                    &mut urls,
+                    &mut inferred_urls,
+                    url.to_owned(),
+                    Location::HeaderSubject,
+                );
+            }
+        }
+        for (part_id, part) in ctx.output.text_parts.iter().enumerate() {
+            let part_id = part_id as u32;
+            let is_body = ctx.input.message.text_body.contains(&part_id)
+                || ctx.input.message.html_body.contains(&part_id);
+            let text_location = if is_body {
+                Location::BodyText
+            } else {
+                Location::Attachment
+            };
+
+            let tokens = match part {
+                TextPart::Plain { tokens, .. } => tokens,
+                TextPart::Html {
+                    html_tokens,
+                    tokens,
+                    ..
+                } => {
+                    for token in html_tokens {
+                        if let HtmlToken::StartTag { attributes, .. } = token {
+                            for (attr, value) in attributes {
+                                match value {
+                                    Some(value) if [HREF, SRC].contains(attr) => {
+                                        collect_url(
+                                            &mut urls,
+                                            &mut inferred_urls,
+                                            UrlParts::new(value.trim().to_string()),
+                                            if is_body {
+                                                Location::BodyHtml
+                                            } else {
+                                                Location::Attachment
+                                            },
+                                        );
+                                    }
+                                    _ => {}
+                                }
+                            }
+                        }
+                    }
+                    tokens
+                }
+                TextPart::None => &[][..],
+            };
+
+            for token in tokens {
+                match token {
+                    TokenType::Url(url) | TokenType::UrlNoScheme(url) => {
+                        if !ctx.input.is_train
+                            && is_body
+                            && !ctx.result.has_tag("RCPT_DOMAIN_IN_BODY")
+                            && let Some(url_parsed) = &url.url_parsed
+                        {
+                            let host = url_parsed.host.sld_or_default();
+                            for rcpt in ctx.output.all_recipients() {
+                                if rcpt.email.domain_part.sld_or_default() == host {
+                                    ctx.result.add_tag("RCPT_DOMAIN_IN_BODY");
+                                    break;
+                                }
+                            }
+                        }
+
+                        collect_url(&mut urls, &mut inferred_urls, url.to_owned(), text_location);
+                    }
+                    _ => {}
+                }
+            }
+
+            if is_body && !ctx.input.is_train {
+                let is_single = match part {
+                    TextPart::Plain { tokens, .. } => is_single_url(tokens),
+                    TextPart::Html {
+                        html_tokens,
+                        tokens,
+                        ..
+                    } => is_single_html_url(html_tokens, tokens),
+                    TextPart::None => false,
+                };
+
+                if is_single {
+                    ctx.result.add_tag("URL_ONLY");
+                }
+            }
+        }
+
+        urls.extend(inferred_urls);
+
+        if !ctx.input.is_train {
+            let mut redirected_urls = HashSet::new();
+            let mut trusted_domains: HashSet<String> = HashSet::new();
+
+            {
+                let mut checked_domains: HashSet<&str> = HashSet::new();
+                for url in &urls {
+                    if let Some(url_parsed) = &url.element.url_parsed {
+                        let host_sld = url_parsed.host.sld_or_default();
+                        if checked_domains.insert(host_sld)
+                            && is_trusted_domain(self, host_sld, ctx.input.span_id).await
+                        {
+                            trusted_domains.insert(host_sld.into());
+                        }
+                    }
+                }
+            }
+
+            for url in &urls {
+                for ch in url.element.url.chars() {
+                    if ch.is_zwsp() {
+                        ctx.result.add_tag("ZERO_WIDTH_SPACE_URL");
+                    }
+
+                    if ch.is_obscured() {
+                        ctx.result.add_tag("SUSPICIOUS_URL");
+                    }
+                }
+
+                // Skip non-URLs such as 'data:' and 'mailto:'
+                if !url.element.url.contains("://") {
+                    continue;
+                }
+
+                // Obtain parse url
+                let url_parsed = if let Some(url_parsed) = &url.element.url_parsed {
+                    url_parsed
+                } else {
+                    // URL could not be parsed
+                    ctx.result.add_tag("UNPARSABLE_URL");
+                    continue;
+                };
+                let host_sld = url_parsed.host.sld_or_default();
+
+                // Skip local and trusted domains
+                if trusted_domains.contains(host_sld) {
+                    continue;
+                }
+
+                if let Some(ip) = url_parsed.host.ip {
+                    // Check IP DNSBL
+                    check_dnsbl(self, ctx, &IpResolver::new(ip), Element::Ip, url.location).await;
+                } else if is_url_redirector(self, host_sld, ctx.input.span_id).await {
+                    // Check for redirectors
+                    ctx.result.add_tag("REDIRECTOR_URL");
+
+                    if !ctx.result.has_tag("URL_REDIRECTOR_NESTED") {
+                        let mut redirect_count = 1;
+                        let mut url_redirect = Cow::Borrowed(url.element.url.as_str());
+
+                        while redirect_count <= 3 {
+                            match http_get_header(
+                                self,
+                                url_redirect.as_ref(),
+                                LOCATION,
+                                Duration::from_secs(5),
+                            )
+                            .await
+                            {
+                                Ok(Some(location)) => {
+                                    let location = UrlParts::new(location);
+                                    if let Some(location_parsed) = &location.url_parsed {
+                                        if is_url_redirector(
+                                            self,
+                                            location_parsed.host.sld_or_default(),
+                                            ctx.input.span_id,
+                                        )
+                                        .await
+                                        {
+                                            url_redirect = Cow::Owned(location.url);
+                                            redirect_count += 1;
+                                            continue;
+                                        } else {
+                                            if is_trusted_domain(
+                                                self,
+                                                location_parsed.host.sld_or_default(),
+                                                ctx.input.span_id,
+                                            )
+                                            .await
+                                            {
+                                                trusted_domains.insert(
+                                                    location_parsed.host.sld_or_default().into(),
+                                                );
+                                            }
+
+                                            redirected_urls.insert(ElementLocation::new(
+                                                location,
+                                                url.location,
+                                            ));
+                                        }
+                                    }
+                                }
+                                Ok(None) => {}
+                                Err(err) => {
+                                    trc::error!(err.span_id(ctx.input.span_id));
+                                }
+                            }
+                            break;
+                        }
+
+                        if redirect_count > 3 {
+                            ctx.result.add_tag("URL_REDIRECTOR_NESTED");
+                        }
+                    }
+                }
+            }
+
+            urls.extend(redirected_urls);
+
+            for (el, url_parsed) in urls.iter().filter_map(|el| {
+                el.element
+                    .url_parsed
+                    .as_ref()
+                    .map(|url_parsed| (el, url_parsed))
+            }) {
+                let host = &url_parsed.host;
+                let is_explicit_link = el.element.is_explicit_link();
+
+                if host.ip.is_none() {
+                    if !host.fqdn.is_ascii() {
+                        if let Ok(cured_host) =
+                            decancer::cure(&host.fqdn, decancer::Options::default())
+                        {
+                            let cured_host = cured_host.to_string();
+                            if cured_host != host.fqdn
+                                && matches!(self.dns_exists_ip(&cured_host).await, Ok(true))
+                            {
+                                ctx.result.add_tag("HOMOGRAPH_URL");
+                            }
+                        }
+
+                        if host.fqdn.is_mixed_charset() {
+                            ctx.result.add_tag("MIXED_CHARSET_URL");
+                        }
+                    }
+
+                    // Check Domain DNSBL
+                    if is_explicit_link
+                        && let Some(sld) = &host.sld
+                        && !trusted_domains.contains(sld.as_str())
+                    {
+                        check_dnsbl(
+                            self,
+                            ctx,
+                            &StringResolver(sld),
+                            Element::Domain,
+                            el.location,
+                        )
+                        .await;
+                    }
+                } else {
+                    // URL is an ip address
+                    ctx.result.add_tag("SUSPICIOUS_URL");
+                }
+
+                // Check URL DNSBL
+                if is_explicit_link {
+                    check_dnsbl(self, ctx, &el.element, Element::Url, el.location).await;
+                }
+            }
+        }
+
+        // Update context
+        ctx.output.urls = urls;
+    }
+}
+
+#[allow(unreachable_code)]
+#[allow(unused_variables)]
+async fn http_get_header(
+    server: &Server,
+    url: &str,
+    header: hyper::header::HeaderName,
+    timeout: Duration,
+) -> trc::Result<Option<String>> {
+    #[cfg(feature = "test_mode")]
+    {
+        return if url.contains("redirect.") {
+            Ok(url.split_once("/?").unwrap().1.to_string().into())
+        } else {
+            Ok(None)
+        };
+    }
+    server
+        .core
+        .spam
+        .url_client
+        .get(url)
+        .timeout(timeout)
+        .send()
+        .await
+        .map_err(|err| {
+            trc::SieveEvent::RuntimeError
+                .into_err()
+                .reason(err)
+                .details("Failed to send request")
+        })
+        .map(|response| {
+            response
+                .headers()
+                .get(header)
+                .and_then(|h| h.to_str().ok())
+                .map(|h| h.to_string())
+        })
+}
+
+fn collect_url(
+    urls: &mut HashSet<ElementLocation<UrlParts<'static>>>,
+    inferred_urls: &mut HashSet<ElementLocation<UrlParts<'static>>>,
+    url: UrlParts<'static>,
+    location: Location,
+) {
+    if url.is_explicit_link() {
+        urls.insert(ElementLocation::new(url, location));
+    } else {
+        inferred_urls.insert(ElementLocation::new(url, location));
+    }
+}
+
+fn is_single_url<T, E, U, I>(tokens: &[TokenType<T, E, U, I>]) -> bool {
+    let mut url_count = 0;
+    let mut word_count = 0;
+
+    for token in tokens {
+        match token {
+            TokenType::Alphabetic(_)
+            | TokenType::Alphanumeric(_)
+            | TokenType::Integer(_)
+            | TokenType::Email(_)
+            | TokenType::Float(_) => {
+                word_count += 1;
+            }
+            TokenType::Url(_) | TokenType::UrlNoScheme(_) => {
+                url_count += 1;
+            }
+            _ => {}
+        }
+    }
+
+    url_count == 1 && word_count <= 1
+}
+
+fn is_single_html_url<T, E, U, I>(
+    html_tokens: &[HtmlToken],
+    tokens: &[TokenType<T, E, U, I>],
+) -> bool {
+    let mut url_count = 0;
+    let mut word_count = 0;
+
+    for token in tokens {
+        match token {
+            TokenType::Alphabetic(_)
+            | TokenType::Alphanumeric(_)
+            | TokenType::Integer(_)
+            | TokenType::Email(_)
+            | TokenType::Float(_) => {
+                word_count += 1;
+            }
+            TokenType::Url(_) | TokenType::UrlNoScheme(_) => {
+                url_count += 1;
+            }
+            _ => {}
+        }
+    }
+
+    if word_count > 1 || url_count != 1 {
+        return false;
+    }
+
+    url_count = 0;
+
+    for token in html_tokens {
+        if matches!(token, HtmlToken::StartTag { name, attributes, .. } if *name == A && attributes.iter().any(|(k, _)| *k == HREF))
+        {
+            url_count += 1;
+        }
+    }
+
+    url_count == 1
+}
+
+impl PartialEq for UrlParts<'_> {
+    fn eq(&self, other: &Self) -> bool {
+        self.url == other.url
+    }
+}
+
+impl Eq for UrlParts<'_> {}
+
+impl Hash for UrlParts<'_> {
+    fn hash<H: Hasher>(&self, state: &mut H) {
+        self.url.hash(state);
+    }
+}
+
+impl<'x> UrlParts<'x> {
+    pub fn new(url: impl Into<Cow<'x, str>>) -> Self {
+        let url_original = url.into();
+        let url = url_original.trim().to_lowercase();
+
+        Self {
+            url_parsed: Self::parse(&url),
+            url,
+            url_original,
+            has_scheme: true,
+        }
+    }
+
+    pub fn no_scheme(url: impl Into<Cow<'x, str>>) -> Self {
+        let url_original = url.into();
+        let host = url_original.trim().to_lowercase();
+        let mut url = String::with_capacity(HTTPS_SCHEME.len() + host.len());
+        url.push_str(HTTPS_SCHEME);
+        url.push_str(&host);
+
+        Self {
+            url_parsed: Self::parse(&url),
+            url,
+            url_original,
+            has_scheme: false,
+        }
+    }
+
+    pub fn is_explicit_link(&self) -> bool {
+        self.has_scheme
+            || self.url_original.contains(['/', '?'])
+            || self
+                .url_parsed
+                .as_ref()
+                .is_some_and(|url| url.host.fqdn.starts_with("www."))
+    }
+
+    fn parse(url: &str) -> Option<UrlParsed> {
+        url.parse::<Uri>().ok().and_then(|parts| {
+            parts
+                .host()
+                .map(Hostname::new)
+                .map(|host| UrlParsed { host, parts })
+        })
+    }
+
+    pub fn to_owned(&self) -> UrlParts<'static> {
+        UrlParts {
+            url: self.url.clone(),
+            url_original: Cow::Owned(self.url_original.clone().into_owned()),
+            url_parsed: self.url_parsed.clone(),
+            has_scheme: self.has_scheme,
+        }
+    }
+}

crates/spam-filter/src/lib.rs

@@ -0,0 +1,261 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
+ */
+
+#![warn(clippy::large_futures)]
+
+pub mod analysis;
+pub mod modules;
+
+use analysis::ElementLocation;
+use analysis::url::UrlParts;
+use mail_auth::{
+    ArcOutput, DkimOutput, DmarcResult, IprevOutput, SpfOutput, dkim2::Dkim2Output, dmarc::Policy,
+};
+use mail_parser::Message;
+use modules::html::HtmlToken;
+use nlp::tokenizers::types::TokenType;
+use std::borrow::Cow;
+use std::collections::HashSet;
+use std::hash::{Hash, Hasher};
+use std::net::{IpAddr, Ipv4Addr};
+use store::ahash::AHashSet;
+
+pub struct SpamFilterInput<'x> {
+    pub message: &'x Message<'x>,
+    pub span_id: u64,
+
+    // Sender authentication
+    pub arc_result: Option<&'x ArcOutput<'x>>,
+    pub spf_ehlo_result: Option<&'x SpfOutput>,
+    pub spf_mail_from_result: Option<&'x SpfOutput>,
+    pub dkim_result: &'x [DkimOutput<'x>],
+    pub dkim2_result: Option<&'x Dkim2Output<'x>>,
+    pub dmarc_result: Option<&'x DmarcResult>,
+    pub dmarc_policy: Option<&'x Policy>,
+    pub iprev_result: Option<&'x IprevOutput>,
+
+    // Session details
+    pub remote_ip: IpAddr,
+    pub ehlo_domain: Option<&'x str>,
+    pub authenticated_as: Option<&'x str>,
+    pub asn: Option<u32>,
+    pub country: Option<&'x str>,
+
+    // TLS
+    pub is_tls: bool,
+
+    // Envelope
+    pub env_from: &'x str,
+    pub env_from_flags: u64,
+    pub env_rcpt_orig_to: Vec<&'x str>,
+    pub env_rcpt_rewritten_to: Vec<&'x str>,
+
+    pub is_train: bool,
+    pub is_test: bool,
+}
+
+pub struct SpamFilterOutput<'x> {
+    pub ehlo_host: Hostname,
+    pub iprev_ptr: Option<String>,
+
+    pub env_from_addr: Email,
+    pub env_from_postmaster: bool,
+    pub env_to_orig_addr: HashSet<Email>,
+    pub env_to_rewritten_addr: HashSet<Email>,
+    pub from: Recipient,
+    pub recipients_to: Vec<Recipient>,
+    pub recipients_cc: Vec<Recipient>,
+    pub recipients_bcc: Vec<Recipient>,
+    pub reply_to: Option<Recipient>,
+
+    pub subject: String,
+    pub subject_lc: String,
+    pub subject_thread: String,
+    pub subject_thread_lc: String,
+    pub subject_tokens: Vec<TokenType<Cow<'x, str>, Email, UrlParts<'x>, IpParts>>,
+
+    pub ips: AHashSet<ElementLocation<IpAddr>>,
+    pub urls: HashSet<ElementLocation<UrlParts<'x>>>,
+    pub emails: HashSet<ElementLocation<Recipient>>,
+    pub domains: HashSet<ElementLocation<String>>,
+
+    pub text_parts: Vec<TextPart<'x>>,
+}
+
+#[derive(Debug)]
+pub struct IpParts {
+    ip: Option<IpAddr>,
+}
+
+pub enum TextPart<'x> {
+    Plain {
+        text_body: &'x str,
+        tokens: Vec<TokenType<Cow<'x, str>, Email, UrlParts<'x>, IpParts>>,
+    },
+    Html {
+        html_tokens: Vec<HtmlToken>,
+        text_body: String,
+        tokens: Vec<TokenType<Cow<'x, str>, Email, UrlParts<'x>, IpParts>>,
+    },
+    None,
+}
+
+#[derive(Debug, Default)]
+pub struct SpamFilterResult {
+    pub tags: AHashSet<String>,
+    pub classifier_confidence: Vec<Option<f32>>,
+    pub score: f32,
+    pub rbl_ip_checks: usize,
+    pub rbl_domain_checks: usize,
+    pub rbl_url_checks: usize,
+    pub rbl_email_checks: usize,
+    pub llm_result: Option<(String, String)>,
+}
+
+pub struct SpamFilterContext<'x> {
+    pub input: SpamFilterInput<'x>,
+    pub output: SpamFilterOutput<'x>,
+    pub result: SpamFilterResult,
+}
+
+#[derive(Debug, Clone)]
+pub struct Hostname {
+    pub fqdn: String,
+    pub ip: Option<IpAddr>,
+    pub sld: Option<String>,
+}
+
+#[derive(Debug, Clone)]
+pub struct Email {
+    pub address: String,
+    pub local_part: String,
+    pub domain_part: Hostname,
+}
+
+#[derive(Debug, Clone)]
+pub struct Recipient {
+    pub email: Email,
+    pub name: Option<String>,
+}
+
+impl<'x> SpamFilterInput<'x> {
+    pub fn from_message(message: &'x Message<'x>, span_id: u64) -> Self {
+        Self {
+            message,
+            span_id,
+            arc_result: None,
+            spf_ehlo_result: None,
+            spf_mail_from_result: None,
+            dkim_result: &[],
+            dkim2_result: None,
+            dmarc_result: None,
+            dmarc_policy: None,
+            iprev_result: None,
+            remote_ip: IpAddr::V4(Ipv4Addr::LOCALHOST),
+            ehlo_domain: None,
+            authenticated_as: None,
+            asn: None,
+            country: None,
+            is_tls: true,
+            env_from: "",
+            env_from_flags: 0,
+            env_rcpt_rewritten_to: vec![],
+            env_rcpt_orig_to: vec![],
+            is_test: false,
+            is_train: false,
+        }
+    }
+
+    pub fn train_mode(mut self) -> Self {
+        self.is_train = true;
+        self
+    }
+}
+
+impl PartialEq for Hostname {
+    fn eq(&self, other: &Self) -> bool {
+        self.fqdn.eq(&other.fqdn)
+    }
+}
+
+impl Eq for Hostname {}
+
+impl PartialEq for Email {
+    fn eq(&self, other: &Self) -> bool {
+        self.local_part.eq(&other.local_part) && self.domain_part.eq(&other.domain_part)
+    }
+}
+
+impl Eq for Email {}
+
+impl Hash for Hostname {
+    fn hash<H: Hasher>(&self, state: &mut H) {
+        self.fqdn.hash(state)
+    }
+}
+
+impl Hash for Email {
+    fn hash<H: Hasher>(&self, state: &mut H) {
+        self.local_part.hash(state);
+        self.domain_part.hash(state);
+    }
+}
+
+impl Email {
+    pub fn classifier_parts(&self) -> Option<(&str, &str)> {
+        // Returns (local@, @domain)
+        if self.is_valid() {
+            let at_pos = self.address.find('@')?;
+            Some((&self.address[..=at_pos], &self.address[at_pos..]))
+        } else {
+            None
+        }
+    }
+
+    pub fn is_valid(&self) -> bool {
+        self.domain_part.sld.is_some() && !self.local_part.is_empty()
+    }
+}
+
+impl PartialEq for Recipient {
+    fn eq(&self, other: &Self) -> bool {
+        self.email.eq(&other.email)
+    }
+}
+
+impl Eq for Recipient {}
+
+impl Hash for Recipient {
+    fn hash<H: Hasher>(&self, state: &mut H) {
+        self.email.hash(state)
+    }
+}
+
+impl PartialOrd for Email {
+    fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
+        Some(self.cmp(other))
+    }
+}
+
+impl PartialOrd for Recipient {
+    fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
+        Some(self.cmp(other))
+    }
+}
+
+impl Ord for Email {
+    fn cmp(&self, other: &Self) -> std::cmp::Ordering {
+        self.local_part
+            .cmp(&other.local_part)
+            .then_with(|| self.domain_part.fqdn.cmp(&other.domain_part.fqdn))
+    }
+}
+
+impl Ord for Recipient {
+    fn cmp(&self, other: &Self) -> std::cmp::Ordering {
+        self.email.cmp(&other.email)
+    }
+}

crates/spam-filter/src/modules/dnsbl.rs

@@ -0,0 +1,196 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
+ */
+
+use super::expression::SpamFilterResolver;
+use crate::SpamFilterContext;
+use common::{
+    Server,
+    config::mailstore::spamfilter::{DnsBlServer, Element, IpResolver, Location},
+    expr::functions::ResolveVariable,
+};
+use mail_auth::{Error, common::resolver::ToFqdn};
+use std::{
+    net::Ipv4Addr,
+    sync::Arc,
+    time::{Duration, Instant},
+};
+use trc::SpamEvent;
+
+pub(crate) async fn check_dnsbl(
+    server: &Server,
+    ctx: &mut SpamFilterContext<'_>,
+    resolver: &impl ResolveVariable,
+    scope: Element,
+    location: Location,
+) {
+    let (mut checks, max_checks) = match scope {
+        Element::Email => (
+            ctx.result.rbl_email_checks,
+            server.core.spam.dnsbl.max_email_checks,
+        ),
+        Element::Ip => (
+            ctx.result.rbl_ip_checks,
+            server.core.spam.dnsbl.max_ip_checks,
+        ),
+        Element::Url => (
+            ctx.result.rbl_url_checks,
+            server.core.spam.dnsbl.max_url_checks,
+        ),
+        Element::Domain => (
+            ctx.result.rbl_domain_checks,
+            server.core.spam.dnsbl.max_domain_checks,
+        ),
+        Element::Header | Element::Body | Element::Any => unreachable!(),
+    };
+
+    for dnsbl in &server.core.spam.dnsbl.servers {
+        if dnsbl.scope == scope
+            && checks < max_checks
+            && let Some(tag) = is_dnsbl(
+                server,
+                dnsbl,
+                SpamFilterResolver::new(ctx, resolver, location),
+                scope,
+                &mut checks,
+            )
+            .await
+        {
+            ctx.result.add_tag(tag);
+        }
+    }
+
+    match scope {
+        Element::Email => ctx.result.rbl_email_checks = checks,
+        Element::Ip => ctx.result.rbl_ip_checks = checks,
+        Element::Url => ctx.result.rbl_url_checks = checks,
+        Element::Domain => ctx.result.rbl_domain_checks = checks,
+        Element::Header | Element::Body | Element::Any => unreachable!(),
+    }
+}
+
+async fn is_dnsbl(
+    server: &Server,
+    config: &DnsBlServer,
+    resolver: SpamFilterResolver<'_, impl ResolveVariable>,
+    element: Element,
+    checks: &mut usize,
+) -> Option<String> {
+    let time = Instant::now();
+    let zone = server
+        .eval_if::<String, _>(&config.zone, &resolver, resolver.ctx.input.span_id)
+        .await?;
+
+    #[cfg(feature = "test_mode")]
+    {
+        if zone.contains(".11.20.") {
+            let parts = zone.split('.').collect::<Vec<_>>();
+
+            return if config.tags.if_then.iter().any(|i| i.expr.items.len() == 3) && parts[0] != "2"
+            {
+                None
+            } else {
+                server
+                    .eval_if(
+                        &config.tags,
+                        &SpamFilterResolver::new(
+                            resolver.ctx,
+                            &IpResolver::new(
+                                format!("127.0.{}.{}", parts[1], parts[0]).parse().unwrap(),
+                            ),
+                            resolver.location,
+                        ),
+                        resolver.ctx.input.span_id,
+                    )
+                    .await
+            };
+        }
+    }
+
+    let result = match server.inner.cache.dns_rbl.get(zone.as_str()) {
+        Some(Some(result)) => result,
+        Some(None) => return None,
+        None => {
+            *checks += 1;
+
+            match server
+                .core
+                .smtp
+                .resolvers
+                .dns
+                .ipv4_lookup_raw(zone.to_fqdn().as_ref())
+                .await
+            {
+                Ok(result) => {
+                    trc::event!(
+                        Spam(SpamEvent::Dnsbl),
+                        Hostname = zone.clone(),
+                        Result = result
+                            .entry
+                            .iter()
+                            .map(|ip| trc::Value::from(ip.to_string()))
+                            .collect::<Vec<_>>(),
+                        Details = element.as_str(),
+                        Elapsed = time.elapsed()
+                    );
+
+                    let entry = Arc::new(IpResolver::new(
+                        result
+                            .entry
+                            .iter()
+                            .copied()
+                            .next()
+                            .unwrap_or(Ipv4Addr::BROADCAST)
+                            .into(),
+                    ));
+
+                    server.inner.cache.dns_rbl.insert_with_expiry(
+                        zone.into(),
+                        Some(entry.clone()),
+                        result.expires,
+                    );
+
+                    entry
+                }
+                Err(Error::Dns(mail_auth::DnsError::RecordNotFound(_))) => {
+                    trc::event!(
+                        Spam(SpamEvent::Dnsbl),
+                        Hostname = zone.clone(),
+                        Result = trc::Value::None,
+                        Details = element.as_str(),
+                        Elapsed = time.elapsed()
+                    );
+
+                    server.inner.cache.dns_rbl.insert(
+                        zone.into(),
+                        None,
+                        Duration::from_secs(86400),
+                    );
+
+                    return None;
+                }
+                Err(err) => {
+                    trc::event!(
+                        Spam(SpamEvent::DnsblError),
+                        Hostname = zone,
+                        Elapsed = time.elapsed(),
+                        Details = element.as_str(),
+                        CausedBy = err.to_string()
+                    );
+
+                    return None;
+                }
+            }
+        }
+    };
+
+    server
+        .eval_if(
+            &config.tags,
+            &SpamFilterResolver::new(resolver.ctx, result.as_ref(), resolver.location),
+            resolver.ctx.input.span_id,
+        )
+        .await
+}

crates/spam-filter/src/modules/expression.rs

@@ -0,0 +1,504 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
+ */
+
+use common::{
+    config::mailstore::spamfilter::*,
+    expr::{StringCow, Variable, functions::ResolveVariable},
+};
+use compact_str::{CompactString, ToCompactString, format_compact};
+use mail_parser::{Header, HeaderValue};
+use nlp::tokenizers::types::TokenType;
+use registry::schema::enums::ExpressionVariable;
+
+use crate::{Recipient, SpamFilterContext, TextPart, analysis::url::UrlParts};
+
+pub(crate) struct SpamFilterResolver<'x, T: ResolveVariable> {
+    pub ctx: &'x SpamFilterContext<'x>,
+    pub item: &'x T,
+    pub location: Location,
+}
+
+impl<'x, T: ResolveVariable> SpamFilterResolver<'x, T> {
+    pub fn new(ctx: &'x SpamFilterContext<'x>, item: &'x T, location: Location) -> Self {
+        Self {
+            ctx,
+            item,
+            location,
+        }
+    }
+}
+
+impl<T: ResolveVariable> ResolveVariable for SpamFilterResolver<'_, T> {
+    fn resolve_variable(&self, variable: ExpressionVariable) -> Variable<'_> {
+        match variable {
+            ExpressionVariable::RemoteIp => self.ctx.input.remote_ip.to_compact_string().into(),
+            ExpressionVariable::RemoteIpPtr => self
+                .ctx
+                .output
+                .iprev_ptr
+                .as_deref()
+                .unwrap_or_default()
+                .into(),
+            ExpressionVariable::HeloDomain => self.ctx.output.ehlo_host.fqdn.as_str().into(),
+            ExpressionVariable::AuthenticatedAs => {
+                self.ctx.input.authenticated_as.unwrap_or_default().into()
+            }
+            ExpressionVariable::Asn => self.ctx.input.asn.unwrap_or_default().into(),
+            ExpressionVariable::Country => self.ctx.input.country.unwrap_or_default().into(),
+            ExpressionVariable::IsTls => self.ctx.input.is_tls.into(),
+            ExpressionVariable::EnvFrom => self.ctx.output.env_from_addr.address.as_str().into(),
+            ExpressionVariable::EnvFromLocal => {
+                self.ctx.output.env_from_addr.local_part.as_str().into()
+            }
+            ExpressionVariable::EnvFromDomain => self
+                .ctx
+                .output
+                .env_from_addr
+                .domain_part
+                .fqdn
+                .as_str()
+                .into(),
+            ExpressionVariable::EnvTo => self
+                .ctx
+                .output
+                .env_to_orig_addr
+                .iter()
+                .map(|e| Variable::from(e.address.as_str()))
+                .collect::<Vec<_>>()
+                .into(),
+            ExpressionVariable::From => self.ctx.output.from.email.address.as_str().into(),
+            ExpressionVariable::FromName => self
+                .ctx
+                .output
+                .from
+                .name
+                .as_deref()
+                .unwrap_or_default()
+                .into(),
+            ExpressionVariable::FromLocal => self.ctx.output.from.email.local_part.as_str().into(),
+            ExpressionVariable::FromDomain => {
+                self.ctx.output.from.email.domain_part.fqdn.as_str().into()
+            }
+            ExpressionVariable::ReplyTo => self
+                .ctx
+                .output
+                .reply_to
+                .as_ref()
+                .map(|r| r.email.address.as_str())
+                .unwrap_or_default()
+                .into(),
+            ExpressionVariable::ReplyToName => self
+                .ctx
+                .output
+                .reply_to
+                .as_ref()
+                .and_then(|r| r.name.as_deref())
+                .unwrap_or_default()
+                .into(),
+            ExpressionVariable::ReplyToLocal => self
+                .ctx
+                .output
+                .reply_to
+                .as_ref()
+                .map(|r| r.email.local_part.as_str())
+                .unwrap_or_default()
+                .into(),
+            ExpressionVariable::ReplyToDomain => self
+                .ctx
+                .output
+                .reply_to
+                .as_ref()
+                .map(|r| r.email.domain_part.fqdn.as_str())
+                .unwrap_or_default()
+                .into(),
+            ExpressionVariable::To => self
+                .ctx
+                .output
+                .recipients_to
+                .iter()
+                .map(|r| Variable::from(r.email.address.as_str()))
+                .collect::<Vec<_>>()
+                .into(),
+            ExpressionVariable::ToName => self
+                .ctx
+                .output
+                .recipients_to
+                .iter()
+                .filter_map(|r| Variable::from(r.name.as_deref()?).into())
+                .collect::<Vec<_>>()
+                .into(),
+            ExpressionVariable::ToLocal => self
+                .ctx
+                .output
+                .recipients_to
+                .iter()
+                .map(|r| Variable::from(r.email.local_part.as_str()))
+                .collect::<Vec<_>>()
+                .into(),
+            ExpressionVariable::ToDomain => self
+                .ctx
+                .output
+                .recipients_to
+                .iter()
+                .map(|r| Variable::from(r.email.domain_part.fqdn.as_str()))
+                .collect::<Vec<_>>()
+                .into(),
+            ExpressionVariable::Cc => self
+                .ctx
+                .output
+                .recipients_cc
+                .iter()
+                .map(|r| Variable::from(r.email.address.as_str()))
+                .collect::<Vec<_>>()
+                .into(),
+            ExpressionVariable::CcName => self
+                .ctx
+                .output
+                .recipients_cc
+                .iter()
+                .filter_map(|r| Variable::from(r.name.as_deref()?).into())
+                .collect::<Vec<_>>()
+                .into(),
+            ExpressionVariable::CcLocal => self
+                .ctx
+                .output
+                .recipients_cc
+                .iter()
+                .map(|r| Variable::from(r.email.local_part.as_str()))
+                .collect::<Vec<_>>()
+                .into(),
+            ExpressionVariable::CcDomain => self
+                .ctx
+                .output
+                .recipients_cc
+                .iter()
+                .map(|r| Variable::from(r.email.domain_part.fqdn.as_str()))
+                .collect::<Vec<_>>()
+                .into(),
+            ExpressionVariable::Bcc => self
+                .ctx
+                .output
+                .recipients_bcc
+                .iter()
+                .map(|r| Variable::from(r.email.address.as_str()))
+                .collect::<Vec<_>>()
+                .into(),
+            ExpressionVariable::BccName => self
+                .ctx
+                .output
+                .recipients_bcc
+                .iter()
+                .filter_map(|r| Variable::from(r.name.as_deref()?).into())
+                .collect::<Vec<_>>()
+                .into(),
+            ExpressionVariable::BccLocal => self
+                .ctx
+                .output
+                .recipients_bcc
+                .iter()
+                .map(|r| Variable::from(r.email.local_part.as_str()))
+                .collect::<Vec<_>>()
+                .into(),
+            ExpressionVariable::BccDomain => self
+                .ctx
+                .output
+                .recipients_bcc
+                .iter()
+                .map(|r| Variable::from(r.email.domain_part.fqdn.as_str()))
+                .collect::<Vec<_>>()
+                .into(),
+            ExpressionVariable::Body | ExpressionVariable::BodyText => {
+                self.ctx.text_body().unwrap_or_default().into()
+            }
+            ExpressionVariable::BodyHtml => self
+                .ctx
+                .input
+                .message
+                .html_body
+                .first()
+                .and_then(|idx| self.ctx.output.text_parts.get(*idx as usize))
+                .map(|part| {
+                    if let TextPart::Html { text_body, .. } = part {
+                        text_body.as_str()
+                    } else {
+                        ""
+                    }
+                })
+                .unwrap_or_default()
+                .into(),
+            ExpressionVariable::BodyRaw => Variable::from(CompactString::from_utf8_lossy(
+                self.ctx.input.message.raw_message(),
+            )),
+            ExpressionVariable::Subject => self.ctx.output.subject_lc.as_str().into(),
+            ExpressionVariable::SubjectThread => self.ctx.output.subject_thread_lc.as_str().into(),
+            ExpressionVariable::Location => self.location.as_str().into(),
+            ExpressionVariable::SubjectWords => self
+                .ctx
+                .output
+                .subject_tokens
+                .iter()
+                .filter_map(|w| match w {
+                    TokenType::Alphabetic(w)
+                    | TokenType::Alphanumeric(w)
+                    | TokenType::Integer(w)
+                    | TokenType::Float(w) => Some(Variable::from(w.as_ref())),
+                    _ => None,
+                })
+                .collect::<Vec<_>>()
+                .into(),
+            ExpressionVariable::BodyWords => self
+                .ctx
+                .input
+                .message
+                .html_body
+                .first()
+                .and_then(|idx| self.ctx.output.text_parts.get(*idx as usize))
+                .map(|part| match part {
+                    TextPart::Plain { tokens, .. } | TextPart::Html { tokens, .. } => tokens
+                        .iter()
+                        .filter_map(|w| match w {
+                            TokenType::Alphabetic(w)
+                            | TokenType::Alphanumeric(w)
+                            | TokenType::Integer(w)
+                            | TokenType::Float(w) => Some(Variable::from(w.as_ref())),
+                            _ => None,
+                        })
+                        .collect::<Vec<_>>(),
+                    TextPart::None => vec![],
+                })
+                .unwrap_or_default()
+                .into(),
+            variable => self.item.resolve_variable(variable),
+        }
+    }
+
+    fn resolve_global(&self, variable: &str) -> Variable<'_> {
+        Variable::Integer(self.ctx.result.tags.contains(variable).into())
+    }
+}
+
+pub(crate) struct EmailHeader<'x> {
+    pub header: &'x Header<'x>,
+    pub raw: &'x str,
+}
+
+impl ResolveVariable for EmailHeader<'_> {
+    fn resolve_variable(&self, variable: ExpressionVariable) -> Variable<'_> {
+        match variable {
+            ExpressionVariable::Name => self.header.name().into(),
+            ExpressionVariable::NameLower => {
+                CompactString::from_str_to_lowercase(self.header.name()).into()
+            }
+            ExpressionVariable::Value
+            | ExpressionVariable::ValueLower
+            | ExpressionVariable::Attributes => match &self.header.value {
+                HeaderValue::Text(text) => {
+                    if variable == ExpressionVariable::ValueLower {
+                        CompactString::from_str_to_lowercase(text).into()
+                    } else {
+                        text.as_ref().into()
+                    }
+                }
+                HeaderValue::TextList(list) => Variable::Array(
+                    list.iter()
+                        .map(|text| {
+                            Variable::String(if variable == ExpressionVariable::ValueLower {
+                                StringCow::Owned(CompactString::from_str_to_lowercase(text))
+                            } else {
+                                StringCow::Borrowed(text.as_ref())
+                            })
+                        })
+                        .collect(),
+                ),
+                HeaderValue::Address(address) => {
+                    Variable::Array(if matches!(variable, ExpressionVariable::ValueLower) {
+                        address
+                            .iter()
+                            .filter_map(|a| {
+                                a.address.as_ref().map(|text| {
+                                    Variable::String(
+                                        if variable == ExpressionVariable::ValueLower {
+                                            StringCow::Owned(CompactString::from_str_to_lowercase(
+                                                text,
+                                            ))
+                                        } else {
+                                            StringCow::Borrowed(text.as_ref())
+                                        },
+                                    )
+                                })
+                            })
+                            .collect()
+                    } else {
+                        address
+                            .iter()
+                            .filter_map(|a| {
+                                a.name.as_ref().map(|text| {
+                                    Variable::String(
+                                        if variable == ExpressionVariable::ValueLower {
+                                            StringCow::Owned(CompactString::from_str_to_lowercase(
+                                                text,
+                                            ))
+                                        } else {
+                                            StringCow::Borrowed(text.as_ref())
+                                        },
+                                    )
+                                })
+                            })
+                            .collect()
+                    })
+                }
+                HeaderValue::DateTime(date_time) => {
+                    CompactString::new(date_time.to_rfc3339()).into()
+                }
+                HeaderValue::ContentType(ct) => {
+                    if variable != ExpressionVariable::Attributes {
+                        if let Some(st) = ct.subtype() {
+                            format_compact!("{}/{}", ct.ctype(), st).into()
+                        } else {
+                            ct.ctype().into()
+                        }
+                    } else {
+                        Variable::Array(
+                            ct.attributes()
+                                .map(|attr| {
+                                    attr.iter()
+                                        .map(|attr| {
+                                            Variable::from(format_compact!(
+                                                "{}={}", attr.name, attr.value
+                                            ))
+                                        })
+                                        .collect::<Vec<_>>()
+                                })
+                                .unwrap_or_default(),
+                        )
+                    }
+                }
+                HeaderValue::Received(_) => {
+                    if variable == ExpressionVariable::ValueLower {
+                        CompactString::from_str_to_lowercase(self.raw.trim()).into()
+                    } else {
+                        self.raw.trim().into()
+                    }
+                }
+                HeaderValue::Empty => "".into(),
+            },
+            ExpressionVariable::Raw => self.raw.into(),
+            ExpressionVariable::RawLower => CompactString::from_str_to_lowercase(self.raw).into(),
+            _ => Variable::Integer(0),
+        }
+    }
+
+    fn resolve_global(&self, _: &str) -> Variable<'_> {
+        Variable::Integer(0)
+    }
+}
+
+impl ResolveVariable for Recipient {
+    fn resolve_variable(&self, variable: ExpressionVariable) -> Variable<'_> {
+        match variable {
+            ExpressionVariable::Email | ExpressionVariable::Value => {
+                Variable::from(self.email.address.as_str())
+            }
+            ExpressionVariable::Name => Variable::from(self.name.as_deref().unwrap_or_default()),
+            ExpressionVariable::Local => Variable::from(self.email.local_part.as_str()),
+            ExpressionVariable::Domain => Variable::from(self.email.domain_part.fqdn.as_str()),
+            ExpressionVariable::Sld => Variable::from(self.email.domain_part.sld_or_default()),
+            _ => Variable::Integer(0),
+        }
+    }
+
+    fn resolve_global(&self, _: &str) -> Variable<'_> {
+        Variable::Integer(0)
+    }
+}
+
+impl ResolveVariable for UrlParts<'_> {
+    fn resolve_variable(&self, variable: ExpressionVariable) -> Variable<'_> {
+        match variable {
+            ExpressionVariable::Url | ExpressionVariable::Value => {
+                Variable::from(self.url.as_str())
+            }
+            ExpressionVariable::UrlOriginal => Variable::from(self.url_original.as_ref()),
+            ExpressionVariable::PathQuery => Variable::from(
+                self.url_parsed
+                    .as_ref()
+                    .and_then(|p| p.parts.path_and_query().map(|p| p.as_str()))
+                    .unwrap_or_default(),
+            ),
+            ExpressionVariable::Path => Variable::from(
+                self.url_parsed
+                    .as_ref()
+                    .map(|p| p.parts.path())
+                    .unwrap_or_default(),
+            ),
+            ExpressionVariable::Query => Variable::from(
+                self.url_parsed
+                    .as_ref()
+                    .and_then(|p| p.parts.query())
+                    .unwrap_or_default(),
+            ),
+            ExpressionVariable::Scheme => Variable::from(
+                self.url_parsed
+                    .as_ref()
+                    .and_then(|p| p.parts.scheme_str())
+                    .unwrap_or_default(),
+            ),
+            ExpressionVariable::Authority => Variable::from(
+                self.url_parsed
+                    .as_ref()
+                    .and_then(|p| p.parts.authority().map(|a| a.as_str()))
+                    .unwrap_or_default(),
+            ),
+            ExpressionVariable::Host => Variable::from(
+                self.url_parsed
+                    .as_ref()
+                    .map(|p| p.host.fqdn.as_str())
+                    .unwrap_or_default(),
+            ),
+            ExpressionVariable::Sld => Variable::from(
+                self.url_parsed
+                    .as_ref()
+                    .map(|p| p.host.sld_or_default())
+                    .unwrap_or_default(),
+            ),
+            ExpressionVariable::Port => Variable::Integer(
+                self.url_parsed
+                    .as_ref()
+                    .and_then(|p| p.parts.port_u16())
+                    .unwrap_or(0) as _,
+            ),
+            _ => Variable::Integer(0),
+        }
+    }
+
+    fn resolve_global(&self, _: &str) -> Variable<'_> {
+        Variable::Integer(0)
+    }
+}
+
+pub struct StringResolver<'x>(pub &'x str);
+
+impl ResolveVariable for StringResolver<'_> {
+    fn resolve_variable(&self, _: ExpressionVariable) -> Variable<'_> {
+        Variable::from(self.0)
+    }
+
+    fn resolve_global(&self, _: &str) -> Variable<'_> {
+        Variable::Integer(0)
+    }
+}
+
+pub struct StringListResolver<'x>(pub &'x [String]);
+
+impl ResolveVariable for StringListResolver<'_> {
+    fn resolve_variable(&self, _: ExpressionVariable) -> Variable<'_> {
+        Variable::Array(self.0.iter().map(|v| Variable::from(v.as_str())).collect())
+    }
+
+    fn resolve_global(&self, _: &str) -> Variable<'_> {
+        Variable::Integer(0)
+    }
+}

crates/spam-filter/src/modules/html.rs

@@ -0,0 +1,448 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
+ */
+
+use mail_parser::decoders::html::add_html_token;
+
+#[derive(Debug, Eq, PartialEq, Clone, serde::Serialize, serde::Deserialize)]
+#[serde(tag = "type")]
+pub enum HtmlToken {
+    StartTag {
+        name: u64,
+        attributes: Vec<(u64, Option<String>)>,
+        is_self_closing: bool,
+    },
+    EndTag {
+        name: u64,
+    },
+    Comment {
+        text: String,
+    },
+    Text {
+        text: String,
+    },
+}
+
+pub(crate) const A: u64 = b'a' as u64;
+pub(crate) const IMG: u64 = (b'i' as u64) | ((b'm' as u64) << 8) | ((b'g' as u64) << 16);
+pub(crate) const HEAD: u64 =
+    (b'h' as u64) | ((b'e' as u64) << 8) | ((b'a' as u64) << 16) | ((b'd' as u64) << 24);
+pub(crate) const BODY: u64 =
+    (b'b' as u64) | ((b'o' as u64) << 8) | ((b'd' as u64) << 16) | ((b'y' as u64) << 24);
+pub(crate) const META: u64 =
+    (b'm' as u64) | ((b'e' as u64) << 8) | ((b't' as u64) << 16) | ((b'a' as u64) << 24);
+pub(crate) const LINK: u64 =
+    (b'l' as u64) | ((b'i' as u64) << 8) | ((b'n' as u64) << 16) | ((b'k' as u64) << 24);
+pub(crate) const ALT: u64 = (b'a' as u64) | ((b'l' as u64) << 8) | ((b't' as u64) << 16);
+pub(crate) const TITLE: u64 = (b't' as u64)
+    | ((b'i' as u64) << 8)
+    | ((b't' as u64) << 16)
+    | ((b'l' as u64) << 24)
+    | ((b'e' as u64) << 32);
+
+pub(crate) const HREF: u64 =
+    (b'h' as u64) | ((b'r' as u64) << 8) | ((b'e' as u64) << 16) | ((b'f' as u64) << 24);
+pub(crate) const SRC: u64 = (b's' as u64) | ((b'r' as u64) << 8) | ((b'c' as u64) << 16);
+pub(crate) const WIDTH: u64 = (b'w' as u64)
+    | ((b'i' as u64) << 8)
+    | ((b'd' as u64) << 16)
+    | ((b't' as u64) << 24)
+    | ((b'h' as u64) << 32);
+pub(crate) const HEIGHT: u64 = (b'h' as u64)
+    | ((b'e' as u64) << 8)
+    | ((b'i' as u64) << 16)
+    | ((b'g' as u64) << 24)
+    | ((b'h' as u64) << 32)
+    | ((b't' as u64) << 40);
+pub(crate) const REL: u64 = (b'r' as u64) | ((b'e' as u64) << 8) | ((b'l' as u64) << 16);
+pub(crate) const CONTENT: u64 = (b'c' as u64)
+    | ((b'o' as u64) << 8)
+    | ((b'n' as u64) << 16)
+    | ((b't' as u64) << 24)
+    | ((b'e' as u64) << 32)
+    | ((b'n' as u64) << 40)
+    | ((b't' as u64) << 48);
+pub(crate) const HTTP_EQUIV: u64 = (b'h' as u64)
+    | ((b't' as u64) << 8)
+    | ((b't' as u64) << 16)
+    | ((b'p' as u64) << 24)
+    | ((b'-' as u64) << 32)
+    | ((b'e' as u64) << 40)
+    | ((b'q' as u64) << 48)
+    | ((b'u' as u64) << 56);
+
+pub fn html_to_tokens(input: &str) -> Vec<HtmlToken> {
+    let input = input.as_bytes();
+    let mut iter = input.iter().enumerate().peekable();
+    let mut tags = vec![];
+
+    let mut is_token_start = true;
+    let mut is_after_space = false;
+    let mut is_new_line = true;
+
+    let mut token_start = 0;
+    let mut token_end = 0;
+
+    let mut text = String::with_capacity(16);
+
+    while let Some((mut pos, &ch)) = iter.next() {
+        match ch {
+            b'<' => {
+                if !is_token_start {
+                    add_html_token(
+                        &mut text,
+                        &input[token_start..token_end + 1],
+                        is_after_space,
+                    );
+                    is_after_space = false;
+                    is_token_start = true;
+                }
+                if !text.is_empty() {
+                    tags.push(HtmlToken::Text {
+                        text: text.as_str().into(),
+                    });
+                    text.clear();
+                }
+
+                while matches!(iter.peek(), Some(&(_, &ch)) if ch.is_ascii_whitespace()) {
+                    pos += 1;
+                    iter.next();
+                }
+
+                if matches!(input.get(pos + 1..pos + 4), Some(b"!--")) {
+                    let mut comment = Vec::new();
+                    let mut last_ch: u8 = 0;
+                    for (_, &ch) in iter.by_ref() {
+                        match ch {
+                            b'>' if comment.len() > 2
+                                && matches!(comment.last(), Some(b'-'))
+                                && matches!(comment.get(comment.len() - 2), Some(b'-')) =>
+                            {
+                                break;
+                            }
+                            b' ' | b'\t' | b'\r' | b'\n' => {
+                                if last_ch != b' ' {
+                                    comment.push(b' ');
+                                } else {
+                                    last_ch = b' ';
+                                }
+                                continue;
+                            }
+                            _ => {
+                                comment.push(ch);
+                            }
+                        }
+                        last_ch = ch;
+                    }
+                    tags.push(HtmlToken::Comment {
+                        text: String::from_utf8(comment).unwrap_or_default(),
+                    });
+                } else {
+                    let mut is_end_tag = false;
+                    loop {
+                        match iter.peek() {
+                            Some(&(_, &b'/')) => {
+                                is_end_tag = true;
+                                //pos += 1;
+                                iter.next();
+                            }
+                            Some((_, ch)) if ch.is_ascii_whitespace() => {
+                                //pos += 1;
+                                iter.next();
+                            }
+                            _ => break,
+                        }
+                    }
+
+                    let mut in_quote = false;
+                    let mut is_self_closing = false;
+
+                    let mut key: u64 = 0;
+                    let mut shift = 0;
+
+                    let mut tag = 0;
+                    let mut attributes: Vec<(u64, Option<String>)> = vec![];
+
+                    'outer: while let Some((_, &ch)) = iter.next() {
+                        match ch {
+                            b'a'..=b'z' | b'0'..=b'9' | b'-' | b'_' if shift < 64 => {
+                                key |= (ch as u64) << shift;
+                                shift += 8;
+                            }
+                            b'A'..=b'Z' if shift < 64 => {
+                                key |= ((ch - b'A' + b'a') as u64) << shift;
+                                shift += 8;
+                            }
+                            b'/' if !in_quote => {
+                                is_self_closing = true;
+                            }
+                            b'>' if !in_quote => {
+                                if shift != 0 {
+                                    if tag == 0 {
+                                        tag = key;
+                                    } else {
+                                        attributes.push((key, None));
+                                    }
+                                }
+                                break;
+                            }
+                            b'"' => {
+                                in_quote = !in_quote;
+                            }
+                            b'=' if !in_quote => {
+                                while matches!(iter.peek(), Some(&(_, &ch)) if ch.is_ascii_whitespace())
+                                {
+                                    iter.next();
+                                }
+
+                                if shift != 0 {
+                                    attributes.push((key, None));
+                                    key = 0;
+                                    shift = 0;
+                                }
+
+                                let mut value = vec![];
+
+                                for (_, &ch) in iter.by_ref() {
+                                    match ch {
+                                        b'>' if !in_quote => {
+                                            if !value.is_empty() {
+                                                let value =
+                                                    String::from_utf8(value).unwrap_or_default();
+                                                if let Some((_, v)) = attributes.last_mut() {
+                                                    *v = value.into();
+                                                } else {
+                                                    // Broken attribute
+                                                    attributes.push((0, Some(value)));
+                                                }
+                                            }
+                                            break 'outer;
+                                        }
+                                        b'"' => {
+                                            if in_quote {
+                                                in_quote = false;
+                                                break;
+                                            } else {
+                                                in_quote = true;
+                                            }
+                                        }
+                                        b' ' | b'\t' | b'\r' | b'\n' if !in_quote => {
+                                            break;
+                                        }
+                                        _ => {
+                                            value.push(ch);
+                                        }
+                                    }
+                                }
+
+                                if !value.is_empty() {
+                                    let value = String::from_utf8(value).unwrap_or_default();
+                                    if let Some((_, v)) = attributes.last_mut() {
+                                        *v = value.into();
+                                    } else {
+                                        // Broken attribute
+                                        attributes.push((0, Some(value)));
+                                    }
+                                }
+                            }
+                            b' ' | b'\t' | b'\r' | b'\n' if shift != 0 => {
+                                if tag == 0 {
+                                    tag = key;
+                                } else {
+                                    attributes.push((key, None));
+                                }
+                                key = 0;
+                                shift = 0;
+                            }
+                            _ => {}
+                        }
+                    }
+
+                    if tag != 0 {
+                        if is_end_tag {
+                            tags.push(HtmlToken::EndTag { name: tag });
+                        } else {
+                            tags.push(HtmlToken::StartTag {
+                                name: tag,
+                                attributes,
+                                is_self_closing,
+                            });
+                        }
+                    }
+                }
+                continue;
+            }
+            b' ' | b'\t' | b'\r' | b'\n' => {
+                if !is_token_start {
+                    add_html_token(
+                        &mut text,
+                        &input[token_start..token_end + 1],
+                        is_after_space && !is_new_line,
+                    );
+                    is_new_line = false;
+                }
+                is_after_space = true;
+                is_token_start = true;
+                continue;
+            }
+            b'&' if !is_token_start => {
+                add_html_token(
+                    &mut text,
+                    &input[token_start..token_end + 1],
+                    is_after_space && !is_new_line,
+                );
+                is_new_line = false;
+                is_token_start = true;
+                is_after_space = false;
+            }
+            b';' if !is_token_start => {
+                add_html_token(
+                    &mut text,
+                    &input[token_start..pos + 1],
+                    is_after_space && !is_new_line,
+                );
+                is_token_start = true;
+                is_after_space = false;
+                is_new_line = false;
+                continue;
+            }
+            _ => (),
+        }
+
+        if is_token_start {
+            token_start = pos;
+            is_token_start = false;
+        }
+        token_end = pos;
+    }
+
+    if !is_token_start {
+        add_html_token(
+            &mut text,
+            &input[token_start..token_end + 1],
+            is_after_space && !is_new_line,
+        );
+    }
+    if !text.is_empty() {
+        tags.push(HtmlToken::Text {
+            text: text.as_str().into(),
+        });
+    }
+
+    tags
+}
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_html_to_tokens_text() {
+        let input = "Hello, world!";
+        let tokens = html_to_tokens(input);
+        assert_eq!(
+            tokens,
+            vec![HtmlToken::Text {
+                text: "Hello, world!".into()
+            }]
+        );
+    }
+
+    #[test]
+    fn test_html_to_tokens_start_tag() {
+        let input = "<div>";
+        let tokens = html_to_tokens(input);
+        assert_eq!(
+            tokens,
+            vec![HtmlToken::StartTag {
+                name: 7760228,
+                attributes: vec![],
+                is_self_closing: false
+            }]
+        );
+    }
+
+    #[test]
+    fn test_html_to_tokens_end_tag() {
+        let input = "</div>";
+        let tokens = html_to_tokens(input);
+        assert_eq!(tokens, vec![HtmlToken::EndTag { name: 7760228 }]);
+    }
+
+    #[test]
+    fn test_html_to_tokens_comment() {
+        let input = "<!-- This is a comment -->";
+        let tokens = html_to_tokens(input);
+        assert_eq!(
+            tokens,
+            vec![HtmlToken::Comment {
+                text: "!-- This is a comment --".into()
+            }]
+        );
+    }
+
+    #[test]
+    fn test_html_to_tokens_mixed() {
+        let input = "<div>Hello, <span>&quot; world &quot; </span>!</div>";
+        let tokens = html_to_tokens(input);
+        assert_eq!(
+            tokens,
+            vec![
+                HtmlToken::StartTag {
+                    name: 7760228,
+                    attributes: vec![],
+                    is_self_closing: false
+                },
+                HtmlToken::Text {
+                    text: "Hello,".into()
+                },
+                HtmlToken::StartTag {
+                    name: 1851879539,
+                    attributes: vec![],
+                    is_self_closing: false
+                },
+                HtmlToken::Text {
+                    text: " \" world \"".into()
+                },
+                HtmlToken::EndTag { name: 1851879539 },
+                HtmlToken::Text { text: " !".into() },
+                HtmlToken::EndTag { name: 7760228 }
+            ]
+        );
+    }
+
+    #[test]
+    fn test_html_to_tokens_with_attributes() {
+        let input = r#"<input type="text" value="test"><single/><one attr/><a b=1 b c="123">"#;
+        let tokens = html_to_tokens(input);
+        assert_eq!(
+            tokens,
+            vec![
+                HtmlToken::StartTag {
+                    name: 500186508905,
+                    attributes: vec![
+                        (1701869940, Some("text".into())),
+                        (435761734006, Some("test".into()))
+                    ],
+                    is_self_closing: false
+                },
+                HtmlToken::StartTag {
+                    name: 111516266162547,
+                    attributes: vec![],
+                    is_self_closing: true
+                },
+                HtmlToken::StartTag {
+                    name: 6647407,
+                    attributes: vec![(1920234593, None)],
+                    is_self_closing: true
+                },
+                HtmlToken::StartTag {
+                    name: 97,
+                    attributes: vec![(98, Some("1".into())), (98, None), (99, Some("123".into()))],
+                    is_self_closing: false
+                }
+            ]
+        );
+    }
+}

crates/spam-filter/src/modules/mod.rs

@@ -0,0 +1,12 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
+ */
+
+pub mod classifier;
+pub mod dnsbl;
+pub mod expression;
+pub mod html;
+pub mod pyzor;
+pub mod sanitize;

crates/spam-filter/src/modules/pyzor.rs

@@ -0,0 +1,789 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
+ */
+
+use common::config::mailstore::spamfilter::PyzorConfig;
+use mail_parser::{Message, PartType, decoders::html::add_html_token};
+use nlp::tokenizers::types::{TokenType, TypesTokenizer};
+use sha1::{Digest, Sha1};
+use std::{
+    borrow::Cow,
+    net::SocketAddr,
+    time::{Duration, SystemTime},
+};
+use tokio::net::UdpSocket;
+use utils::HexEncode;
+
+const MIN_LINE_LENGTH: usize = 8;
+const ATOMIC_NUM_LINES: usize = 4;
+const DIGEST_SPEC: &[(usize, usize)] = &[(20, 3), (60, 3)];
+
+#[derive(Default, Debug, PartialEq, Eq)]
+pub(crate) struct PyzorResponse {
+    pub code: u32,
+    pub count: u64,
+    pub wl_count: u64,
+}
+
+pub(crate) async fn pyzor_check(
+    message: &Message<'_>,
+    config: &PyzorConfig,
+) -> trc::Result<Option<PyzorResponse>> {
+    // Make sure there is at least one text part
+    if !message
+        .parts
+        .iter()
+        .any(|p| matches!(p.body, PartType::Text(_) | PartType::Html(_)))
+    {
+        return Ok(None);
+    }
+
+    // Hash message
+    let request = message.pyzor_check_message();
+
+    #[cfg(feature = "test_mode")]
+    {
+        if request.contains("b5b476f0b5ba6e1c038361d3ded5818dd39c90a2") {
+            return Ok(PyzorResponse {
+                code: 200,
+                count: 1000,
+                wl_count: 0,
+            }
+            .into());
+        } else if request.contains("d67d4b8bfc3860449e3418bb6017e2612f3e2a99") {
+            return Ok(PyzorResponse {
+                code: 200,
+                count: 60,
+                wl_count: 10,
+            }
+            .into());
+        } else if request.contains("81763547012b75e57a20d18ce0b93014208cdfdb") {
+            return Ok(PyzorResponse {
+                code: 200,
+                count: 50,
+                wl_count: 20,
+            }
+            .into());
+        }
+    }
+
+    // Send message to address
+    pyzor_send_message(config.address, config.timeout, &request)
+        .await
+        .map(Into::into)
+        .map_err(|err| {
+            trc::SpamEvent::PyzorError
+                .into_err()
+                .ctx(trc::Key::Url, config.address.to_string())
+                .reason(err)
+                .details("Pyzor failed")
+        })
+}
+
+async fn pyzor_send_message(
+    addr: SocketAddr,
+    timeout: Duration,
+    message: &str,
+) -> std::io::Result<PyzorResponse> {
+    let socket = UdpSocket::bind("0.0.0.0:0").await?;
+    tokio::time::timeout(timeout, socket.send_to(message.as_bytes(), addr)).await??;
+
+    let mut buffer = vec![0u8; 1024];
+    let (size, _) = tokio::time::timeout(timeout, socket.recv_from(&mut buffer)).await??;
+
+    let raw_response = std::str::from_utf8(&buffer[..size])
+        .map_err(|err| std::io::Error::new(std::io::ErrorKind::InvalidData, err))?;
+    let mut response = PyzorResponse {
+        code: u32::MAX,
+        count: u64::MAX,
+        wl_count: u64::MAX,
+    };
+
+    for line in raw_response.lines() {
+        if let Some((k, v)) = line.split_once(':') {
+            if k.eq_ignore_ascii_case("code") {
+                response.code = v.trim().parse().map_err(|_| {
+                    std::io::Error::new(
+                        std::io::ErrorKind::InvalidData,
+                        format!("Invalid line: {raw_response}"),
+                    )
+                })?;
+            } else if k.eq_ignore_ascii_case("count") {
+                response.count = v.trim().parse().map_err(|_| {
+                    std::io::Error::new(
+                        std::io::ErrorKind::InvalidData,
+                        format!("Invalid line: {raw_response}"),
+                    )
+                })?;
+            } else if k.eq_ignore_ascii_case("wl-count") {
+                response.wl_count = v.trim().parse().map_err(|_| {
+                    std::io::Error::new(
+                        std::io::ErrorKind::InvalidData,
+                        format!("Invalid line: {raw_response}"),
+                    )
+                })?;
+            }
+        }
+    }
+
+    if response.code != u32::MAX && response.count != u64::MAX && response.wl_count != u64::MAX {
+        Ok(response)
+    } else {
+        Err(std::io::Error::new(
+            std::io::ErrorKind::InvalidData,
+            format!("Invalid response: {raw_response}"),
+        ))
+    }
+}
+
+trait PyzorWrite {
+    fn write_all(&mut self, data: &[u8]);
+}
+
+impl PyzorWrite for Vec<u8> {
+    fn write_all(&mut self, data: &[u8]) {
+        self.extend_from_slice(data);
+    }
+}
+
+impl PyzorWrite for Sha1 {
+    fn write_all(&mut self, data: &[u8]) {
+        self.update(data);
+    }
+}
+
+trait PyzorDigest<W: PyzorWrite> {
+    fn pyzor_digest(&self, writer: W) -> W;
+}
+
+pub trait PyzorCheck {
+    fn pyzor_check_message(&self) -> String;
+}
+
+impl<W: PyzorWrite> PyzorDigest<W> for Message<'_> {
+    fn pyzor_digest(&self, writer: W) -> W {
+        let parts = self
+            .parts
+            .iter()
+            .filter_map(|part| match &part.body {
+                PartType::Text(text) => Some(text.as_ref().into()),
+                PartType::Html(html) => Some(html_to_text(html.as_ref()).into()),
+                _ => None,
+            })
+            .collect::<Vec<Cow<str>>>();
+
+        pyzor_digest(writer, parts.iter().flat_map(|text| text.lines()))
+    }
+}
+
+impl PyzorCheck for Message<'_> {
+    fn pyzor_check_message(&self) -> String {
+        let time = SystemTime::now()
+            .duration_since(SystemTime::UNIX_EPOCH)
+            .map_or(0, |d| d.as_secs());
+
+        pyzor_create_message(
+            self,
+            time,
+            (time & 0xFFFF) as u16 ^ ((time >> 16) & 0xFFFF) as u16,
+        )
+    }
+}
+
+fn pyzor_create_message(message: &Message<'_>, time: u64, thread: u16) -> String {
+    // Hash message
+    let hash = message.pyzor_digest(Sha1::new()).finalize().hex_encode();
+    // Hash key
+    let mut hash_key = Sha1::new();
+    hash_key.update("anonymous:".as_bytes());
+    let hash_key = hash_key.finalize().hex_encode();
+
+    // Hash message
+    let message = format!(
+        "Op: check\nOp-Digest: {hash}\nThread: {thread}\nPV: 2.1\nUser: anonymous\nTime: {time}"
+    );
+    let mut msg_hash = Sha1::new();
+    msg_hash.update(message.as_bytes());
+    let msg_hash = msg_hash.finalize();
+
+    // Sign
+    let mut sig = Sha1::new();
+    sig.update(msg_hash);
+    sig.update(format!(":{time}:{hash_key}"));
+    let sig = sig.finalize().hex_encode();
+
+    format!("{message}\nSig: {sig}\n")
+}
+
+fn pyzor_digest<'x, I, W>(mut writer: W, lines: I) -> W
+where
+    I: Iterator<Item = &'x str>,
+    W: PyzorWrite,
+{
+    let mut result = Vec::with_capacity(16);
+
+    for line in lines {
+        let mut clean_line = String::with_capacity(line.len());
+        let mut token_start = usize::MAX;
+        let mut token_end = usize::MAX;
+
+        let add_line = |line: &mut String, span: &str| {
+            if !span.contains(char::from(0)) {
+                if span.len() < 10 {
+                    line.push_str(span);
+                }
+            } else {
+                let span = span.replace(char::from(0), "");
+                if span.len() < 10 {
+                    line.push_str(&span);
+                }
+            }
+        };
+
+        for token in TypesTokenizer::new(line) {
+            match token.word {
+                TokenType::Alphabetic(_)
+                | TokenType::Alphanumeric(_)
+                | TokenType::Integer(_)
+                | TokenType::Float(_)
+                | TokenType::Other(_)
+                | TokenType::Punctuation(_) => {
+                    if token_start == usize::MAX {
+                        token_start = token.from;
+                    }
+                    token_end = token.to;
+                }
+                TokenType::Space
+                | TokenType::Url(_)
+                | TokenType::UrlNoScheme(_)
+                | TokenType::UrlNoHost(_)
+                | TokenType::IpAddr(_)
+                | TokenType::Email(_) => {
+                    if token_start != usize::MAX {
+                        add_line(&mut clean_line, &line[token_start..token_end]);
+                        token_start = usize::MAX;
+                        token_end = usize::MAX;
+                    }
+                }
+            }
+        }
+
+        if token_start != usize::MAX {
+            add_line(&mut clean_line, &line[token_start..token_end]);
+        }
+
+        if clean_line.len() >= MIN_LINE_LENGTH {
+            result.push(clean_line);
+        }
+    }
+
+    if result.len() > ATOMIC_NUM_LINES {
+        for (offset, length) in DIGEST_SPEC {
+            for i in 0..*length {
+                if let Some(line) = result.get((*offset * result.len() / 100) + i) {
+                    writer.write_all(line.as_bytes());
+                }
+            }
+        }
+    } else {
+        for line in result {
+            writer.write_all(line.as_bytes());
+        }
+    }
+
+    writer
+}
+
+fn html_to_text(input: &str) -> String {
+    let mut result = String::with_capacity(input.len());
+    let input = input.as_bytes();
+
+    let mut in_tag = false;
+    let mut in_comment = false;
+    let mut in_style = false;
+    let mut in_script = false;
+
+    let mut is_token_start = true;
+    let mut is_after_space = false;
+    let mut is_tag_close = false;
+
+    let mut token_start = 0;
+    let mut token_end = 0;
+
+    let mut tag_token_pos = 0;
+    let mut comment_pos = 0;
+
+    for (pos, ch) in input.iter().enumerate() {
+        if !in_comment {
+            match ch {
+                b'<' => {
+                    if !(in_tag || in_style || in_script || is_token_start) {
+                        add_html_token(
+                            &mut result,
+                            &input[token_start..token_end + 1],
+                            is_after_space,
+                        );
+                        is_after_space = false;
+                    }
+
+                    tag_token_pos = 0;
+                    in_tag = true;
+                    is_token_start = true;
+                    is_tag_close = false;
+                    continue;
+                }
+                b'>' if in_tag => {
+                    if tag_token_pos == 1
+                        && let Some(tag) = input.get(token_start..token_end + 1)
+                    {
+                        if tag.eq_ignore_ascii_case(b"style") {
+                            in_style = !is_tag_close;
+                        } else if tag.eq_ignore_ascii_case(b"script") {
+                            in_script = !is_tag_close;
+                        }
+                    }
+
+                    in_tag = false;
+                    is_token_start = true;
+                    is_after_space = !result.is_empty();
+
+                    continue;
+                }
+                b'/' if in_tag => {
+                    if tag_token_pos == 0 {
+                        is_tag_close = true;
+                    }
+                    continue;
+                }
+                b'!' if in_tag && tag_token_pos == 0 => {
+                    if let Some(b"--") = input.get(pos + 1..pos + 3) {
+                        in_comment = true;
+                        continue;
+                    }
+                }
+                b' ' | b'\t' | b'\r' | b'\n' => {
+                    if !(in_tag || in_style || in_script) {
+                        if !is_token_start {
+                            add_html_token(
+                                &mut result,
+                                &input[token_start..token_end + 1],
+                                is_after_space,
+                            );
+                        }
+                        is_after_space = true;
+                    }
+
+                    is_token_start = true;
+                    continue;
+                }
+                b'&' if !(in_tag || is_token_start || in_style || in_script) => {
+                    add_html_token(
+                        &mut result,
+                        &input[token_start..token_end + 1],
+                        is_after_space,
+                    );
+                    is_token_start = true;
+                    is_after_space = false;
+                }
+                b';' if !(in_tag || is_token_start || in_style || in_script) => {
+                    add_html_token(&mut result, &input[token_start..pos + 1], is_after_space);
+                    is_token_start = true;
+                    is_after_space = false;
+                    continue;
+                }
+                _ => (),
+            }
+            if is_token_start {
+                token_start = pos;
+                is_token_start = false;
+                if in_tag {
+                    tag_token_pos += 1;
+                }
+            }
+            token_end = pos;
+        } else {
+            match ch {
+                b'-' => comment_pos += 1,
+                b'>' if comment_pos == 2 => {
+                    comment_pos = 0;
+                    in_comment = false;
+                    in_tag = false;
+                    is_token_start = true;
+                }
+                _ => comment_pos = 0,
+            }
+        }
+    }
+
+    if !(in_tag || is_token_start || in_style || in_script) {
+        add_html_token(
+            &mut result,
+            &input[token_start..token_end + 1],
+            is_after_space,
+        );
+    }
+
+    result.shrink_to_fit();
+    result
+}
+
+#[cfg(test)]
+mod test {
+    use std::time::Duration;
+
+    use mail_parser::MessageParser;
+    use sha1::Digest;
+    use sha1::Sha1;
+    use utils::HexEncode;
+
+    use super::pyzor_create_message;
+    use super::pyzor_send_message;
+    use super::{PyzorDigest, html_to_text, pyzor_digest};
+
+    use super::PyzorResponse;
+
+    #[ignore]
+    #[tokio::test]
+    async fn send_message() {
+        assert_eq!(
+            pyzor_send_message(
+                "public.pyzor.org:24441".parse().unwrap(),
+                Duration::from_secs(10),
+                concat!(
+                    "Op: check\n",
+                    "Op-Digest: b2c27325a034c581df0c9ef37e4a0d63208a3e7e\n",
+                    "Thread: 49005\n",
+                    "PV: 2.1\n",
+                    "User: anonymous\n",
+                    "Time: 1697468672\n",
+                    "Sig: 9cf4571b85d3887fdd0d4f444fd0c164e0290722\n"
+                ),
+            )
+            .await
+            .unwrap(),
+            PyzorResponse {
+                code: 200,
+                count: 0,
+                wl_count: 0
+            }
+        );
+    }
+
+    #[test]
+    fn message_pyzor() {
+        let message = pyzor_create_message(
+            &MessageParser::new().parse(HTML_TEXT_STYLE_SCRIPT).unwrap(),
+            1697468672,
+            49005,
+        );
+
+        assert_eq!(
+            message,
+            concat!(
+                "Op: check\n",
+                "Op-Digest: b2c27325a034c581df0c9ef37e4a0d63208a3e7e\n",
+                "Thread: 49005\n",
+                "PV: 2.1\n",
+                "User: anonymous\n",
+                "Time: 1697468672\n",
+                "Sig: 9cf4571b85d3887fdd0d4f444fd0c164e0290722\n"
+            )
+        );
+    }
+
+    #[test]
+    fn digest_pyzor() {
+        // HTML stripping
+        assert_eq!(html_to_text(HTML_RAW), HTML_RAW_STRIPED);
+
+        // Token stripping
+        for strip_me in [
+            "[email protected]",
+            "[email protected]",
+            "[email protected]",
+            "[email protected]",
+            "0A2D3f%a#S",
+            "3sddkf9jdkd9",
+            "@@#@@@@@@@@@",
+            "http://spammer.com/special-offers?buy=now",
+        ] {
+            assert_eq!(
+                String::from_utf8(pyzor_digest(
+                    Vec::new(),
+                    format!("Test {strip_me} Test2").lines(),
+                ))
+                .unwrap(),
+                "TestTest2"
+            );
+        }
+
+        // Test short lines
+        assert_eq!(
+            String::from_utf8(pyzor_digest(
+                Vec::new(),
+                concat!("This line is included\n", "not this\n", "This also").lines(),
+            ))
+            .unwrap(),
+            "ThislineisincludedThisalso"
+        );
+
+        // Test atomic
+        assert_eq!(
+            String::from_utf8(pyzor_digest(
+                Vec::new(),
+                "All this message\nShould be included\nIn the digest".lines(),
+            ))
+            .unwrap(),
+            "AllthismessageShouldbeincludedInthedigest"
+        );
+
+        // Test spec
+        let mut text = String::new();
+        for i in 0..100 {
+            text += format!("Line{i} test test test\n").as_str();
+        }
+        let mut expected = String::new();
+        for i in [20, 21, 22, 60, 61, 62] {
+            expected += format!("Line{i}testtesttest").as_str();
+        }
+        assert_eq!(
+            String::from_utf8(pyzor_digest(Vec::new(), text.lines(),)).unwrap(),
+            expected
+        );
+
+        // Test email parsing
+        for (input, expected) in [
+            (
+                HTML_TEXT,
+                concat!(
+                    "Emailspam,alsoknownasjunkemailorbulkemail,isasubset",
+                    "ofspaminvolvingnearlyidenticalmessagessenttonumerous",
+                    "byemail.Clickingonlinksinspamemailmaysendusersto",
+                    "byemail.Clickingonlinksinspamemailmaysendusersto",
+                    "phishingwebsitesorsitesthatarehostingmalware.",
+                    "Emailspam.Emailspam,alsoknownasjunkemailorbulkemail,",
+                    "isasubsetofspaminvolvingnearlyidenticalmessage",
+                    "ssenttonumerousbyemail.Clickingonlinksinspamemailmaysenduse",
+                    "rstophishingwebsitesorsitesthatarehostingmalware."
+                ),
+            ),
+            (HTML_TEXT_STYLE_SCRIPT, "Thisisatest.Thisisatest."),
+            (TEXT_ATTACHMENT, "Thisisatestmailing"),
+            (TEXT_ATTACHMENT_W_NULL, "Thisisatestmailing"),
+            (TEXT_ATTACHMENT_W_MULTIPLE_NULLS, "Thisisatestmailing"),
+            (TEXT_ATTACHMENT_W_SUBJECT_NULL, "Thisisatestmailing"),
+            (TEXT_ATTACHMENT_W_CONTENTTYPE_NULL, "Thisisatestmailing"),
+        ] {
+            assert_eq!(
+                String::from_utf8(
+                    MessageParser::new()
+                        .parse(input)
+                        .unwrap()
+                        .pyzor_digest(Vec::new(),)
+                )
+                .unwrap(),
+                expected,
+                "failed for {input}"
+            )
+        }
+
+        // Test SHA hash
+        assert_eq!(
+            MessageParser::new()
+                .parse(HTML_TEXT_STYLE_SCRIPT)
+                .unwrap()
+                .pyzor_digest(Sha1::new(),)
+                .finalize()
+                .hex_encode(),
+            "b2c27325a034c581df0c9ef37e4a0d63208a3e7e",
+        )
+    }
+
+    const HTML_TEXT: &str = r#"MIME-Version: 1.0
+Sender: [email protected]
+Received: by 10.216.157.70 with HTTP; Thu, 16 Jan 2014 00:43:31 -0800 (PST)
+Date: Thu, 16 Jan 2014 10:43:31 +0200
+Delivered-To: [email protected]
+X-Google-Sender-Auth: ybCmONS9U9D6ZUfjx-9_tY-hF2Q
+Message-ID: <CAK-mJS8sE-V6qtspzzZ+bZ1eSUE_FNMt3K-5kBOG-z3NMgU_Rg@mail.gmail.com>
+Subject: Test
+From: Alexandru Chirila <[email protected]>
+To: Alexandru Chirila <[email protected]>
+Content-Type: multipart/alternative; boundary=001a11c25ff293069304f0126bfd
+
+--001a11c25ff293069304f0126bfd
+Content-Type: text/plain; charset=ISO-8859-1
+
+Email spam.
+
+Email spam, also known as junk email or unsolicited bulk email, is a subset
+of electronic spam involving nearly identical messages sent to numerous
+recipients by email. Clicking on links in spam email may send users to
+phishing web sites or sites that are hosting malware.
+
+--001a11c25ff293069304f0126bfd
+Content-Type: text/html; charset=ISO-8859-1
+Content-Transfer-Encoding: quoted-printable
+
+<div dir=3D"ltr"><div>Email spam.</div><div><br></div><div>Email spam, also=
+ known as junk email or unsolicited bulk email, is a subset of electronic s=
+pam involving nearly identical messages sent to numerous recipients by emai=
+l. Clicking on links in spam email may send users to phishing web sites or =
+sites that are hosting malware.</div>
+</div>
+
+--001a11c25ff293069304f0126bfd--
+"#;
+
+    const HTML_TEXT_STYLE_SCRIPT: &str = r#"MIME-Version: 1.0
+Sender: [email protected]
+Received: by 10.216.157.70 with HTTP; Thu, 16 Jan 2014 00:43:31 -0800 (PST)
+Date: Thu, 16 Jan 2014 10:43:31 +0200
+Delivered-To: [email protected]
+X-Google-Sender-Auth: ybCmONS9U9D6ZUfjx-9_tY-hF2Q
+Message-ID: <CAK-mJS8sE-V6qtspzzZ+bZ1eSUE_FNMt3K-5kBOG-z3NMgU_Rg@mail.gmail.com>
+Subject: Test
+From: Alexandru Chirila <[email protected]>
+To: Alexandru Chirila <[email protected]>
+Content-Type: multipart/alternative; boundary=001a11c25ff293069304f0126bfd
+
+--001a11c25ff293069304f0126bfd
+Content-Type: text/plain; charset=ISO-8859-1
+
+This is a test.
+
+--001a11c25ff293069304f0126bfd
+Content-Type: text/html; charset=ISO-8859-1
+Content-Transfer-Encoding: quoted-printable
+
+<div dir=3D"ltr">
+<style> This is my style.</style>
+<script> This is my script.</script>
+<div>This is a test.</div>
+</div>
+
+--001a11c25ff293069304f0126bfd--
+"#;
+
+    const TEXT_ATTACHMENT: &str = r#"MIME-Version: 1.0
+Received: by 10.76.127.40 with HTTP; Fri, 17 Jan 2014 02:21:43 -0800 (PST)
+Date: Fri, 17 Jan 2014 12:21:43 +0200
+Delivered-To: [email protected]
+Message-ID: <CALTHOsuHFaaatiXJKU=LdDCo4NmD_h49yvG2RDsWw17D0-NXJg@mail.gmail.com>
+Subject: Test
+From: Alexandru Chirila <[email protected]>
+To: Alexandru Chirila <[email protected]>
+Content-Type: multipart/mixed; boundary=f46d040a62c49bb1c804f027e8cc
+
+--f46d040a62c49bb1c804f027e8cc
+Content-Type: multipart/alternative; boundary=f46d040a62c49bb1c404f027e8ca
+
+--f46d040a62c49bb1c404f027e8ca
+Content-Type: text/plain; charset=ISO-8859-1
+
+This is a test mailing
+
+--f46d040a62c49bb1c404f027e8ca--
+--f46d040a62c49bb1c804f027e8cc
+Content-Type: image/png; name="tar.png"
+Content-Disposition: attachment; filename="tar.png"
+Content-Transfer-Encoding: base64
+X-Attachment-Id: f_hqjas5ad0
+
+iVBORw0KGgoAAAANSUhEUgAAAskAAADlCAAAAACErzVVAAAACXBIWXMAAAsTAAALEwEAmpwYAAAD
+QmCC
+--f46d040a62c49bb1c804f027e8cc--"#;
+
+    const TEXT_ATTACHMENT_W_NULL: &str = "MIME-Version: 1.0
+Received: by 10.76.127.40 with HTTP; Fri, 17 Jan 2014 02:21:43 -0800 (PST)
+Date: Fri, 17 Jan 2014 12:21:43 +0200
+Delivered-To: [email protected]
+Message-ID: <CALTHOsuHFaaatiXJKU=LdDCo4NmD_h49yvG2RDsWw17D0-NXJg@mail.gmail.com>
+Subject: Test
+From: Alexandru Chirila <[email protected]>
+To: Alexandru Chirila <[email protected]>
+Content-Type: multipart/mixed; boundary=f46d040a62c49bb1c804f027e8cc
+
+--f46d040a62c49bb1c804f027e8cc
+Content-Type: multipart/alternative; boundary=f46d040a62c49bb1c404f027e8ca
+
+--f46d040a62c49bb1c404f027e8ca
+Content-Type: text/plain; charset=ISO-8859-1
+
+This is a test ma\0iling
+--f46d040a62c49bb1c804f027e8cc--";
+
+    const TEXT_ATTACHMENT_W_MULTIPLE_NULLS: &str = "MIME-Version: 1.0
+Received: by 10.76.127.40 with HTTP; Fri, 17 Jan 2014 02:21:43 -0800 (PST)
+Date: Fri, 17 Jan 2014 12:21:43 +0200
+Delivered-To: [email protected]
+Message-ID: <CALTHOsuHFaaatiXJKU=LdDCo4NmD_h49yvG2RDsWw17D0-NXJg@mail.gmail.com>
+Subject: Test
+From: Alexandru Chirila <[email protected]>
+To: Alexandru Chirila <[email protected]>
+Content-Type: multipart/mixed; boundary=f46d040a62c49bb1c804f027e8cc
+
+--f46d040a62c49bb1c804f027e8cc
+Content-Type: multipart/alternative; boundary=f46d040a62c49bb1c404f027e8ca
+
+--f46d040a62c49bb1c404f027e8ca
+Content-Type: text/plain; charset=ISO-8859-1
+
+This is a test ma\0\0\0iling
+--f46d040a62c49bb1c804f027e8cc--";
+
+    const TEXT_ATTACHMENT_W_SUBJECT_NULL: &str = "MIME-Version: 1.0
+Received: by 10.76.127.40 with HTTP; Fri, 17 Jan 2014 02:21:43 -0800 (PST)
+Date: Fri, 17 Jan 2014 12:21:43 +0200
+Delivered-To: [email protected]
+Message-ID: <CALTHOsuHFaaatiXJKU=LdDCo4NmD_h49yvG2RDsWw17D0-NXJg@mail.gmail.com>
+Subject: Te\0\0\0st
+From: Alexandru Chirila <[email protected]>
+To: Alexandru Chirila <[email protected]>
+Content-Type: multipart/mixed; boundary=f46d040a62c49bb1c804f027e8cc
+
+--f46d040a62c49bb1c804f027e8cc
+Content-Type: multipart/alternative; boundary=f46d040a62c49bb1c404f027e8ca
+
+--f46d040a62c49bb1c404f027e8ca
+Content-Type: text/plain; charset=ISO-8859-1
+
+This is a test mailing
+--f46d040a62c49bb1c804f027e8cc--";
+
+    const TEXT_ATTACHMENT_W_CONTENTTYPE_NULL: &str = "MIME-Version: 1.0
+Received: by 10.76.127.40 with HTTP; Fri, 17 Jan 2014 02:21:43 -0800 (PST)
+Date: Fri, 17 Jan 2014 12:21:43 +0200
+Delivered-To: [email protected]
+Message-ID: <CALTHOsuHFaaatiXJKU=LdDCo4NmD_h49yvG2RDsWw17D0-NXJg@mail.gmail.com>
+Subject: Test
+From: Alexandru Chirila <[email protected]>
+To: Alexandru Chirila <[email protected]>
+Content-Type: multipart/mixed; boundary=f46d040a62c49bb1c804f027e8cc
+
+--f46d040a62c49bb1c804f027e8cc
+Content-Type: multipart/alternative; boundary=f46d040a62c49bb1c404f027e8ca
+
+--f46d040a62c49bb1c404f027e8ca
+Content-Type: text/plain; charset=\"iso-8859-1\0\0\0\"
+
+This is a test mailing
+--f46d040a62c49bb1c804f027e8cc--";
+
+    const HTML_RAW: &str = r#"<html><head><title>Email spam</title></head><body>
+<p><b>Email spam</b>, also known as <b>junk email</b> 
+or <b>unsolicited bulk email</b> (<i>UBE</i>), is a subset of 
+<a href="/wiki/Spam_(electronic)" title="Spam (electronic)">electronic spam</a> 
+involving nearly identical messages sent to numerous recipients by <a href="/wiki/Email" title="Email">
+email</a>. Clicking on <a href="/wiki/Html_email#Security_vulnerabilities" title="Html email" class="mw-redirect">
+links in spam email</a> may send users to <a href="/wiki/Phishing" title="Phishing">phishing</a> 
+web sites or sites that are hosting <a href="/wiki/Malware" title="Malware">malware</a>.</body></html>"#;
+
+    const HTML_RAW_STRIPED: &str = concat!(
+        "Email spam Email spam , also known as junk email or unsolicited bulk email ( UBE ),",
+        " is a subset of electronic spam involving nearly identical messages sent to numerous recipients by email",
+        " . Clicking on links in spam email may send users to phishing web sites or sites that are hosting malware ."
+    );
+}

crates/spam-filter/src/modules/sanitize.rs

@@ -0,0 +1,133 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
+ */
+
+use std::net::IpAddr;
+
+use crate::{Email, Hostname};
+
+impl Hostname {
+    pub fn new(host: &str) -> Self {
+        let mut fqdn = host.trim_end_matches('.').to_lowercase();
+
+        // Decode punycode
+        if fqdn.contains("xn--") {
+            let mut decoded = String::with_capacity(fqdn.len());
+
+            for part in fqdn.split('.') {
+                if !decoded.is_empty() {
+                    decoded.push('.');
+                }
+
+                if let Some(puny) = part
+                    .strip_prefix("xn--")
+                    .and_then(idna::punycode::decode_to_string)
+                    .filter(|puny| {
+                        idna::domain_to_ascii(puny).is_ok_and(|reencoded| reencoded == part)
+                    })
+                {
+                    decoded.push_str(&puny);
+                } else {
+                    decoded.push_str(part);
+                }
+            }
+
+            fqdn = decoded;
+        }
+
+        let ip = fqdn
+            .strip_prefix('[')
+            .and_then(|ip| ip.strip_suffix(']'))
+            .unwrap_or(&fqdn)
+            .parse::<IpAddr>()
+            .ok();
+
+        Hostname {
+            sld: if ip.is_none() {
+                psl::domain(fqdn.as_bytes()).and_then(|domain| {
+                    if domain.suffix().typ().is_some() {
+                        std::str::from_utf8(domain.as_bytes()).ok().map(Into::into)
+                    } else {
+                        None
+                    }
+                })
+            } else {
+                None
+            },
+            ip,
+            fqdn,
+        }
+    }
+}
+
+impl Email {
+    pub fn new(address: &str) -> Self {
+        let address = address.to_lowercase();
+        let (local_part, domain) = address.rsplit_once('@').unwrap_or((address.as_str(), ""));
+
+        Email {
+            local_part: local_part.into(),
+            domain_part: Hostname::new(domain),
+            address,
+        }
+    }
+}
+
+impl Hostname {
+    pub fn sld_or_default(&self) -> &str {
+        self.sld.as_deref().unwrap_or(self.fqdn.as_str())
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use crate::{Email, Hostname};
+
+    #[test]
+    fn hostname_punycode_round_trip() {
+        for (host, fqdn, sld) in [
+            ("mail.example.com", "mail.example.com", Some("example.com")),
+            (
+                "MAIL.Example.CO.UK.",
+                "mail.example.co.uk",
+                Some("example.co.uk"),
+            ),
+            (
+                "mail.xn--eebajf.xn--9dbq2a",
+                "mail.\u{5de}\u{5d9}\u{5d9}\u{5dc}.\u{5e7}\u{5d5}\u{5dd}",
+                Some("\u{5de}\u{5d9}\u{5d9}\u{5dc}.\u{5e7}\u{5d5}\u{5dd}"),
+            ),
+            ("xn--gmail-.com", "xn--gmail-.com", Some("xn--gmail-.com")),
+            (
+                "xn--example-.org",
+                "xn--example-.org",
+                Some("xn--example-.org"),
+            ),
+            ("xn--.com", "xn--.com", Some("xn--.com")),
+            ("127.0.0.1", "127.0.0.1", None),
+        ] {
+            let parsed = Hostname::new(host);
+            assert_eq!(parsed.fqdn, fqdn, "fqdn of {host:?}");
+            assert_eq!(parsed.sld.as_deref(), sld, "sld of {host:?}");
+        }
+    }
+
+    #[test]
+    fn email_a_label_and_u_label_are_equal() {
+        assert_eq!(
+            Email::new("[email protected]"),
+            Email::new("bill@\u{5de}\u{5d9}\u{5d9}\u{5dc}.\u{5e7}\u{5d5}\u{5dd}")
+        );
+        assert_ne!(
+            Email::new("[email protected]"),
+            Email::new("[email protected]")
+        );
+        assert_ne!(Email::new("postmaster"), Email::new("mailer-daemon"));
+        assert_ne!(
+            Email::new("[email protected]"),
+            Email::new("[email protected]")
+        );
+    }
+}