Import upstream v0.16.22, stripped

Upstream commit: 474dd0229cb20cf513036619781ed97bd8073c3f Enterprise-only files removed or emptied: 63 Enterprise-only snippets removed: 117 in 50 files Dangling module declarations removed: 5 Cargo edits turning enterprise off: 14 Verification: clean Enterprise feature gates left for rebuilt features: 19 in 18 files Produced by tools/fork/strip.py. The full report is in docs/fork/strip-reports/ on main.
2026-09-18 10:21:56 -07:00
commit 7dae9b29fd
1650 changed files with 485521 additions and 0 deletions
@@ -0,0 +1,25 @@
+[package]
+name = "nlp"
+version = "0.16.22"
+edition = "2024"
+
+[dependencies]
+xxhash-rust = { version = "0.8.18", features = ["xxh3"] }
+serde = { version = "1.0", features = ["derive"]}
+ahash = { version = "0.8.12", features = ["serde"] }
+whatlang = "0.18" # Language detection
+rust-stemmers = "1.2" # Stemmers
+jieba-rs = "0.10" # Chinese stemmer
+psl = "2"
+maplit = "1.0.2"
+hashify = "0.2.9"
+rand = "0.10.2"
+rkyv = { version = "0.8.18", features = ["little_endian"] }
+
+[features]
+test_mode = []
+
+[dev-dependencies]
+
+[lints]
+workspace = true
@@ -0,0 +1,111 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
+ */
+
+use crate::classifier::{Optimizer, model::FhClassifier};
+
+pub struct Adam {
+    parameters: Vec<f32>,
+    bias: f32,
+    learning_rate: f32,
+    beta1: f32,
+    beta2: f32,
+    epsilon: f32,
+    t: f32,
+    m0: Vec<f32>,
+    v0: Vec<f32>,
+    m_bias: f32,
+    v_bias: f32,
+
+    // Step info
+    bias2_sqrt: f32,
+    alpha_t: f32,
+}
+
+impl Adam {
+    pub fn new(n_parameters: usize, learning_rate: f32) -> Self {
+        Adam {
+            parameters: vec![0.0; n_parameters],
+            learning_rate,
+            beta1: 0.9,
+            beta2: 0.999,
+            epsilon: 1e-8,
+            t: 0.0,
+            m0: vec![0.0; n_parameters],
+            v0: vec![0.0; n_parameters],
+            m_bias: 0.0,
+            v_bias: 0.0,
+            bias: 0.0,
+            bias2_sqrt: 0.0,
+            alpha_t: 0.0,
+        }
+    }
+
+    pub fn with_hyperparams(mut self, beta1: f32, beta2: f32, epsilon: f32) -> Self {
+        self.beta1 = beta1;
+        self.beta2 = beta2;
+        self.epsilon = epsilon;
+        self
+    }
+
+    pub fn with_initial_weights(self, value: f32) -> Self {
+        Adam {
+            parameters: vec![value; self.parameters.len()],
+            ..self
+        }
+    }
+}
+
+impl Optimizer for Adam {
+    #[inline(always)]
+    fn step(&mut self) {
+        self.t += 1.0;
+        let bias1 = 1.0 - self.beta1.powf(self.t);
+        self.bias2_sqrt = (1.0 - self.beta2.powf(self.t)).sqrt();
+        self.alpha_t = self.learning_rate / bias1;
+    }
+
+    #[inline(always)]
+    fn update_param(&mut self, i: usize, g: f32) {
+        self.m0[i] = self.beta1 * self.m0[i] + (1.0 - self.beta1) * g;
+        self.v0[i] = self.beta2 * self.v0[i] + (1.0 - self.beta2) * g * g;
+        self.parameters[i] -=
+            self.alpha_t * self.m0[i] / (self.v0[i].sqrt() / self.bias2_sqrt + self.epsilon);
+    }
+
+    #[inline(always)]
+    fn update_bias(&mut self, g: f32) {
+        self.m_bias = self.beta1 * self.m_bias + (1.0 - self.beta1) * g;
+        self.v_bias = self.beta2 * self.v_bias + (1.0 - self.beta2) * g * g;
+        self.bias -=
+            self.alpha_t * self.m_bias / (self.v_bias.sqrt() / self.bias2_sqrt + self.epsilon);
+    }
+
+    #[inline(always)]
+    fn get_param(&self, idx: usize) -> f32 {
+        self.parameters[idx]
+    }
+
+    #[inline(always)]
+    fn get_bias(&self) -> f32 {
+        self.bias
+    }
+
+    #[inline(always)]
+    fn get_param_mut(&mut self, idx: usize) -> &mut f32 {
+        &mut self.parameters[idx]
+    }
+
+    fn build_classifier(&self) -> FhClassifier {
+        FhClassifier {
+            parameters: self.parameters.clone(),
+            bias: self.bias,
+        }
+    }
+
+    fn num_parameters(&self) -> usize {
+        self.parameters.len()
+    }
+}
@@ -0,0 +1,175 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
+ */
+
+use std::collections::HashMap;
+use xxhash_rust::xxh3::xxh3_64_with_seed;
+
+#[derive(Debug)]
+pub struct Sample<T> {
+    pub features: Vec<T>,
+    pub class: f32,
+}
+
+pub struct FhFeatureBuilder {
+    pub(super) weight_mask: u64,
+}
+
+#[derive(Debug)]
+pub struct FhFeature {
+    pub idx: usize,
+    pub weight: f32,
+}
+
+#[derive(Debug)]
+pub struct CcfhFeature {
+    pub idx_w1: usize,
+    pub idx_w2: usize,
+    pub idx_i: usize,
+    pub weight: f32,
+}
+
+pub struct CcfhFeatureBuilder {
+    pub(super) weight_mask: u64,
+    pub(super) indicator_mask: u64,
+}
+
+pub trait FeatureWeight {
+    fn idx(&self) -> usize;
+    fn weight(&self) -> f32;
+    fn weight_mut(&mut self) -> &mut f32;
+}
+
+pub trait UnprocessedFeature {
+    fn prefix(&self) -> u16;
+    fn value(&self) -> &[u8];
+}
+
+impl FeatureWeight for FhFeature {
+    fn weight(&self) -> f32 {
+        self.weight
+    }
+
+    fn weight_mut(&mut self) -> &mut f32 {
+        &mut self.weight
+    }
+
+    fn idx(&self) -> usize {
+        self.idx
+    }
+}
+
+impl FeatureWeight for CcfhFeature {
+    fn weight(&self) -> f32 {
+        self.weight
+    }
+
+    fn weight_mut(&mut self) -> &mut f32 {
+        &mut self.weight
+    }
+
+    fn idx(&self) -> usize {
+        self.idx_w1
+    }
+}
+
+impl FeatureBuilder for FhFeatureBuilder {
+    type Feature = FhFeature;
+
+    fn build_feature(&self, bytes: &[u8], weight: f32) -> FhFeature {
+        let hash1 = xxh3_64_with_seed(bytes, 0);
+        let sign = if hash1 & (1 << 63) == 0 { 1.0 } else { -1.0 };
+
+        FhFeature {
+            idx: (hash1 & self.weight_mask) as usize,
+            weight: sign * weight,
+        }
+    }
+}
+
+impl FeatureBuilder for CcfhFeatureBuilder {
+    type Feature = CcfhFeature;
+
+    fn build_feature(&self, bytes: &[u8], weight: f32) -> CcfhFeature {
+        let hash1 = xxh3_64_with_seed(bytes, 0);
+        let hash2 = xxh3_64_with_seed(bytes, 0x9E3779B97F4A7C15);
+        let hash3 = xxh3_64_with_seed(bytes, 0x517CC1B727220A95);
+        let sign = if hash3 & (1 << 63) == 0 { 1.0 } else { -1.0 };
+
+        CcfhFeature {
+            idx_w1: (hash1 & self.weight_mask) as usize,
+            idx_w2: (hash2 & self.weight_mask) as usize,
+            idx_i: (hash3 & self.indicator_mask) as usize,
+            weight: sign * weight,
+        }
+    }
+}
+
+pub trait FeatureBuilder {
+    // Feature type associated type
+    type Feature: FeatureWeight;
+
+    fn build_feature(&self, bytes: &[u8], weight: f32) -> Self::Feature;
+
+    fn scale<I: UnprocessedFeature>(&self, features: &mut HashMap<I, f32>) {
+        // Log frequency scaling
+        for x in features.values_mut() {
+            *x = x.ln_1p();
+        }
+    }
+
+    fn build<I: UnprocessedFeature>(
+        &self,
+        features_in: &HashMap<I, f32>,
+        account_id: Option<u32>,
+        l2_normalize: bool,
+    ) -> Vec<Self::Feature> {
+        let mut features_out = Vec::with_capacity(features_in.len());
+        let mut buf = Vec::with_capacity(2 + 4 + 63);
+        for (feature, count) in features_in {
+            buf.extend_from_slice(&feature.prefix().to_be_bytes());
+            buf.extend_from_slice(feature.value());
+            features_out.push(self.build_feature(&buf, *count));
+
+            if let Some(account_id) = account_id {
+                buf.extend_from_slice(&account_id.to_be_bytes());
+                features_out.push(self.build_feature(&buf, *count));
+            }
+
+            buf.clear();
+        }
+
+        // L2 normalization
+        if l2_normalize {
+            let sum_of_squares = features_out
+                .iter()
+                .map(|f| f.weight() as f64 * f.weight() as f64)
+                .sum::<f64>();
+            if sum_of_squares > 0.0 {
+                let norm = sum_of_squares.sqrt() as f32;
+                for feature in &mut features_out {
+                    *feature.weight_mut() /= norm;
+                }
+            }
+        }
+
+        features_out
+    }
+}
+
+impl<T> Sample<T> {
+    pub fn new(features: Vec<T>, class: bool) -> Self {
+        Self {
+            features,
+            class: if class { 1.0 } else { 0.0 },
+        }
+    }
+}
+
+impl<T> AsRef<Sample<T>> for Sample<T> {
+    fn as_ref(&self) -> &Sample<T> {
+        self
+    }
+}
@@ -0,0 +1,132 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
+ */
+
+use crate::classifier::{Optimizer, model::FhClassifier};
+
+#[derive(rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Debug)]
+pub struct Ftrl {
+    alpha: f64,
+    beta: f64,
+    l1_ratio: f64,
+    l2_ratio: f64,
+    zn: Vec<Zn>,
+    zn_bias: Zn,
+}
+
+#[derive(rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Clone, Copy, Debug, Default)]
+pub struct Zn {
+    z: f32,
+    n: f64,
+}
+
+impl Ftrl {
+    pub fn new(n_features: usize) -> Self {
+        Ftrl {
+            alpha: 2.0,
+            beta: 1.0,
+            l1_ratio: 0.001,
+            l2_ratio: 0.0001,
+            zn: vec![Zn::default(); n_features],
+            zn_bias: Zn::default(),
+        }
+    }
+
+    pub fn with_hyperparams(mut self, alpha: f64, beta: f64, l1_ratio: f64, l2_ratio: f64) -> Self {
+        self.alpha = alpha;
+        self.beta = beta;
+        self.l1_ratio = l1_ratio;
+        self.l2_ratio = l2_ratio;
+        self
+    }
+
+    pub fn set_hyperparams(&mut self, alpha: f64, beta: f64, l1_ratio: f64, l2_ratio: f64) {
+        self.alpha = alpha;
+        self.beta = beta;
+        self.l1_ratio = l1_ratio;
+        self.l2_ratio = l2_ratio;
+    }
+
+    pub fn with_initial_weights(self, value: f32) -> Self {
+        Ftrl {
+            zn: vec![Zn { z: value, n: 0.0 }; self.zn.len()],
+            ..self
+        }
+    }
+}
+
+impl Optimizer for Ftrl {
+    #[inline(always)]
+    fn update_param(&mut self, idx: usize, grad: f32) {
+        let zn = &mut self.zn[idx];
+        let current_w = if zn.z.abs() as f64 <= self.l1_ratio {
+            0.0
+        } else {
+            -(zn.z - zn.z.signum() * self.l1_ratio as f32)
+                / (self.l2_ratio + (self.beta + zn.n.sqrt()) / self.alpha) as f32
+        };
+        let grad = grad as f64;
+        let grad_sq = grad * grad;
+        let sigma = ((zn.n + grad_sq).sqrt() - zn.n.sqrt()) / self.alpha;
+        zn.z += (grad - sigma * current_w as f64) as f32;
+        zn.n += grad_sq;
+    }
+
+    #[inline(always)]
+    fn update_bias(&mut self, grad: f32) {
+        let current_bias = -self.zn_bias.z
+            / ((self.zn_bias.n.sqrt() + self.beta) / self.alpha + self.l2_ratio) as f32;
+        let grad = grad as f64;
+        let grad_sq = grad * grad;
+        let sigma = ((self.zn_bias.n + grad_sq).sqrt() - self.zn_bias.n.sqrt()) / self.alpha;
+        self.zn_bias.z += (grad - sigma * current_bias as f64) as f32;
+        self.zn_bias.n += grad_sq;
+    }
+
+    #[inline(always)]
+    fn get_param(&self, idx: usize) -> f32 {
+        let zn = self.zn[idx];
+        if zn.z.abs() as f64 <= self.l1_ratio {
+            0.0
+        } else {
+            -(zn.z - zn.z.signum() * self.l1_ratio as f32)
+                / (self.l2_ratio + (self.beta + zn.n.sqrt()) / self.alpha) as f32
+        }
+    }
+
+    #[inline(always)]
+    fn get_bias(&self) -> f32 {
+        -self.zn_bias.z / ((self.zn_bias.n.sqrt() + self.beta) / self.alpha + self.l2_ratio) as f32
+    }
+
+    fn step(&mut self) {}
+
+    #[inline(always)]
+    fn get_param_mut(&mut self, idx: usize) -> &mut f32 {
+        &mut self.zn[idx].z
+    }
+
+    fn build_classifier(&self) -> FhClassifier {
+        FhClassifier {
+            parameters: self
+                .zn
+                .iter()
+                .map(|zn| {
+                    if zn.z.abs() as f64 <= self.l1_ratio {
+                        0.0
+                    } else {
+                        -(zn.z - zn.z.signum() * self.l1_ratio as f32)
+                            / (self.l2_ratio + (self.beta + zn.n.sqrt()) / self.alpha) as f32
+                    }
+                })
+                .collect(),
+            bias: self.get_bias(),
+        }
+    }
+
+    fn num_parameters(&self) -> usize {
+        self.zn.len()
+    }
+}
@@ -0,0 +1,49 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
+ */
+
+use crate::classifier::model::FhClassifier;
+
+pub mod adam;
+pub mod feature;
+pub mod ftrl;
+pub mod model;
+pub mod reservoir;
+pub mod sgd;
+pub mod train;
+
+const MAX_DLOSS: f32 = 1e4;
+
+pub trait Optimizer {
+    fn step(&mut self);
+    fn update_param(&mut self, i: usize, g: f32);
+    fn update_bias(&mut self, g: f32);
+    fn get_param(&self, idx: usize) -> f32;
+    fn get_param_mut(&mut self, idx: usize) -> &mut f32;
+    fn get_bias(&self) -> f32;
+    fn build_classifier(&self) -> FhClassifier;
+    fn num_parameters(&self) -> usize;
+}
+
+#[inline(always)]
+fn sigmoid(z: f32) -> f32 {
+    let z = z.clamp(-35.0, 35.0);
+    if z >= 0.0 {
+        1.0 / (1.0 + (-z).exp())
+    } else {
+        let exp_z = z.exp();
+        exp_z / (1.0 + exp_z)
+    }
+}
+
+#[inline(always)]
+fn gradient(y: f32, p: f32) -> f32 {
+    if p > -16.0 {
+        let exp_tmp = (-p).exp();
+        ((1.0 - y) - y * exp_tmp) / (1.0 + exp_tmp)
+    } else {
+        p.exp() - y
+    }
+}
@@ -0,0 +1,109 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
+ */
+
+use crate::classifier::{
+    feature::{CcfhFeature, CcfhFeatureBuilder, FhFeature, FhFeatureBuilder},
+    sigmoid,
+};
+
+#[derive(rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Debug, Default)]
+pub struct FhClassifier {
+    pub(crate) parameters: Vec<f32>,
+    pub(crate) bias: f32,
+}
+
+#[derive(rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Debug, Default)]
+pub struct CcfhClassifier {
+    pub(crate) parameters: Vec<f32>,
+    pub(crate) indicators: Vec<f32>,
+    pub(crate) bias: f32,
+}
+
+impl FhClassifier {
+    pub fn predict_proba_sample(&self, features: &[FhFeature]) -> f32 {
+        let mut z: f32 = 0.0;
+
+        for f in features {
+            z += self.parameters[f.idx] * f.weight;
+        }
+
+        sigmoid(z + self.bias)
+    }
+
+    pub fn predict(&self, features: &[FhFeature]) -> f32 {
+        if self.predict_proba_sample(features) > 0.7 {
+            1.0
+        } else {
+            0.0
+        }
+    }
+
+    pub fn predict_batch<I>(&self, test: I) -> Vec<f32>
+    where
+        I: IntoIterator,
+        I::Item: AsRef<Vec<FhFeature>>,
+    {
+        test.into_iter()
+            .map(|features| self.predict(features.as_ref()))
+            .collect()
+    }
+
+    pub fn feature_builder(&self) -> FhFeatureBuilder {
+        FhFeatureBuilder {
+            weight_mask: (self.parameters.len() - 1) as u64,
+        }
+    }
+
+    pub fn parameters(&self) -> &[f32] {
+        &self.parameters
+    }
+
+    pub fn bias(&self) -> f32 {
+        self.bias
+    }
+}
+
+impl CcfhClassifier {
+    pub fn predict_proba_sample(&self, features: &[CcfhFeature]) -> f32 {
+        let mut z: f32 = 0.0;
+        for f in features {
+            let q = self.indicators[f.idx_i];
+            let v1 = self.parameters[f.idx_w1];
+            let v2 = self.parameters[f.idx_w2];
+            z += (q * v1 + (1.0 - q) * v2) * f.weight;
+        }
+        sigmoid(z + self.bias)
+    }
+
+    pub fn predict(&self, features: &[CcfhFeature]) -> f32 {
+        if self.predict_proba_sample(features) >= 0.5 {
+            1.0
+        } else {
+            0.0
+        }
+    }
+
+    pub fn predict_batch<I>(&self, test: I) -> Vec<f32>
+    where
+        I: IntoIterator,
+        I::Item: AsRef<Vec<CcfhFeature>>,
+    {
+        test.into_iter()
+            .map(|features| self.predict(features.as_ref()))
+            .collect()
+    }
+
+    pub fn feature_builder(&self) -> CcfhFeatureBuilder {
+        CcfhFeatureBuilder {
+            weight_mask: (self.parameters.len() - 1) as u64,
+            indicator_mask: (self.indicators.len() - 1) as u64,
+        }
+    }
+
+    pub fn is_active(&self) -> bool {
+        !self.parameters.is_empty()
+    }
+}
@@ -0,0 +1,91 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
+ */
+
+use rand::{RngExt, seq::IndexedRandom};
+
+#[derive(rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Debug)]
+pub struct SampleReservoir<T> {
+    pub spam: SampleReservoirClass<T>,
+    pub ham: SampleReservoirClass<T>,
+}
+
+#[derive(rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Debug)]
+pub struct SampleReservoirClass<T> {
+    pub buffer: Vec<T>,
+    pub total_seen: u64,
+}
+
+impl<T: Clone + Eq> SampleReservoir<T> {
+    pub fn update_reservoir(&mut self, item: &T, is_spam: bool, capacity: usize) {
+        let class = if is_spam {
+            &mut self.spam
+        } else {
+            &mut self.ham
+        };
+
+        class.total_seen += 1;
+
+        if class.buffer.len() < capacity {
+            class.buffer.push(item.clone());
+        } else if let Some(buf) = class
+            .buffer
+            .get_mut(rand::rng().random_range(0..class.total_seen as usize))
+        {
+            *buf = item.clone();
+        }
+    }
+
+    pub fn update_counts(&mut self, is_spam: bool) {
+        let class = if is_spam {
+            &mut self.spam
+        } else {
+            &mut self.ham
+        };
+
+        class.total_seen += 1;
+    }
+
+    pub fn replay_samples(
+        &mut self,
+        count_needed: usize,
+        is_spam: bool,
+    ) -> impl Iterator<Item = &T> {
+        (if is_spam {
+            &mut self.spam
+        } else {
+            &mut self.ham
+        })
+        .buffer
+        .sample(&mut rand::rng(), count_needed)
+    }
+
+    pub fn remove_sample(&mut self, item: &T, is_spam: bool) {
+        let class = if is_spam {
+            &mut self.spam
+        } else {
+            &mut self.ham
+        };
+
+        if let Some(pos) = class.buffer.iter().position(|x| x == item) {
+            class.buffer.swap_remove(pos);
+        }
+    }
+}
+
+impl<T> Default for SampleReservoir<T> {
+    fn default() -> Self {
+        SampleReservoir {
+            spam: SampleReservoirClass {
+                buffer: Vec::new(),
+                total_seen: 0,
+            },
+            ham: SampleReservoirClass {
+                buffer: Vec::new(),
+                total_seen: 0,
+            },
+        }
+    }
+}
@@ -0,0 +1,451 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
+ */
+
+use crate::classifier::{Optimizer, gradient, model::FhClassifier};
+
+#[derive(rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Debug, Default)]
+pub struct Sgd {
+    parameters: Vec<f32>,
+    bias: f32,
+    alpha: f64,
+    l1_ratio: f64,
+    l2_ratio: f64,
+    t: f64,
+    w_scale: f32,
+    optimal_init: f64,
+    eta: f32,
+    u: f32,
+    q: Vec<f32>,
+}
+
+impl Sgd {
+    pub fn new(n_features: usize, alpha: f64, l1_ratio: f64, l2_ratio: f64) -> Self {
+        let typw = (1.0 / alpha.sqrt()).sqrt();
+        let initial_eta0 = typw / 1.0_f64.max(gradient(1.0, -typw as f32) as f64);
+        let optimal_init = 1.0 / (initial_eta0 * alpha);
+
+        Sgd {
+            parameters: vec![0.0; n_features],
+            bias: 0.0,
+            alpha,
+            l1_ratio,
+            l2_ratio,
+            t: 0.0,
+            w_scale: 1.0,
+            optimal_init,
+            eta: initial_eta0 as f32,
+            u: 0.0,
+            q: vec![0.0; n_features],
+        }
+    }
+
+    pub fn with_initial_parameters(self, value: f32) -> Self {
+        Sgd {
+            parameters: vec![value; self.parameters.len()],
+            ..self
+        }
+    }
+
+    fn maybe_rescale(&mut self) {
+        if !(1e-6..=1e6).contains(&self.w_scale) {
+            for w in &mut self.parameters {
+                *w *= self.w_scale;
+            }
+            self.w_scale = 1.0;
+        }
+    }
+
+    #[inline(always)]
+    fn apply_l1_penalty(&mut self) {
+        if self.l1_ratio > 0.0 {
+            for (z, q) in self.parameters.iter_mut().zip(self.q.iter_mut()) {
+                let z_orig = *z;
+                let scaled_z = *z * self.w_scale;
+                if scaled_z > 0.0 {
+                    *z = (*z - (self.u + *q) / self.w_scale).max(0.0);
+                } else if scaled_z < 0.0 {
+                    *z = (*z + (self.u - *q) / self.w_scale).min(0.0);
+                }
+                *q += self.w_scale * (z_orig - *z);
+            }
+        }
+    }
+}
+
+impl Optimizer for Sgd {
+    fn step(&mut self) {
+        self.t += 1.0;
+        self.eta = (1.0 / ((self.alpha) * (self.optimal_init + self.t - 1.0))) as f32;
+        self.w_scale *= 1.0 - ((1.0 - self.l1_ratio) as f32 * self.eta * self.l2_ratio as f32);
+        self.u += self.eta * self.l1_ratio as f32 * self.alpha as f32;
+    }
+
+    fn update_param(&mut self, i: usize, g: f32) {
+        self.parameters[i] += (-self.eta * g) / self.w_scale;
+    }
+
+    fn update_bias(&mut self, g: f32) {
+        self.bias += -self.eta * g;
+        self.maybe_rescale();
+        self.apply_l1_penalty();
+    }
+
+    #[inline(always)]
+    fn get_param(&self, idx: usize) -> f32 {
+        self.parameters[idx] * self.w_scale
+    }
+
+    #[inline(always)]
+    fn get_bias(&self) -> f32 {
+        self.bias
+    }
+
+    #[inline(always)]
+    fn get_param_mut(&mut self, idx: usize) -> &mut f32 {
+        &mut self.parameters[idx]
+    }
+
+    fn build_classifier(&self) -> FhClassifier {
+        FhClassifier {
+            parameters: self.parameters.iter().map(|w| w * self.w_scale).collect(),
+            bias: self.bias,
+        }
+    }
+
+    fn num_parameters(&self) -> usize {
+        self.parameters.len()
+    }
+}
+
+#[cfg(test)]
+pub mod tests {
+    use crate::classifier::{
+        Optimizer,
+        adam::Adam,
+        feature::{
+            CcfhFeature, CcfhFeatureBuilder, FeatureBuilder, FhFeature, FhFeatureBuilder, Sample,
+            UnprocessedFeature,
+        },
+        ftrl::Ftrl,
+        train::{CcfhTrainer, FhTrainer},
+    };
+    use rand::{SeedableRng, rngs::StdRng, seq::SliceRandom};
+    use std::{
+        collections::HashMap,
+        fs::File,
+        io::{BufRead, BufReader},
+        time::Instant,
+    };
+
+    #[ignore]
+    #[test]
+    fn text_classifier() {
+        let reader = BufReader::new(
+            File::open("/Users/me/code/playground/phishing_email.csv")
+                .expect("Could not open file"),
+        );
+        let mut samples = Vec::with_capacity(1024);
+
+        let time = Instant::now();
+
+        for line in reader.lines().skip(1) {
+            let line = line.unwrap();
+            let (text, class) = line.trim().rsplit_once(',').unwrap();
+            //let (class, text) = line.trim().split_once(',').unwrap();
+            let text = text.trim_start_matches('"').trim_end_matches('"');
+
+            samples.push((text.to_string(), class == "1"));
+        }
+
+        println!("Loaded {} samples in {:?}", samples.len(), time.elapsed());
+
+        samples.shuffle(&mut StdRng::seed_from_u64(42));
+
+        let (train_samples, test_samples) = train_test_split(&samples, 0.2);
+
+        println!(
+            "Training samples: {}, Testing samples: {}",
+            train_samples.len(),
+            test_samples.len()
+        );
+
+        const FH_SIZE: usize = 16;
+        const CCFH_SIZE: usize = FH_SIZE - 2;
+        let mut rng = StdRng::seed_from_u64(42);
+
+        let fh_builder = FhFeatureBuilder {
+            weight_mask: (1 << FH_SIZE) - 1,
+        };
+        let mut fh_train_samples = build_fh_samples(train_samples.as_slice(), &fh_builder);
+        fh_train_samples.shuffle(&mut rng);
+        let fh_test_samples = build_fh_samples(test_samples.as_slice(), &fh_builder);
+        let ccfh_builder = CcfhFeatureBuilder {
+            weight_mask: (1 << FH_SIZE) - 1,
+            indicator_mask: (1 << CCFH_SIZE) - 1,
+        };
+        let mut ccfh_train_samples = build_ccfh_samples(train_samples.as_slice(), &ccfh_builder);
+        ccfh_train_samples.shuffle(&mut rng);
+        let ccfh_test_samples = build_ccfh_samples(test_samples.as_slice(), &ccfh_builder);
+
+        fh_model_stats(
+            "FTRL",
+            FhTrainer::new(Ftrl::new(1 << FH_SIZE)),
+            &fh_train_samples,
+            &fh_test_samples,
+        );
+
+        ccfh_model_stats(
+            "FTRL + FTRL",
+            CcfhTrainer::new(
+                Ftrl::new(1 << FH_SIZE),
+                Ftrl::new(1 << CCFH_SIZE).with_initial_weights(0.5),
+            ),
+            &ccfh_train_samples,
+            &ccfh_test_samples,
+        );
+
+        fh_model_stats(
+            "Adam",
+            FhTrainer::new(Adam::new(1 << FH_SIZE, 0.01)),
+            &fh_train_samples,
+            &fh_test_samples,
+        );
+
+        ccfh_model_stats(
+            "Adam + Adam",
+            CcfhTrainer::new(
+                Adam::new(1 << FH_SIZE, 0.01),
+                Adam::new(1 << CCFH_SIZE, 0.01).with_initial_weights(0.5),
+            ),
+            &ccfh_train_samples,
+            &ccfh_test_samples,
+        );
+
+        /*fh_model_stats(
+            "SGD",
+            FhTrainer::new(Sgd::new(1 << FH_SIZE, 0.0001, 0.0, 0.0001)),
+            &fh_train_samples,
+            &fh_test_samples,
+        );
+
+        ccfh_model_stats(
+            "FTRL + SGD",
+            CcfhTrainer::new(
+                Ftrl::new(1 << FH_SIZE),
+                Sgd::new(1 << CCFH_SIZE, 0.0001, 0.0, 0.0001).with_initial_parameters(0.5),
+            ),
+            &ccfh_train_samples,
+            &ccfh_test_samples,
+        );*/
+    }
+
+    fn fh_model_stats(
+        name: &str,
+        mut model: FhTrainer<impl Optimizer>,
+        train_samples: &[Sample<FhFeature>],
+        test_samples: &[Sample<FhFeature>],
+    ) {
+        print!("⏳ Training {}... ", name);
+        let time = Instant::now();
+        let mut batch = Vec::new();
+        for sample in train_samples {
+            batch.push(sample);
+            if batch.len() == 128 {
+                model.fit(&mut batch, 5);
+                batch.clear();
+            }
+        }
+        if !batch.is_empty() {
+            model.fit(&mut batch, 5);
+        }
+        println!(" trained in {:?}", time.elapsed());
+        let y_pred = model
+            .build_classifier()
+            .predict_batch(test_samples.iter().map(|s| &s.features));
+        let y_train: Vec<f32> = test_samples.iter().map(|s| s.class).collect();
+        println!("Accuracy: {:.4}", accuracy_score(&y_train, &y_pred));
+        println!("Precision: {:.4}", precision_score(&y_train, &y_pred, 1.0));
+        println!("Recall: {:.4}", recall_score(&y_train, &y_pred, 1.0));
+        println!("F1 Score: {:.4}", f1_score(&y_train, &y_pred, 1.0));
+    }
+
+    fn ccfh_model_stats(
+        name: &str,
+        mut model: CcfhTrainer<impl Optimizer, impl Optimizer>,
+        train_samples: &[Sample<CcfhFeature>],
+        test_samples: &[Sample<CcfhFeature>],
+    ) {
+        print!("⏳ Training {}... ", name);
+        let time = Instant::now();
+        let mut batch = Vec::new();
+        for sample in train_samples {
+            batch.push(sample);
+            if batch.len() == 128 {
+                model.fit(&mut batch, 5);
+                batch.clear();
+            }
+        }
+        if !batch.is_empty() {
+            model.fit(&mut batch, 5);
+        }
+        println!(" trained in {:?}", time.elapsed());
+        let y_pred = model
+            .build_classifier()
+            .predict_batch(test_samples.iter().map(|s| &s.features));
+        let y_train: Vec<f32> = test_samples.iter().map(|s| s.class).collect();
+        println!("Accuracy: {:.4}", accuracy_score(&y_train, &y_pred));
+        println!("Precision: {:.4}", precision_score(&y_train, &y_pred, 1.0));
+        println!("Recall: {:.4}", recall_score(&y_train, &y_pred, 1.0));
+        println!("F1 Score: {:.4}", f1_score(&y_train, &y_pred, 1.0));
+    }
+
+    fn accuracy_score(y_true: &[f32], y_pred: &[f32]) -> f32 {
+        y_true
+            .iter()
+            .zip(y_pred.iter())
+            .filter(|(true_val, pred_val)| **true_val == **pred_val)
+            .count() as f32
+            / y_true.len() as f32
+    }
+
+    fn precision_score(y_true: &[f32], y_pred: &[f32], positive_class: f32) -> f32 {
+        let true_positives = y_true
+            .iter()
+            .zip(y_pred.iter())
+            .filter(|(true_val, pred_val)| {
+                **pred_val == positive_class && **true_val == positive_class
+            })
+            .count() as f32;
+
+        let predicted_positives = y_pred
+            .iter()
+            .filter(|pred_val| **pred_val == positive_class)
+            .count() as f32;
+
+        if predicted_positives == 0.0 {
+            0.0
+        } else {
+            true_positives / predicted_positives
+        }
+    }
+
+    fn recall_score(y_true: &[f32], y_pred: &[f32], positive_class: f32) -> f32 {
+        let true_positives = y_true
+            .iter()
+            .zip(y_pred.iter())
+            .filter(|(true_val, pred_val)| {
+                **pred_val == positive_class && **true_val == positive_class
+            })
+            .count() as f32;
+
+        let actual_positives = y_true
+            .iter()
+            .filter(|true_val| **true_val == positive_class)
+            .count() as f32;
+
+        if actual_positives == 0.0 {
+            0.0
+        } else {
+            true_positives / actual_positives
+        }
+    }
+
+    fn f1_score(y_true: &[f32], y_pred: &[f32], positive_class: f32) -> f32 {
+        let precision = precision_score(y_true, y_pred, positive_class);
+        let recall = recall_score(y_true, y_pred, positive_class);
+
+        if precision + recall == 0.0 {
+            0.0
+        } else {
+            2.0 * (precision * recall) / (precision + recall)
+        }
+    }
+
+    #[allow(clippy::type_complexity)]
+    pub fn train_test_split(
+        data: &[(String, bool)],
+        test_size: f32,
+    ) -> (Vec<(&String, bool)>, Vec<(&String, bool)>) {
+        let mut class_0: Vec<(&String, bool)> = Vec::new();
+        let mut class_1: Vec<(&String, bool)> = Vec::new();
+
+        for (sample, class) in data {
+            if !*class {
+                class_0.push((sample, *class));
+            } else {
+                class_1.push((sample, *class));
+            }
+        }
+
+        let test_count_0 = (class_0.len() as f32 * test_size).round() as usize;
+        let test_count_1 = (class_1.len() as f32 * test_size).round() as usize;
+
+        let (test_0, train_0) = class_0.split_at(test_count_0);
+        let (test_1, train_1) = class_1.split_at(test_count_1);
+
+        let mut train = Vec::new();
+        let mut test = Vec::new();
+
+        train.extend_from_slice(train_0);
+        train.extend_from_slice(train_1);
+        test.extend_from_slice(test_0);
+        test.extend_from_slice(test_1);
+
+        (train, test)
+    }
+
+    pub fn build_fh_samples(
+        data: &[(&String, bool)],
+        builder: &FhFeatureBuilder,
+    ) -> Vec<Sample<FhFeature>> {
+        let mut samples = Vec::with_capacity(data.len());
+
+        for (text, class) in data {
+            let mut sample: HashMap<String, f32> = HashMap::new();
+            for word in text.split_whitespace() {
+                *sample.entry(word.to_string()).or_default() += 1.0;
+            }
+            builder.scale(&mut sample);
+            samples.push(Sample {
+                features: builder.build(&sample, 12345.into(), true),
+                class: if *class { 1.0 } else { 0.0 },
+            });
+        }
+
+        samples
+    }
+
+    pub fn build_ccfh_samples(
+        data: &[(&String, bool)],
+        builder: &CcfhFeatureBuilder,
+    ) -> Vec<Sample<CcfhFeature>> {
+        let mut samples = Vec::with_capacity(data.len());
+
+        for (text, class) in data {
+            let mut sample: HashMap<String, f32> = HashMap::new();
+            for word in text.split_whitespace() {
+                *sample.entry(word.to_string()).or_default() += 1.0;
+            }
+            builder.scale(&mut sample);
+            samples.push(Sample {
+                features: builder.build(&sample, 12345.into(), true),
+                class: if *class { 1.0 } else { 0.0 },
+            });
+        }
+
+        samples
+    }
+
+    impl UnprocessedFeature for String {
+        fn prefix(&self) -> u16 {
+            0
+        }
+
+        fn value(&self) -> &[u8] {
+            self.as_bytes()
+        }
+    }
+}
@@ -0,0 +1,157 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
+ */
+
+use crate::classifier::{
+    MAX_DLOSS, Optimizer,
+    feature::{CcfhFeature, CcfhFeatureBuilder, FhFeature, FhFeatureBuilder, Sample},
+    gradient,
+    model::{CcfhClassifier, FhClassifier},
+};
+use rand::{SeedableRng, rngs::StdRng, seq::SliceRandom};
+
+#[derive(rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Debug, Default)]
+pub struct FhTrainer<T: Optimizer> {
+    pub optimizer: T,
+}
+
+#[derive(rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Debug, Default)]
+pub struct CcfhTrainer<W: Optimizer, I: Optimizer> {
+    pub w_optimizer: W,
+    pub i_optimizer: I,
+}
+
+impl<T: Optimizer> FhTrainer<T> {
+    pub fn new(optimizer: T) -> Self {
+        FhTrainer { optimizer }
+    }
+
+    pub fn fit(&mut self, samples: &mut [impl AsRef<Sample<FhFeature>>], num_epochs: usize) {
+        for _ in 0..num_epochs {
+            samples.shuffle(&mut StdRng::seed_from_u64(42));
+
+            for sample in samples.iter() {
+                let sample = sample.as_ref();
+                let mut dot: f32 = 0.0;
+                for f in &sample.features {
+                    dot += self.optimizer.get_param(f.idx) * f.weight;
+                }
+                let p = dot + self.optimizer.get_bias();
+                let dloss = gradient(sample.class, p).clamp(-MAX_DLOSS, MAX_DLOSS);
+
+                self.optimizer.step();
+
+                for f in &sample.features {
+                    self.optimizer.update_param(f.idx, dloss * f.weight);
+                }
+
+                self.optimizer.update_bias(dloss);
+            }
+        }
+    }
+
+    pub fn feature_builder(&self) -> FhFeatureBuilder {
+        FhFeatureBuilder {
+            weight_mask: (self.optimizer.num_parameters() - 1) as u64,
+        }
+    }
+
+    pub fn build_classifier(&self) -> FhClassifier {
+        self.optimizer.build_classifier()
+    }
+
+    pub fn optimizer(&self) -> &T {
+        &self.optimizer
+    }
+
+    pub fn optimizer_mut(&mut self) -> &mut T {
+        &mut self.optimizer
+    }
+}
+
+impl<W: Optimizer, I: Optimizer> CcfhTrainer<W, I> {
+    pub fn new(w_optimizer: W, i_optimizer: I) -> Self {
+        CcfhTrainer {
+            w_optimizer,
+            i_optimizer,
+        }
+    }
+
+    pub fn fit(&mut self, samples: &mut [impl AsRef<Sample<CcfhFeature>>], num_epochs: usize) {
+        for _ in 0..num_epochs {
+            samples.shuffle(&mut StdRng::seed_from_u64(42));
+
+            for sample in samples.iter() {
+                let sample = sample.as_ref();
+                let mut dot: f32 = 0.0;
+                for f in &sample.features {
+                    let q = self.i_optimizer.get_param(f.idx_i);
+                    let v1 = self.w_optimizer.get_param(f.idx_w1);
+                    let v2 = self.w_optimizer.get_param(f.idx_w2);
+                    dot += (q * v1 + (1.0 - q) * v2) * f.weight;
+                }
+                let p = dot + self.w_optimizer.get_bias();
+                let dloss = gradient(sample.class, p).clamp(-MAX_DLOSS, MAX_DLOSS);
+
+                self.w_optimizer.step();
+                self.i_optimizer.step();
+
+                for f in &sample.features {
+                    let q = self.i_optimizer.get_param(f.idx_i);
+                    let v1 = self.w_optimizer.get_param(f.idx_w1);
+                    let v2 = self.w_optimizer.get_param(f.idx_w2);
+
+                    // Update weights
+                    let d_v1 = f.weight * q;
+                    let d_v2 = f.weight * (1.0 - q);
+                    self.w_optimizer.update_param(f.idx_w1, dloss * d_v1);
+                    self.w_optimizer.update_param(f.idx_w2, dloss * d_v2);
+
+                    // Update indicator
+                    let d_q = (v1 - v2) * f.weight;
+                    self.i_optimizer.update_param(f.idx_i, dloss * d_q);
+                    let fi = self.i_optimizer.get_param_mut(f.idx_i);
+                    *fi = fi.clamp(0.0, 1.0);
+                }
+
+                self.w_optimizer.update_bias(dloss);
+            }
+        }
+    }
+
+    pub fn feature_builder(&self) -> CcfhFeatureBuilder {
+        CcfhFeatureBuilder {
+            weight_mask: (self.w_optimizer.num_parameters() - 1) as u64,
+            indicator_mask: (self.i_optimizer.num_parameters() - 1) as u64,
+        }
+    }
+
+    pub fn build_classifier(&self) -> CcfhClassifier {
+        let w_classifier = self.w_optimizer.build_classifier();
+        let i_classifier = self.i_optimizer.build_classifier();
+
+        CcfhClassifier {
+            parameters: w_classifier.parameters,
+            indicators: i_classifier.parameters,
+            bias: w_classifier.bias,
+        }
+    }
+
+    pub fn w_optimizer(&self) -> &W {
+        &self.w_optimizer
+    }
+
+    pub fn w_optimizer_mut(&mut self) -> &mut W {
+        &mut self.w_optimizer
+    }
+
+    pub fn i_optimizer(&self) -> &I {
+        &self.i_optimizer
+    }
+
+    pub fn i_optimizer_mut(&mut self) -> &mut I {
+        &mut self.i_optimizer
+    }
+}
@@ -0,0 +1,236 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
+ */
+
+use super::Language;
+use ahash::AHashMap;
+use whatlang::{Lang, detect};
+
+pub const MIN_LANGUAGE_SCORE: f64 = 0.6;
+
+#[derive(Debug)]
+struct WeightedAverage {
+    weight: usize,
+    occurrences: usize,
+    confidence: f64,
+}
+
+#[derive(Debug)]
+pub struct LanguageDetector {
+    lang_detected: AHashMap<Language, WeightedAverage>,
+}
+
+impl Default for LanguageDetector {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl LanguageDetector {
+    pub fn new() -> LanguageDetector {
+        LanguageDetector {
+            lang_detected: AHashMap::default(),
+        }
+    }
+
+    pub fn detect(&mut self, text: &str, min_score: f64) -> Language {
+        if let Some((language, confidence)) = LanguageDetector::detect_single(text) {
+            let w = self
+                .lang_detected
+                .entry(language)
+                .or_insert_with(|| WeightedAverage {
+                    weight: 0,
+                    confidence: 0.0,
+                    occurrences: 0,
+                });
+            w.occurrences += 1;
+            w.weight += text.len();
+            w.confidence += confidence * text.len() as f64;
+            if confidence < min_score {
+                Language::Unknown
+            } else {
+                language
+            }
+        } else {
+            Language::Unknown
+        }
+    }
+
+    pub fn most_frequent_language(&self) -> Option<Language> {
+        self.lang_detected
+            .iter()
+            .filter(|(l, _)| !matches!(l, Language::None))
+            .max_by(|(_, a), (_, b)| {
+                ((a.confidence / a.weight as f64) * a.occurrences as f64)
+                    .partial_cmp(&((b.confidence / b.weight as f64) * b.occurrences as f64))
+                    .unwrap_or(std::cmp::Ordering::Less)
+            })
+            .map(|(l, _)| *l)
+    }
+
+    pub fn detect_single(text: &str) -> Option<(Language, f64)> {
+        detect(text).map(|info| {
+            (
+                match info.lang() {
+                    Lang::Epo => Language::Esperanto,
+                    Lang::Eng => Language::English,
+                    Lang::Rus => Language::Russian,
+                    Lang::Cmn => Language::Mandarin,
+                    Lang::Spa => Language::Spanish,
+                    Lang::Por => Language::Portuguese,
+                    Lang::Ita => Language::Italian,
+                    Lang::Ben => Language::Bengali,
+                    Lang::Fra => Language::French,
+                    Lang::Deu => Language::German,
+                    Lang::Ukr => Language::Ukrainian,
+                    Lang::Kat => Language::Georgian,
+                    Lang::Ara => Language::Arabic,
+                    Lang::Hin => Language::Hindi,
+                    Lang::Jpn => Language::Japanese,
+                    Lang::Heb => Language::Hebrew,
+                    Lang::Yid => Language::Yiddish,
+                    Lang::Pol => Language::Polish,
+                    Lang::Amh => Language::Amharic,
+                    Lang::Jav => Language::Javanese,
+                    Lang::Kor => Language::Korean,
+                    Lang::Nob => Language::Bokmal,
+                    Lang::Dan => Language::Danish,
+                    Lang::Swe => Language::Swedish,
+                    Lang::Fin => Language::Finnish,
+                    Lang::Tur => Language::Turkish,
+                    Lang::Nld => Language::Dutch,
+                    Lang::Hun => Language::Hungarian,
+                    Lang::Ces => Language::Czech,
+                    Lang::Ell => Language::Greek,
+                    Lang::Bul => Language::Bulgarian,
+                    Lang::Bel => Language::Belarusian,
+                    Lang::Mar => Language::Marathi,
+                    Lang::Kan => Language::Kannada,
+                    Lang::Ron => Language::Romanian,
+                    Lang::Slv => Language::Slovene,
+                    Lang::Hrv => Language::Croatian,
+                    Lang::Srp => Language::Serbian,
+                    Lang::Mkd => Language::Macedonian,
+                    Lang::Lit => Language::Lithuanian,
+                    Lang::Lav => Language::Latvian,
+                    Lang::Est => Language::Estonian,
+                    Lang::Tam => Language::Tamil,
+                    Lang::Vie => Language::Vietnamese,
+                    Lang::Urd => Language::Urdu,
+                    Lang::Tha => Language::Thai,
+                    Lang::Guj => Language::Gujarati,
+                    Lang::Uzb => Language::Uzbek,
+                    Lang::Pan => Language::Punjabi,
+                    Lang::Aze => Language::Azerbaijani,
+                    Lang::Ind => Language::Indonesian,
+                    Lang::Tel => Language::Telugu,
+                    Lang::Pes => Language::Persian,
+                    Lang::Mal => Language::Malayalam,
+                    Lang::Ori => Language::Oriya,
+                    Lang::Mya => Language::Burmese,
+                    Lang::Nep => Language::Nepali,
+                    Lang::Sin => Language::Sinhalese,
+                    Lang::Khm => Language::Khmer,
+                    Lang::Tuk => Language::Turkmen,
+                    Lang::Aka => Language::Akan,
+                    Lang::Zul => Language::Zulu,
+                    Lang::Sna => Language::Shona,
+                    Lang::Afr => Language::Afrikaans,
+                    Lang::Lat => Language::Latin,
+                    Lang::Slk => Language::Slovak,
+                    Lang::Cat => Language::Catalan,
+                    Lang::Tgl => Language::Tagalog,
+                    Lang::Hye => Language::Armenian,
+                    _ => Language::Unknown,
+                },
+                info.confidence(),
+            )
+        })
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn detect_languages() {
+        let inputs = [
+            (
+                "The quick brown fox jumps over the lazy dog",
+                Language::English,
+            ),
+            (
+                "Jovencillo emponzoñado de whisky: ¡qué figurota exhibe!",
+                Language::Spanish,
+            ),
+            (
+                "Ma la volpe col suo balzo ha raggiunto il quieto Fido",
+                Language::Italian,
+            ),
+            (
+                "Jaz em prisão bota que vexa dez cegonhas felizes",
+                Language::Portuguese,
+            ),
+            (
+                "Zwölf Boxkämpfer jagten Victor quer über den großen Sylter Deich",
+                Language::German,
+            ),
+            ("עטלף אבק נס דרך מזגן שהתפוצץ כי חם", Language::Hebrew),
+            (
+                "Съешь ещё этих мягких французских булок, да выпей же чаю",
+                Language::Russian,
+            ),
+            (
+                "Чуєш їх, доцю, га? Кумедна ж ти, прощайся без ґольфів!",
+                Language::Ukrainian,
+            ),
+            (
+                "Љубазни фењерџија чађавог лица хоће да ми покаже штос",
+                Language::Serbian,
+            ),
+            (
+                "Pijamalı hasta yağız şoföre çabucak güvendi",
+                Language::Turkish,
+            ),
+            ("己所不欲,勿施于人。", Language::Mandarin),
+            ("井の中の蛙大海を知らず", Language::Japanese),
+            ("시작이 반이다", Language::Korean),
+        ];
+
+        let mut detector = LanguageDetector::new();
+
+        for input in inputs.iter() {
+            assert_eq!(detector.detect(input.0, 0.0), input.1);
+        }
+    }
+
+    #[test]
+    fn weighted_language() {
+        let mut detector = LanguageDetector::new();
+        for lang in [
+            (Language::Spanish, 0.5, 70),
+            (Language::Japanese, 0.2, 100),
+            (Language::Japanese, 0.3, 100),
+            (Language::Japanese, 0.4, 200),
+            (Language::English, 0.7, 50),
+        ]
+        .iter()
+        {
+            let w = detector
+                .lang_detected
+                .entry(lang.0)
+                .or_insert_with(|| WeightedAverage {
+                    weight: 0,
+                    confidence: 0.0,
+                    occurrences: 0,
+                });
+            w.occurrences += 1;
+            w.weight += lang.2;
+            w.confidence += lang.1 * lang.2 as f64;
+        }
+        assert_eq!(detector.most_frequent_language(), Some(Language::Japanese));
+    }
+}
@@ -0,0 +1,271 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
+ */
+
+pub mod detect;
+pub mod search_snippet;
+pub mod stemmer;
+pub mod stopwords;
+
+use self::detect::LanguageDetector;
+use crate::tokenizers::{
+    Token, chinese::ChineseTokenizer, japanese::JapaneseTokenizer, space::SpaceTokenizer,
+    word::WordTokenizer,
+};
+use std::borrow::Cow;
+
+pub type LanguageTokenizer<'x> = Box<dyn Iterator<Item = Token<Cow<'x, str>>> + 'x + Sync + Send>;
+
+impl Language {
+    pub fn tokenize_text<'x>(
+        &self,
+        text: &'x str,
+        max_token_length: usize,
+    ) -> LanguageTokenizer<'x> {
+        match self {
+            Language::Japanese => Box::new(
+                JapaneseTokenizer::new(WordTokenizer::new(text, usize::MAX))
+                    .filter(move |t| t.word.len() <= max_token_length),
+            ),
+            Language::Mandarin => Box::new(
+                ChineseTokenizer::new(WordTokenizer::new(text, usize::MAX))
+                    .filter(move |t| t.word.len() <= max_token_length),
+            ),
+            Language::None => {
+                Box::new(
+                    SpaceTokenizer::new(text, max_token_length).map(|word| Token {
+                        word: word.into(),
+                        from: 0,
+                        to: 0,
+                    }),
+                )
+            }
+            _ => Box::new(WordTokenizer::new(text, max_token_length)),
+        }
+    }
+}
+
+#[derive(
+    Debug, PartialEq, Clone, Copy, Hash, Eq, serde::Serialize, serde::Deserialize, Default,
+)]
+pub enum Language {
+    Esperanto = 0,
+    #[default]
+    English = 1,
+    Russian = 2,
+    Mandarin = 3,
+    Spanish = 4,
+    Portuguese = 5,
+    Italian = 6,
+    Bengali = 7,
+    French = 8,
+    German = 9,
+    Ukrainian = 10,
+    Georgian = 11,
+    Arabic = 12,
+    Hindi = 13,
+    Japanese = 14,
+    Hebrew = 15,
+    Yiddish = 16,
+    Polish = 17,
+    Amharic = 18,
+    Javanese = 19,
+    Korean = 20,
+    Bokmal = 21,
+    Danish = 22,
+    Swedish = 23,
+    Finnish = 24,
+    Turkish = 25,
+    Dutch = 26,
+    Hungarian = 27,
+    Czech = 28,
+    Greek = 29,
+    Bulgarian = 30,
+    Belarusian = 31,
+    Marathi = 32,
+    Kannada = 33,
+    Romanian = 34,
+    Slovene = 35,
+    Croatian = 36,
+    Serbian = 37,
+    Macedonian = 38,
+    Lithuanian = 39,
+    Latvian = 40,
+    Estonian = 41,
+    Tamil = 42,
+    Vietnamese = 43,
+    Urdu = 44,
+    Thai = 45,
+    Gujarati = 46,
+    Uzbek = 47,
+    Punjabi = 48,
+    Azerbaijani = 49,
+    Indonesian = 50,
+    Telugu = 51,
+    Persian = 52,
+    Malayalam = 53,
+    Oriya = 54,
+    Burmese = 55,
+    Nepali = 56,
+    Sinhalese = 57,
+    Khmer = 58,
+    Turkmen = 59,
+    Akan = 60,
+    Zulu = 61,
+    Shona = 62,
+    Afrikaans = 63,
+    Latin = 64,
+    Slovak = 65,
+    Catalan = 66,
+    Tagalog = 67,
+    Armenian = 68,
+    Unknown = 69,
+    None = 70,
+}
+
+impl Language {
+    pub fn is_unknown(&self) -> bool {
+        matches!(self, Language::Unknown)
+    }
+
+    pub fn from_iso_639(code: &str) -> Option<Self> {
+        hashify::map!(
+            code.split_once(['-', '_']).map(|c| c.0).unwrap_or(code).as_bytes(),
+            Language,
+            "en" => Language::English,
+            "es" => Language::Spanish,
+            "pt" => Language::Portuguese,
+            "it" => Language::Italian,
+            "fr" => Language::French,
+            "de" => Language::German,
+            "da" => Language::Danish,
+            "ru" => Language::Russian,
+            "zh" => Language::Mandarin,
+            "ja" => Language::Japanese,
+            "ar" => Language::Arabic,
+            "hi" => Language::Hindi,
+            "ko" => Language::Korean,
+            "bn" => Language::Bengali,
+            "he" => Language::Hebrew,
+            "ur" => Language::Urdu,
+            "fa" => Language::Persian,
+            "ml" => Language::Malayalam,
+            "or" => Language::Oriya,
+            "my" => Language::Burmese,
+            "ne" => Language::Nepali,
+            "si" => Language::Sinhalese,
+            "km" => Language::Khmer,
+            "tk" => Language::Turkmen,
+            "am" => Language::Amharic,
+            "az" => Language::Azerbaijani,
+            "id" => Language::Indonesian,
+            "te" => Language::Telugu,
+            "ta" => Language::Tamil,
+            "vi" => Language::Vietnamese,
+            "gu" => Language::Gujarati,
+            "pa" => Language::Punjabi,
+            "uz" => Language::Uzbek,
+            "hy" => Language::Armenian,
+            "ka" => Language::Georgian,
+            "la" => Language::Latin,
+            "sl" => Language::Slovene,
+            "hr" => Language::Croatian,
+            "sr" => Language::Serbian,
+            "mk" => Language::Macedonian,
+            "lt" => Language::Lithuanian,
+            "lv" => Language::Latvian,
+            "et" => Language::Estonian,
+            "tl" => Language::Tagalog,
+            "af" => Language::Afrikaans,
+            "zu" => Language::Zulu,
+            "sn" => Language::Shona,
+            "ak" => Language::Akan,
+            "ca" => Language::Catalan,
+            "el" => Language::Greek,
+            "sv" => Language::Swedish,
+            "pl" => Language::Polish,
+            "nl" => Language::Dutch,
+            "fi" => Language::Finnish,
+            "hu" => Language::Hungarian,
+            "tr" => Language::Turkish,
+            "ro" => Language::Romanian,
+            "nb" => Language::Bokmal,
+            "no" => Language::Bokmal,
+            "yi" => Language::Yiddish,
+            "eo" => Language::Esperanto,
+            "uk" => Language::Ukrainian,
+            "cs" => Language::Czech,
+            "sk" => Language::Slovak,
+            "bg" => Language::Bulgarian,
+            "be" => Language::Belarusian,
+            "th" => Language::Thai,
+            "mr" => Language::Marathi,
+            "kn" => Language::Kannada,
+            "jv" => Language::Javanese
+        )
+        .copied()
+    }
+}
+
+impl Language {
+    pub fn detect(text: String, default: Language) -> (String, Language) {
+        if let Some((l, t)) = text
+            .split_once(':')
+            .and_then(|(l, t)| (Language::from_iso_639(l)?, t).into())
+        {
+            (t.to_string(), l)
+        } else {
+            let l = LanguageDetector::detect_single(&text)
+                .and_then(|(l, c)| if c > 0.3 { Some(l) } else { None })
+                .unwrap_or(default);
+            (text, l)
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn iso_639_codes() {
+        for (code, expected) in [
+            ("nl", Language::Dutch),
+            ("fi", Language::Finnish),
+            ("hu", Language::Hungarian),
+            ("tr", Language::Turkish),
+            ("ro", Language::Romanian),
+            ("nb", Language::Bokmal),
+            ("no", Language::Bokmal),
+            ("yi", Language::Yiddish),
+            ("eo", Language::Esperanto),
+            ("uk", Language::Ukrainian),
+            ("cs", Language::Czech),
+            ("sk", Language::Slovak),
+            ("bg", Language::Bulgarian),
+            ("be", Language::Belarusian),
+            ("th", Language::Thai),
+            ("mr", Language::Marathi),
+            ("kn", Language::Kannada),
+            ("jv", Language::Javanese),
+            ("en", Language::English),
+            ("pl", Language::Polish),
+        ] {
+            assert_eq!(Language::from_iso_639(code), Some(expected), "{code}");
+        }
+    }
+
+    #[test]
+    fn iso_639_locale_suffixes() {
+        for code in ["nl_NL", "nl_BE", "nl_NL@euro", "nl-NL"] {
+            assert_eq!(
+                Language::from_iso_639(code),
+                Some(Language::Dutch),
+                "{code}"
+            );
+        }
+        assert_eq!(Language::from_iso_639("xx_XX"), None);
+    }
+}
@@ -0,0 +1,271 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
+ */
+
+use super::Language;
+
+fn escape_char(c: char, string: &mut String) {
+    match c {
+        '&' => string.push_str("&amp;"),
+        '<' => string.push_str("&lt;"),
+        '>' => string.push_str("&gt;"),
+        '"' => string.push_str("&quot;"),
+        '\n' | '\r' => string.push(' '),
+        _ => string.push(c),
+    }
+}
+
+fn escape_char_len(c: char) -> usize {
+    match c {
+        '&' => "&amp;".len(),
+        '<' => "&lt;".len(),
+        '>' => "&gt;".len(),
+        '"' => "&quot;".len(),
+        '\r' | '\n' => 1,
+        _ => c.len_utf8(),
+    }
+}
+
+pub struct Term {
+    offset: usize,
+    len: usize,
+}
+
+pub fn generate_snippet(
+    text: &str,
+    needles: &[impl AsRef<str>],
+    language: Language,
+    is_exact: bool,
+) -> Option<String> {
+    let mut terms = Vec::new();
+    if is_exact {
+        let tokens = language.tokenize_text(text, 200).collect::<Vec<_>>();
+        for tokens in tokens.windows(needles.len()) {
+            if needles
+                .iter()
+                .zip(tokens)
+                .all(|(needle, token)| needle.as_ref() == token.word.as_ref())
+            {
+                for token in tokens {
+                    terms.push(Term {
+                        offset: token.from,
+                        len: token.to - token.from,
+                    });
+                }
+            }
+        }
+    } else {
+        for token in language.tokenize_text(text, 200) {
+            if needles.iter().any(|needle| {
+                let needle = needle.as_ref();
+                needle == token.word.as_ref() || needle.len() > 2 && token.word.contains(needle)
+            }) {
+                terms.push(Term {
+                    offset: token.from,
+                    len: token.to - token.from,
+                });
+            }
+        }
+    }
+    if terms.is_empty() {
+        return None;
+    }
+
+    let mut snippet = String::with_capacity(text.len());
+    let start_offset = terms.first()?.offset;
+
+    if start_offset > 0 {
+        let mut word_count = 0;
+        let mut from_offset = 0;
+        let mut last_is_space = false;
+
+        if text.len() > 240 {
+            for (pos, char) in text.get(0..start_offset)?.char_indices().rev() {
+                // Add up to 2 words or 40 characters of context
+                if char.is_whitespace() {
+                    if !last_is_space {
+                        word_count += 1;
+                        if word_count == 3 {
+                            break;
+                        }
+                        last_is_space = true;
+                    }
+                } else {
+                    last_is_space = false;
+                }
+                from_offset = pos;
+                if start_offset - from_offset >= 40 {
+                    break;
+                }
+            }
+        }
+
+        last_is_space = false;
+        for char in text.get(from_offset..start_offset)?.chars() {
+            if !char.is_whitespace() {
+                last_is_space = false;
+            } else {
+                if last_is_space {
+                    continue;
+                }
+                last_is_space = true;
+            }
+            escape_char(char, &mut snippet);
+        }
+    }
+
+    let mut terms = terms.iter().peekable();
+
+    'outer: while let Some(term) = terms.next() {
+        if snippet.len() + ("<mark>".len() * 2) + term.len + 1 > 255 {
+            break;
+        }
+
+        snippet.push_str("<mark>");
+        snippet.push_str(text.get(term.offset..term.offset + term.len)?);
+        snippet.push_str("</mark>");
+
+        let next_offset = if let Some(next_term) = terms.peek() {
+            next_term.offset
+        } else {
+            text.len()
+        };
+
+        let mut last_is_space = false;
+        for char in text.get(term.offset + term.len..next_offset)?.chars() {
+            if !char.is_whitespace() {
+                last_is_space = false;
+            } else {
+                if last_is_space {
+                    continue;
+                }
+                last_is_space = true;
+            }
+
+            if snippet.len() + escape_char_len(char) <= 255 {
+                escape_char(char, &mut snippet);
+            } else {
+                break 'outer;
+            }
+        }
+    }
+
+    Some(snippet)
+}
+
+#[cfg(test)]
+mod tests {
+    use crate::language::{Language, search_snippet::generate_snippet};
+
+    #[test]
+    fn search_snippets() {
+        let inputs = [
+            (
+                vec![
+                    "Help a friend from Abidjan Côte d'Ivoire",
+                    concat!(
+                        "When my mother died when she was given birth to me, my father took me so ",
+                        "special because I am motherless. Before the death of my late father on 22nd June ",
+                        "2013 in a private hospital here in Abidjan Côte d'Ivoire. He secretly called me on his ",
+                        "bedside and told me that he has a sum of $7.5M (Seven Million five Hundred ",
+                        "Thousand Dollars) left in a suspense account in a local bank here in Abidjan Côte ",
+                        "d'Ivoire, that he used my name as his only daughter for the next of kin in deposit of ",
+                        "the fund. ",
+                        "I am 24year old. Dear I am honorably seeking your assistance in the following ways. ",
+                        "1) To provide any bank account where this money would be transferred into. ",
+                        "2) To serve as the guardian of this fund. ",
+                        "3) To make arrangement for me to come over to your country to further my ",
+                        "education and to secure a residential permit for me in your country. ",
+                        "Moreover, I am willing to offer you 30 percent of the total sum as compensation for ",
+                        "your effort input after the successful transfer of this fund to your nominated ",
+                        "account overseas."
+                    ),
+                ],
+                vec![
+                    (
+                        vec!["côte"],
+                        vec![
+                            "Help a friend from Abidjan <mark>Côte</mark> d'Ivoire",
+                            concat!(
+                                "in Abidjan <mark>Côte</mark> d'Ivoire. He secretly called me on his bedside ",
+                                "and told me that he has a sum of $7.5M (Seven Million five Hundred Thousand ",
+                                "Dollars) left in a suspense account in a local bank here in Abidjan ",
+                                "<mark>Côte</mark> d'Ivoire, that "
+                            ),
+                        ],
+                    ),
+                    (
+                        vec!["your", "country"],
+                        vec![concat!(
+                            "honorably seeking <mark>your</mark> assistance in the following ways. ",
+                            "1) To provide any bank account where this money would be transferred into. 2) ",
+                            "To serve as the guardian of this fund. 3) To make arrangement for me to come ",
+                            "over to <mark>your</mark> "
+                        )],
+                    ),
+                    (
+                        vec!["overseas"],
+                        vec!["nominated account <mark>overseas</mark>."],
+                    ),
+                ],
+            ),
+            (
+                vec![
+                    "孫子兵法",
+                    concat!(
+                        "<\"孫子兵法：\">",
+                        "孫子曰：兵者，國之大事，死生之地，存亡之道，不可不察也。",
+                        "孫子曰：凡用兵之法，馳車千駟，革車千乘，帶甲十萬；千里饋糧，則內外之費賓客之用，膠漆之材，",
+                        "車甲之奉，日費千金，然後十萬之師舉矣。",
+                        "孫子曰：凡用兵之法，全國為上，破國次之；全旅為上，破旅次之；全卒為上，破卒次之；全伍為上，破伍次之。",
+                        "是故百戰百勝，非善之善者也；不戰而屈人之兵，善之善者也。",
+                        "孫子曰：昔之善戰者，先為不可勝，以待敵之可勝，不可勝在己，可勝在敵。故善戰者，能為不可勝，不能使敵必可勝。",
+                        "故曰：勝可知，而不可為。",
+                        "兵者，詭道也。故能而示之不能，用而示之不用，近而示之遠，遠而示之近。利而誘之，亂而取之，實而備之，強而避之，",
+                        "怒而撓之，卑而驕之，佚而勞之，親而離之。攻其無備，出其不意，此兵家之勝，不可先傳也。",
+                        "夫未戰而廟算勝者，得算多也；未戰而廟算不勝者，得算少也；多算勝，少算不勝，而況於無算乎？吾以此觀之，勝負見矣。",
+                        "孫子曰：凡治眾如治寡，分數是也。鬥眾如鬥寡，形名是也。三軍之眾，可使必受敵而無敗者，奇正是也。兵之所加，",
+                        "如以碬投卵者，虛實是也。",
+                    ),
+                ],
+                vec![
+                    (
+                        vec!["孫子兵法"],
+                        vec![
+                            "<mark>孫子兵法</mark>",
+                            concat!(
+                                "&lt;&quot;<mark>孫子兵法</mark>：&quot;&gt;孫子曰：兵者，國之大事，死生之地，存亡之道，",
+                                "不可不察也。孫子曰：凡用兵之法，馳車千駟，革車千乘，帶甲十萬；千里饋糧，則內外之費賓客之用，膠"
+                            ),
+                        ],
+                    ),
+                    (
+                        vec!["孫子曰"],
+                        vec![concat!(
+                            "&lt;&quot;孫子兵法：&quot;&gt;<mark>孫子曰</mark>：兵者，國之大事，死生之地，存亡之道，",
+                            "不可不察也。<mark>孫子曰</mark>：凡用兵之法，馳車千駟，革車千乘，帶甲十萬；千里饋糧，則內外之費賓",
+                        )],
+                    ),
+                ],
+            ),
+        ];
+
+        for (parts, tests) in inputs {
+            for (needles, snippets) in tests {
+                let mut results = Vec::new();
+
+                for part in &parts {
+                    if let Some(matched) =
+                        generate_snippet(part, &needles, Language::English, false)
+                    {
+                        results.push(matched);
+                    }
+                }
+
+                assert_eq!(snippets, results);
+            }
+        }
+    }
+}
@@ -0,0 +1,152 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
+ */
+
+use std::borrow::Cow;
+
+use rust_stemmers::Algorithm;
+
+use super::{Language, LanguageTokenizer};
+
+#[derive(Debug, PartialEq, Eq)]
+pub struct StemmedToken<'x> {
+    pub word: Cow<'x, str>,
+    pub stemmed_word: Option<Cow<'x, str>>,
+    pub from: usize, // Word offset in the text part
+    pub to: usize,   // Word length
+}
+
+pub struct Stemmer<'x> {
+    stemmer: Option<rust_stemmers::Stemmer>,
+    tokenizer: LanguageTokenizer<'x>,
+}
+
+impl<'x> Stemmer<'x> {
+    pub fn new(text: &'x str, language: Language, max_token_length: usize) -> Stemmer<'x> {
+        Stemmer {
+            tokenizer: language.tokenize_text(text, max_token_length),
+            stemmer: STEMMER_MAP[language as usize].map(rust_stemmers::Stemmer::create),
+        }
+    }
+}
+
+impl<'x> Iterator for Stemmer<'x> {
+    type Item = StemmedToken<'x>;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        let token = self.tokenizer.next()?;
+        Some(StemmedToken {
+            stemmed_word: self.stemmer.as_ref().and_then(|stemmer| {
+                match stemmer.stem(&token.word) {
+                    Cow::Owned(text) if text.len() != token.word.len() || text != token.word => {
+                        Some(text.into())
+                    }
+                    _ => None,
+                }
+            }),
+            word: token.word,
+            from: token.from,
+            to: token.to,
+        })
+    }
+}
+
+pub static STEMMER_MAP: &[Option<Algorithm>] = &[
+    None,                        // Esperanto = 0,
+    Some(Algorithm::English),    // English = 1,
+    Some(Algorithm::Russian),    // Russian = 2,
+    None,                        // Mandarin = 3,
+    Some(Algorithm::Spanish),    // Spanish = 4,
+    Some(Algorithm::Portuguese), // Portuguese = 5,
+    Some(Algorithm::Italian),    // Italian = 6,
+    None,                        // Bengali = 7,
+    Some(Algorithm::French),     // French = 8,
+    Some(Algorithm::German),     // German = 9,
+    None,                        // Ukrainian = 10,
+    None,                        // Georgian = 11,
+    Some(Algorithm::Arabic),     // Arabic = 12,
+    None,                        // Hindi = 13,
+    None,                        // Japanese = 14,
+    None,                        // Hebrew = 15,
+    None,                        // Yiddish = 16,
+    None,                        // Polish = 17,
+    None,                        // Amharic = 18,
+    None,                        // Javanese = 19,
+    None,                        // Korean = 20,
+    Some(Algorithm::Norwegian),  // Bokmal = 21,
+    Some(Algorithm::Danish),     // Danish = 22,
+    Some(Algorithm::Swedish),    // Swedish = 23,
+    Some(Algorithm::Finnish),    // Finnish = 24,
+    Some(Algorithm::Turkish),    // Turkish = 25,
+    Some(Algorithm::Dutch),      // Dutch = 26,
+    Some(Algorithm::Hungarian),  // Hungarian = 27,
+    None,                        // Czech = 28,
+    Some(Algorithm::Greek),      // Greek = 29,
+    None,                        // Bulgarian = 30,
+    None,                        // Belarusian = 31,
+    None,                        // Marathi = 32,
+    None,                        // Kannada = 33,
+    Some(Algorithm::Romanian),   // Romanian = 34,
+    None,                        // Slovene = 35,
+    None,                        // Croatian = 36,
+    None,                        // Serbian = 37,
+    None,                        // Macedonian = 38,
+    None,                        // Lithuanian = 39,
+    None,                        // Latvian = 40,
+    None,                        // Estonian = 41,
+    Some(Algorithm::Tamil),      // Tamil = 42,
+    None,                        // Vietnamese = 43,
+    None,                        // Urdu = 44,
+    None,                        // Thai = 45,
+    None,                        // Gujarati = 46,
+    None,                        // Uzbek = 47,
+    None,                        // Punjabi = 48,
+    None,                        // Azerbaijani = 49,
+    None,                        // Indonesian = 50,
+    None,                        // Telugu = 51,
+    None,                        // Persian = 52,
+    None,                        // Malayalam = 53,
+    None,                        // Oriya = 54,
+    None,                        // Burmese = 55,
+    None,                        // Nepali = 56,
+    None,                        // Sinhalese = 57,
+    None,                        // Khmer = 58,
+    None,                        // Turkmen = 59,
+    None,                        // Akan = 60,
+    None,                        // Zulu = 61,
+    None,                        // Shona = 62,
+    None,                        // Afrikaans = 63,
+    None,                        // Latin = 64,
+    None,                        // Slovak = 65,
+    None,                        // Catalan = 66,
+    None,                        // Tagalog = 67,
+    None,                        // Armenian = 68,
+    None,                        // Unknown = 69,
+    None,                        // None = 70,
+];
+
+#[cfg(test)]
+mod tests {
+
+    use super::*;
+
+    #[test]
+    fn stemmer() {
+        let inputs = [
+            (
+                "love loving lovingly loved lovely",
+                Language::English,
+                "love",
+            ),
+            ("querer queremos quer", Language::Spanish, "quer"),
+        ];
+
+        for (input, language, result) in inputs {
+            for token in Stemmer::new(input, language, 40) {
+                assert_eq!(token.stemmed_word.unwrap_or(token.word), result);
+            }
+        }
+    }
+}
@@ -0,0 +1,9 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
+ */
+
+pub mod classifier;
+pub mod language;
+pub mod tokenizers;
@@ -0,0 +1,202 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
+ */
+
+use std::{borrow::Cow, sync::LazyLock, vec::IntoIter};
+
+use jieba_rs::Jieba;
+
+use super::{InnerToken, Token};
+
+pub(crate) static JIEBA: LazyLock<Jieba> = LazyLock::new(Jieba::new);
+
+pub struct ChineseTokenizer<'x, T, I>
+where
+    T: Iterator<Item = Token<I>>,
+    I: InnerToken<'x>,
+{
+    tokenizer: T,
+    tokens: IntoIter<Token<I>>,
+    phantom: std::marker::PhantomData<&'x str>,
+}
+
+impl<'x, T, I> ChineseTokenizer<'x, T, I>
+where
+    T: Iterator<Item = Token<I>>,
+    I: InnerToken<'x>,
+{
+    pub fn new(tokenizer: T) -> Self {
+        ChineseTokenizer {
+            tokenizer,
+            tokens: Vec::new().into_iter(),
+            phantom: std::marker::PhantomData,
+        }
+    }
+}
+
+impl<'x, T, I> Iterator for ChineseTokenizer<'x, T, I>
+where
+    T: Iterator<Item = Token<I>>,
+    I: InnerToken<'x>,
+{
+    type Item = Token<I>;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        loop {
+            if let Some(token) = self.tokens.next() {
+                return Some(token);
+            } else {
+                let token = self.tokenizer.next()?;
+                if token.word.is_alphabetic_8bit() {
+                    let mut token_to = token.from;
+                    match token.word.unwrap_alphabetic() {
+                        Cow::Borrowed(word) => {
+                            self.tokens = JIEBA
+                                .cut(word, false)
+                                .into_iter()
+                                .map(|cut| {
+                                    let token_from = token_to;
+                                    token_to += cut.word.len();
+                                    Token {
+                                        word: I::new_alphabetic(cut.word),
+                                        from: token_from,
+                                        to: token_to,
+                                    }
+                                })
+                                .collect::<Vec<_>>()
+                                .into_iter();
+                        }
+                        Cow::Owned(word) => {
+                            self.tokens = JIEBA
+                                .cut(&word, false)
+                                .into_iter()
+                                .map(|cut| {
+                                    let token_from = token_to;
+                                    token_to += cut.word.len();
+                                    Token {
+                                        word: I::new_alphabetic(cut.word.to_string()),
+                                        from: token_from,
+                                        to: token_to,
+                                    }
+                                })
+                                .collect::<Vec<_>>()
+                                .into_iter();
+                        }
+                    }
+                } else {
+                    return token.into();
+                }
+            }
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use crate::tokenizers::{Token, chinese::ChineseTokenizer, word::WordTokenizer};
+
+    #[test]
+    fn chinese_tokenizer() {
+        assert_eq!(
+            ChineseTokenizer::new(WordTokenizer::new(
+                "孫子曰：兵者，國之大事，死生之地，存亡之道，不可不察也。",
+                40
+            ),)
+            .collect::<Vec<_>>(),
+            vec![
+                Token {
+                    word: "孫".into(),
+                    from: 0,
+                    to: 3
+                },
+                Token {
+                    word: "子".into(),
+                    from: 3,
+                    to: 6
+                },
+                Token {
+                    word: "曰".into(),
+                    from: 6,
+                    to: 9
+                },
+                Token {
+                    word: "兵".into(),
+                    from: 12,
+                    to: 15
+                },
+                Token {
+                    word: "者".into(),
+                    from: 15,
+                    to: 18
+                },
+                Token {
+                    word: "國".into(),
+                    from: 21,
+                    to: 24
+                },
+                Token {
+                    word: "之".into(),
+                    from: 24,
+                    to: 27
+                },
+                Token {
+                    word: "大事".into(),
+                    from: 27,
+                    to: 33
+                },
+                Token {
+                    word: "死".into(),
+                    from: 36,
+                    to: 39
+                },
+                Token {
+                    word: "生".into(),
+                    from: 39,
+                    to: 42
+                },
+                Token {
+                    word: "之".into(),
+                    from: 42,
+                    to: 45
+                },
+                Token {
+                    word: "地".into(),
+                    from: 45,
+                    to: 48
+                },
+                Token {
+                    word: "存亡".into(),
+                    from: 51,
+                    to: 57
+                },
+                Token {
+                    word: "之".into(),
+                    from: 57,
+                    to: 60
+                },
+                Token {
+                    word: "道".into(),
+                    from: 60,
+                    to: 63
+                },
+                Token {
+                    word: "不可不".into(),
+                    from: 66,
+                    to: 75
+                },
+                Token {
+                    word: "察".into(),
+                    from: 75,
+                    to: 78
+                },
+                Token {
+                    word: "也".into(),
+                    from: 78,
+                    to: 81
+                }
+            ]
+        );
+    }
+}
@@ -0,0 +1,407 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
+ */
+
+use super::{InnerToken, Token};
+use maplit::hashmap;
+use std::collections::HashMap;
+use std::vec::IntoIter;
+use std::{hash::Hash, sync::LazyLock};
+
+pub struct JapaneseTokenizer<'x, T, I>
+where
+    T: Iterator<Item = Token<I>>,
+    I: InnerToken<'x>,
+{
+    tokenizer: T,
+    tokens: IntoIter<Token<I>>,
+    phantom: std::marker::PhantomData<&'x str>,
+}
+
+impl<'x, T, I> JapaneseTokenizer<'x, T, I>
+where
+    T: Iterator<Item = Token<I>>,
+    I: InnerToken<'x>,
+{
+    pub fn new(tokenizer: T) -> Self {
+        JapaneseTokenizer {
+            tokenizer,
+            tokens: Vec::new().into_iter(),
+            phantom: std::marker::PhantomData,
+        }
+    }
+}
+
+impl<'x, T, I> Iterator for JapaneseTokenizer<'x, T, I>
+where
+    T: Iterator<Item = Token<I>>,
+    I: InnerToken<'x>,
+{
+    type Item = Token<I>;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        loop {
+            if let Some(token) = self.tokens.next() {
+                return Some(token);
+            } else {
+                let token = self.tokenizer.next()?;
+                if token.word.is_alphabetic_8bit() {
+                    let mut token_to = token.from;
+                    self.tokens = tokenize(token.word.unwrap_alphabetic().as_ref())
+                        .into_iter()
+                        .map(|word| {
+                            let token_from = token_to;
+                            token_to += word.len();
+                            Token {
+                                word: I::new_alphabetic(word.to_string()),
+                                from: token_from,
+                                to: token_to,
+                            }
+                        })
+                        .collect::<Vec<_>>()
+                        .into_iter();
+                } else {
+                    return token.into();
+                }
+            }
+        }
+    }
+}
+
+// Ported from https://github.com/woxtu/rust-tinysegmenter, MIT license
+
+const BIAS: i32 = -332;
+
+fn get_score<T: Eq + Hash>(d: &HashMap<T, i32>, s: &T) -> i32 {
+    d.get(s).cloned().unwrap_or(0)
+}
+
+fn get_ctype(c: char) -> char {
+    match c as u32 {
+        0x4E00 | 0x4E8C | 0x4E09 | 0x56DB | 0x4E94 | 0x516D | 0x4E03 | 0x516B | 0x4E5D | 0x5341 => {
+            'M'
+        }
+        0x767E | 0x5343 | 0x4E07 | 0x5104 | 0x5146 => 'M',
+        0x4E00..=0x9FA0 | 0x3005 | 0x3006 | 0x30F5 | 0x30F6 => 'H',
+        0x3041..=0x3093 => 'I',
+        0x30A1..=0x30F4 | 0x30FC | 0xFF71..=0xFF9D | 0xFF9E | 0xFF70 => 'K',
+        0x61..=0x7A | 0x41..=0x5A | 0xFF41..=0xFF5A | 0xFF21..=0xFF3A => 'A',
+        0x30..=0x3a | 0xFF10..=0xFF19 => 'N',
+        _ => 'O',
+    }
+}
+
+pub fn tokenize(s: &str) -> Vec<String> {
+    if s.is_empty() {
+        return Vec::new();
+    }
+
+    let mut result = Vec::with_capacity(s.chars().count());
+
+    let segments = [B3, B2, B1]
+        .into_iter()
+        .chain(s.chars())
+        .chain([E1, E2, E3])
+        .collect::<Vec<_>>();
+
+    let ctypes = ['O'; 3]
+        .into_iter()
+        .chain(s.chars().map(get_ctype))
+        .chain(['O'; 3])
+        .collect::<Vec<_>>();
+
+    let mut word = segments[3].to_string();
+    let mut p = vec!['U'; 3];
+
+    for index in 4..segments.len() - 3 {
+        let mut score = BIAS;
+        let w = &segments[index - 3..index + 3];
+        let c = &ctypes[index - 3..index + 3];
+
+        score += get_score(&*UP1, &p[0]);
+        score += get_score(&*UP2, &p[1]);
+        score += get_score(&*UP3, &p[2]);
+        score += get_score(&*BP1, &(p[0], p[1]));
+        score += get_score(&*BP2, &(p[1], p[2]));
+        score += get_score(&*UW1, &w[0]);
+        score += get_score(&*UW2, &w[1]);
+        score += get_score(&*UW3, &w[2]);
+        score += get_score(&*UW4, &w[3]);
+        score += get_score(&*UW5, &w[4]);
+        score += get_score(&*UW6, &w[5]);
+        score += get_score(&*BW1, &(w[1], w[2]));
+        score += get_score(&*BW2, &(w[2], w[3]));
+        score += get_score(&*BW3, &(w[3], w[4]));
+        score += get_score(&*TW1, &(w[0], w[1], w[2]));
+        score += get_score(&*TW2, &(w[1], w[2], w[3]));
+        score += get_score(&*TW3, &(w[2], w[3], w[4]));
+        score += get_score(&*TW4, &(w[3], w[4], w[5]));
+        score += get_score(&*UC1, &c[0]);
+        score += get_score(&*UC2, &c[1]);
+        score += get_score(&*UC3, &c[2]);
+        score += get_score(&*UC4, &c[3]);
+        score += get_score(&*UC5, &c[4]);
+        score += get_score(&*UC6, &c[5]);
+        score += get_score(&*BC1, &(c[1], c[2]));
+        score += get_score(&*BC2, &(c[2], c[3]));
+        score += get_score(&*BC3, &(c[3], c[4]));
+        score += get_score(&*TC1, &(c[0], c[1], c[2]));
+        score += get_score(&*TC2, &(c[1], c[2], c[3]));
+        score += get_score(&*TC3, &(c[2], c[3], c[4]));
+        score += get_score(&*TC4, &(c[3], c[4], c[5]));
+        score += get_score(&*UQ1, &(p[0], c[0]));
+        score += get_score(&*UQ2, &(p[1], c[1]));
+        score += get_score(&*UQ3, &(p[2], c[2]));
+        score += get_score(&*BQ1, &(p[1], c[1], c[2]));
+        score += get_score(&*BQ2, &(p[1], c[2], c[3]));
+        score += get_score(&*BQ3, &(p[2], c[1], c[2]));
+        score += get_score(&*BQ4, &(p[2], c[2], c[3]));
+        score += get_score(&*TQ1, &(p[1], c[0], c[1], c[2]));
+        score += get_score(&*TQ2, &(p[1], c[1], c[2], c[3]));
+        score += get_score(&*TQ3, &(p[2], c[0], c[1], c[2]));
+        score += get_score(&*TQ4, &(p[2], c[1], c[2], c[3]));
+
+        p.remove(0);
+        p.push(if score < 0 { 'O' } else { 'B' });
+
+        if 0 < score {
+            result.push(word.clone());
+            word.clear();
+        }
+        word.push(segments[index]);
+    }
+
+    result.push(word.clone());
+    result
+}
+
+const B1: char = '\u{F0000}';
+const B2: char = '\u{F0001}';
+const B3: char = '\u{F0002}';
+const E1: char = '\u{F0003}';
+const E2: char = '\u{F0004}';
+const E3: char = '\u{F0005}';
+
+static BC1: LazyLock<HashMap<(char, char), i32>> = LazyLock::new(|| {
+    hashmap! { ('H', 'H') => 6, ('I', 'I') => 2461, ('K', 'H') => 406, ('O', 'H') => -1378, }
+});
+static BC2: LazyLock<HashMap<(char, char), i32>> = LazyLock::new(|| {
+    hashmap! { ('A', 'A') => -3267, ('A', 'I') => 2744, ('A', 'N') => -878, ('H', 'H') => -4070, ('H', 'M') => -1711, ('H', 'N') => 4012, ('H', 'O') => 3761, ('I', 'A') => 1327, ('I', 'H') => -1184, ('I', 'I') => -1332, ('I', 'K') => 1721, ('I', 'O') => 5492, ('K', 'I') => 3831, ('K', 'K') => -8741, ('M', 'H') => -3132, ('M', 'K') => 3334, ('O', 'O') => -2920, }
+});
+static BC3: LazyLock<HashMap<(char, char), i32>> = LazyLock::new(|| {
+    hashmap! { ('H', 'H') => 996, ('H', 'I') => 626, ('H', 'K') => -721, ('H', 'N') => -1307, ('H', 'O') => -836, ('I', 'H') => -301, ('K', 'K') => 2762, ('M', 'K') => 1079, ('M', 'M') => 4034, ('O', 'A') => -1652, ('O', 'H') => 266, }
+});
+static BP1: LazyLock<HashMap<(char, char), i32>> = LazyLock::new(|| {
+    hashmap! { ('B', 'B') => 295, ('O', 'B') => 304, ('O', 'O') => -125, ('U', 'B') => 352, }
+});
+static BP2: LazyLock<HashMap<(char, char), i32>> = LazyLock::new(|| {
+    hashmap! { ('B', 'O') => 60, ('O', 'O') => -1762, }
+});
+static BQ1: LazyLock<HashMap<(char, char, char), i32>> = LazyLock::new(|| {
+    hashmap! { ('B', 'H', 'H') => 1150, ('B', 'H', 'M') => 1521, ('B', 'I', 'I') => -1158, ('B', 'I', 'M') => 886, ('B', 'M', 'H') => 1208, ('B', 'N', 'H') => 449, ('B', 'O', 'H') => -91, ('B', 'O', 'O') => -2597, ('O', 'H', 'I') => 451, ('O', 'I', 'H') => -296, ('O', 'K', 'A') => 1851, ('O', 'K', 'H') => -1020, ('O', 'K', 'K') => 904, ('O', 'O', 'O') => 2965, }
+});
+static BQ2: LazyLock<HashMap<(char, char, char), i32>> = LazyLock::new(|| {
+    hashmap! { ('B', 'H', 'H') => 118, ('B', 'H', 'I') => -1159, ('B', 'H', 'M') => 466, ('B', 'I', 'H') => -919, ('B', 'K', 'K') => -1720, ('B', 'K', 'O') => 864, ('O', 'H', 'H') => -1139, ('O', 'H', 'M') => -181, ('O', 'I', 'H') => 153, ('U', 'H', 'I') => -1146, }
+});
+static BQ3: LazyLock<HashMap<(char, char, char), i32>> = LazyLock::new(|| {
+    hashmap! { ('B', 'H', 'H') => -792, ('B', 'H', 'I') => 2664, ('B', 'I', 'I') => -299, ('B', 'K', 'I') => 419, ('B', 'M', 'H') => 937, ('B', 'M', 'M') => 8335, ('B', 'N', 'N') => 998, ('B', 'O', 'H') => 775, ('O', 'H', 'H') => 2174, ('O', 'H', 'M') => 439, ('O', 'I', 'I') => 280, ('O', 'K', 'H') => 1798, ('O', 'K', 'I') => -793, ('O', 'K', 'O') => -2242, ('O', 'M', 'H') => -2402, ('O', 'O', 'O') => 11699, }
+});
+static BQ4: LazyLock<HashMap<(char, char, char), i32>> = LazyLock::new(|| {
+    hashmap! { ('B', 'H', 'H') => -3895, ('B', 'I', 'H') => 3761, ('B', 'I', 'I') => -4654, ('B', 'I', 'K') => 1348, ('B', 'K', 'K') => -1806, ('B', 'M', 'I') => -3385, ('B', 'O', 'O') => -12396, ('O', 'A', 'H') => 926, ('O', 'H', 'H') => 266, ('O', 'H', 'K') => -2036, ('O', 'N', 'N') => -973, }
+});
+static BW1: LazyLock<HashMap<(char, char), i32>> = LazyLock::new(|| {
+    hashmap! { (',', 'と') => 660, (',', '同') => 727, (B1, 'あ') => 1404, (B1, '同') => 542, ('、', 'と') => 660, ('、', '同') => 727, ('」', 'と') => 1682, ('あ', 'っ') => 1505, ('い', 'う') => 1743, ('い', 'っ') => -2055, ('い', 'る') => 672, ('う', 'し') => -4817, ('う', 'ん') => 665, ('か', 'ら') => 3472, ('が', 'ら') => 600, ('こ', 'う') => -790, ('こ', 'と') => 2083, ('こ', 'ん') => -1262, ('さ', 'ら') => -4143, ('さ', 'ん') => 4573, ('し', 'た') => 2641, ('し', 'て') => 1104, ('す', 'で') => -3399, ('そ', 'こ') => 1977, ('そ', 'れ') => -871, ('た', 'ち') => 1122, ('た', 'め') => 601, ('っ', 'た') => 3463, ('つ', 'い') => -802, ('て', 'い') => 805, ('て', 'き') => 1249, ('で', 'き') => 1127, ('で', 'す') => 3445, ('で', 'は') => 844, ('と', 'い') => -4915, ('と', 'み') => 1922, ('ど', 'こ') => 3887, ('な', 'い') => 5713, ('な', 'っ') => 3015, ('な', 'ど') => 7379, ('な', 'ん') => -1113, ('に', 'し') => 2468, ('に', 'は') => 1498, ('に', 'も') => 1671, ('に', '対') => -912, ('の', '一') => -501, ('の', '中') => 741, ('ま', 'せ') => 2448, ('ま', 'で') => 1711, ('ま', 'ま') => 2600, ('ま', 'る') => -2155, ('や', 'む') => -1947, ('よ', 'っ') => -2565, ('れ', 'た') => 2369, ('れ', 'で') => -913, ('を', 'し') => 1860, ('を', '見') => 731, ('亡', 'く') => -1886, ('京', '都') => 2558, ('取', 'り') => -2784, ('大', 'き') => -2604, ('大', '阪') => 1497, ('平', '方') => -2314, ('引', 'き') => -1336, ('日', '本') => -195, ('本', '当') => -2423, ('毎', '日') => -2113, ('目', '指') => -724, ('｣', 'と') => 1682, }
+});
+static BW2: LazyLock<HashMap<(char, char), i32>> = LazyLock::new(|| {
+    hashmap! { ('.', '.') => -11822, ('1', '1') => -669, ('―', '―') => -5730, ('−', '−') => -13175, ('い', 'う') => -1609, ('う', 'か') => 2490, ('か', 'し') => -1350, ('か', 'も') => -602, ('か', 'ら') => -7194, ('か', 'れ') => 4612, ('が', 'い') => 853, ('が', 'ら') => -3198, ('き', 'た') => 1941, ('く', 'な') => -1597, ('こ', 'と') => -8392, ('こ', 'の') => -4193, ('さ', 'せ') => 4533, ('さ', 'れ') => 13168, ('さ', 'ん') => -3977, ('し', 'い') => -1819, ('し', 'か') => -545, ('し', 'た') => 5078, ('し', 'て') => 972, ('し', 'な') => 939, ('そ', 'の') => -3744, ('た', 'い') => -1253, ('た', 'た') => -662, ('た', 'だ') => -3857, ('た', 'ち') => -786, ('た', 'と') => 1224, ('た', 'は') => -939, ('っ', 'た') => 4589, ('っ', 'て') => 1647, ('っ', 'と') => -2094, ('て', 'い') => 6144, ('て', 'き') => 3640, ('て', 'く') => 2551, ('て', 'は') => -3110, ('て', 'も') => -3065, ('で', 'い') => 2666, ('で', 'き') => -1528, ('で', 'し') => -3828, ('で', 'す') => -4761, ('で', 'も') => -4203, ('と', 'い') => 1890, ('と', 'こ') => -1746, ('と', 'と') => -2279, ('と', 'の') => 720, ('と', 'み') => 5168, ('と', 'も') => -3941, ('な', 'い') => -2488, ('な', 'が') => -1313, ('な', 'ど') => -6509, ('な', 'の') => 2614, ('な', 'ん') => 3099, ('に', 'お') => -1615, ('に', 'し') => 2748, ('に', 'な') => 2454, ('に', 'よ') => -7236, ('に', '対') => -14943, ('に', '従') => -4688, ('に', '関') => -11388, ('の', 'か') => 2093, ('の', 'で') => -7059, ('の', 'に') => -6041, ('の', 'の') => -6125, ('は', 'い') => 1073, ('は', 'が') => -1033, ('は', 'ず') => -2532, ('ば', 'れ') => 1813, ('ま', 'し') => -1316, ('ま', 'で') => -6621, ('ま', 'れ') => 5409, ('め', 'て') => -3153, ('も', 'い') => 2230, ('も', 'の') => -10713, ('ら', 'か') => -944, ('ら', 'し') => -1611, ('ら', 'に') => -1897, ('り', 'し') => 651, ('り', 'ま') => 1620, ('れ', 'た') => 4270, ('れ', 'て') => 849, ('れ', 'ば') => 4114, ('ろ', 'う') => 6067, ('わ', 'れ') => 7901, ('を', '通') => -11877, ('ん', 'だ') => 728, ('ん', 'な') => -4115, ('一', '人') => 602, ('一', '方') => -1375, ('一', '日') => 970, ('一', '部') => -1051, ('上', 'が') => -4479, ('会', '社') => -1116, ('出', 'て') => 2163, ('分', 'の') => -7758, ('同', '党') => 970, ('同', '日') => -913, ('大', '阪') => -2471, ('委', '員') => -1250, ('少', 'な') => -1050, ('年', '度') => -8669, ('年', '間') => -1626, ('府', '県') => -2363, ('手', '権') => -1982, ('新', '聞') => -4066, ('日', '新') => -722, ('日', '本') => -7068, ('日', '米') => 3372, ('曜', '日') => -601, ('朝', '鮮') => -2355, ('本', '人') => -2697, ('東', '京') => -1543, ('然', 'と') => -1384, ('社', '会') => -1276, ('立', 'て') => -990, ('第', 'に') => -1612, ('米', '国') => -4268, ('１', '１') => -669, ('ｸ', 'ﾞ') => 1319,}
+});
+static BW3: LazyLock<HashMap<(char, char), i32>> = LazyLock::new(|| {
+    hashmap! { ('あ', 'た') => -2194, ('あ', 'り') => 719, ('あ', 'る') => 3846, ('い', '.') => -1185, ('い', '。') => -1185, ('い', 'い') => 5308, ('い', 'え') => 2079, ('い', 'く') => 3029, ('い', 'た') => 2056, ('い', 'っ') => 1883, ('い', 'る') => 5600, ('い', 'わ') => 1527, ('う', 'ち') => 1117, ('う', 'と') => 4798, ('え', 'と') => 1454, ('か', '.') => 2857, ('か', '。') => 2857, ('か', 'け') => -743, ('か', 'っ') => -4098, ('か', 'に') => -669, ('か', 'ら') => 6520, ('か', 'り') => -2670, ('が', ',') => 1816, ('が', '、') => 1816, ('が', 'き') => -4855, ('が', 'け') => -1127, ('が', 'っ') => -913, ('が', 'ら') => -4977, ('が', 'り') => -2064, ('き', 'た') => 1645, ('け', 'ど') => 1374, ('こ', 'と') => 7397, ('こ', 'の') => 1542, ('こ', 'ろ') => -2757, ('さ', 'い') => -714, ('さ', 'を') => 976, ('し', ',') => 1557, ('し', '、') => 1557, ('し', 'い') => -3714, ('し', 'た') => 3562, ('し', 'て') => 1449, ('し', 'な') => 2608, ('し', 'ま') => 1200, ('す', '.') => -1310, ('す', '。') => -1310, ('す', 'る') => 6521, ('ず', ',') => 3426, ('ず', '、') => 3426, ('ず', 'に') => 841, ('そ', 'う') => 428, ('た', '.') => 8875, ('た', '。') => 8875, ('た', 'い') => -594, ('た', 'の') => 812, ('た', 'り') => -1183, ('た', 'る') => -853, ('だ', '.') => 4098, ('だ', '。') => 4098, ('だ', 'っ') => 1004, ('っ', 'た') => -4748, ('っ', 'て') => 300, ('て', 'い') => 6240, ('て', 'お') => 855, ('て', 'も') => 302, ('で', 'す') => 1437, ('で', 'に') => -1482, ('で', 'は') => 2295, ('と', 'う') => -1387, ('と', 'し') => 2266, ('と', 'の') => 541, ('と', 'も') => -3543, ('ど', 'う') => 4664, ('な', 'い') => 1796, ('な', 'く') => -903, ('な', 'ど') => 2135, ('に', ',') => -1021, ('に', '、') => -1021, ('に', 'し') => 1771, ('に', 'な') => 1906, ('に', 'は') => 2644, ('の', ',') => -724, ('の', '、') => -724, ('の', '子') => -1000, ('は', ',') => 1337, ('は', '、') => 1337, ('べ', 'き') => 2181, ('ま', 'し') => 1113, ('ま', 'す') => 6943, ('ま', 'っ') => -1549, ('ま', 'で') => 6154, ('ま', 'れ') => -793, ('ら', 'し') => 1479, ('ら', 'れ') => 6820, ('る', 'る') => 3818, ('れ', ',') => 854, ('れ', '、') => 854, ('れ', 'た') => 1850, ('れ', 'て') => 1375, ('れ', 'ば') => -3246, ('れ', 'る') => 1091, ('わ', 'れ') => -605, ('ん', 'だ') => 606, ('ん', 'で') => 798, ('カ', '月') => 990, ('会', '議') => 860, ('入', 'り') => 1232, ('大', '会') => 2217, ('始', 'め') => 1681, ('市', ' ') => 965, ('新', '聞') => -5055, ('日', ',') => 974, ('日', '、') => 974, ('社', '会') => 2024, ('ｶ', '月') => 990, }
+});
+
+static TC1: LazyLock<HashMap<(char, char, char), i32>> = LazyLock::new(|| {
+    hashmap! { ('A', 'A', 'A') => 1093, ('H', 'H', 'H') => 1029, ('H', 'H', 'M') => 580, ('H', 'I', 'I') => 998, ('H', 'O', 'H') => -390, ('H', 'O', 'M') => -331, ('I', 'H', 'I') => 1169, ('I', 'O', 'H') => -142, ('I', 'O', 'I') => -1015, ('I', 'O', 'M') => 467, ('M', 'M', 'H') => 187, ('O', 'O', 'I') => -1832, }
+});
+static TC2: LazyLock<HashMap<(char, char, char), i32>> = LazyLock::new(|| {
+    hashmap! { ('H', 'H', 'O') => 2088, ('H', 'I', 'I') => -1023, ('H', 'M', 'M') => -1154, ('I', 'H', 'I') => -1965, ('K', 'K', 'H') => 703, ('O', 'I', 'I') => -2649, }
+});
+static TC3: LazyLock<HashMap<(char, char, char), i32>> = LazyLock::new(|| {
+    hashmap! { ('A', 'A', 'A') => -294, ('H', 'H', 'H') => 346, ('H', 'H', 'I') => -341, ('H', 'I', 'I') => -1088, ('H', 'I', 'K') => 731, ('H', 'O', 'H') => -1486, ('I', 'H', 'H') => 128, ('I', 'H', 'I') => -3041, ('I', 'H', 'O') => -1935, ('I', 'I', 'H') => -825, ('I', 'I', 'M') => -1035, ('I', 'O', 'I') => -542, ('K', 'H', 'H') => -1216, ('K', 'K', 'A') => 491, ('K', 'K', 'H') => -1217, ('K', 'O', 'K') => -1009, ('M', 'H', 'H') => -2694, ('M', 'H', 'M') => -457, ('M', 'H', 'O') => 123, ('M', 'M', 'H') => -471, ('N', 'N', 'H') => -1689, ('N', 'N', 'O') => 662, ('O', 'H', 'O') => -3393, }
+});
+static TC4: LazyLock<HashMap<(char, char, char), i32>> = LazyLock::new(|| {
+    hashmap! { ('H', 'H', 'H') => -203, ('H', 'H', 'I') => 1344, ('H', 'H', 'K') => 365, ('H', 'H', 'M') => -122, ('H', 'H', 'N') => 182, ('H', 'H', 'O') => 669, ('H', 'I', 'H') => 804, ('H', 'I', 'I') => 679, ('H', 'O', 'H') => 446, ('I', 'H', 'H') => 695, ('I', 'H', 'O') => -2324, ('I', 'I', 'H') => 321, ('I', 'I', 'I') => 1497, ('I', 'I', 'O') => 656, ('I', 'O', 'O') => 54, ('K', 'A', 'K') => 4845, ('K', 'K', 'A') => 3386, ('K', 'K', 'K') => 3065, ('M', 'H', 'H') => -405, ('M', 'H', 'I') => 201, ('M', 'M', 'H') => -241, ('M', 'M', 'M') => 661, ('M', 'O', 'M') => 841, }
+});
+static TQ1: LazyLock<HashMap<(char, char, char, char), i32>> = LazyLock::new(|| {
+    hashmap! { ('B', 'H', 'H', 'H') => -227, ('B', 'H', 'H', 'I') => 316, ('B', 'H', 'I', 'H') => -132, ('B', 'I', 'H', 'H') => 60, ('B', 'I', 'I', 'I') => 1595, ('B', 'N', 'H', 'H') => -744, ('B', 'O', 'H', 'H') => 225, ('B', 'O', 'O', 'O') => -908, ('O', 'A', 'K', 'K') => 482, ('O', 'H', 'H', 'H') => 281, ('O', 'H', 'I', 'H') => 249, ('O', 'I', 'H', 'I') => 200, ('O', 'I', 'I', 'H') => -68, }
+});
+static TQ2: LazyLock<HashMap<(char, char, char, char), i32>> = LazyLock::new(|| {
+    hashmap! { ('B', 'I', 'H', 'H') => -1401, ('B', 'I', 'I', 'I') => -1033, ('B', 'K', 'A', 'K') => -543, ('B', 'O', 'O', 'O') => -5591, }
+});
+static TQ3: LazyLock<HashMap<(char, char, char, char), i32>> = LazyLock::new(|| {
+    hashmap! { ('B', 'H', 'H', 'H') => 478, ('B', 'H', 'H', 'M') => -1073, ('B', 'H', 'I', 'H') => 222, ('B', 'H', 'I', 'I') => -504, ('B', 'I', 'I', 'H') => -116, ('B', 'I', 'I', 'I') => -105, ('B', 'M', 'H', 'I') => -863, ('B', 'M', 'H', 'M') => -464, ('B', 'O', 'M', 'H') => 620, ('O', 'H', 'H', 'H') => 346, ('O', 'H', 'H', 'I') => 1729, ('O', 'H', 'I', 'I') => 997, ('O', 'H', 'M', 'H') => 481, ('O', 'I', 'H', 'H') => 623, ('O', 'I', 'I', 'H') => 1344, ('O', 'K', 'A', 'K') => 2792, ('O', 'K', 'H', 'H') => 587, ('O', 'K', 'K', 'A') => 679, ('O', 'O', 'H', 'H') => 110, ('O', 'O', 'I', 'I') => -685, }
+});
+static TQ4: LazyLock<HashMap<(char, char, char, char), i32>> = LazyLock::new(|| {
+    hashmap! { ('B', 'H', 'H', 'H') => -721, ('B', 'H', 'H', 'M') => -3604, ('B', 'H', 'I', 'I') => -966, ('B', 'I', 'I', 'H') => -607, ('B', 'I', 'I', 'I') => -2181, ('O', 'A', 'A', 'A') => -2763, ('O', 'A', 'K', 'K') => 180, ('O', 'H', 'H', 'H') => -294, ('O', 'H', 'H', 'I') => 2446, ('O', 'H', 'H', 'O') => 480, ('O', 'H', 'I', 'H') => -1573, ('O', 'I', 'H', 'H') => 1935, ('O', 'I', 'H', 'I') => -493, ('O', 'I', 'I', 'H') => 626, ('O', 'I', 'I', 'I') => -4007, ('O', 'K', 'A', 'K') => -8156, }
+});
+static TW1: LazyLock<HashMap<(char, char, char), i32>> = LazyLock::new(|| {
+    hashmap! { ('に', 'つ', 'い') => -4681, ('東', '京', '都') => 2026, }
+});
+static TW2: LazyLock<HashMap<(char, char, char), i32>> = LazyLock::new(|| {
+    hashmap! { ('あ', 'る', '程') => -2049, ('い', 'っ', 'た') => -1256, ('こ', 'ろ', 'が') => -2434, ('し', 'ょ', 'う') => 3873, ('そ', 'の', '後') => -4430, ('だ', 'っ', 'て') => -1049, ('て', 'い', 'た') => 1833, ('と', 'し', 'て') => -4657, ('と', 'も', 'に') => -4517, ('も', 'の', 'で') => 1882, ('一', '気', 'に') => -792, ('初', 'め', 'て') => -1512, ('同', '時', 'に') => -8097, ('大', 'き', 'な') => -1255, ('対', 'し', 'て') => -2721, ('社', '会', '党') => -3216, }
+});
+static TW3: LazyLock<HashMap<(char, char, char), i32>> = LazyLock::new(|| {
+    hashmap! { ('い', 'た', 'だ') => -1734, ('し', 'て', 'い') => 1314, ('と', 'し', 'て') => -4314, ('に', 'つ', 'い') => -5483, ('に', 'と', 'っ') => -5989, ('に', '当', 'た') => -6247, ('の', 'で', ',') => -727, ('の', 'で', '、') => -727, ('の', 'も', 'の') => -600, ('れ', 'か', 'ら') => -3752, ('十', '二', '月') => -2287, }
+});
+static TW4: LazyLock<HashMap<(char, char, char), i32>> = LazyLock::new(|| {
+    hashmap! { ('い', 'う', '.') => 8576, ('い', 'う', '。') => 8576, ('か', 'ら', 'な') => -2348, ('し', 'て', 'い') => 2958, ('た', 'が', ',') => 1516, ('た', 'が', '、') => 1516, ('て', 'い', 'る') => 1538, ('と', 'い', 'う') => 1349, ('ま', 'し', 'た') => 5543, ('ま', 'せ', 'ん') => 1097, ('よ', 'う', 'と') => -4258, ('よ', 'る', 'と') => 5865, }
+});
+
+static UC1: LazyLock<HashMap<char, i32>> = LazyLock::new(|| {
+    hashmap! { 'A' => 484, 'K' => 93, 'M' => 645, 'O' => -505, }
+});
+static UC2: LazyLock<HashMap<char, i32>> = LazyLock::new(|| {
+    hashmap! { 'A' => 819, 'H' => 1059, 'I' => 409, 'M' => 3987, 'N' => 5775, 'O' => 646, }
+});
+static UC3: LazyLock<HashMap<char, i32>> = LazyLock::new(|| {
+    hashmap! { 'A' => -1370, 'I' => 2311, }
+});
+static UC4: LazyLock<HashMap<char, i32>> = LazyLock::new(|| {
+    hashmap! { 'A' => -2643, 'H' => 1809, 'I' => -1032, 'K' => -3450, 'M' => 3565, 'N' => 3876, 'O' => 6646, }
+});
+static UC5: LazyLock<HashMap<char, i32>> = LazyLock::new(|| {
+    hashmap! { 'H' => 313, 'I' => -1238, 'K' => -799, 'M' => 539, 'O' => -831, }
+});
+static UC6: LazyLock<HashMap<char, i32>> = LazyLock::new(|| {
+    hashmap! { 'H' => -506, 'I' => -253, 'K' => 87, 'M' => 247, 'O' => -387, }
+});
+static UP1: LazyLock<HashMap<char, i32>> = LazyLock::new(|| {
+    hashmap! { 'O' => -214, }
+});
+static UP2: LazyLock<HashMap<char, i32>> = LazyLock::new(|| {
+    hashmap! { 'B' => 69, 'O' => 935, }
+});
+static UP3: LazyLock<HashMap<char, i32>> = LazyLock::new(|| {
+    hashmap! { 'B' => 189, }
+});
+static UQ1: LazyLock<HashMap<(char, char), i32>> = LazyLock::new(|| {
+    hashmap! { ('B', 'H') => 21, ('B', 'I') => -12, ('B', 'K') => -99, ('B', 'N') => 142, ('B', 'O') => -56, ('O', 'H') => -95, ('O', 'I') => 477, ('O', 'K') => 410, ('O', 'O') => -2422, }
+});
+static UQ2: LazyLock<HashMap<(char, char), i32>> = LazyLock::new(|| {
+    hashmap! { ('B', 'H') => 216, ('B', 'I') => 113, ('O', 'K') => 1759, }
+});
+static UQ3: LazyLock<HashMap<(char, char), i32>> = LazyLock::new(|| {
+    hashmap! { ('B', 'A') => -479, ('B', 'H') => 42, ('B', 'I') => 1913, ('B', 'K') => -7198, ('B', 'M') => 3160, ('B', 'N') => 6427, ('B', 'O') => 14761, ('O', 'I') => -827, ('O', 'N') => -3212, }
+});
+static UW1: LazyLock<HashMap<char, i32>> = LazyLock::new(|| {
+    hashmap! { ',' => 156, '、' => 156, '「' => -463, 'あ' => -941, 'う' => -127, 'が' => -553, 'き' => 121, 'こ' => 505, 'で' => -201, 'と' => -547, 'ど' => -123, 'に' => -789, 'の' => -185, 'は' => -847, 'も' => -466, 'や' => -470, 'よ' => 182, 'ら' => -292, 'り' => 208, 'れ' => 169, 'を' => -446, 'ん' => -137, '・' => -135, '主' => -402, '京' => -268, '区' => -912, '午' => 871, '国' => -460, '大' => 561, '委' => 729, '市' => -411, '日' => -141, '理' => 361, '生' => -408, '県' => -386, '都' => -718, '｢' => -463, '･' => -135, }
+});
+static UW2: LazyLock<HashMap<char, i32>> = LazyLock::new(|| {
+    hashmap! { ',' => -829, '、' => -829, '〇' => 892, '「' => -645, '」' => 3145, 'あ' => -538, 'い' => 505, 'う' => 134, 'お' => -502, 'か' => 1454, 'が' => -856, 'く' => -412, 'こ' => 1141, 'さ' => 878, 'ざ' => 540, 'し' => 1529, 'す' => -675, 'せ' => 300, 'そ' => -1011, 'た' => 188, 'だ' => 1837, 'つ' => -949, 'て' => -291, 'で' => -268, 'と' => -981, 'ど' => 1273, 'な' => 1063, 'に' => -1764, 'の' => 130, 'は' => -409, 'ひ' => -1273, 'べ' => 1261, 'ま' => 600, 'も' => -1263, 'や' => -402, 'よ' => 1639, 'り' => -579, 'る' => -694, 'れ' => 571, 'を' => -2516, 'ん' => 2095, 'ア' => -587, 'カ' => 306, 'キ' => 568, 'ッ' => 831, '三' => -758, '不' => -2150, '世' => -302, '中' => -968, '主' => -861, '事' => 492, '人' => -123, '会' => 978, '保' => 362, '入' => 548, '初' => -3025, '副' => -1566, '北' => -3414, '区' => -422, '大' => -1769, '天' => -865, '太' => -483, '子' => -1519, '学' => 760, '実' => 1023, '小' => -2009, '市' => -813, '年' => -1060, '強' => 1067, '手' => -1519, '揺' => -1033, '政' => 1522, '文' => -1355, '新' => -1682, '日' => -1815, '明' => -1462, '最' => -630, '朝' => -1843, '本' => -1650, '東' => -931, '果' => -665, '次' => -2378, '民' => -180, '気' => -1740, '理' => 752, '発' => 529, '目' => -1584, '相' => -242, '県' => -1165, '立' => -763, '第' => 810, '米' => 509, '自' => -1353, '行' => 838, '西' => -744, '見' => -3874, '調' => 1010, '議' => 1198, '込' => 3041, '開' => 1758, '間' => -1257, '｢' => -645, '｣' => 3145, 'ｯ' => 831, 'ｱ' => -587, 'ｶ' => 306, 'ｷ' => 568, }
+});
+static UW3: LazyLock<HashMap<char, i32>> = LazyLock::new(|| {
+    hashmap! { ',' => 4889, '1' => -800, '−' => -1723, '、' => 4889, '々' => -2311, '〇' => 5827, '」' => 2670, '〓' => -3573, 'あ' => -2696, 'い' => 1006, 'う' => 2342, 'え' => 1983, 'お' => -4864, 'か' => -1163, 'が' => 3271, 'く' => 1004, 'け' => 388, 'げ' => 401, 'こ' => -3552, 'ご' => -3116, 'さ' => -1058, 'し' => -395, 'す' => 584, 'せ' => 3685, 'そ' => -5228, 'た' => 842, 'ち' => -521, 'っ' => -1444, 'つ' => -1081, 'て' => 6167, 'で' => 2318, 'と' => 1691, 'ど' => -899, 'な' => -2788, 'に' => 2745, 'の' => 4056, 'は' => 4555, 'ひ' => -2171, 'ふ' => -1798, 'へ' => 1199, 'ほ' => -5516, 'ま' => -4384, 'み' => -120, 'め' => 1205, 'も' => 2323, 'や' => -788, 'よ' => -202, 'ら' => 727, 'り' => 649, 'る' => 5905, 'れ' => 2773, 'わ' => -1207, 'を' => 6620, 'ん' => -518, 'ア' => 551, 'グ' => 1319, 'ス' => 874, 'ッ' => -1350, 'ト' => 521, 'ム' => 1109, 'ル' => 1591, 'ロ' => 2201, 'ン' => 278, '・' => -3794, '一' => -1619, '下' => -1759, '世' => -2087, '両' => 3815, '中' => 653, '主' => -758, '予' => -1193, '二' => 974, '人' => 2742, '今' => 792, '他' => 1889, '以' => -1368, '低' => 811, '何' => 4265, '作' => -361, '保' => -2439, '元' => 4858, '党' => 3593, '全' => 1574, '公' => -3030, '六' => 755, '共' => -1880, '円' => 5807, '再' => 3095, '分' => 457, '初' => 2475, '別' => 1129, '前' => 2286, '副' => 4437, '力' => 365, '動' => -949, '務' => -1872, '化' => 1327, '北' => -1038, '区' => 4646, '千' => -2309, '午' => -783, '協' => -1006, '口' => 483, '右' => 1233, '各' => 3588, '合' => -241, '同' => 3906, '和' => -837, '員' => 4513, '国' => 642, '型' => 1389, '場' => 1219, '外' => -241, '妻' => 2016, '学' => -1356, '安' => -423, '実' => -1008, '家' => 1078, '小' => -513, '少' => -3102, '州' => 1155, '市' => 3197, '平' => -1804, '年' => 2416, '広' => -1030, '府' => 1605, '度' => 1452, '建' => -2352, '当' => -3885, '得' => 1905, '思' => -1291, '性' => 1822, '戸' => -488, '指' => -3973, '政' => -2013, '教' => -1479, '数' => 3222, '文' => -1489, '新' => 1764, '日' => 2099, '旧' => 5792, '昨' => -661, '時' => -1248, '曜' => -951, '最' => -937, '月' => 4125, '期' => 360, '李' => 3094, '村' => 364, '東' => -805, '核' => 5156, '森' => 2438, '業' => 484, '氏' => 2613, '民' => -1694, '決' => -1073, '法' => 1868, '海' => -495, '無' => 979, '物' => 461, '特' => -3850, '生' => -273, '用' => 914, '町' => 1215, '的' => 7313, '直' => -1835, '省' => 792, '県' => 6293, '知' => -1528, '私' => 4231, '税' => 401, '立' => -960, '第' => 1201, '米' => 7767, '系' => 3066, '約' => 3663, '級' => 1384, '統' => -4229, '総' => 1163, '線' => 1255, '者' => 6457, '能' => 725, '自' => -2869, '英' => 785, '見' => 1044, '調' => -562, '財' => -733, '費' => 1777, '車' => 1835, '軍' => 1375, '込' => -1504, '通' => -1136, '選' => -681, '郎' => 1026, '郡' => 4404, '部' => 1200, '金' => 2163, '長' => 421, '開' => -1432, '間' => 1302, '関' => -1282, '雨' => 2009, '電' => -1045, '非' => 2066, '駅' => 1620, '１' => -800, '｣' => 2670, '･' => -3794, 'ｯ' => -1350, 'ｱ' => 551, 'ｽ' => 874, 'ﾄ' => 521, 'ﾑ' => 1109, 'ﾙ' => 1591, 'ﾛ' => 2201, 'ﾝ' => 278, }
+});
+static UW4: LazyLock<HashMap<char, i32>> = LazyLock::new(|| {
+    hashmap! { ',' => 3930, '.' => 3508, '―' => -4841, '、' => 3930, '。' => 3508, '〇' => 4999, '「' => 1895, '」' => 3798, '〓' => -5156, 'あ' => 4752, 'い' => -3435, 'う' => -640, 'え' => -2514, 'お' => 2405, 'か' => 530, 'が' => 6006, 'き' => -4482, 'ぎ' => -3821, 'く' => -3788, 'け' => -4376, 'げ' => -4734, 'こ' => 2255, 'ご' => 1979, 'さ' => 2864, 'し' => -843, 'じ' => -2506, 'す' => -731, 'ず' => 1251, 'せ' => 181, 'そ' => 4091, 'た' => 5034, 'だ' => 5408, 'ち' => -3654, 'っ' => -5882, 'つ' => -1659, 'て' => 3994, 'で' => 7410, 'と' => 4547, 'な' => 5433, 'に' => 6499, 'ぬ' => 1853, 'ね' => 1413, 'の' => 7396, 'は' => 8578, 'ば' => 1940, 'ひ' => 4249, 'び' => -4134, 'ふ' => 1345, 'へ' => 6665, 'べ' => -744, 'ほ' => 1464, 'ま' => 1051, 'み' => -2082, 'む' => -882, 'め' => -5046, 'も' => 4169, 'ゃ' => -2666, 'や' => 2795, 'ょ' => -1544, 'よ' => 3351, 'ら' => -2922, 'り' => -9726, 'る' => -14896, 'れ' => -2613, 'ろ' => -4570, 'わ' => -1783, 'を' => 13150, 'ん' => -2352, 'カ' => 2145, 'コ' => 1789, 'セ' => 1287, 'ッ' => -724, 'ト' => -403, 'メ' => -1635, 'ラ' => -881, 'リ' => -541, 'ル' => -856, 'ン' => -3637, '・' => -4371, 'ー' => -11870, '一' => -2069, '中' => 2210, '予' => 782, '事' => -190, '井' => -1768, '人' => 1036, '以' => 544, '会' => 950, '体' => -1286, '作' => 530, '側' => 4292, '先' => 601, '党' => -2006, '共' => -1212, '内' => 584, '円' => 788, '初' => 1347, '前' => 1623, '副' => 3879, '力' => -302, '動' => -740, '務' => -2715, '化' => 776, '区' => 4517, '協' => 1013, '参' => 1555, '合' => -1834, '和' => -681, '員' => -910, '器' => -851, '回' => 1500, '国' => -619, '園' => -1200, '地' => 866, '場' => -1410, '塁' => -2094, '士' => -1413, '多' => 1067, '大' => 571, '子' => -4802, '学' => -1397, '定' => -1057, '寺' => -809, '小' => 1910, '屋' => -1328, '山' => -1500, '島' => -2056, '川' => -2667, '市' => 2771, '年' => 374, '庁' => -4556, '後' => 456, '性' => 553, '感' => 916, '所' => -1566, '支' => 856, '改' => 787, '政' => 2182, '教' => 704, '文' => 522, '方' => -856, '日' => 1798, '時' => 1829, '最' => 845, '月' => -9066, '木' => -485, '来' => -442, '校' => -360, '業' => -1043, '氏' => 5388, '民' => -2716, '気' => -910, '沢' => -939, '済' => -543, '物' => -735, '率' => 672, '球' => -1267, '生' => -1286, '産' => -1101, '田' => -2900, '町' => 1826, '的' => 2586, '目' => 922, '省' => -3485, '県' => 2997, '空' => -867, '立' => -2112, '第' => 788, '米' => 2937, '系' => 786, '約' => 2171, '経' => 1146, '統' => -1169, '総' => 940, '線' => -994, '署' => 749, '者' => 2145, '能' => -730, '般' => -852, '行' => -792, '規' => 792, '警' => -1184, '議' => -244, '谷' => -1000, '賞' => 730, '車' => -1481, '軍' => 1158, '輪' => -1433, '込' => -3370, '近' => 929, '道' => -1291, '選' => 2596, '郎' => -4866, '都' => 1192, '野' => -1100, '銀' => -2213, '長' => 357, '間' => -2344, '院' => -2297, '際' => -2604, '電' => -878, '領' => -1659, '題' => -792, '館' => -1984, '首' => 1749, '高' => 2120, '｢' => 1895, '｣' => 3798, '･' => -4371, 'ｯ' => -724, 'ｰ' => -11870, 'ｶ' => 2145, 'ｺ' => 1789, 'ｾ' => 1287, 'ﾄ' => -403, 'ﾒ' => -1635, 'ﾗ' => -881, 'ﾘ' => -541, 'ﾙ' => -856, 'ﾝ' => -3637, }
+});
+static UW5: LazyLock<HashMap<char, i32>> = LazyLock::new(|| {
+    hashmap! { ',' => 465, '.' => -299, '1' => -514, E2 => -32768, ']' => -2762, '、' => 465, '。' => -299, '「' => 363, 'あ' => 1655, 'い' => 331, 'う' => -503, 'え' => 1199, 'お' => 527, 'か' => 647, 'が' => -421, 'き' => 1624, 'ぎ' => 1971, 'く' => 312, 'げ' => -983, 'さ' => -1537, 'し' => -1371, 'す' => -852, 'だ' => -1186, 'ち' => 1093, 'っ' => 52, 'つ' => 921, 'て' => -18, 'で' => -850, 'と' => -127, 'ど' => 1682, 'な' => -787, 'に' => -1224, 'の' => -635, 'は' => -578, 'べ' => 1001, 'み' => 502, 'め' => 865, 'ゃ' => 3350, 'ょ' => 854, 'り' => -208, 'る' => 429, 'れ' => 504, 'わ' => 419, 'を' => -1264, 'ん' => 327, 'イ' => 241, 'ル' => 451, 'ン' => -343, '中' => -871, '京' => 722, '会' => -1153, '党' => -654, '務' => 3519, '区' => -901, '告' => 848, '員' => 2104, '大' => -1296, '学' => -548, '定' => 1785, '嵐' => -1304, '市' => -2991, '席' => 921, '年' => 1763, '思' => 872, '所' => -814, '挙' => 1618, '新' => -1682, '日' => 218, '月' => -4353, '査' => 932, '格' => 1356, '機' => -1508, '氏' => -1347, '田' => 240, '町' => -3912, '的' => -3149, '相' => 1319, '省' => -1052, '県' => -4003, '研' => -997, '社' => -278, '空' => -813, '統' => 1955, '者' => -2233, '表' => 663, '語' => -1073, '議' => 1219, '選' => -1018, '郎' => -368, '長' => 786, '間' => 1191, '題' => 2368, '館' => -689, '１' => -514, '｢' => 363, 'ｲ' => 241, 'ﾙ' => 451, 'ﾝ' => -343, }
+});
+static UW6: LazyLock<HashMap<char, i32>> = LazyLock::new(|| {
+    hashmap! { ',' => 227, '.' => 808, '1' => -270, E1 => 306, '、' => 227, '。' => 808, 'あ' => -307, 'う' => 189, 'か' => 241, 'が' => -73, 'く' => -121, 'こ' => -200, 'じ' => 1782, 'す' => 383, 'た' => -428, 'っ' => 573, 'て' => -1014, 'で' => 101, 'と' => -105, 'な' => -253, 'に' => -149, 'の' => -417, 'は' => -236, 'も' => -206, 'り' => 187, 'る' => -135, 'を' => 195, 'ル' => -673, 'ン' => -496, '一' => -277, '中' => 201, '件' => -800, '会' => 624, '前' => 302, '区' => 1792, '員' => -1212, '委' => 798, '学' => -960, '市' => 887, '広' => -695, '後' => 535, '業' => -697, '相' => 753, '社' => -507, '福' => 974, '空' => -822, '者' => 1811, '連' => 463, '郎' => 1082, '１' => -270, 'ﾙ' => -673, 'ﾝ' => -496, }
+});
+
+#[cfg(test)]
+mod tests {
+    use crate::tokenizers::{Token, japanese::JapaneseTokenizer, word::WordTokenizer};
+
+    #[test]
+    fn japanese_tokenizer() {
+        assert_eq!(
+            JapaneseTokenizer::new(WordTokenizer::new(
+                "お先に失礼します あなたの名前は何ですか 123 abc-872",
+                40
+            ))
+            .collect::<Vec<_>>(),
+            vec![
+                Token {
+                    word: "お先".into(),
+                    from: 0,
+                    to: 6
+                },
+                Token {
+                    word: "に".into(),
+                    from: 6,
+                    to: 9
+                },
+                Token {
+                    word: "失礼".into(),
+                    from: 9,
+                    to: 15
+                },
+                Token {
+                    word: "し".into(),
+                    from: 15,
+                    to: 18
+                },
+                Token {
+                    word: "ます".into(),
+                    from: 18,
+                    to: 24
+                },
+                Token {
+                    word: "あなた".into(),
+                    from: 25,
+                    to: 34
+                },
+                Token {
+                    word: "の".into(),
+                    from: 34,
+                    to: 37
+                },
+                Token {
+                    word: "名前".into(),
+                    from: 37,
+                    to: 43
+                },
+                Token {
+                    word: "は".into(),
+                    from: 43,
+                    to: 46
+                },
+                Token {
+                    word: "何".into(),
+                    from: 46,
+                    to: 49
+                },
+                Token {
+                    word: "です".into(),
+                    from: 49,
+                    to: 55
+                },
+                Token {
+                    word: "か".into(),
+                    from: 55,
+                    to: 58
+                },
+                Token {
+                    word: "123".into(),
+                    from: 59,
+                    to: 62
+                },
+                Token {
+                    word: "abc".into(),
+                    from: 63,
+                    to: 66
+                },
+                Token {
+                    word: "872".into(),
+                    from: 67,
+                    to: 70
+                }
+            ]
+        );
+    }
+}
@@ -0,0 +1,58 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
+ */
+
+pub mod chinese;
+pub mod japanese;
+pub mod space;
+pub mod stream;
+pub mod types;
+pub mod word;
+
+use std::borrow::Cow;
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct Token<T> {
+    pub word: T,
+    pub from: usize,
+    pub to: usize,
+}
+
+pub trait InnerToken<'x>: Sized {
+    fn new_alphabetic(value: impl Into<Cow<'x, str>>) -> Self;
+    fn unwrap_alphabetic(self) -> Cow<'x, str>;
+    fn is_alphabetic(&self) -> bool;
+    fn is_alphabetic_8bit(&self) -> bool;
+}
+
+impl<'x> InnerToken<'x> for Cow<'x, str> {
+    fn new_alphabetic(value: impl Into<Cow<'x, str>>) -> Self {
+        value.into()
+    }
+
+    fn is_alphabetic(&self) -> bool {
+        true
+    }
+
+    fn is_alphabetic_8bit(&self) -> bool {
+        !self.is_ascii()
+    }
+
+    fn unwrap_alphabetic(self) -> Cow<'x, str> {
+        self
+    }
+}
+
+impl<T> Token<T> {
+    pub fn new(offset: usize, len: usize, word: T) -> Token<T> {
+        debug_assert!(offset <= u32::MAX as usize);
+        debug_assert!(len <= u8::MAX as usize);
+        Token {
+            from: offset,
+            to: offset + len,
+            word,
+        }
+    }
+}
@@ -0,0 +1,57 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
+ */
+
+use std::str::Chars;
+
+pub struct SpaceTokenizer<'x> {
+    iterator: Chars<'x>,
+    token: String,
+    max_token_length: usize,
+}
+
+impl SpaceTokenizer<'_> {
+    pub fn new(text: &'_ str, max_token_length: usize) -> SpaceTokenizer<'_> {
+        SpaceTokenizer {
+            iterator: text.chars(),
+            token: String::new(),
+            max_token_length,
+        }
+    }
+}
+
+impl Iterator for SpaceTokenizer<'_> {
+    type Item = String;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        for ch in self.iterator.by_ref() {
+            if ch.is_alphanumeric() {
+                if ch.is_uppercase() {
+                    for ch in ch.to_lowercase() {
+                        self.token.push(ch);
+                    }
+                } else {
+                    self.token.push(ch);
+                }
+            } else if !self.token.is_empty() {
+                if self.token.len() < self.max_token_length {
+                    return Some(std::mem::take(&mut self.token));
+                } else {
+                    self.token.clear();
+                }
+            }
+        }
+
+        if !self.token.is_empty() {
+            if self.token.len() < self.max_token_length {
+                return Some(std::mem::take(&mut self.token));
+            } else {
+                self.token.clear();
+            }
+        }
+
+        None
+    }
+}
@@ -0,0 +1,180 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
+ */
+
+use crate::{
+    language::{
+        Language,
+        detect::{LanguageDetector, MIN_LANGUAGE_SCORE},
+        stemmer::STEMMER_MAP,
+        stopwords::{STOP_WORDS, StopwordFnc},
+    },
+    tokenizers::{chinese::JIEBA, japanese},
+};
+use std::borrow::Cow;
+
+pub struct WordStemTokenizer {
+    stemmer: Stemmer,
+    stop_words: Option<StopwordFnc>,
+}
+
+enum Stemmer {
+    IndoEuropean(rust_stemmers::Stemmer),
+    Mandarin,
+    Japanese,
+    None,
+}
+
+impl WordStemTokenizer {
+    pub fn new(text: &str) -> Self {
+        // Detect language
+        let (mut language, score) =
+            LanguageDetector::detect_single(text).unwrap_or((Language::English, 1.0));
+        if score < MIN_LANGUAGE_SCORE {
+            language = Language::English;
+        }
+
+        Self {
+            stemmer: match language {
+                Language::Mandarin => Stemmer::Mandarin,
+                Language::Japanese => Stemmer::Japanese,
+                _ => STEMMER_MAP[language as usize]
+                    .map(|algo| Stemmer::IndoEuropean(rust_stemmers::Stemmer::create(algo)))
+                    .unwrap_or(Stemmer::None),
+            },
+            stop_words: STOP_WORDS[language as usize],
+        }
+    }
+
+    pub fn tokenize<'x>(&self, word: &'x str, mut cb: impl FnMut(Cow<'x, str>)) {
+        if self.stop_words.is_some_and(|sw| sw(word)) {
+            return;
+        }
+        match &self.stemmer {
+            Stemmer::IndoEuropean(stemmer) => {
+                cb(stemmer.stem(word));
+            }
+            Stemmer::Mandarin => {
+                for token in JIEBA.cut(word, false) {
+                    cb(Cow::from(token.word));
+                }
+            }
+            Stemmer::Japanese => {
+                for word in japanese::tokenize(word) {
+                    cb(Cow::from(word));
+                }
+            }
+            Stemmer::None => {
+                cb(Cow::from(word));
+            }
+        }
+    }
+}
+
+#[cfg(test)]
+pub mod tests {
+    use crate::tokenizers::{
+        stream::WordStemTokenizer,
+        types::{TokenType, TypesTokenizer},
+    };
+
+    #[test]
+    fn stream_tokenizer() {
+        let inputs = [
+            (
+                "The quick brown fox jumps over the lazy dog",
+                vec!["quick", "brown", "fox", "jump", "lazi", "dog"],
+            ),
+            (
+                "Jovencillo emponzoñado de whisky: ¡qué figurota exhibe!",
+                vec!["jovencill", "emponzoñ", "whisky", "figurot", "exhib"],
+            ),
+            (
+                "Ma la volpe col suo balzo ha raggiunto il quieto Fido",
+                vec!["volp", "balz", "raggiunt", "quiet", "fid"],
+            ),
+            (
+                "Jaz em prisão bota que vexa dez cegonhas felizes",
+                vec!["jaz", "prisã", "bot", "vex", "dez", "cegonh", "feliz"],
+            ),
+            (
+                "Zwölf Boxkämpfer jagten Victor quer über den großen Sylter Deich",
+                vec![
+                    "zwolf", "boxkampf", "jagt", "victor", "quer", "gross", "sylt", "deich",
+                ],
+            ),
+            (
+                "עטלף אבק נס דרך מזגן שהתפוצץ כי חם",
+                vec!["עטלף", "אבק", "נס", "דרך", "מזגן", "שהתפוצץ", "כי", "חם"],
+            ),
+            (
+                "Съешь ещё этих мягких французских булок, да выпей же чаю",
+                vec![
+                    "съеш",
+                    "ещё",
+                    "эт",
+                    "мягк",
+                    "французск",
+                    "булок",
+                    "вып",
+                    "ча",
+                ],
+            ),
+            (
+                "Чуєш їх, доцю, га? Кумедна ж ти, прощайся без ґольфів!",
+                vec![
+                    "чуєш",
+                    "їх",
+                    "доцю",
+                    "га",
+                    "кумедна",
+                    "ж",
+                    "ти",
+                    "прощайся",
+                    "без",
+                    "ґольфів",
+                ],
+            ),
+            (
+                "Љубазни фењерџија чађавог лица хоће да ми покаже штос",
+                vec![
+                    "љубазни",
+                    "фењерџија",
+                    "чађавог",
+                    "лица",
+                    "хоће",
+                    "да",
+                    "ми",
+                    "покаже",
+                    "штос",
+                ],
+            ),
+            (
+                "Pijamalı hasta yağız şoföre çabucak güvendi",
+                vec!["pijamalı", "hasta", "yağız", "şoför", "çabucak", "güvendi"],
+            ),
+            ("己所不欲,勿施于人。", vec!["己所不欲", "勿施于人"]),
+            (
+                "井の中の蛙大海を知らず",
+                vec!["井", "の", "中", "の", "蛙大", "海", "を", "知ら", "ず"],
+            ),
+            ("시작이 반이다", vec!["시작이", "반이다"]),
+        ];
+
+        for (input, expect) in inputs.iter() {
+            let tokenizer = WordStemTokenizer::new(input);
+            let mut result = Vec::new();
+            for token in TypesTokenizer::new(&input.to_lowercase()) {
+                if let TokenType::Alphabetic(word) = token.word {
+                    tokenizer.tokenize(word, |t| {
+                        result.push(t.into_owned());
+                    });
+                }
+            }
+
+            assert_eq!(&result, expect,);
+        }
+    }
+}
@@ -0,0 +1,150 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
+ */
+
+use std::{borrow::Cow, str::CharIndices};
+
+use super::Token;
+
+pub struct WordTokenizer<'x> {
+    max_token_length: usize,
+    text: &'x str,
+    iterator: CharIndices<'x>,
+}
+
+impl WordTokenizer<'_> {
+    pub fn new(text: &'_ str, max_token_length: usize) -> WordTokenizer<'_> {
+        WordTokenizer {
+            max_token_length,
+            text,
+            iterator: text.char_indices(),
+        }
+    }
+}
+
+/// Parses indo-european text into lowercase tokens.
+impl<'x> Iterator for WordTokenizer<'x> {
+    type Item = Token<Cow<'x, str>>;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        while let Some((token_start, ch)) = self.iterator.next() {
+            if ch.is_alphanumeric() {
+                let mut is_uppercase = ch.is_uppercase();
+                let token_end = (&mut self.iterator)
+                    .filter_map(|(pos, ch)| {
+                        if ch.is_alphanumeric() {
+                            if !is_uppercase && ch.is_uppercase() {
+                                is_uppercase = true;
+                            }
+                            None
+                        } else {
+                            pos.into()
+                        }
+                    })
+                    .next()
+                    .unwrap_or(self.text.len());
+
+                let token_len = token_end - token_start;
+                if token_end > token_start && token_len <= self.max_token_length {
+                    return Token::new(
+                        token_start,
+                        token_len,
+                        if is_uppercase {
+                            self.text[token_start..token_end].to_lowercase().into()
+                        } else {
+                            self.text[token_start..token_end].into()
+                        },
+                    )
+                    .into();
+                }
+            }
+        }
+        None
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn indo_european_tokenizer() {
+        let inputs = [
+            (
+                "The quick brown fox jumps over the lazy dog",
+                vec![
+                    Token::new(0, 3, "the".into()),
+                    Token::new(4, 5, "quick".into()),
+                    Token::new(10, 5, "brown".into()),
+                    Token::new(16, 3, "fox".into()),
+                    Token::new(20, 5, "jumps".into()),
+                    Token::new(26, 4, "over".into()),
+                    Token::new(31, 3, "the".into()),
+                    Token::new(35, 4, "lazy".into()),
+                    Token::new(40, 3, "dog".into()),
+                ],
+            ),
+            (
+                "Jovencillo EMPONZOÑADO de whisky: ¡qué figurota exhibe!",
+                vec![
+                    Token::new(0, 10, "jovencillo".into()),
+                    Token::new(11, 12, "emponzoñado".into()),
+                    Token::new(24, 2, "de".into()),
+                    Token::new(27, 6, "whisky".into()),
+                    Token::new(37, 4, "qué".into()),
+                    Token::new(42, 8, "figurota".into()),
+                    Token::new(51, 6, "exhibe".into()),
+                ],
+            ),
+            (
+                "ZWÖLF Boxkämpfer jagten Victor quer über den großen Sylter Deich",
+                vec![
+                    Token::new(0, 6, "zwölf".into()),
+                    Token::new(7, 11, "boxkämpfer".into()),
+                    Token::new(19, 6, "jagten".into()),
+                    Token::new(26, 6, "victor".into()),
+                    Token::new(33, 4, "quer".into()),
+                    Token::new(38, 5, "über".into()),
+                    Token::new(44, 3, "den".into()),
+                    Token::new(48, 7, "großen".into()),
+                    Token::new(56, 6, "sylter".into()),
+                    Token::new(63, 5, "deich".into()),
+                ],
+            ),
+            (
+                "Съешь ещё этих мягких французских булок, да выпей же чаю",
+                vec![
+                    Token::new(0, 10, "съешь".into()),
+                    Token::new(11, 6, "ещё".into()),
+                    Token::new(18, 8, "этих".into()),
+                    Token::new(27, 12, "мягких".into()),
+                    Token::new(40, 22, "французских".into()),
+                    Token::new(63, 10, "булок".into()),
+                    Token::new(75, 4, "да".into()),
+                    Token::new(80, 10, "выпей".into()),
+                    Token::new(91, 4, "же".into()),
+                    Token::new(96, 6, "чаю".into()),
+                ],
+            ),
+            (
+                "Pijamalı hasta yağız şoföre çabucak güvendi",
+                vec![
+                    Token::new(0, 9, "pijamalı".into()),
+                    Token::new(10, 5, "hasta".into()),
+                    Token::new(16, 7, "yağız".into()),
+                    Token::new(24, 8, "şoföre".into()),
+                    Token::new(33, 8, "çabucak".into()),
+                    Token::new(42, 8, "güvendi".into()),
+                ],
+            ),
+        ];
+
+        for (input, tokens) in inputs.iter() {
+            for (pos, token) in WordTokenizer::new(input, 40).enumerate() {
+                assert_eq!(token, tokens[pos]);
+            }
+        }
+    }
+}