Import upstream v0.16.22, stripped

Upstream commit: 474dd0229cb20cf513036619781ed97bd8073c3f
Enterprise-only files removed or emptied: 63
Enterprise-only snippets removed: 117 in 50 files
Dangling module declarations removed: 5
Cargo edits turning enterprise off: 14
Verification: clean
Enterprise feature gates left for rebuilt features: 19 in 18 files

Produced by tools/fork/strip.py. The full report is in docs/fork/strip-reports/ on main.

crates/nlp/src/tokenizers/mod.rs

@@ -0,0 +1,58 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
+ */
+
+pub mod chinese;
+pub mod japanese;
+pub mod space;
+pub mod stream;
+pub mod types;
+pub mod word;
+
+use std::borrow::Cow;
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct Token<T> {
+    pub word: T,
+    pub from: usize,
+    pub to: usize,
+}
+
+pub trait InnerToken<'x>: Sized {
+    fn new_alphabetic(value: impl Into<Cow<'x, str>>) -> Self;
+    fn unwrap_alphabetic(self) -> Cow<'x, str>;
+    fn is_alphabetic(&self) -> bool;
+    fn is_alphabetic_8bit(&self) -> bool;
+}
+
+impl<'x> InnerToken<'x> for Cow<'x, str> {
+    fn new_alphabetic(value: impl Into<Cow<'x, str>>) -> Self {
+        value.into()
+    }
+
+    fn is_alphabetic(&self) -> bool {
+        true
+    }
+
+    fn is_alphabetic_8bit(&self) -> bool {
+        !self.is_ascii()
+    }
+
+    fn unwrap_alphabetic(self) -> Cow<'x, str> {
+        self
+    }
+}
+
+impl<T> Token<T> {
+    pub fn new(offset: usize, len: usize, word: T) -> Token<T> {
+        debug_assert!(offset <= u32::MAX as usize);
+        debug_assert!(len <= u8::MAX as usize);
+        Token {
+            from: offset,
+            to: offset + len,
+            word,
+        }
+    }
+}