Import upstream v0.16.22, stripped

Upstream commit: 474dd0229cb20cf513036619781ed97bd8073c3f Enterprise-only files removed or emptied: 63 Enterprise-only snippets removed: 117 in 50 files Dangling module declarations removed: 5 Cargo edits turning enterprise off: 14 Verification: clean Enterprise feature gates left for rebuilt features: 19 in 18 files Produced by tools/fork/strip.py. The full report is in docs/fork/strip-reports/ on main.
2026-09-18 10:21:56 -07:00
commit 7dae9b29fd
1650 changed files with 485521 additions and 0 deletions
@@ -0,0 +1,202 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
+ */
+
+use std::{borrow::Cow, sync::LazyLock, vec::IntoIter};
+
+use jieba_rs::Jieba;
+
+use super::{InnerToken, Token};
+
+pub(crate) static JIEBA: LazyLock<Jieba> = LazyLock::new(Jieba::new);
+
+pub struct ChineseTokenizer<'x, T, I>
+where
+    T: Iterator<Item = Token<I>>,
+    I: InnerToken<'x>,
+{
+    tokenizer: T,
+    tokens: IntoIter<Token<I>>,
+    phantom: std::marker::PhantomData<&'x str>,
+}
+
+impl<'x, T, I> ChineseTokenizer<'x, T, I>
+where
+    T: Iterator<Item = Token<I>>,
+    I: InnerToken<'x>,
+{
+    pub fn new(tokenizer: T) -> Self {
+        ChineseTokenizer {
+            tokenizer,
+            tokens: Vec::new().into_iter(),
+            phantom: std::marker::PhantomData,
+        }
+    }
+}
+
+impl<'x, T, I> Iterator for ChineseTokenizer<'x, T, I>
+where
+    T: Iterator<Item = Token<I>>,
+    I: InnerToken<'x>,
+{
+    type Item = Token<I>;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        loop {
+            if let Some(token) = self.tokens.next() {
+                return Some(token);
+            } else {
+                let token = self.tokenizer.next()?;
+                if token.word.is_alphabetic_8bit() {
+                    let mut token_to = token.from;
+                    match token.word.unwrap_alphabetic() {
+                        Cow::Borrowed(word) => {
+                            self.tokens = JIEBA
+                                .cut(word, false)
+                                .into_iter()
+                                .map(|cut| {
+                                    let token_from = token_to;
+                                    token_to += cut.word.len();
+                                    Token {
+                                        word: I::new_alphabetic(cut.word),
+                                        from: token_from,
+                                        to: token_to,
+                                    }
+                                })
+                                .collect::<Vec<_>>()
+                                .into_iter();
+                        }
+                        Cow::Owned(word) => {
+                            self.tokens = JIEBA
+                                .cut(&word, false)
+                                .into_iter()
+                                .map(|cut| {
+                                    let token_from = token_to;
+                                    token_to += cut.word.len();
+                                    Token {
+                                        word: I::new_alphabetic(cut.word.to_string()),
+                                        from: token_from,
+                                        to: token_to,
+                                    }
+                                })
+                                .collect::<Vec<_>>()
+                                .into_iter();
+                        }
+                    }
+                } else {
+                    return token.into();
+                }
+            }
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use crate::tokenizers::{Token, chinese::ChineseTokenizer, word::WordTokenizer};
+
+    #[test]
+    fn chinese_tokenizer() {
+        assert_eq!(
+            ChineseTokenizer::new(WordTokenizer::new(
+                "孫子曰：兵者，國之大事，死生之地，存亡之道，不可不察也。",
+                40
+            ),)
+            .collect::<Vec<_>>(),
+            vec![
+                Token {
+                    word: "孫".into(),
+                    from: 0,
+                    to: 3
+                },
+                Token {
+                    word: "子".into(),
+                    from: 3,
+                    to: 6
+                },
+                Token {
+                    word: "曰".into(),
+                    from: 6,
+                    to: 9
+                },
+                Token {
+                    word: "兵".into(),
+                    from: 12,
+                    to: 15
+                },
+                Token {
+                    word: "者".into(),
+                    from: 15,
+                    to: 18
+                },
+                Token {
+                    word: "國".into(),
+                    from: 21,
+                    to: 24
+                },
+                Token {
+                    word: "之".into(),
+                    from: 24,
+                    to: 27
+                },
+                Token {
+                    word: "大事".into(),
+                    from: 27,
+                    to: 33
+                },
+                Token {
+                    word: "死".into(),
+                    from: 36,
+                    to: 39
+                },
+                Token {
+                    word: "生".into(),
+                    from: 39,
+                    to: 42
+                },
+                Token {
+                    word: "之".into(),
+                    from: 42,
+                    to: 45
+                },
+                Token {
+                    word: "地".into(),
+                    from: 45,
+                    to: 48
+                },
+                Token {
+                    word: "存亡".into(),
+                    from: 51,
+                    to: 57
+                },
+                Token {
+                    word: "之".into(),
+                    from: 57,
+                    to: 60
+                },
+                Token {
+                    word: "道".into(),
+                    from: 60,
+                    to: 63
+                },
+                Token {
+                    word: "不可不".into(),
+                    from: 66,
+                    to: 75
+                },
+                Token {
+                    word: "察".into(),
+                    from: 75,
+                    to: 78
+                },
+                Token {
+                    word: "也".into(),
+                    from: 78,
+                    to: 81
+                }
+            ]
+        );
+    }
+}