inbuxa/inbuxa-server
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases 3 Wiki Activity

Import upstream v0.16.22, stripped

Browse Source 
Upstream commit: 474dd0229cb20cf513036619781ed97bd8073c3f
Enterprise-only files removed or emptied: 63
Enterprise-only snippets removed: 117 in 50 files
Dangling module declarations removed: 5
Cargo edits turning enterprise off: 14
Verification: clean
Enterprise feature gates left for rebuilt features: 19 in 18 files

Produced by tools/fork/strip.py. The full report is in docs/fork/strip-reports/ on main.
This commit is contained in:
jcoffey-dev
2026-09-18 10:21:56 -07:00
commit 7dae9b29fd
1650 changed files with 485521 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/nlp/src/language/detect.rs
+236
View File
@@ -0,0 +1,236 @@
/*
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
*
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
*/
use super::Language;
use ahash::AHashMap;
use whatlang::{Lang, detect};
pub const MIN_LANGUAGE_SCORE: f64 = 0.6;
#[derive(Debug)]
struct WeightedAverage {
weight: usize,
occurrences: usize,
confidence: f64,
}
#[derive(Debug)]
pub struct LanguageDetector {
lang_detected: AHashMap<Language, WeightedAverage>,
}
impl Default for LanguageDetector {
fn default() -> Self {
Self::new()
}
}
impl LanguageDetector {
pub fn new() -> LanguageDetector {
LanguageDetector {
lang_detected: AHashMap::default(),
}
}
pub fn detect(&mut self, text: &str, min_score: f64) -> Language {
if let Some((language, confidence)) = LanguageDetector::detect_single(text) {
let w = self
.lang_detected
.entry(language)
.or_insert_with(|| WeightedAverage {
weight: 0,
confidence: 0.0,
occurrences: 0,
});
w.occurrences += 1;
w.weight += text.len();
w.confidence += confidence * text.len() as f64;
if confidence < min_score {
Language::Unknown
} else {
language
}
} else {
Language::Unknown
}
}
pub fn most_frequent_language(&self) -> Option<Language> {
self.lang_detected
.iter()
.filter(|(l, _)| !matches!(l, Language::None))
.max_by(|(_, a), (_, b)| {
((a.confidence / a.weight as f64) * a.occurrences as f64)
.partial_cmp(&((b.confidence / b.weight as f64) * b.occurrences as f64))
.unwrap_or(std::cmp::Ordering::Less)
})
.map(|(l, _)| *l)
}
pub fn detect_single(text: &str) -> Option<(Language, f64)> {
detect(text).map(|info| {
(
match info.lang() {
Lang::Epo => Language::Esperanto,
Lang::Eng => Language::English,
Lang::Rus => Language::Russian,
Lang::Cmn => Language::Mandarin,
Lang::Spa => Language::Spanish,
Lang::Por => Language::Portuguese,
Lang::Ita => Language::Italian,
Lang::Ben => Language::Bengali,
Lang::Fra => Language::French,
Lang::Deu => Language::German,
Lang::Ukr => Language::Ukrainian,
Lang::Kat => Language::Georgian,
Lang::Ara => Language::Arabic,
Lang::Hin => Language::Hindi,
Lang::Jpn => Language::Japanese,
Lang::Heb => Language::Hebrew,
Lang::Yid => Language::Yiddish,
Lang::Pol => Language::Polish,
Lang::Amh => Language::Amharic,
Lang::Jav => Language::Javanese,
Lang::Kor => Language::Korean,
Lang::Nob => Language::Bokmal,
Lang::Dan => Language::Danish,
Lang::Swe => Language::Swedish,
Lang::Fin => Language::Finnish,
Lang::Tur => Language::Turkish,
Lang::Nld => Language::Dutch,
Lang::Hun => Language::Hungarian,
Lang::Ces => Language::Czech,
Lang::Ell => Language::Greek,
Lang::Bul => Language::Bulgarian,
Lang::Bel => Language::Belarusian,
Lang::Mar => Language::Marathi,
Lang::Kan => Language::Kannada,
Lang::Ron => Language::Romanian,
Lang::Slv => Language::Slovene,
Lang::Hrv => Language::Croatian,
Lang::Srp => Language::Serbian,
Lang::Mkd => Language::Macedonian,
Lang::Lit => Language::Lithuanian,
Lang::Lav => Language::Latvian,
Lang::Est => Language::Estonian,
Lang::Tam => Language::Tamil,
Lang::Vie => Language::Vietnamese,
Lang::Urd => Language::Urdu,
Lang::Tha => Language::Thai,
Lang::Guj => Language::Gujarati,
Lang::Uzb => Language::Uzbek,
Lang::Pan => Language::Punjabi,
Lang::Aze => Language::Azerbaijani,
Lang::Ind => Language::Indonesian,
Lang::Tel => Language::Telugu,
Lang::Pes => Language::Persian,
Lang::Mal => Language::Malayalam,
Lang::Ori => Language::Oriya,
Lang::Mya => Language::Burmese,
Lang::Nep => Language::Nepali,
Lang::Sin => Language::Sinhalese,
Lang::Khm => Language::Khmer,
Lang::Tuk => Language::Turkmen,
Lang::Aka => Language::Akan,
Lang::Zul => Language::Zulu,
Lang::Sna => Language::Shona,
Lang::Afr => Language::Afrikaans,
Lang::Lat => Language::Latin,
Lang::Slk => Language::Slovak,
Lang::Cat => Language::Catalan,
Lang::Tgl => Language::Tagalog,
Lang::Hye => Language::Armenian,
_ => Language::Unknown,
},
info.confidence(),
)
})
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn detect_languages() {
let inputs = [
(
"The quick brown fox jumps over the lazy dog",
Language::English,
),
(
"Jovencillo emponzoñado de whisky: ¡qué figurota exhibe!",
Language::Spanish,
),
(
"Ma la volpe col suo balzo ha raggiunto il quieto Fido",
Language::Italian,
),
(
"Jaz em prisão bota que vexa dez cegonhas felizes",
Language::Portuguese,
),
(
"Zwölf Boxkämpfer jagten Victor quer über den großen Sylter Deich",
Language::German,
),
("עטלף אבק נס דרך מזגן שהתפוצץ כי חם", Language::Hebrew),
(
"Съешь ещё этих мягких французских булок, да выпей же чаю",
Language::Russian,
),
(
"Чуєш їх, доцю, га? Кумедна ж ти, прощайся без ґольфів!",
Language::Ukrainian,
),
(
"Љубазни фењерџија чађавог лица хоће да ми покаже штос",
Language::Serbian,
),
(
"Pijamalı hasta yağız şoföre çabucak güvendi",
Language::Turkish,
),
("己所不欲,勿施于人。", Language::Mandarin),
("井の中の蛙大海を知らず", Language::Japanese),
("시작이 반이다", Language::Korean),
];
let mut detector = LanguageDetector::new();
for input in inputs.iter() {
assert_eq!(detector.detect(input.0, 0.0), input.1);
}
}
#[test]
fn weighted_language() {
let mut detector = LanguageDetector::new();
for lang in [
(Language::Spanish, 0.5, 70),
(Language::Japanese, 0.2, 100),
(Language::Japanese, 0.3, 100),
(Language::Japanese, 0.4, 200),
(Language::English, 0.7, 50),
]
.iter()
{
let w = detector
.lang_detected
.entry(lang.0)
.or_insert_with(|| WeightedAverage {
weight: 0,
confidence: 0.0,
occurrences: 0,
});
w.occurrences += 1;
w.weight += lang.2;
w.confidence += lang.1 * lang.2 as f64;
}
assert_eq!(detector.most_frequent_language(), Some(Language::Japanese));
}
}
crates/nlp/src/language/mod.rs
+271
View File
@@ -0,0 +1,271 @@
/*
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
*
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
*/
pub mod detect;
pub mod search_snippet;
pub mod stemmer;
pub mod stopwords;
use self::detect::LanguageDetector;
use crate::tokenizers::{
Token, chinese::ChineseTokenizer, japanese::JapaneseTokenizer, space::SpaceTokenizer,
word::WordTokenizer,
};
use std::borrow::Cow;
pub type LanguageTokenizer<'x> = Box<dyn Iterator<Item = Token<Cow<'x, str>>> + 'x + Sync + Send>;
impl Language {
pub fn tokenize_text<'x>(
&self,
text: &'x str,
max_token_length: usize,
) -> LanguageTokenizer<'x> {
match self {
Language::Japanese => Box::new(
JapaneseTokenizer::new(WordTokenizer::new(text, usize::MAX))
.filter(move |t| t.word.len() <= max_token_length),
),
Language::Mandarin => Box::new(
ChineseTokenizer::new(WordTokenizer::new(text, usize::MAX))
.filter(move |t| t.word.len() <= max_token_length),
),
Language::None => {
Box::new(
SpaceTokenizer::new(text, max_token_length).map(|word| Token {
word: word.into(),
from: 0,
to: 0,
}),
)
}
_ => Box::new(WordTokenizer::new(text, max_token_length)),
}
}
}
#[derive(
Debug, PartialEq, Clone, Copy, Hash, Eq, serde::Serialize, serde::Deserialize, Default,
)]
pub enum Language {
Esperanto = 0,
#[default]
English = 1,
Russian = 2,
Mandarin = 3,
Spanish = 4,
Portuguese = 5,
Italian = 6,
Bengali = 7,
French = 8,
German = 9,
Ukrainian = 10,
Georgian = 11,
Arabic = 12,
Hindi = 13,
Japanese = 14,
Hebrew = 15,
Yiddish = 16,
Polish = 17,
Amharic = 18,
Javanese = 19,
Korean = 20,
Bokmal = 21,
Danish = 22,
Swedish = 23,
Finnish = 24,
Turkish = 25,
Dutch = 26,
Hungarian = 27,
Czech = 28,
Greek = 29,
Bulgarian = 30,
Belarusian = 31,
Marathi = 32,
Kannada = 33,
Romanian = 34,
Slovene = 35,
Croatian = 36,
Serbian = 37,
Macedonian = 38,
Lithuanian = 39,
Latvian = 40,
Estonian = 41,
Tamil = 42,
Vietnamese = 43,
Urdu = 44,
Thai = 45,
Gujarati = 46,
Uzbek = 47,
Punjabi = 48,
Azerbaijani = 49,
Indonesian = 50,
Telugu = 51,
Persian = 52,
Malayalam = 53,
Oriya = 54,
Burmese = 55,
Nepali = 56,
Sinhalese = 57,
Khmer = 58,
Turkmen = 59,
Akan = 60,
Zulu = 61,
Shona = 62,
Afrikaans = 63,
Latin = 64,
Slovak = 65,
Catalan = 66,
Tagalog = 67,
Armenian = 68,
Unknown = 69,
None = 70,
}
impl Language {
pub fn is_unknown(&self) -> bool {
matches!(self, Language::Unknown)
}
pub fn from_iso_639(code: &str) -> Option<Self> {
hashify::map!(
code.split_once(['-', '_']).map(|c| c.0).unwrap_or(code).as_bytes(),
Language,
"en" => Language::English,
"es" => Language::Spanish,
"pt" => Language::Portuguese,
"it" => Language::Italian,
"fr" => Language::French,
"de" => Language::German,
"da" => Language::Danish,
"ru" => Language::Russian,
"zh" => Language::Mandarin,
"ja" => Language::Japanese,
"ar" => Language::Arabic,
"hi" => Language::Hindi,
"ko" => Language::Korean,
"bn" => Language::Bengali,
"he" => Language::Hebrew,
"ur" => Language::Urdu,
"fa" => Language::Persian,
"ml" => Language::Malayalam,
"or" => Language::Oriya,
"my" => Language::Burmese,
"ne" => Language::Nepali,
"si" => Language::Sinhalese,
"km" => Language::Khmer,
"tk" => Language::Turkmen,
"am" => Language::Amharic,
"az" => Language::Azerbaijani,
"id" => Language::Indonesian,
"te" => Language::Telugu,
"ta" => Language::Tamil,
"vi" => Language::Vietnamese,
"gu" => Language::Gujarati,
"pa" => Language::Punjabi,
"uz" => Language::Uzbek,
"hy" => Language::Armenian,
"ka" => Language::Georgian,
"la" => Language::Latin,
"sl" => Language::Slovene,
"hr" => Language::Croatian,
"sr" => Language::Serbian,
"mk" => Language::Macedonian,
"lt" => Language::Lithuanian,
"lv" => Language::Latvian,
"et" => Language::Estonian,
"tl" => Language::Tagalog,
"af" => Language::Afrikaans,
"zu" => Language::Zulu,
"sn" => Language::Shona,
"ak" => Language::Akan,
"ca" => Language::Catalan,
"el" => Language::Greek,
"sv" => Language::Swedish,
"pl" => Language::Polish,
"nl" => Language::Dutch,
"fi" => Language::Finnish,
"hu" => Language::Hungarian,
"tr" => Language::Turkish,
"ro" => Language::Romanian,
"nb" => Language::Bokmal,
"no" => Language::Bokmal,
"yi" => Language::Yiddish,
"eo" => Language::Esperanto,
"uk" => Language::Ukrainian,
"cs" => Language::Czech,
"sk" => Language::Slovak,
"bg" => Language::Bulgarian,
"be" => Language::Belarusian,
"th" => Language::Thai,
"mr" => Language::Marathi,
"kn" => Language::Kannada,
"jv" => Language::Javanese
)
.copied()
}
}
impl Language {
pub fn detect(text: String, default: Language) -> (String, Language) {
if let Some((l, t)) = text
.split_once(':')
.and_then(|(l, t)| (Language::from_iso_639(l)?, t).into())
{
(t.to_string(), l)
} else {
let l = LanguageDetector::detect_single(&text)
.and_then(|(l, c)| if c > 0.3 { Some(l) } else { None })
.unwrap_or(default);
(text, l)
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn iso_639_codes() {
for (code, expected) in [
("nl", Language::Dutch),
("fi", Language::Finnish),
("hu", Language::Hungarian),
("tr", Language::Turkish),
("ro", Language::Romanian),
("nb", Language::Bokmal),
("no", Language::Bokmal),
("yi", Language::Yiddish),
("eo", Language::Esperanto),
("uk", Language::Ukrainian),
("cs", Language::Czech),
("sk", Language::Slovak),
("bg", Language::Bulgarian),
("be", Language::Belarusian),
("th", Language::Thai),
("mr", Language::Marathi),
("kn", Language::Kannada),
("jv", Language::Javanese),
("en", Language::English),
("pl", Language::Polish),
] {
assert_eq!(Language::from_iso_639(code), Some(expected), "{code}");
}
}
#[test]
fn iso_639_locale_suffixes() {
for code in ["nl_NL", "nl_BE", "nl_NL@euro", "nl-NL"] {
assert_eq!(
Language::from_iso_639(code),
Some(Language::Dutch),
"{code}"
);
}
assert_eq!(Language::from_iso_639("xx_XX"), None);
}
}
crates/nlp/src/language/search_snippet.rs
+271
View File
@@ -0,0 +1,271 @@
/*
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
*
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
*/
use super::Language;
fn escape_char(c: char, string: &mut String) {
match c {
'&' => string.push_str("&amp;"),
'<' => string.push_str("&lt;"),
'>' => string.push_str("&gt;"),
'"' => string.push_str("&quot;"),
'\n' | '\r' => string.push(' '),
_ => string.push(c),
}
}
fn escape_char_len(c: char) -> usize {
match c {
'&' => "&amp;".len(),
'<' => "&lt;".len(),
'>' => "&gt;".len(),
'"' => "&quot;".len(),
'\r' | '\n' => 1,
_ => c.len_utf8(),
}
}
pub struct Term {
offset: usize,
len: usize,
}
pub fn generate_snippet(
text: &str,
needles: &[impl AsRef<str>],
language: Language,
is_exact: bool,
) -> Option<String> {
let mut terms = Vec::new();
if is_exact {
let tokens = language.tokenize_text(text, 200).collect::<Vec<_>>();
for tokens in tokens.windows(needles.len()) {
if needles
.iter()
.zip(tokens)
.all(|(needle, token)| needle.as_ref() == token.word.as_ref())
{
for token in tokens {
terms.push(Term {
offset: token.from,
len: token.to - token.from,
});
}
}
}
} else {
for token in language.tokenize_text(text, 200) {
if needles.iter().any(|needle| {
let needle = needle.as_ref();
needle == token.word.as_ref() || needle.len() > 2 && token.word.contains(needle)
}) {
terms.push(Term {
offset: token.from,
len: token.to - token.from,
});
}
}
}
if terms.is_empty() {
return None;
}
let mut snippet = String::with_capacity(text.len());
let start_offset = terms.first()?.offset;
if start_offset > 0 {
let mut word_count = 0;
let mut from_offset = 0;
let mut last_is_space = false;
if text.len() > 240 {
for (pos, char) in text.get(0..start_offset)?.char_indices().rev() {
// Add up to 2 words or 40 characters of context
if char.is_whitespace() {
if !last_is_space {
word_count += 1;
if word_count == 3 {
break;
}
last_is_space = true;
}
} else {
last_is_space = false;
}
from_offset = pos;
if start_offset - from_offset >= 40 {
break;
}
}
}
last_is_space = false;
for char in text.get(from_offset..start_offset)?.chars() {
if !char.is_whitespace() {
last_is_space = false;
} else {
if last_is_space {
continue;
}
last_is_space = true;
}
escape_char(char, &mut snippet);
}
}
let mut terms = terms.iter().peekable();
'outer: while let Some(term) = terms.next() {
if snippet.len() + ("<mark>".len() * 2) + term.len + 1 > 255 {
break;
}
snippet.push_str("<mark>");
snippet.push_str(text.get(term.offset..term.offset + term.len)?);
snippet.push_str("</mark>");
let next_offset = if let Some(next_term) = terms.peek() {
next_term.offset
} else {
text.len()
};
let mut last_is_space = false;
for char in text.get(term.offset + term.len..next_offset)?.chars() {
if !char.is_whitespace() {
last_is_space = false;
} else {
if last_is_space {
continue;
}
last_is_space = true;
}
if snippet.len() + escape_char_len(char) <= 255 {
escape_char(char, &mut snippet);
} else {
break 'outer;
}
}
}
Some(snippet)
}
#[cfg(test)]
mod tests {
use crate::language::{Language, search_snippet::generate_snippet};
#[test]
fn search_snippets() {
let inputs = [
(
vec![
"Help a friend from Abidjan Côte d'Ivoire",
concat!(
"When my mother died when she was given birth to me, my father took me so ",
"special because I am motherless. Before the death of my late father on 22nd June ",
"2013 in a private hospital here in Abidjan Côte d'Ivoire. He secretly called me on his ",
"bedside and told me that he has a sum of $7.5M (Seven Million five Hundred ",
"Thousand Dollars) left in a suspense account in a local bank here in Abidjan Côte ",
"d'Ivoire, that he used my name as his only daughter for the next of kin in deposit of ",
"the fund. ",
"I am 24year old. Dear I am honorably seeking your assistance in the following ways. ",
"1) To provide any bank account where this money would be transferred into. ",
"2) To serve as the guardian of this fund. ",
"3) To make arrangement for me to come over to your country to further my ",
"education and to secure a residential permit for me in your country. ",
"Moreover, I am willing to offer you 30 percent of the total sum as compensation for ",
"your effort input after the successful transfer of this fund to your nominated ",
"account overseas."
),
],
vec![
(
vec!["côte"],
vec![
"Help a friend from Abidjan <mark>Côte</mark> d'Ivoire",
concat!(
"in Abidjan <mark>Côte</mark> d'Ivoire. He secretly called me on his bedside ",
"and told me that he has a sum of $7.5M (Seven Million five Hundred Thousand ",
"Dollars) left in a suspense account in a local bank here in Abidjan ",
"<mark>Côte</mark> d'Ivoire, that "
),
],
),
(
vec!["your", "country"],
vec![concat!(
"honorably seeking <mark>your</mark> assistance in the following ways. ",
"1) To provide any bank account where this money would be transferred into. 2) ",
"To serve as the guardian of this fund. 3) To make arrangement for me to come ",
"over to <mark>your</mark> "
)],
),
(
vec!["overseas"],
vec!["nominated account <mark>overseas</mark>."],
),
],
),
(
vec![
"孫子兵法",
concat!(
"<\"孫子兵法：\">",
"孫子曰：兵者，國之大事，死生之地，存亡之道，不可不察也。",
"孫子曰：凡用兵之法，馳車千駟，革車千乘，帶甲十萬；千里饋糧，則內外之費賓客之用，膠漆之材，",
"車甲之奉，日費千金，然後十萬之師舉矣。",
"孫子曰：凡用兵之法，全國為上，破國次之；全旅為上，破旅次之；全卒為上，破卒次之；全伍為上，破伍次之。",
"是故百戰百勝，非善之善者也；不戰而屈人之兵，善之善者也。",
"孫子曰：昔之善戰者，先為不可勝，以待敵之可勝，不可勝在己，可勝在敵。故善戰者，能為不可勝，不能使敵必可勝。",
"故曰：勝可知，而不可為。",
"兵者，詭道也。故能而示之不能，用而示之不用，近而示之遠，遠而示之近。利而誘之，亂而取之，實而備之，強而避之，",
"怒而撓之，卑而驕之，佚而勞之，親而離之。攻其無備，出其不意，此兵家之勝，不可先傳也。",
"夫未戰而廟算勝者，得算多也；未戰而廟算不勝者，得算少也；多算勝，少算不勝，而況於無算乎？吾以此觀之，勝負見矣。",
"孫子曰：凡治眾如治寡，分數是也。鬥眾如鬥寡，形名是也。三軍之眾，可使必受敵而無敗者，奇正是也。兵之所加，",
"如以碬投卵者，虛實是也。",
),
],
vec![
(
vec!["孫子兵法"],
vec![
"<mark>孫子兵法</mark>",
concat!(
"&lt;&quot;<mark>孫子兵法</mark>：&quot;&gt;孫子曰：兵者，國之大事，死生之地，存亡之道，",
"不可不察也。孫子曰：凡用兵之法，馳車千駟，革車千乘，帶甲十萬；千里饋糧，則內外之費賓客之用，膠"
),
],
),
(
vec!["孫子曰"],
vec![concat!(
"&lt;&quot;孫子兵法：&quot;&gt;<mark>孫子曰</mark>：兵者，國之大事，死生之地，存亡之道，",
"不可不察也。<mark>孫子曰</mark>：凡用兵之法，馳車千駟，革車千乘，帶甲十萬；千里饋糧，則內外之費賓",
)],
),
],
),
];
for (parts, tests) in inputs {
for (needles, snippets) in tests {
let mut results = Vec::new();
for part in &parts {
if let Some(matched) =
generate_snippet(part, &needles, Language::English, false)
{
results.push(matched);
}
}
assert_eq!(snippets, results);
}
}
}
}
crates/nlp/src/language/stemmer.rs
+152
View File
@@ -0,0 +1,152 @@
/*
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
*
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
*/
use std::borrow::Cow;
use rust_stemmers::Algorithm;
use super::{Language, LanguageTokenizer};
#[derive(Debug, PartialEq, Eq)]
pub struct StemmedToken<'x> {
pub word: Cow<'x, str>,
pub stemmed_word: Option<Cow<'x, str>>,
pub from: usize, // Word offset in the text part
pub to: usize, // Word length
}
pub struct Stemmer<'x> {
stemmer: Option<rust_stemmers::Stemmer>,
tokenizer: LanguageTokenizer<'x>,
}
impl<'x> Stemmer<'x> {
pub fn new(text: &'x str, language: Language, max_token_length: usize) -> Stemmer<'x> {
Stemmer {
tokenizer: language.tokenize_text(text, max_token_length),
stemmer: STEMMER_MAP[language as usize].map(rust_stemmers::Stemmer::create),
}
}
}
impl<'x> Iterator for Stemmer<'x> {
type Item = StemmedToken<'x>;
fn next(&mut self) -> Option<Self::Item> {
let token = self.tokenizer.next()?;
Some(StemmedToken {
stemmed_word: self.stemmer.as_ref().and_then(|stemmer| {
match stemmer.stem(&token.word) {
Cow::Owned(text) if text.len() != token.word.len() || text != token.word => {
Some(text.into())
}
_ => None,
}
}),
word: token.word,
from: token.from,
to: token.to,
})
}
}
pub static STEMMER_MAP: &[Option<Algorithm>] = &[
None, // Esperanto = 0,
Some(Algorithm::English), // English = 1,
Some(Algorithm::Russian), // Russian = 2,
None, // Mandarin = 3,
Some(Algorithm::Spanish), // Spanish = 4,
Some(Algorithm::Portuguese), // Portuguese = 5,
Some(Algorithm::Italian), // Italian = 6,
None, // Bengali = 7,
Some(Algorithm::French), // French = 8,
Some(Algorithm::German), // German = 9,
None, // Ukrainian = 10,
None, // Georgian = 11,
Some(Algorithm::Arabic), // Arabic = 12,
None, // Hindi = 13,
None, // Japanese = 14,
None, // Hebrew = 15,
None, // Yiddish = 16,
None, // Polish = 17,
None, // Amharic = 18,
None, // Javanese = 19,
None, // Korean = 20,
Some(Algorithm::Norwegian), // Bokmal = 21,
Some(Algorithm::Danish), // Danish = 22,
Some(Algorithm::Swedish), // Swedish = 23,
Some(Algorithm::Finnish), // Finnish = 24,
Some(Algorithm::Turkish), // Turkish = 25,
Some(Algorithm::Dutch), // Dutch = 26,
Some(Algorithm::Hungarian), // Hungarian = 27,
None, // Czech = 28,
Some(Algorithm::Greek), // Greek = 29,
None, // Bulgarian = 30,
None, // Belarusian = 31,
None, // Marathi = 32,
None, // Kannada = 33,
Some(Algorithm::Romanian), // Romanian = 34,
None, // Slovene = 35,
None, // Croatian = 36,
None, // Serbian = 37,
None, // Macedonian = 38,
None, // Lithuanian = 39,
None, // Latvian = 40,
None, // Estonian = 41,
Some(Algorithm::Tamil), // Tamil = 42,
None, // Vietnamese = 43,
None, // Urdu = 44,
None, // Thai = 45,
None, // Gujarati = 46,
None, // Uzbek = 47,
None, // Punjabi = 48,
None, // Azerbaijani = 49,
None, // Indonesian = 50,
None, // Telugu = 51,
None, // Persian = 52,
None, // Malayalam = 53,
None, // Oriya = 54,
None, // Burmese = 55,
None, // Nepali = 56,
None, // Sinhalese = 57,
None, // Khmer = 58,
None, // Turkmen = 59,
None, // Akan = 60,
None, // Zulu = 61,
None, // Shona = 62,
None, // Afrikaans = 63,
None, // Latin = 64,
None, // Slovak = 65,
None, // Catalan = 66,
None, // Tagalog = 67,
None, // Armenian = 68,
None, // Unknown = 69,
None, // None = 70,
];
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn stemmer() {
let inputs = [
(
"love loving lovingly loved lovely",
Language::English,
"love",
),
("querer queremos quer", Language::Spanish, "quer"),
];
for (input, language, result) in inputs {
for token in Stemmer::new(input, language, 40) {
assert_eq!(token.stemmed_word.unwrap_or(token.word), result);
}
}
}
}
crates/nlp/src/language/stopwords.rs
+4234
View File
File diff suppressed because it is too large Load Diff