jcoffey-dev
7dae9b29fd
Upstream commit: 474dd0229cb20cf513036619781ed97bd8073c3f Enterprise-only files removed or emptied: 63 Enterprise-only snippets removed: 117 in 50 files Dangling module declarations removed: 5 Cargo edits turning enterprise off: 14 Verification: clean Enterprise feature gates left for rebuilt features: 19 in 18 files Produced by tools/fork/strip.py. The full report is in docs/fork/strip-reports/ on main.
59 lines
1.2 KiB
Rust
59 lines
1.2 KiB
Rust
/*
|
|
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
|
|
*
|
|
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
|
*/
|
|
|
|
pub mod chinese;
|
|
pub mod japanese;
|
|
pub mod space;
|
|
pub mod stream;
|
|
pub mod types;
|
|
pub mod word;
|
|
|
|
use std::borrow::Cow;
|
|
|
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
|
pub struct Token<T> {
|
|
pub word: T,
|
|
pub from: usize,
|
|
pub to: usize,
|
|
}
|
|
|
|
pub trait InnerToken<'x>: Sized {
|
|
fn new_alphabetic(value: impl Into<Cow<'x, str>>) -> Self;
|
|
fn unwrap_alphabetic(self) -> Cow<'x, str>;
|
|
fn is_alphabetic(&self) -> bool;
|
|
fn is_alphabetic_8bit(&self) -> bool;
|
|
}
|
|
|
|
impl<'x> InnerToken<'x> for Cow<'x, str> {
|
|
fn new_alphabetic(value: impl Into<Cow<'x, str>>) -> Self {
|
|
value.into()
|
|
}
|
|
|
|
fn is_alphabetic(&self) -> bool {
|
|
true
|
|
}
|
|
|
|
fn is_alphabetic_8bit(&self) -> bool {
|
|
!self.is_ascii()
|
|
}
|
|
|
|
fn unwrap_alphabetic(self) -> Cow<'x, str> {
|
|
self
|
|
}
|
|
}
|
|
|
|
impl<T> Token<T> {
|
|
pub fn new(offset: usize, len: usize, word: T) -> Token<T> {
|
|
debug_assert!(offset <= u32::MAX as usize);
|
|
debug_assert!(len <= u8::MAX as usize);
|
|
Token {
|
|
from: offset,
|
|
to: offset + len,
|
|
word,
|
|
}
|
|
}
|
|
}
|