semantic_router.tokenizers

​

BaseTokenizer Objects

class BaseTokenizer()

​

vocab_size

@property
def vocab_size() -> int

​

config

@property
def config() -> dict

​

save

def save(path: str | Path) -> None

• tokenizer.json: saved configuration of the tokenizer

• path (str, :class:pathlib.Path“): Path to save the tokenizer to

​

load

@classmethod
def load(cls, path: str | Path) -> "BaseTokenizer"

• tokenizer.json: saved configuration of the tokenizer

• path (str, :class:pathlib.Path“): Path to load the tokenizer from

​

PretrainedTokenizer Objects

class PretrainedTokenizer(BaseTokenizer)

• tokenizer (class:tokenizers.Tokenizer“): Binding for HuggingFace Rust tokenizers
• add_special_tokens (bool): Whether to accept special tokens from the tokenizer (i.e. [PAD])
• pad (bool): Whether to pad the input to a consistent length (using [PAD] tokens)
• tokenizer0 (tokenizer1): HuggingFace ID of the model (i.e. tokenizer2)

​

__init__

def __init__(model_ident: str,
             custom_normalizer: Any = None,
             add_special_tokens: bool = False,
             pad: bool = True) -> None

​

vocab_size

@property
def vocab_size()

​

config

@property
def config() -> dict

​

tokenize

def tokenize(texts: str | list[str], pad: bool = True) -> np.ndarray

• texts (str, list): Texts to be tokenized
• pad (bool): unused here (configured in the constructor)

​

TokenizerFactory Objects

class TokenizerFactory()

​

get

@staticmethod
def get(type_: str, **tokenizer_kwargs) -> BaseTokenizer

• type_ (str): Tokenizer type to instantiate
• \**kwargs: kwargs to be passed to Tokenizer constructor