LICENSE
README.md
pyproject.toml
tests/test_byte_embeddings.py
tests/test_char_causal_lm.py
tests/test_char_embeddings.py
tests/test_chat_template.py
tests/test_control.py
tests/test_logits_processor.py
tests/test_tokenizer.py
tests/test_utils.py
utf8_tokenizer/__init__.py
utf8_tokenizer/byt5_comparison.py
utf8_tokenizer/byte_embeddings.py
utf8_tokenizer/char_causal_lm.py
utf8_tokenizer/char_embeddings.py
utf8_tokenizer/chat_template.jinja
utf8_tokenizer/control.py
utf8_tokenizer/logits_processor.py
utf8_tokenizer/tokenizer.py
utf8_tokenizer/utils.py
utf8_tokenizer.egg-info/PKG-INFO
utf8_tokenizer.egg-info/SOURCES.txt
utf8_tokenizer.egg-info/dependency_links.txt
utf8_tokenizer.egg-info/requires.txt
utf8_tokenizer.egg-info/top_level.txt