*No copyright* Apache License
-----------------------------
python-tokenizers-0.22.2-build/tokenizers-0.22.2/PKG-INFO
python-tokenizers-0.22.2-build/tokenizers-0.22.2/pyproject.toml

*No copyright* Apache License 2.0
---------------------------------
python-tokenizers-0.22.2-build/tokenizers-0.22.2/tokenizers/LICENSE

Unknown or generated
--------------------
python-tokenizers-0.22.2-build/tokenizers-0.22.2/bindings/python/.cargo/config.toml
python-tokenizers-0.22.2-build/tokenizers-0.22.2/bindings/python/CHANGELOG.md
python-tokenizers-0.22.2-build/tokenizers-0.22.2/bindings/python/Cargo.lock
python-tokenizers-0.22.2-build/tokenizers-0.22.2/bindings/python/Cargo.toml
python-tokenizers-0.22.2-build/tokenizers-0.22.2/bindings/python/MANIFEST.in
python-tokenizers-0.22.2-build/tokenizers-0.22.2/bindings/python/Makefile
python-tokenizers-0.22.2-build/tokenizers-0.22.2/bindings/python/README.md
python-tokenizers-0.22.2-build/tokenizers-0.22.2/bindings/python/benches/test_tiktoken.py
python-tokenizers-0.22.2-build/tokenizers-0.22.2/bindings/python/conftest.py
python-tokenizers-0.22.2-build/tokenizers-0.22.2/bindings/python/docs/pyo3.md
python-tokenizers-0.22.2-build/tokenizers-0.22.2/bindings/python/examples/custom_components.py
python-tokenizers-0.22.2-build/tokenizers-0.22.2/bindings/python/examples/example.py
python-tokenizers-0.22.2-build/tokenizers-0.22.2/bindings/python/examples/train_bert_wordpiece.py
python-tokenizers-0.22.2-build/tokenizers-0.22.2/bindings/python/examples/train_bytelevel_bpe.py
python-tokenizers-0.22.2-build/tokenizers-0.22.2/bindings/python/examples/train_with_datasets.py
python-tokenizers-0.22.2-build/tokenizers-0.22.2/bindings/python/examples/using_the_visualizer.ipynb
python-tokenizers-0.22.2-build/tokenizers-0.22.2/bindings/python/py_src/tokenizers/__init__.py
python-tokenizers-0.22.2-build/tokenizers-0.22.2/bindings/python/py_src/tokenizers/__init__.pyi
python-tokenizers-0.22.2-build/tokenizers-0.22.2/bindings/python/py_src/tokenizers/decoders/__init__.py
python-tokenizers-0.22.2-build/tokenizers-0.22.2/bindings/python/py_src/tokenizers/decoders/__init__.pyi
python-tokenizers-0.22.2-build/tokenizers-0.22.2/bindings/python/py_src/tokenizers/implementations/__init__.py
python-tokenizers-0.22.2-build/tokenizers-0.22.2/bindings/python/py_src/tokenizers/implementations/base_tokenizer.py
python-tokenizers-0.22.2-build/tokenizers-0.22.2/bindings/python/py_src/tokenizers/implementations/bert_wordpiece.py
python-tokenizers-0.22.2-build/tokenizers-0.22.2/bindings/python/py_src/tokenizers/implementations/byte_level_bpe.py
python-tokenizers-0.22.2-build/tokenizers-0.22.2/bindings/python/py_src/tokenizers/implementations/char_level_bpe.py
python-tokenizers-0.22.2-build/tokenizers-0.22.2/bindings/python/py_src/tokenizers/implementations/sentencepiece_bpe.py
python-tokenizers-0.22.2-build/tokenizers-0.22.2/bindings/python/py_src/tokenizers/implementations/sentencepiece_unigram.py
python-tokenizers-0.22.2-build/tokenizers-0.22.2/bindings/python/py_src/tokenizers/models/__init__.py
python-tokenizers-0.22.2-build/tokenizers-0.22.2/bindings/python/py_src/tokenizers/models/__init__.pyi
python-tokenizers-0.22.2-build/tokenizers-0.22.2/bindings/python/py_src/tokenizers/normalizers/__init__.py
python-tokenizers-0.22.2-build/tokenizers-0.22.2/bindings/python/py_src/tokenizers/normalizers/__init__.pyi
python-tokenizers-0.22.2-build/tokenizers-0.22.2/bindings/python/py_src/tokenizers/pre_tokenizers/__init__.py
python-tokenizers-0.22.2-build/tokenizers-0.22.2/bindings/python/py_src/tokenizers/pre_tokenizers/__init__.pyi
python-tokenizers-0.22.2-build/tokenizers-0.22.2/bindings/python/py_src/tokenizers/processors/__init__.py
python-tokenizers-0.22.2-build/tokenizers-0.22.2/bindings/python/py_src/tokenizers/processors/__init__.pyi
python-tokenizers-0.22.2-build/tokenizers-0.22.2/bindings/python/py_src/tokenizers/tokenizers.pyi
python-tokenizers-0.22.2-build/tokenizers-0.22.2/bindings/python/py_src/tokenizers/tools/__init__.py
python-tokenizers-0.22.2-build/tokenizers-0.22.2/bindings/python/py_src/tokenizers/tools/visualizer-styles.css
python-tokenizers-0.22.2-build/tokenizers-0.22.2/bindings/python/py_src/tokenizers/tools/visualizer.py
python-tokenizers-0.22.2-build/tokenizers-0.22.2/bindings/python/py_src/tokenizers/trainers/__init__.py
python-tokenizers-0.22.2-build/tokenizers-0.22.2/bindings/python/py_src/tokenizers/trainers/__init__.pyi
python-tokenizers-0.22.2-build/tokenizers-0.22.2/bindings/python/rust-toolchain
python-tokenizers-0.22.2-build/tokenizers-0.22.2/bindings/python/scripts/convert.py
python-tokenizers-0.22.2-build/tokenizers-0.22.2/bindings/python/scripts/sentencepiece_extractor.py
python-tokenizers-0.22.2-build/tokenizers-0.22.2/bindings/python/scripts/spm_parity_check.py
python-tokenizers-0.22.2-build/tokenizers-0.22.2/bindings/python/setup.cfg
python-tokenizers-0.22.2-build/tokenizers-0.22.2/bindings/python/src/decoders.rs
python-tokenizers-0.22.2-build/tokenizers-0.22.2/bindings/python/src/encoding.rs
python-tokenizers-0.22.2-build/tokenizers-0.22.2/bindings/python/src/error.rs
python-tokenizers-0.22.2-build/tokenizers-0.22.2/bindings/python/src/lib.rs
python-tokenizers-0.22.2-build/tokenizers-0.22.2/bindings/python/src/models.rs
python-tokenizers-0.22.2-build/tokenizers-0.22.2/bindings/python/src/normalizers.rs
python-tokenizers-0.22.2-build/tokenizers-0.22.2/bindings/python/src/pre_tokenizers.rs
python-tokenizers-0.22.2-build/tokenizers-0.22.2/bindings/python/src/processors.rs
python-tokenizers-0.22.2-build/tokenizers-0.22.2/bindings/python/src/token.rs
python-tokenizers-0.22.2-build/tokenizers-0.22.2/bindings/python/src/tokenizer.rs
python-tokenizers-0.22.2-build/tokenizers-0.22.2/bindings/python/src/trainers.rs
python-tokenizers-0.22.2-build/tokenizers-0.22.2/bindings/python/src/utils/iterators.rs
python-tokenizers-0.22.2-build/tokenizers-0.22.2/bindings/python/src/utils/mod.rs
python-tokenizers-0.22.2-build/tokenizers-0.22.2/bindings/python/src/utils/normalization.rs
python-tokenizers-0.22.2-build/tokenizers-0.22.2/bindings/python/src/utils/pretokenization.rs
python-tokenizers-0.22.2-build/tokenizers-0.22.2/bindings/python/src/utils/regex.rs
python-tokenizers-0.22.2-build/tokenizers-0.22.2/bindings/python/src/utils/serde_pyo3.rs
python-tokenizers-0.22.2-build/tokenizers-0.22.2/bindings/python/stub.py
python-tokenizers-0.22.2-build/tokenizers-0.22.2/bindings/python/test.txt
python-tokenizers-0.22.2-build/tokenizers-0.22.2/bindings/python/tests/bindings/test_decoders.py
python-tokenizers-0.22.2-build/tokenizers-0.22.2/bindings/python/tests/bindings/test_encoding.py
python-tokenizers-0.22.2-build/tokenizers-0.22.2/bindings/python/tests/bindings/test_models.py
python-tokenizers-0.22.2-build/tokenizers-0.22.2/bindings/python/tests/bindings/test_normalizers.py
python-tokenizers-0.22.2-build/tokenizers-0.22.2/bindings/python/tests/bindings/test_pre_tokenizers.py
python-tokenizers-0.22.2-build/tokenizers-0.22.2/bindings/python/tests/bindings/test_processors.py
python-tokenizers-0.22.2-build/tokenizers-0.22.2/bindings/python/tests/bindings/test_tokenizer.py
python-tokenizers-0.22.2-build/tokenizers-0.22.2/bindings/python/tests/bindings/test_trainers.py
python-tokenizers-0.22.2-build/tokenizers-0.22.2/bindings/python/tests/documentation/test_pipeline.py
python-tokenizers-0.22.2-build/tokenizers-0.22.2/bindings/python/tests/documentation/test_quicktour.py
python-tokenizers-0.22.2-build/tokenizers-0.22.2/bindings/python/tests/documentation/test_tutorial_train_from_iterators.py
python-tokenizers-0.22.2-build/tokenizers-0.22.2/bindings/python/tests/implementations/test_base_tokenizer.py
python-tokenizers-0.22.2-build/tokenizers-0.22.2/bindings/python/tests/implementations/test_bert_wordpiece.py
python-tokenizers-0.22.2-build/tokenizers-0.22.2/bindings/python/tests/implementations/test_byte_level_bpe.py
python-tokenizers-0.22.2-build/tokenizers-0.22.2/bindings/python/tests/implementations/test_char_bpe.py
python-tokenizers-0.22.2-build/tokenizers-0.22.2/bindings/python/tests/implementations/test_sentencepiece.py
python-tokenizers-0.22.2-build/tokenizers-0.22.2/bindings/python/tests/test_serialization.py
python-tokenizers-0.22.2-build/tokenizers-0.22.2/bindings/python/tests/utils.py
python-tokenizers-0.22.2-build/tokenizers-0.22.2/py_src/tokenizers/__init__.py
python-tokenizers-0.22.2-build/tokenizers-0.22.2/py_src/tokenizers/__init__.pyi
python-tokenizers-0.22.2-build/tokenizers-0.22.2/py_src/tokenizers/decoders/__init__.py
python-tokenizers-0.22.2-build/tokenizers-0.22.2/py_src/tokenizers/decoders/__init__.pyi
python-tokenizers-0.22.2-build/tokenizers-0.22.2/py_src/tokenizers/implementations/__init__.py
python-tokenizers-0.22.2-build/tokenizers-0.22.2/py_src/tokenizers/implementations/base_tokenizer.py
python-tokenizers-0.22.2-build/tokenizers-0.22.2/py_src/tokenizers/implementations/bert_wordpiece.py
python-tokenizers-0.22.2-build/tokenizers-0.22.2/py_src/tokenizers/implementations/byte_level_bpe.py
python-tokenizers-0.22.2-build/tokenizers-0.22.2/py_src/tokenizers/implementations/char_level_bpe.py
python-tokenizers-0.22.2-build/tokenizers-0.22.2/py_src/tokenizers/implementations/sentencepiece_bpe.py
python-tokenizers-0.22.2-build/tokenizers-0.22.2/py_src/tokenizers/implementations/sentencepiece_unigram.py
python-tokenizers-0.22.2-build/tokenizers-0.22.2/py_src/tokenizers/models/__init__.py
python-tokenizers-0.22.2-build/tokenizers-0.22.2/py_src/tokenizers/models/__init__.pyi
python-tokenizers-0.22.2-build/tokenizers-0.22.2/py_src/tokenizers/normalizers/__init__.py
python-tokenizers-0.22.2-build/tokenizers-0.22.2/py_src/tokenizers/normalizers/__init__.pyi
python-tokenizers-0.22.2-build/tokenizers-0.22.2/py_src/tokenizers/pre_tokenizers/__init__.py
python-tokenizers-0.22.2-build/tokenizers-0.22.2/py_src/tokenizers/pre_tokenizers/__init__.pyi
python-tokenizers-0.22.2-build/tokenizers-0.22.2/py_src/tokenizers/processors/__init__.py
python-tokenizers-0.22.2-build/tokenizers-0.22.2/py_src/tokenizers/processors/__init__.pyi
python-tokenizers-0.22.2-build/tokenizers-0.22.2/py_src/tokenizers/tokenizers.pyi
python-tokenizers-0.22.2-build/tokenizers-0.22.2/py_src/tokenizers/tools/__init__.py
python-tokenizers-0.22.2-build/tokenizers-0.22.2/py_src/tokenizers/tools/visualizer-styles.css
python-tokenizers-0.22.2-build/tokenizers-0.22.2/py_src/tokenizers/tools/visualizer.py
python-tokenizers-0.22.2-build/tokenizers-0.22.2/py_src/tokenizers/trainers/__init__.py
python-tokenizers-0.22.2-build/tokenizers-0.22.2/py_src/tokenizers/trainers/__init__.pyi
python-tokenizers-0.22.2-build/tokenizers-0.22.2/tokenizers/CHANGELOG.md
python-tokenizers-0.22.2-build/tokenizers-0.22.2/tokenizers/Cargo.toml
python-tokenizers-0.22.2-build/tokenizers-0.22.2/tokenizers/Makefile
python-tokenizers-0.22.2-build/tokenizers-0.22.2/tokenizers/README.md
python-tokenizers-0.22.2-build/tokenizers-0.22.2/tokenizers/README.tpl
python-tokenizers-0.22.2-build/tokenizers-0.22.2/tokenizers/benches/added_vocab_deserialize.rs
python-tokenizers-0.22.2-build/tokenizers-0.22.2/tokenizers/benches/bert_benchmark.rs
python-tokenizers-0.22.2-build/tokenizers-0.22.2/tokenizers/benches/bpe_benchmark.rs
python-tokenizers-0.22.2-build/tokenizers-0.22.2/tokenizers/benches/common/mod.rs
python-tokenizers-0.22.2-build/tokenizers-0.22.2/tokenizers/benches/layout_benchmark.rs
python-tokenizers-0.22.2-build/tokenizers-0.22.2/tokenizers/benches/llama3_benchmark.rs
python-tokenizers-0.22.2-build/tokenizers-0.22.2/tokenizers/benches/unigram_benchmark.rs
python-tokenizers-0.22.2-build/tokenizers-0.22.2/tokenizers/examples/encode_batch.rs
python-tokenizers-0.22.2-build/tokenizers-0.22.2/tokenizers/examples/serialization.rs
python-tokenizers-0.22.2-build/tokenizers-0.22.2/tokenizers/src/decoders/bpe.rs
python-tokenizers-0.22.2-build/tokenizers-0.22.2/tokenizers/src/decoders/byte_fallback.rs
python-tokenizers-0.22.2-build/tokenizers-0.22.2/tokenizers/src/decoders/ctc.rs
python-tokenizers-0.22.2-build/tokenizers-0.22.2/tokenizers/src/decoders/fuse.rs
python-tokenizers-0.22.2-build/tokenizers-0.22.2/tokenizers/src/decoders/mod.rs
python-tokenizers-0.22.2-build/tokenizers-0.22.2/tokenizers/src/decoders/sequence.rs
python-tokenizers-0.22.2-build/tokenizers-0.22.2/tokenizers/src/decoders/strip.rs
python-tokenizers-0.22.2-build/tokenizers-0.22.2/tokenizers/src/decoders/wordpiece.rs
python-tokenizers-0.22.2-build/tokenizers-0.22.2/tokenizers/src/lib.rs
python-tokenizers-0.22.2-build/tokenizers-0.22.2/tokenizers/src/models/bpe/mod.rs
python-tokenizers-0.22.2-build/tokenizers-0.22.2/tokenizers/src/models/bpe/model.rs
python-tokenizers-0.22.2-build/tokenizers-0.22.2/tokenizers/src/models/bpe/serialization.rs
python-tokenizers-0.22.2-build/tokenizers-0.22.2/tokenizers/src/models/bpe/trainer.rs
python-tokenizers-0.22.2-build/tokenizers-0.22.2/tokenizers/src/models/bpe/word.rs
python-tokenizers-0.22.2-build/tokenizers-0.22.2/tokenizers/src/models/mod.rs
python-tokenizers-0.22.2-build/tokenizers-0.22.2/tokenizers/src/models/unigram/lattice.rs
python-tokenizers-0.22.2-build/tokenizers-0.22.2/tokenizers/src/models/unigram/mod.rs
python-tokenizers-0.22.2-build/tokenizers-0.22.2/tokenizers/src/models/unigram/model.rs
python-tokenizers-0.22.2-build/tokenizers-0.22.2/tokenizers/src/models/unigram/serialization.rs
python-tokenizers-0.22.2-build/tokenizers-0.22.2/tokenizers/src/models/unigram/trainer.rs
python-tokenizers-0.22.2-build/tokenizers-0.22.2/tokenizers/src/models/unigram/trie.rs
python-tokenizers-0.22.2-build/tokenizers-0.22.2/tokenizers/src/models/wordlevel/mod.rs
python-tokenizers-0.22.2-build/tokenizers-0.22.2/tokenizers/src/models/wordlevel/serialization.rs
python-tokenizers-0.22.2-build/tokenizers-0.22.2/tokenizers/src/models/wordlevel/trainer.rs
python-tokenizers-0.22.2-build/tokenizers-0.22.2/tokenizers/src/models/wordpiece/mod.rs
python-tokenizers-0.22.2-build/tokenizers-0.22.2/tokenizers/src/models/wordpiece/serialization.rs
python-tokenizers-0.22.2-build/tokenizers-0.22.2/tokenizers/src/models/wordpiece/trainer.rs
python-tokenizers-0.22.2-build/tokenizers-0.22.2/tokenizers/src/normalizers/bert.rs
python-tokenizers-0.22.2-build/tokenizers-0.22.2/tokenizers/src/normalizers/byte_level.rs
python-tokenizers-0.22.2-build/tokenizers-0.22.2/tokenizers/src/normalizers/mod.rs
python-tokenizers-0.22.2-build/tokenizers-0.22.2/tokenizers/src/normalizers/precompiled.rs
python-tokenizers-0.22.2-build/tokenizers-0.22.2/tokenizers/src/normalizers/prepend.rs
python-tokenizers-0.22.2-build/tokenizers-0.22.2/tokenizers/src/normalizers/replace.rs
python-tokenizers-0.22.2-build/tokenizers-0.22.2/tokenizers/src/normalizers/strip.rs
python-tokenizers-0.22.2-build/tokenizers-0.22.2/tokenizers/src/normalizers/unicode.rs
python-tokenizers-0.22.2-build/tokenizers-0.22.2/tokenizers/src/normalizers/utils.rs
python-tokenizers-0.22.2-build/tokenizers-0.22.2/tokenizers/src/pre_tokenizers/bert.rs
python-tokenizers-0.22.2-build/tokenizers-0.22.2/tokenizers/src/pre_tokenizers/byte_level.rs
python-tokenizers-0.22.2-build/tokenizers-0.22.2/tokenizers/src/pre_tokenizers/delimiter.rs
python-tokenizers-0.22.2-build/tokenizers-0.22.2/tokenizers/src/pre_tokenizers/digits.rs
python-tokenizers-0.22.2-build/tokenizers-0.22.2/tokenizers/src/pre_tokenizers/fixed_length.rs
python-tokenizers-0.22.2-build/tokenizers-0.22.2/tokenizers/src/pre_tokenizers/metaspace.rs
python-tokenizers-0.22.2-build/tokenizers-0.22.2/tokenizers/src/pre_tokenizers/mod.rs
python-tokenizers-0.22.2-build/tokenizers-0.22.2/tokenizers/src/pre_tokenizers/punctuation.rs
python-tokenizers-0.22.2-build/tokenizers-0.22.2/tokenizers/src/pre_tokenizers/sequence.rs
python-tokenizers-0.22.2-build/tokenizers-0.22.2/tokenizers/src/pre_tokenizers/split.rs
python-tokenizers-0.22.2-build/tokenizers-0.22.2/tokenizers/src/pre_tokenizers/unicode_scripts/mod.rs
python-tokenizers-0.22.2-build/tokenizers-0.22.2/tokenizers/src/pre_tokenizers/unicode_scripts/pre_tokenizer.rs
python-tokenizers-0.22.2-build/tokenizers-0.22.2/tokenizers/src/pre_tokenizers/unicode_scripts/scripts.rs
python-tokenizers-0.22.2-build/tokenizers-0.22.2/tokenizers/src/pre_tokenizers/whitespace.rs
python-tokenizers-0.22.2-build/tokenizers-0.22.2/tokenizers/src/processors/bert.rs
python-tokenizers-0.22.2-build/tokenizers-0.22.2/tokenizers/src/processors/mod.rs
python-tokenizers-0.22.2-build/tokenizers-0.22.2/tokenizers/src/processors/roberta.rs
python-tokenizers-0.22.2-build/tokenizers-0.22.2/tokenizers/src/processors/sequence.rs
python-tokenizers-0.22.2-build/tokenizers-0.22.2/tokenizers/src/processors/template.rs
python-tokenizers-0.22.2-build/tokenizers-0.22.2/tokenizers/src/tokenizer/added_vocabulary.rs
python-tokenizers-0.22.2-build/tokenizers-0.22.2/tokenizers/src/tokenizer/encoding.rs
python-tokenizers-0.22.2-build/tokenizers-0.22.2/tokenizers/src/tokenizer/mod.rs
python-tokenizers-0.22.2-build/tokenizers-0.22.2/tokenizers/src/tokenizer/normalizer.rs
python-tokenizers-0.22.2-build/tokenizers-0.22.2/tokenizers/src/tokenizer/pattern.rs
python-tokenizers-0.22.2-build/tokenizers-0.22.2/tokenizers/src/tokenizer/pre_tokenizer.rs
python-tokenizers-0.22.2-build/tokenizers-0.22.2/tokenizers/src/tokenizer/serialization.rs
python-tokenizers-0.22.2-build/tokenizers-0.22.2/tokenizers/src/utils/cache.rs
python-tokenizers-0.22.2-build/tokenizers-0.22.2/tokenizers/src/utils/fancy.rs
python-tokenizers-0.22.2-build/tokenizers-0.22.2/tokenizers/src/utils/from_pretrained.rs
python-tokenizers-0.22.2-build/tokenizers-0.22.2/tokenizers/src/utils/iter.rs
python-tokenizers-0.22.2-build/tokenizers-0.22.2/tokenizers/src/utils/mod.rs
python-tokenizers-0.22.2-build/tokenizers-0.22.2/tokenizers/src/utils/onig.rs
python-tokenizers-0.22.2-build/tokenizers-0.22.2/tokenizers/src/utils/padding.rs
python-tokenizers-0.22.2-build/tokenizers-0.22.2/tokenizers/src/utils/parallelism.rs
python-tokenizers-0.22.2-build/tokenizers-0.22.2/tokenizers/src/utils/progress.rs
python-tokenizers-0.22.2-build/tokenizers-0.22.2/tokenizers/src/utils/truncation.rs
python-tokenizers-0.22.2-build/tokenizers-0.22.2/tokenizers/tests/added_tokens.rs
python-tokenizers-0.22.2-build/tokenizers-0.22.2/tokenizers/tests/common/mod.rs
python-tokenizers-0.22.2-build/tokenizers-0.22.2/tokenizers/tests/documentation.rs
python-tokenizers-0.22.2-build/tokenizers-0.22.2/tokenizers/tests/from_pretrained.rs
python-tokenizers-0.22.2-build/tokenizers-0.22.2/tokenizers/tests/offsets.rs
python-tokenizers-0.22.2-build/tokenizers-0.22.2/tokenizers/tests/serialization.rs
python-tokenizers-0.22.2-build/tokenizers-0.22.2/tokenizers/tests/stream.rs
python-tokenizers-0.22.2-build/tokenizers-0.22.2/tokenizers/tests/training.rs
python-tokenizers-0.22.2-build/tokenizers-0.22.2/tokenizers/tests/unigram.rs