tokenizers-0.15.2.dist-info/METADATA,sha256=x_CpzOdWRWGoO6-Mwgpyu25gkbKNMrBwICA128lmWdY,6848
tokenizers-0.15.2.dist-info/WHEEL,sha256=tZi1scbRgKK_pT9C-iwwvGEMU0MzRNBBY7FEboCqBwE,90
tokenizers/decoders/__init__.py,sha256=xffrQt9YxtuKLOBEkJnZZPu03ozf3sm-x1KgxF-f0DQ,349
tokenizers/decoders/__init__.pyi,sha256=Bmoj3V9FRg7U3nzzJBoaDWXjDDYU54e_7qHTO4J2v64,7311
tokenizers/implementations/base_tokenizer.py,sha256=nOUjtXTgDJSdIjDDnBzoWELUnzpRy6yLyVhUOL5t5xY,14624
tokenizers/implementations/bert_wordpiece.py,sha256=jRUMSYU1C7nVEDZMKdyG3Vp7-CUongE1kGJhvOP_EHM,5671
tokenizers/implementations/byte_level_bpe.py,sha256=iCCZSWW9nwQgfeXeOiwVEX7sLNfPsQxO7CCuEmTaHXw,4411
tokenizers/implementations/char_level_bpe.py,sha256=AQaZFT_RK7SW1ubGsFtXOnftit_vWp0fj4-fJnzZn7c,5616
tokenizers/implementations/sentencepiece_bpe.py,sha256=WzbvXIt9DNjj5uANx249wxRf6x9uM9bB_a9hg6ybPuw,3781
tokenizers/implementations/sentencepiece_unigram.py,sha256=nEsscERNWT-ZdEEH9okt17jsUoh_awBHZhETz3htmHw,7656
tokenizers/implementations/__init__.py,sha256=9ZI2cJHPCUCZXP37GNmRHtBzIYiKWzyqky1rtcYdrPw,316
tokenizers/models/__init__.py,sha256=qE73qycPAKcey_pS8ov1FIz10L2Ybzy1E-1KGee27Qg,184
tokenizers/models/__init__.pyi,sha256=ShlzeGeZW0zJWW-yYpzpa-y0cLEGg5TV3mOv4mZRP74,17311
tokenizers/normalizers/__init__.py,sha256=HdGKGl2qjqQN520YXGgBn6Ti3qMQ-grvNH9QgzeA-wY,837
tokenizers/normalizers/__init__.pyi,sha256=QosdCP3B-MN6bhDBeVcuX07Ztturcvz9x67X33lXDGg,20168
tokenizers/pre_tokenizers/__init__.py,sha256=ZWLPWG2nhoTojLyHIDUPRsa1V8drqWufJVjzbRmR7UQ,572
tokenizers/pre_tokenizers/__init__.pyi,sha256=uWlLCivyHDGnUp8T6dLMk3qoSlq3kmRwYnS73y3hp5Y,23635
tokenizers/processors/__init__.py,sha256=X1OKKtr3IUeede85mI1cLF0MqmDRlr7LNFMYdEy5KMU,316
tokenizers/processors/__init__.pyi,sha256=cBI1jIP2DPfF9xDyw-a2OHLLQcLAANFplCjxotcxmZ8,11689
tokenizers/tools/visualizer-styles.css,sha256=g98RJucUCjYnPLwXycfcV-dAbHMzqJpnZ7NoP_cCUBk,5020
tokenizers/tools/visualizer.py,sha256=Er3CRAqTshUzZttZGsC4b36AGNh7ryNDuTWKHD39tfA,15024
tokenizers/tools/__init__.py,sha256=mJLVrHN1FP23Cf4_EJWoYUdh8yTb5BbJFNfa8Ec5vHM,56
tokenizers/trainers/__init__.py,sha256=v9CdtoVauwD6b1wAA6R5goSLZslBbPE5vZHM2d8sgKM,256
tokenizers/trainers/__init__.pyi,sha256=ZPBCBM4u9PLnJyXklRKTiRRjp2qs_pmI-GR1qr5MTBM,5542
tokenizers/__init__.py,sha256=AJ5NdKOV_dZ_SOw9oFF1M4ln0fh1hod-voqzeYHS71U,2715
tokenizers/__init__.pyi,sha256=O7p6xv31eVIrojoB5NBFuY9GVG-o7bdx8wbwtIBoskU,39582
tokenizers/tokenizers.pyd,sha256=O_IDysy-_i_QWpIjG9O5zna6k_tIMydp1pSwyMQEI9k,4901376
tokenizers-0.15.2.dist-info/RECORD,,
