pragaash commited on
Commit
72e0b3e
1 Parent(s): 29194e9

Replace tokenizer class with auto map for relative import of byte tokenizer.

Browse files
Files changed (1) hide show
  1. tokenizer_config.json +3 -1
tokenizer_config.json CHANGED
@@ -1,9 +1,11 @@
1
  {
2
  "added_tokens_decoder": {},
 
 
 
3
  "byte_level": true,
4
  "clean_up_tokenization_spaces": true,
5
  "model_max_length": 1000000000000000019884624838656,
6
  "padding_side": "left",
7
- "tokenizer_class": "ByteTokenizer",
8
  "truncation_side": "left"
9
  }
 
1
  {
2
  "added_tokens_decoder": {},
3
+ "auto_map": {
4
+ "AutoTokenizer": "tokenizer.ByteTokenizer"
5
+ },
6
  "byte_level": true,
7
  "clean_up_tokenization_spaces": true,
8
  "model_max_length": 1000000000000000019884624838656,
9
  "padding_side": "left",
 
10
  "truncation_side": "left"
11
  }