monson commited on
Commit
dc651de
1 Parent(s): be0bd1a

Upload tokenizer

Browse files
special_tokens_map.json CHANGED
@@ -18,6 +18,7 @@
18
  "rstrip": false,
19
  "single_word": false
20
  },
 
21
  "unk_token": {
22
  "content": "<unk>",
23
  "lstrip": false,
 
18
  "rstrip": false,
19
  "single_word": false
20
  },
21
+ "pad_token": "</s>",
22
  "unk_token": {
23
  "content": "<unk>",
24
  "lstrip": false,
tokenizer.json CHANGED
@@ -62,6 +62,12 @@
62
  "id": "A",
63
  "type_id": 0
64
  }
 
 
 
 
 
 
65
  }
66
  ],
67
  "pair": [
@@ -77,6 +83,12 @@
77
  "type_id": 0
78
  }
79
  },
 
 
 
 
 
 
80
  {
81
  "SpecialToken": {
82
  "id": "<s>",
@@ -88,9 +100,24 @@
88
  "id": "B",
89
  "type_id": 1
90
  }
 
 
 
 
 
 
91
  }
92
  ],
93
  "special_tokens": {
 
 
 
 
 
 
 
 
 
94
  "<s>": {
95
  "id": "<s>",
96
  "ids": [
 
62
  "id": "A",
63
  "type_id": 0
64
  }
65
+ },
66
+ {
67
+ "SpecialToken": {
68
+ "id": "</s>",
69
+ "type_id": 0
70
+ }
71
  }
72
  ],
73
  "pair": [
 
83
  "type_id": 0
84
  }
85
  },
86
+ {
87
+ "SpecialToken": {
88
+ "id": "</s>",
89
+ "type_id": 0
90
+ }
91
+ },
92
  {
93
  "SpecialToken": {
94
  "id": "<s>",
 
100
  "id": "B",
101
  "type_id": 1
102
  }
103
+ },
104
+ {
105
+ "SpecialToken": {
106
+ "id": "</s>",
107
+ "type_id": 1
108
+ }
109
  }
110
  ],
111
  "special_tokens": {
112
+ "</s>": {
113
+ "id": "</s>",
114
+ "ids": [
115
+ 2
116
+ ],
117
+ "tokens": [
118
+ "</s>"
119
+ ]
120
+ },
121
  "<s>": {
122
  "id": "<s>",
123
  "ids": [
tokenizer_config.json CHANGED
@@ -1,6 +1,6 @@
1
  {
2
  "add_bos_token": true,
3
- "add_eos_token": false,
4
  "added_tokens_decoder": {
5
  "0": {
6
  "content": "<unk>",
@@ -37,7 +37,7 @@
37
  "eos_token": "</s>",
38
  "legacy": true,
39
  "model_max_length": 1000000000000000019884624838656,
40
- "pad_token": null,
41
  "sp_model_kwargs": {},
42
  "spaces_between_special_tokens": false,
43
  "tokenizer_class": "LlamaTokenizer",
 
1
  {
2
  "add_bos_token": true,
3
+ "add_eos_token": true,
4
  "added_tokens_decoder": {
5
  "0": {
6
  "content": "<unk>",
 
37
  "eos_token": "</s>",
38
  "legacy": true,
39
  "model_max_length": 1000000000000000019884624838656,
40
+ "pad_token": "</s>",
41
  "sp_model_kwargs": {},
42
  "spaces_between_special_tokens": false,
43
  "tokenizer_class": "LlamaTokenizer",