Upload tokenizer
Browse files- added_tokens.json +48 -0
- tokenizer.json +432 -0
added_tokens.json
CHANGED
@@ -1114,10 +1114,58 @@
|
|
1114 |
"v_tok_2": 50267,
|
1115 |
"v_tok_20": 50285,
|
1116 |
"v_tok_200": 50465,
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1117 |
"v_tok_201": 50466,
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1118 |
"v_tok_202": 50467,
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1119 |
"v_tok_203": 50468,
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1120 |
"v_tok_204": 50469,
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1121 |
"v_tok_205": 50470,
|
1122 |
"v_tok_206": 50471,
|
1123 |
"v_tok_207": 50472,
|
|
|
1114 |
"v_tok_2": 50267,
|
1115 |
"v_tok_20": 50285,
|
1116 |
"v_tok_200": 50465,
|
1117 |
+
"v_tok_2000": 52265,
|
1118 |
+
"v_tok_2001": 52266,
|
1119 |
+
"v_tok_2002": 52267,
|
1120 |
+
"v_tok_2003": 52268,
|
1121 |
+
"v_tok_2004": 52269,
|
1122 |
+
"v_tok_2005": 52270,
|
1123 |
+
"v_tok_2006": 52271,
|
1124 |
+
"v_tok_2007": 52272,
|
1125 |
+
"v_tok_2008": 52273,
|
1126 |
+
"v_tok_2009": 52274,
|
1127 |
"v_tok_201": 50466,
|
1128 |
+
"v_tok_2010": 52275,
|
1129 |
+
"v_tok_2011": 52276,
|
1130 |
+
"v_tok_2012": 52277,
|
1131 |
+
"v_tok_2013": 52278,
|
1132 |
+
"v_tok_2014": 52279,
|
1133 |
+
"v_tok_2015": 52280,
|
1134 |
+
"v_tok_2016": 52281,
|
1135 |
+
"v_tok_2017": 52282,
|
1136 |
+
"v_tok_2018": 52283,
|
1137 |
+
"v_tok_2019": 52284,
|
1138 |
"v_tok_202": 50467,
|
1139 |
+
"v_tok_2020": 52285,
|
1140 |
+
"v_tok_2021": 52286,
|
1141 |
+
"v_tok_2022": 52287,
|
1142 |
+
"v_tok_2023": 52288,
|
1143 |
+
"v_tok_2024": 52289,
|
1144 |
+
"v_tok_2025": 52290,
|
1145 |
+
"v_tok_2026": 52291,
|
1146 |
+
"v_tok_2027": 52292,
|
1147 |
+
"v_tok_2028": 52293,
|
1148 |
+
"v_tok_2029": 52294,
|
1149 |
"v_tok_203": 50468,
|
1150 |
+
"v_tok_2030": 52295,
|
1151 |
+
"v_tok_2031": 52296,
|
1152 |
+
"v_tok_2032": 52297,
|
1153 |
+
"v_tok_2033": 52298,
|
1154 |
+
"v_tok_2034": 52299,
|
1155 |
+
"v_tok_2035": 52300,
|
1156 |
+
"v_tok_2036": 52301,
|
1157 |
+
"v_tok_2037": 52302,
|
1158 |
+
"v_tok_2038": 52303,
|
1159 |
+
"v_tok_2039": 52304,
|
1160 |
"v_tok_204": 50469,
|
1161 |
+
"v_tok_2040": 52305,
|
1162 |
+
"v_tok_2041": 52306,
|
1163 |
+
"v_tok_2042": 52307,
|
1164 |
+
"v_tok_2043": 52308,
|
1165 |
+
"v_tok_2044": 52309,
|
1166 |
+
"v_tok_2045": 52310,
|
1167 |
+
"v_tok_2046": 52311,
|
1168 |
+
"v_tok_2047": 52312,
|
1169 |
"v_tok_205": 50470,
|
1170 |
"v_tok_206": 50471,
|
1171 |
"v_tok_207": 50472,
|
tokenizer.json
CHANGED
@@ -18047,6 +18047,438 @@
|
|
18047 |
"rstrip": false,
|
18048 |
"normalized": true,
|
18049 |
"special": false
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
18050 |
}
|
18051 |
],
|
18052 |
"normalizer": null,
|
|
|
18047 |
"rstrip": false,
|
18048 |
"normalized": true,
|
18049 |
"special": false
|
18050 |
+
},
|
18051 |
+
{
|
18052 |
+
"id": 52265,
|
18053 |
+
"content": "v_tok_2000",
|
18054 |
+
"single_word": false,
|
18055 |
+
"lstrip": false,
|
18056 |
+
"rstrip": false,
|
18057 |
+
"normalized": true,
|
18058 |
+
"special": false
|
18059 |
+
},
|
18060 |
+
{
|
18061 |
+
"id": 52266,
|
18062 |
+
"content": "v_tok_2001",
|
18063 |
+
"single_word": false,
|
18064 |
+
"lstrip": false,
|
18065 |
+
"rstrip": false,
|
18066 |
+
"normalized": true,
|
18067 |
+
"special": false
|
18068 |
+
},
|
18069 |
+
{
|
18070 |
+
"id": 52267,
|
18071 |
+
"content": "v_tok_2002",
|
18072 |
+
"single_word": false,
|
18073 |
+
"lstrip": false,
|
18074 |
+
"rstrip": false,
|
18075 |
+
"normalized": true,
|
18076 |
+
"special": false
|
18077 |
+
},
|
18078 |
+
{
|
18079 |
+
"id": 52268,
|
18080 |
+
"content": "v_tok_2003",
|
18081 |
+
"single_word": false,
|
18082 |
+
"lstrip": false,
|
18083 |
+
"rstrip": false,
|
18084 |
+
"normalized": true,
|
18085 |
+
"special": false
|
18086 |
+
},
|
18087 |
+
{
|
18088 |
+
"id": 52269,
|
18089 |
+
"content": "v_tok_2004",
|
18090 |
+
"single_word": false,
|
18091 |
+
"lstrip": false,
|
18092 |
+
"rstrip": false,
|
18093 |
+
"normalized": true,
|
18094 |
+
"special": false
|
18095 |
+
},
|
18096 |
+
{
|
18097 |
+
"id": 52270,
|
18098 |
+
"content": "v_tok_2005",
|
18099 |
+
"single_word": false,
|
18100 |
+
"lstrip": false,
|
18101 |
+
"rstrip": false,
|
18102 |
+
"normalized": true,
|
18103 |
+
"special": false
|
18104 |
+
},
|
18105 |
+
{
|
18106 |
+
"id": 52271,
|
18107 |
+
"content": "v_tok_2006",
|
18108 |
+
"single_word": false,
|
18109 |
+
"lstrip": false,
|
18110 |
+
"rstrip": false,
|
18111 |
+
"normalized": true,
|
18112 |
+
"special": false
|
18113 |
+
},
|
18114 |
+
{
|
18115 |
+
"id": 52272,
|
18116 |
+
"content": "v_tok_2007",
|
18117 |
+
"single_word": false,
|
18118 |
+
"lstrip": false,
|
18119 |
+
"rstrip": false,
|
18120 |
+
"normalized": true,
|
18121 |
+
"special": false
|
18122 |
+
},
|
18123 |
+
{
|
18124 |
+
"id": 52273,
|
18125 |
+
"content": "v_tok_2008",
|
18126 |
+
"single_word": false,
|
18127 |
+
"lstrip": false,
|
18128 |
+
"rstrip": false,
|
18129 |
+
"normalized": true,
|
18130 |
+
"special": false
|
18131 |
+
},
|
18132 |
+
{
|
18133 |
+
"id": 52274,
|
18134 |
+
"content": "v_tok_2009",
|
18135 |
+
"single_word": false,
|
18136 |
+
"lstrip": false,
|
18137 |
+
"rstrip": false,
|
18138 |
+
"normalized": true,
|
18139 |
+
"special": false
|
18140 |
+
},
|
18141 |
+
{
|
18142 |
+
"id": 52275,
|
18143 |
+
"content": "v_tok_2010",
|
18144 |
+
"single_word": false,
|
18145 |
+
"lstrip": false,
|
18146 |
+
"rstrip": false,
|
18147 |
+
"normalized": true,
|
18148 |
+
"special": false
|
18149 |
+
},
|
18150 |
+
{
|
18151 |
+
"id": 52276,
|
18152 |
+
"content": "v_tok_2011",
|
18153 |
+
"single_word": false,
|
18154 |
+
"lstrip": false,
|
18155 |
+
"rstrip": false,
|
18156 |
+
"normalized": true,
|
18157 |
+
"special": false
|
18158 |
+
},
|
18159 |
+
{
|
18160 |
+
"id": 52277,
|
18161 |
+
"content": "v_tok_2012",
|
18162 |
+
"single_word": false,
|
18163 |
+
"lstrip": false,
|
18164 |
+
"rstrip": false,
|
18165 |
+
"normalized": true,
|
18166 |
+
"special": false
|
18167 |
+
},
|
18168 |
+
{
|
18169 |
+
"id": 52278,
|
18170 |
+
"content": "v_tok_2013",
|
18171 |
+
"single_word": false,
|
18172 |
+
"lstrip": false,
|
18173 |
+
"rstrip": false,
|
18174 |
+
"normalized": true,
|
18175 |
+
"special": false
|
18176 |
+
},
|
18177 |
+
{
|
18178 |
+
"id": 52279,
|
18179 |
+
"content": "v_tok_2014",
|
18180 |
+
"single_word": false,
|
18181 |
+
"lstrip": false,
|
18182 |
+
"rstrip": false,
|
18183 |
+
"normalized": true,
|
18184 |
+
"special": false
|
18185 |
+
},
|
18186 |
+
{
|
18187 |
+
"id": 52280,
|
18188 |
+
"content": "v_tok_2015",
|
18189 |
+
"single_word": false,
|
18190 |
+
"lstrip": false,
|
18191 |
+
"rstrip": false,
|
18192 |
+
"normalized": true,
|
18193 |
+
"special": false
|
18194 |
+
},
|
18195 |
+
{
|
18196 |
+
"id": 52281,
|
18197 |
+
"content": "v_tok_2016",
|
18198 |
+
"single_word": false,
|
18199 |
+
"lstrip": false,
|
18200 |
+
"rstrip": false,
|
18201 |
+
"normalized": true,
|
18202 |
+
"special": false
|
18203 |
+
},
|
18204 |
+
{
|
18205 |
+
"id": 52282,
|
18206 |
+
"content": "v_tok_2017",
|
18207 |
+
"single_word": false,
|
18208 |
+
"lstrip": false,
|
18209 |
+
"rstrip": false,
|
18210 |
+
"normalized": true,
|
18211 |
+
"special": false
|
18212 |
+
},
|
18213 |
+
{
|
18214 |
+
"id": 52283,
|
18215 |
+
"content": "v_tok_2018",
|
18216 |
+
"single_word": false,
|
18217 |
+
"lstrip": false,
|
18218 |
+
"rstrip": false,
|
18219 |
+
"normalized": true,
|
18220 |
+
"special": false
|
18221 |
+
},
|
18222 |
+
{
|
18223 |
+
"id": 52284,
|
18224 |
+
"content": "v_tok_2019",
|
18225 |
+
"single_word": false,
|
18226 |
+
"lstrip": false,
|
18227 |
+
"rstrip": false,
|
18228 |
+
"normalized": true,
|
18229 |
+
"special": false
|
18230 |
+
},
|
18231 |
+
{
|
18232 |
+
"id": 52285,
|
18233 |
+
"content": "v_tok_2020",
|
18234 |
+
"single_word": false,
|
18235 |
+
"lstrip": false,
|
18236 |
+
"rstrip": false,
|
18237 |
+
"normalized": true,
|
18238 |
+
"special": false
|
18239 |
+
},
|
18240 |
+
{
|
18241 |
+
"id": 52286,
|
18242 |
+
"content": "v_tok_2021",
|
18243 |
+
"single_word": false,
|
18244 |
+
"lstrip": false,
|
18245 |
+
"rstrip": false,
|
18246 |
+
"normalized": true,
|
18247 |
+
"special": false
|
18248 |
+
},
|
18249 |
+
{
|
18250 |
+
"id": 52287,
|
18251 |
+
"content": "v_tok_2022",
|
18252 |
+
"single_word": false,
|
18253 |
+
"lstrip": false,
|
18254 |
+
"rstrip": false,
|
18255 |
+
"normalized": true,
|
18256 |
+
"special": false
|
18257 |
+
},
|
18258 |
+
{
|
18259 |
+
"id": 52288,
|
18260 |
+
"content": "v_tok_2023",
|
18261 |
+
"single_word": false,
|
18262 |
+
"lstrip": false,
|
18263 |
+
"rstrip": false,
|
18264 |
+
"normalized": true,
|
18265 |
+
"special": false
|
18266 |
+
},
|
18267 |
+
{
|
18268 |
+
"id": 52289,
|
18269 |
+
"content": "v_tok_2024",
|
18270 |
+
"single_word": false,
|
18271 |
+
"lstrip": false,
|
18272 |
+
"rstrip": false,
|
18273 |
+
"normalized": true,
|
18274 |
+
"special": false
|
18275 |
+
},
|
18276 |
+
{
|
18277 |
+
"id": 52290,
|
18278 |
+
"content": "v_tok_2025",
|
18279 |
+
"single_word": false,
|
18280 |
+
"lstrip": false,
|
18281 |
+
"rstrip": false,
|
18282 |
+
"normalized": true,
|
18283 |
+
"special": false
|
18284 |
+
},
|
18285 |
+
{
|
18286 |
+
"id": 52291,
|
18287 |
+
"content": "v_tok_2026",
|
18288 |
+
"single_word": false,
|
18289 |
+
"lstrip": false,
|
18290 |
+
"rstrip": false,
|
18291 |
+
"normalized": true,
|
18292 |
+
"special": false
|
18293 |
+
},
|
18294 |
+
{
|
18295 |
+
"id": 52292,
|
18296 |
+
"content": "v_tok_2027",
|
18297 |
+
"single_word": false,
|
18298 |
+
"lstrip": false,
|
18299 |
+
"rstrip": false,
|
18300 |
+
"normalized": true,
|
18301 |
+
"special": false
|
18302 |
+
},
|
18303 |
+
{
|
18304 |
+
"id": 52293,
|
18305 |
+
"content": "v_tok_2028",
|
18306 |
+
"single_word": false,
|
18307 |
+
"lstrip": false,
|
18308 |
+
"rstrip": false,
|
18309 |
+
"normalized": true,
|
18310 |
+
"special": false
|
18311 |
+
},
|
18312 |
+
{
|
18313 |
+
"id": 52294,
|
18314 |
+
"content": "v_tok_2029",
|
18315 |
+
"single_word": false,
|
18316 |
+
"lstrip": false,
|
18317 |
+
"rstrip": false,
|
18318 |
+
"normalized": true,
|
18319 |
+
"special": false
|
18320 |
+
},
|
18321 |
+
{
|
18322 |
+
"id": 52295,
|
18323 |
+
"content": "v_tok_2030",
|
18324 |
+
"single_word": false,
|
18325 |
+
"lstrip": false,
|
18326 |
+
"rstrip": false,
|
18327 |
+
"normalized": true,
|
18328 |
+
"special": false
|
18329 |
+
},
|
18330 |
+
{
|
18331 |
+
"id": 52296,
|
18332 |
+
"content": "v_tok_2031",
|
18333 |
+
"single_word": false,
|
18334 |
+
"lstrip": false,
|
18335 |
+
"rstrip": false,
|
18336 |
+
"normalized": true,
|
18337 |
+
"special": false
|
18338 |
+
},
|
18339 |
+
{
|
18340 |
+
"id": 52297,
|
18341 |
+
"content": "v_tok_2032",
|
18342 |
+
"single_word": false,
|
18343 |
+
"lstrip": false,
|
18344 |
+
"rstrip": false,
|
18345 |
+
"normalized": true,
|
18346 |
+
"special": false
|
18347 |
+
},
|
18348 |
+
{
|
18349 |
+
"id": 52298,
|
18350 |
+
"content": "v_tok_2033",
|
18351 |
+
"single_word": false,
|
18352 |
+
"lstrip": false,
|
18353 |
+
"rstrip": false,
|
18354 |
+
"normalized": true,
|
18355 |
+
"special": false
|
18356 |
+
},
|
18357 |
+
{
|
18358 |
+
"id": 52299,
|
18359 |
+
"content": "v_tok_2034",
|
18360 |
+
"single_word": false,
|
18361 |
+
"lstrip": false,
|
18362 |
+
"rstrip": false,
|
18363 |
+
"normalized": true,
|
18364 |
+
"special": false
|
18365 |
+
},
|
18366 |
+
{
|
18367 |
+
"id": 52300,
|
18368 |
+
"content": "v_tok_2035",
|
18369 |
+
"single_word": false,
|
18370 |
+
"lstrip": false,
|
18371 |
+
"rstrip": false,
|
18372 |
+
"normalized": true,
|
18373 |
+
"special": false
|
18374 |
+
},
|
18375 |
+
{
|
18376 |
+
"id": 52301,
|
18377 |
+
"content": "v_tok_2036",
|
18378 |
+
"single_word": false,
|
18379 |
+
"lstrip": false,
|
18380 |
+
"rstrip": false,
|
18381 |
+
"normalized": true,
|
18382 |
+
"special": false
|
18383 |
+
},
|
18384 |
+
{
|
18385 |
+
"id": 52302,
|
18386 |
+
"content": "v_tok_2037",
|
18387 |
+
"single_word": false,
|
18388 |
+
"lstrip": false,
|
18389 |
+
"rstrip": false,
|
18390 |
+
"normalized": true,
|
18391 |
+
"special": false
|
18392 |
+
},
|
18393 |
+
{
|
18394 |
+
"id": 52303,
|
18395 |
+
"content": "v_tok_2038",
|
18396 |
+
"single_word": false,
|
18397 |
+
"lstrip": false,
|
18398 |
+
"rstrip": false,
|
18399 |
+
"normalized": true,
|
18400 |
+
"special": false
|
18401 |
+
},
|
18402 |
+
{
|
18403 |
+
"id": 52304,
|
18404 |
+
"content": "v_tok_2039",
|
18405 |
+
"single_word": false,
|
18406 |
+
"lstrip": false,
|
18407 |
+
"rstrip": false,
|
18408 |
+
"normalized": true,
|
18409 |
+
"special": false
|
18410 |
+
},
|
18411 |
+
{
|
18412 |
+
"id": 52305,
|
18413 |
+
"content": "v_tok_2040",
|
18414 |
+
"single_word": false,
|
18415 |
+
"lstrip": false,
|
18416 |
+
"rstrip": false,
|
18417 |
+
"normalized": true,
|
18418 |
+
"special": false
|
18419 |
+
},
|
18420 |
+
{
|
18421 |
+
"id": 52306,
|
18422 |
+
"content": "v_tok_2041",
|
18423 |
+
"single_word": false,
|
18424 |
+
"lstrip": false,
|
18425 |
+
"rstrip": false,
|
18426 |
+
"normalized": true,
|
18427 |
+
"special": false
|
18428 |
+
},
|
18429 |
+
{
|
18430 |
+
"id": 52307,
|
18431 |
+
"content": "v_tok_2042",
|
18432 |
+
"single_word": false,
|
18433 |
+
"lstrip": false,
|
18434 |
+
"rstrip": false,
|
18435 |
+
"normalized": true,
|
18436 |
+
"special": false
|
18437 |
+
},
|
18438 |
+
{
|
18439 |
+
"id": 52308,
|
18440 |
+
"content": "v_tok_2043",
|
18441 |
+
"single_word": false,
|
18442 |
+
"lstrip": false,
|
18443 |
+
"rstrip": false,
|
18444 |
+
"normalized": true,
|
18445 |
+
"special": false
|
18446 |
+
},
|
18447 |
+
{
|
18448 |
+
"id": 52309,
|
18449 |
+
"content": "v_tok_2044",
|
18450 |
+
"single_word": false,
|
18451 |
+
"lstrip": false,
|
18452 |
+
"rstrip": false,
|
18453 |
+
"normalized": true,
|
18454 |
+
"special": false
|
18455 |
+
},
|
18456 |
+
{
|
18457 |
+
"id": 52310,
|
18458 |
+
"content": "v_tok_2045",
|
18459 |
+
"single_word": false,
|
18460 |
+
"lstrip": false,
|
18461 |
+
"rstrip": false,
|
18462 |
+
"normalized": true,
|
18463 |
+
"special": false
|
18464 |
+
},
|
18465 |
+
{
|
18466 |
+
"id": 52311,
|
18467 |
+
"content": "v_tok_2046",
|
18468 |
+
"single_word": false,
|
18469 |
+
"lstrip": false,
|
18470 |
+
"rstrip": false,
|
18471 |
+
"normalized": true,
|
18472 |
+
"special": false
|
18473 |
+
},
|
18474 |
+
{
|
18475 |
+
"id": 52312,
|
18476 |
+
"content": "v_tok_2047",
|
18477 |
+
"single_word": false,
|
18478 |
+
"lstrip": false,
|
18479 |
+
"rstrip": false,
|
18480 |
+
"normalized": true,
|
18481 |
+
"special": false
|
18482 |
}
|
18483 |
],
|
18484 |
"normalizer": null,
|