danielhanchen commited on
Commit
c1bbedc
1 Parent(s): 8fffded

Upload tokenizer

Browse files
Files changed (2) hide show
  1. tokenizer.json +2 -2
  2. tokenizer_config.json +2 -2
tokenizer.json CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:2f1f411054535bfe0e360e3431e7340cfc795d0d4ed1ebd426b01993831d49e4
3
- size 17525359
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3f289bc05132635a8bc7aca7aa21255efd5e18f3710f43e3cdb96bcd41be4922
3
+ size 17525357
tokenizer_config.json CHANGED
@@ -856,7 +856,7 @@
856
  "normalized": false,
857
  "rstrip": false,
858
  "single_word": false,
859
- "special": false
860
  },
861
  "107": {
862
  "content": "<end_of_turn>",
@@ -864,7 +864,7 @@
864
  "normalized": false,
865
  "rstrip": false,
866
  "single_word": false,
867
- "special": false
868
  },
869
  "108": {
870
  "content": "\n",
 
856
  "normalized": false,
857
  "rstrip": false,
858
  "single_word": false,
859
+ "special": true
860
  },
861
  "107": {
862
  "content": "<end_of_turn>",
 
864
  "normalized": false,
865
  "rstrip": false,
866
  "single_word": false,
867
+ "special": true
868
  },
869
  "108": {
870
  "content": "\n",