Undi95 commited on
Commit
9f49b22
1 Parent(s): 1cd0963

Update special_tokens_map.json

Browse files
Files changed (1) hide show
  1. special_tokens_map.json +5 -2
special_tokens_map.json CHANGED
@@ -2,9 +2,12 @@
2
  "additional_special_tokens": [
3
  "<unk>",
4
  "<s>",
5
- "</s>"
 
 
6
  ],
7
  "bos_token": "<s>",
8
- "eos_token": "</s>",
 
9
  "unk_token": "<unk>"
10
  }
 
2
  "additional_special_tokens": [
3
  "<unk>",
4
  "<s>",
5
+ "</s>",
6
+ "<|im_end|>",
7
+ "<|im_start|>"
8
  ],
9
  "bos_token": "<s>",
10
+ "eos_token": "<|im_end|>",
11
+ "pad_token": "[PAD]",
12
  "unk_token": "<unk>"
13
  }