Upload tokenizer

added_tokens.json

@@ -0,0 +1,12 @@
+{
+  "<|endofpiece|>": 50007,
+  "<|endoftext|>": 50000,
+  "<|startofpiece|>": 50006,
+  "[CLS]": 50002,
+  "[MASK]": 50003,
+  "[SEP]": 50001,
+  "[UNUSED1]": 50004,
+  "[UNUSED2]": 50005,
+  "[gMASK]": 50009,
+  "[sMASK]": 50008
+}

cog-pretrain.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6ea6f4164152bc58d23e24e48f7bf4187aad72a32e97ec4b3acc832fe183cbc2
+size 1021864

special_tokens_map.json

@@ -0,0 +1,13 @@
+{
+  "additional_special_tokens": [
+    "<|startofpiece|>",
+    "<|endofpiece|>",
+    "[gMASK]",
+    "[sMASK]"
+  ],
+  "cls_token": "[CLS]",
+  "eos_token": "<|endoftext|>",
+  "mask_token": "[MASK]",
+  "pad_token": "<|endoftext|>",
+  "unk_token": "[UNK]"
+}

tokenizer_config.json

@@ -0,0 +1,24 @@
+{
+  "add_prefix_space": false,
+  "additional_special_tokens": [
+    "<|startofpiece|>",
+    "<|endofpiece|>",
+    "[gMASK]",
+    "[sMASK]"
+  ],
+  "auto_map": {
+    "AutoTokenizer": [
+      "THUDM/glm-10b-chinese--tokenization_glm.GLMChineseTokenizer",
+      null
+    ]
+  },
+  "clean_up_tokenization_spaces": true,
+  "cls_token": "[CLS]",
+  "eos_token": "<|endoftext|>",
+  "mask_token": "[MASK]",
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<|endoftext|>",
+  "tokenizer_class": "GLMChineseTokenizer",
+  "unk_token": "[UNK]",
+  "use_fast": false
+}