fix convert_tokens_to_string

README_en.md

@@ -135,7 +135,6 @@ sampling_params = SamplingParams(temperature=0.95, max_tokens=1024, stop_token_i
 
 inputs = tokenizer.apply_chat_template(prompt, tokenize=False, add_generation_prompt=True)
 outputs = llm.generate(prompts=inputs, sampling_params=sampling_params)
-
 print(outputs[0].outputs[0].text)
 ```
 

tokenization_chatglm.py

@@ -63,22 +63,22 @@ class ChatGLM4Tokenizer(PreTrainedTokenizer):
         vocab.update(self.added_tokens_encoder)
         return vocab
 
-    def convert_tokens_to_string(self, tokens: List[Union[bytes, str]]) -> str:
+    def convert_tokens_to_string(self, tokens: List[Union[bytes, str, int]]) -> str:
         """
         Converts a sequence of tokens in a single string.
         """
         text = ""
         temp = b""
         for t in tokens:
+            if isinstance(t, int):
+                t = chr(t)
             if isinstance(t, str):
                 if temp:
                     text += temp.decode("utf-8", errors="replace")
-                    temp = b""
-                text += t
             elif isinstance(t, bytes):
                 temp += t
             else:
-                raise TypeError("token should only be of type types or str")
+                raise TypeError("token should only be of type int, bytes or str")
         if temp:
             text += temp.decode("utf-8", errors="replace")
         return text