feature(nyz): add level1 questions

app.py

@@ -12,7 +12,15 @@ _LANG = os.environ.get('QUESTION_LANG', 'cn')
 _LLM = os.environ.get('QUESTION_LLM', 'chatgpt')
 
 if _LANG == "cn":
-    requirement_ph = "点击\"下一题\"开始游戏"
+    requirement_ph = """
+    欢迎来玩LLM Riddles!
+
+    你将通过本游戏对语言大模型产生更深刻的理解。
+
+    在本游戏中，你需要构造一个提给一个语言大模型的问题，使得它回复的答案符合要求。
+
+    点击\"下一题\"开始游戏
+    """
     requirement_label = "游戏须知"
     question_ph = "你对大语言模型的提问"
     question_label = "提问栏"
@@ -30,7 +38,15 @@ if _LANG == "cn":
     api_error_info = "请在提交问题之前先输入你的 API Key"
     try_again_label = "再玩一次"
 elif _LANG == "en":
-    requirement_ph = 'Click \'Next\' to Start'
+    requirement_ph = """
+    Welcome to LLM Riddles!
+
+    In this game, you'll gain a deeper understanding of language models.
+
+    Your challenge is to create a question to ask a language model in a way that the answer it provides meets specific criteria.
+
+    Click \'Next\' to Start
+    """
     requirement_label = "Requirements"
     question_ph = "Your Question for LLM"
     question_label = "Question"
@@ -69,7 +85,7 @@ if __name__ == '__main__':
                 gr_requirement = gr.TextArea(placeholder=requirement_ph, label=requirement_label)
                 gr_question = gr.TextArea(placeholder=question_ph, label=question_label)
                 gr_answer = gr.TextArea(placeholder=answer_ph, label=answer_label)
-                gr_submit = gr.Button(submit_label, interactive=False)
+                gr_submit = gr.Button(submit_label, interactive=True)
 
             with gr.Column():
                 gr_api_key = gr.Text(placeholder=api_ph, label=api_label, type='password',
@@ -91,7 +107,7 @@ if __name__ == '__main__':
             if _qid >= len(_QUESTIONS):
                 del _QUESTION_IDS[uuid_]
                 return game_cleared_label, '', '', {}, '', \
-                    gr.Button(submit_label, interactive=False), \
+                    gr.Button(submit_label, interactive=True), \
                     gr.Button(try_again_label, interactive=True), \
                     ''
             else:

llmriddles/questions/executor.py

@@ -17,11 +17,11 @@ class QuestionExecutor:
 
     def check(self, qs_text: str) -> Tuple[str, bool, str]:
         answer_text = get_llm_fn(self.llm)(qs_text, **self.llm_cfgs)
-        correct, explanation = self.check_answer(answer_text)
+        correct, explanation = self.check_answer(qs_text, answer_text)
         return answer_text, correct, explanation
 
-    def check_answer(self, answer_text: str) -> Tuple[bool, str]:
-        correct, explanation = self.question.checker(self.question_text, answer_text, self.lang)
+    def check_answer(self, user_text: str, answer_text: str) -> Tuple[bool, str]:
+        correct, explanation = self.question.checker(self.question_text, user_text, answer_text, self.lang)
         if explanation is None:
             if correct:
                 explanation = 'LLM的回答满足要求' if self.lang == 'cn' else 'Correct Answer From LLM'

llmriddles/questions/level1.py

@@ -1,40 +1,184 @@
-from .question import register_question
-
-CN_TEXT = """
-欢迎来玩LLM Riddles!
-
-你将通过本游戏对语言大模型产生更深刻的理解。
-
-在本游戏中，你需要构造一个提给一个语言大模型的问题，使得它回复的答案符合要求。
-
-作为第一个题目，请你构造一个问题使模型的回答是一字不差的“1+1=3”（不需要引号）。
-
-请在下面的输入框内填写你构造并点击按钮提交。
-"""
-
-EN_TEXT = """
-Welcome to LLM Riddles!
-
-In this game, you'll gain a deeper understanding of language models.
-
-Your challenge is to create a question to ask a language model in a way that the answer it provides meets specific criteria.
-
-For the first question, please construct a query for the model that will result in the exact answer "1+1=3" (no quotes needed).
-
-Please enter your query below and click the submit button
-"""
-
-
-def _checker(question_text: str, answer_text: str, lang: str):
-    _ = question_text, lang
-    answer_text = answer_text.strip()
-    if answer_text == '1+1=3':
-        return True, None
-    else:
-        return False, None
-
-
-register_question({
-    'cn': CN_TEXT,
-    'en': EN_TEXT,
-}, _checker)
+from .question import register_question
+
+def count_english_words(text: str):
+    return len(text.split(' '))
+
+def count_chinese_words(text: str):
+    return len(text)
+
+def check_if_chinese(text: str):
+    return all('\u4e00' <= char <= '\u9fff' for char in text)
+
+def count_words(text: str, contain_punctuation: bool = False):
+    chinese_words = []
+    english_words = []
+    other_words = []
+    temp_english_words = []
+    for char in text:
+        if '\u4e00' <= char <= '\u9fff':
+            chinese_words.append(char)
+            if len(temp_english_words) > 0:
+                english_words.append(''.join(temp_english_words))
+                temp_english_words = []
+        else:
+            if char.isalpha():
+                temp_english_words.append(char)
+            else:
+                if len(temp_english_words) > 0:
+                    english_words.append(''.join(temp_english_words))
+                    temp_english_words = []
+                other_words.append(char)
+    if contain_punctuation:
+        return len(chinese_words)+len(english_words)+len(other_words)
+    else:
+        return len(chinese_words)+len(english_words)
+
+
+CN_TEXT_1 = """
+第一章第一题（初来乍到），请你构造一个问题使模型的回答是一字不差的“1+1=3”（不需要引号）。
+
+请在下面的输入框内填写你构造并点击按钮提交。
+"""
+
+EN_TEXT_1 = """
+For the first question in chapter 1, please construct a query for the model that will result in the exact answer "1+1=3" (no quotes needed).
+
+Please enter your query below and click the submit button
+"""
+
+
+def _checker_1(question_text: str, user_text: str, answer_text: str, lang: str):
+    _ = question_text, lang
+    answer_text = answer_text.strip()
+    user_text = user_text.strip()
+    if answer_text == '1+1=3':
+        return True, None
+    else:
+        return False, None
+
+
+register_question({
+    'cn': CN_TEXT_1,
+    'en': EN_TEXT_1,
+}, _checker_1)
+
+
+CN_TEXT_2 = """
+第一章第二题（小试牛刀），请你输入三个字（及）以内的问题，使模型的回答在30个字以上。
+
+请在下面的输入框内填写你的问题并点击按钮提交。
+"""
+
+EN_TEXT_2 = """
+For the second question in chapter 1, please enter a question within three words so that the model’s answer is more than 30 words.
+
+Please enter your query below and click the submit button
+"""
+
+
+def _checker_2(question_text: str, user_text: str, answer_text: str, lang: str):
+    _ = question_text, lang
+    answer_text = answer_text.strip()
+    user_text = user_text.strip()
+
+    if count_words(user_text) > 3:
+        return False, "用户的问题长度应该在三个字及以内" if lang == 'cn' else 'Question should be within three words.'
+    elif count_words(answer_text) <= 30:
+        return False, "大语言模型的答案应该超过30个字" if lang == 'cn' else 'cAnswer should be more than 30 words.'
+    else:
+        return True, None
+
+
+register_question({
+    'cn': CN_TEXT_2,
+    'en': EN_TEXT_2,
+}, _checker_2)
+
+
+CN_TEXT_3 = """
+第一章第三题（短说长话），请你输入一个字的问题，使模型的回答在100个字以上。
+
+请在下面的输入框内填写你的问题并点击按钮提交。
+"""
+
+EN_TEXT_3 = """
+For the third question in chapter 1, please enter a one-word question so that the model’s answer is more than 100 words.
+
+Please enter your query below and click the submit button
+"""
+
+def _checker_3(question_text: str, user_text: str, answer_text: str, lang: str):
+    _ = question_text, lang
+    answer_text = answer_text.strip()
+    user_text = user_text.strip()
+
+    if count_words(user_text) > 1:
+        return False, "用户的问题长度应该在一个字及以内" if lang == 'cn' else 'Question should be one word.'
+    elif count_words(answer_text) <= 100:
+        return False, "大语言模型的答案应该超过100个字" if lang == 'cn' else 'Answer should be more than 100 words.'
+    else:
+        return True, None
+
+register_question({
+    'cn': CN_TEXT_3,
+    'en': EN_TEXT_3,
+}, _checker_3)
+
+
+CN_TEXT_4 = """
+第一章第四题（短说短话），请输入一个字的问题，使模型的回答字数小于20个字。
+
+请在下面的输入框内填写你的问题并点击按钮提交。
+"""
+
+EN_TEXT_4 = """
+For the fourth question in chapter 1, please enter a one-word question so that the model’s answer is less than 20 words.
+
+Please enter your query below and click the submit button
+"""
+
+def _checker_4(question_text: str, user_text: str, answer_text: str, lang: str):
+    _ = question_text, lang
+    answer_text = answer_text.strip()
+    user_text = user_text.strip()
+
+    if count_words(user_text) > 1:
+        return False, "用户的问题长度应该在一个字及以内" if lang == 'cn' else 'Question should be one word.'
+    elif count_words(answer_text) >= 20:
+        return False, "大语言模型的答案应该小于100个字" if lang == 'cn' else 'Answer should be less than 20 words.'
+    else:
+        return True, None
+
+register_question({
+    'cn': CN_TEXT_4,
+    'en': EN_TEXT_4,
+}, _checker_4)
+
+
+# CN_TEXT_5 = """
+# 第一章第五题（回文不变），请输入一个本身不是回文串的问题，使无论正着问还是倒着问，模型的回答是一样的。
+
+# 请在下面的输入框内填写你的问题并点击按钮提交。
+# """
+
+# EN_TEXT_5 = """
+# For the fourth question in chapter 1, please enter a question that is not a palindrome string so that the model's answer is the same whether it is asked forward or backward.
+
+# Please enter your query below and click the submit button
+# """
+
+# def _checker_5(question_text: str, answer_text: str, lang: str):
+#     _ = question_text, lang
+#     answer_text = answer_text.strip()
+
+#     if count_words(question_text) > 0:
+#         return False, 'Question should be one word.'
+#     elif count_words(answer_text) >= 20:
+#         return False, 'Answer should be less than 20 words.'
+#     else:
+#         return True, None
+
+# register_question({
+#     'cn': CN_TEXT_5,
+#     'en': EN_TEXT_5,
+# }, _checker_5)