dev(hansbug): add names for questions

llmriddles/questions/executor.py

@@ -15,6 +15,10 @@ class QuestionExecutor:
     def question_text(self):
         return self.question.texts[self.lang]
 
+    @property
+    def question_name(self):
+        return self.question.names[self.lang]
+
     def check(self, qs_text: str) -> Tuple[str, bool, str]:
         answer_text = get_llm_fn(self.llm)(qs_text, **self.llm_cfgs)
         correct, explanation = self.check_answer(qs_text, answer_text)

llmriddles/questions/level1.py

@@ -1,14 +1,18 @@
 from .question import register_question
 
+
 def count_english_words(text: str):
     return len(text.split(' '))
 
+
 def count_chinese_words(text: str):
     return len(text)
 
+
 def check_if_chinese(text: str):
     return all('\u4e00' <= char <= '\u9fff' for char in text)
 
+
 def count_words(text: str, contain_punctuation: bool = False):
     chinese_words = []
     english_words = []
@@ -29,9 +33,9 @@ def count_words(text: str, contain_punctuation: bool = False):
                     temp_english_words = []
                 other_words.append(char)
     if contain_punctuation:
-        return len(chinese_words)+len(english_words)+len(other_words)
+        return len(chinese_words) + len(english_words) + len(other_words)
     else:
-        return len(chinese_words)+len(english_words)
+        return len(chinese_words) + len(english_words)
 
 
 CN_TEXT_1 = """
@@ -57,11 +61,14 @@ def _checker_1(question_text: str, user_text: str, answer_text: str, lang: str):
         return False, None
 
 
-register_question({
-    'cn': CN_TEXT_1,
-    'en': EN_TEXT_1,
-}, _checker_1)
-
+register_question(
+    {
+        'cn': CN_TEXT_1,
+        'en': EN_TEXT_1,
+    },
+    checkers=_checker_1,
+    name={'cn': '1-1 初来乍到', 'en': '1-1'},
+)
 
 CN_TEXT_2 = """
 第一章第二题（小试牛刀），请你输入三个字（及）以内的问题，使模型的回答在30个字以上。
@@ -89,11 +96,14 @@ def _checker_2(question_text: str, user_text: str, answer_text: str, lang: str):
         return True, None
 
 
-register_question({
-    'cn': CN_TEXT_2,
-    'en': EN_TEXT_2,
-}, _checker_2)
-
+register_question(
+    {
+        'cn': CN_TEXT_2,
+        'en': EN_TEXT_2,
+    },
+    checkers=_checker_2,
+    name={'cn': '1-2 小试牛刀', 'en': '1-2'},
+)
 
 CN_TEXT_3 = """
 第一章第三题（短说长话），请你输入一个字的问题，使模型的回答在100个字以上。
@@ -107,6 +117,7 @@ For the third question in chapter 1, please enter a one-word question so that th
 Please enter your query below and click the submit button
 """
 
+
 def _checker_3(question_text: str, user_text: str, answer_text: str, lang: str):
     _ = question_text, lang
     answer_text = answer_text.strip()
@@ -119,11 +130,15 @@ def _checker_3(question_text: str, user_text: str, answer_text: str, lang: str):
     else:
         return True, None
 
-register_question({
-    'cn': CN_TEXT_3,
-    'en': EN_TEXT_3,
-}, _checker_3)
 
+register_question(
+    {
+        'cn': CN_TEXT_3,
+        'en': EN_TEXT_3,
+    },
+    checkers=_checker_3,
+    name={'cn': '1-3 短说长话', 'en': '1-3'}
+)
 
 CN_TEXT_4 = """
 第一章第四题（短说短话），请输入一个字的问题，使模型的回答字数小于20个字。
@@ -137,6 +152,7 @@ For the fourth question in chapter 1, please enter a one-word question so that t
 Please enter your query below and click the submit button
 """
 
+
 def _checker_4(question_text: str, user_text: str, answer_text: str, lang: str):
     _ = question_text, lang
     answer_text = answer_text.strip()
@@ -149,11 +165,15 @@ def _checker_4(question_text: str, user_text: str, answer_text: str, lang: str):
     else:
         return True, None
 
-register_question({
-    'cn': CN_TEXT_4,
-    'en': EN_TEXT_4,
-}, _checker_4)
 
+register_question(
+    {
+        'cn': CN_TEXT_4,
+        'en': EN_TEXT_4,
+    },
+    checkers=_checker_4,
+    name={'cn': '1-4 短说短话', 'en': '1-4'},
+)
 
 # CN_TEXT_5 = """
 # 第一章第五题（回文不变），请输入一个本身不是回文串的问题，使无论正着问还是倒着问，模型的回答是一样的。

llmriddles/questions/level2.py

@@ -54,13 +54,18 @@ def _en_checker_1(question_text: str, user_text: str, answer_text: str) -> Tuple
     return True, None
 
 
-register_question({
-    'cn': CN_TEXT_1,
-    'en': EN_TEXT_1,
-}, checkers={
-    'cn': _cn_checker_1,
-    'en': _en_checker_1,
-})
+register_question(
+    {
+        'cn': CN_TEXT_1,
+        'en': EN_TEXT_1,
+    },
+    checkers={
+        'cn': _cn_checker_1,
+        'en': _en_checker_1,
+    },
+    name={'cn': '2-1 质数长度', 'en': '2-1'},
+    level=2
+)
 
 CN_TEXT_2 = """
 第二章第二题（越说越大），请输入一个大于一的正整数作为问题，使回答里包含至少比它大一千的数。
@@ -102,13 +107,18 @@ def _en_checker_2(question_text: str, user_text: str, answer_text: str) -> Tuple
     return False, f'Did not detect a number of at least {value + 1000} in the output'
 
 
-register_question({
-    'cn': CN_TEXT_2,
-    'en': EN_TEXT_2,
-}, checkers={
-    'cn': _cn_checker_2,
-    'en': _en_checker_2,
-})
+register_question(
+    {
+        'cn': CN_TEXT_2,
+        'en': EN_TEXT_2,
+    },
+    checkers={
+        'cn': _cn_checker_2,
+        'en': _en_checker_2,
+    },
+    name={'cn': '2-2 越说越大', 'en': '2-2'},
+    level=2
+)
 
 CN_TEXT_3 = """
 第二章第三题（越说越小），请输入一个大于一的正整数作为问题，使回答里包含至少10个不同的至少比它小一千的自然数。
@@ -161,13 +171,18 @@ def _en_checker_3(question_text: str, user_text: str, answer_text: str) -> Tuple
                       f'but it doesn\'t reach 10: {collected_values!r}'
 
 
-register_question({
-    'cn': CN_TEXT_3,
-    'en': EN_TEXT_3,
-}, checkers={
-    'cn': _cn_checker_3,
-    'en': _en_checker_3,
-})
+register_question(
+    {
+        'cn': CN_TEXT_3,
+        'en': EN_TEXT_3,
+    },
+    checkers={
+        'cn': _cn_checker_3,
+        'en': _en_checker_3,
+    },
+    name={'cn': '2-3 越说越小', 'en': '2-3'},
+    level=2,
+)
 
 CN_TEXT_4 = """
 第二章第四题（警世名言），请在不提及1,4,5这三个字符的情况下让模型回答114514，不要有多余的字符。
@@ -191,13 +206,18 @@ def _en_checker_4(question_text: str, user_text: str, answer_text: str) -> Tuple
     return answer_text.strip() == '114514', None
 
 
-register_question({
-    'cn': CN_TEXT_4,
-    'en': EN_TEXT_4,
-}, checkers={
-    'cn': _cn_checker_4,
-    'en': _en_checker_4,
-})
+register_question(
+    {
+        'cn': CN_TEXT_4,
+        'en': EN_TEXT_4,
+    },
+    checkers={
+        'cn': _cn_checker_4,
+        'en': _en_checker_4,
+    },
+    name={'cn': '2-4 警世名言', 'en': '2-4'},
+    level=2,
+)
 
 CN_TEXT_5 = """
 第二章第五题（开平方数），请输入一个大于一的平方数作为问题，使回答里包含它的平方根对应的数。
@@ -245,13 +265,18 @@ def _en_checker_5(question_text: str, user_text: str, answer_text: str) -> Tuple
         return False, f'Square root {sq} not detected'
 
 
-register_question({
-    'cn': CN_TEXT_5,
-    'en': EN_TEXT_5,
-}, checkers={
-    'cn': _cn_checker_5,
-    'en': _en_checker_5,
-})
+register_question(
+    {
+        'cn': CN_TEXT_5,
+        'en': EN_TEXT_5,
+    },
+    checkers={
+        'cn': _cn_checker_5,
+        'en': _en_checker_5,
+    },
+    name={'cn': '2-5 开平方数', 'en': '2-5'},
+    level=2,
+)
 
 CN_TEXT_6 = """
 第二章第六题（得寸进狗），请提一个不包含“狗”这个字的10个字以内的问题，使回答中“狗”这个字出现的次数至少是问题字数的2倍。
@@ -290,10 +315,15 @@ def _en_checker_6(question_text: str, user_text: str, answer_text: str) -> Tuple
                       f'which is less than {len(q_words) * 2} times.'
 
 
-register_question({
-    'cn': CN_TEXT_6,
-    'en': EN_TEXT_6,
-}, checkers={
-    'cn': _cn_checker_6,
-    'en': _en_checker_6,
-})
+register_question(
+    {
+        'cn': CN_TEXT_6,
+        'en': EN_TEXT_6,
+    },
+    checkers={
+        'cn': _cn_checker_6,
+        'en': _en_checker_6,
+    },
+    name={'cn': '2-6 得寸进狗', 'en': '2-6'},
+    level=2
+)

llmriddles/questions/level3.py

@@ -1,5 +1,6 @@
 from .question import register_question
 
+
 def count_words(text: str, contain_punctuation: bool = False):
     chinese_words = []
     english_words = []
@@ -20,9 +21,10 @@ def count_words(text: str, contain_punctuation: bool = False):
                     temp_english_words = []
                 other_words.append(char)
     if contain_punctuation:
-        return len(chinese_words)+len(english_words)+len(other_words)
+        return len(chinese_words) + len(english_words) + len(other_words)
     else:
-        return len(chinese_words)+len(english_words)
+        return len(chinese_words) + len(english_words)
+
 
 CN_TEXT_1 = """
 第三章第一题（自说自话），请你输入一个至少10个字的提问，使模型的回答和提问完全一样。
@@ -36,23 +38,30 @@ For the first question in chapter 3, please enter a question of at least ten wor
 Please enter your query below and click the submit button
 """
 
+
 def _checker_1(question_text: str, user_text: str, answer_text: str, lang: str):
     _ = question_text, lang
     answer_text = answer_text.strip()
     user_text = user_text.strip()
 
     if count_words(user_text) < 10:
-        return False, "用户的问题长度应该至少10个字" if lang=='cn' else 'Question should be no less than 10 words.'
+        return False, "用户的问题长度应该至少10个字" if lang == 'cn' else 'Question should be no less than 10 words.'
 
     if answer_text == user_text:
         return True, None
     else:
         return False, None
 
-register_question({
-    'cn': CN_TEXT_1,
-    'en': EN_TEXT_1,
-}, _checker_1, level=3)
+
+register_question(
+    {
+        'cn': CN_TEXT_1,
+        'en': EN_TEXT_1,
+    },
+    checkers=_checker_1,
+    name={'cn': '3-1 自说自话', 'en': '3-1'},
+    level=3,
+)
 
 CN_TEXT_2 = """
 第三章第二题（话自说自），请你输入一个至少10个字的提问，使模型的回答是问题的反序。
@@ -66,15 +75,16 @@ For the second question in chapter 3, please enter a question of at least ten wo
 Please enter your query below and click the submit button
 """
 
+
 def _checker_2(question_text: str, user_text: str, answer_text: str, lang: str):
     _ = question_text, lang
     answer_text = answer_text.strip()
     user_text = user_text.strip()
 
     if count_words(user_text) < 10:
-        return False, "用户的问题长度应该至少10个字" if lang=='cn' else 'Question should be no less than 10 words.'
+        return False, "用户的问题长度应该至少10个字" if lang == 'cn' else 'Question should be no less than 10 words.'
 
-    if lang=='cn':
+    if lang == 'cn':
         user_text = user_text[::-1]
     else:
         words = user_text.split()
@@ -86,10 +96,16 @@ def _checker_2(question_text: str, user_text: str, answer_text: str, lang: str):
     else:
         return False, None
 
-register_question({
-    'cn': CN_TEXT_2,
-    'en': EN_TEXT_2,
-}, _checker_2, level=3)
+
+register_question(
+    {
+        'cn': CN_TEXT_2,
+        'en': EN_TEXT_2,
+    },
+    checkers=_checker_2,
+    name={'cn': '3-2 话自说自', 'en': '3-2'},
+    level=3,
+)
 
 CN_TEXT_3 = """
 第三章第三题（就是敢说），请你输入一个不包含“1”和“3”的提问，使模型的回答是一字不差的“1+1=3”（不需要引号）。
@@ -103,23 +119,30 @@ For the third question in chapter 3, please enter a question not contain "1" or
 Please enter your query below and click the submit button
 """
 
+
 def _checker_3(question_text: str, user_text: str, answer_text: str, lang: str):
     _ = question_text, lang
     answer_text = answer_text.strip()
     user_text = user_text.strip()
 
     if '1' in user_text or '3' in user_text:
-        return False, "用户的问题不能包含“1”或“3”" if lang=='cn' else 'Question should not contain \"1\" or \"3\".'
+        return False, "用户的问题不能包含“1”或“3”" if lang == 'cn' else 'Question should not contain \"1\" or \"3\".'
 
     if answer_text == '1+1=3':
         return True, None
     else:
         return False, None
 
-register_question({
-    'cn': CN_TEXT_3,
-    'en': EN_TEXT_3,
-}, _checker_3, level=3)
+
+register_question(
+    {
+        'cn': CN_TEXT_3,
+        'en': EN_TEXT_3,
+    },
+    checkers=_checker_3,
+    name={'cn': '3-3 就是敢说', 'en': '3-3'},
+    level=3,
+)
 
 # CN_TEXT_4 = """
 # 第三章第四题（回文协变），请你输入一个本身不是回文串的问题，使得正着问和倒着问时，模型的回答本身不是回文且也是逆序。

llmriddles/questions/level4.py

@@ -1,6 +1,7 @@
-from .question import register_question
 import re
 
+from .question import register_question
+
 
 def check_if_is_number(text: str):
     try:
@@ -85,13 +86,19 @@ def _checker_3(question_text: str, user_text: str, answer_text: str, lang: str):
         return False, "问题应该是一个正整数" if lang == 'cn' else 'Question should be a positive integer.'
     elif int(question_text) == 1:
         return False, "问题应该是一个大于1的正整数" if lang == 'cn' else 'Question should be a positive integer greater than 1.'
-    elif int(question_text)-1 not in get_all_numbers_in_a_sentence(answer_text) or int(question_text)+1 not in get_all_numbers_in_a_sentence(answer_text):
+    elif int(question_text) - 1 not in get_all_numbers_in_a_sentence(answer_text) or int(
+            question_text) + 1 not in get_all_numbers_in_a_sentence(answer_text):
         return False, "回答中应该包含一个与问题相差1的数字" if lang == 'cn' else 'Answer should contain a number that is exactly 1 different from the question.'
     else:
         return True, None
 
 
-register_question({
-    'cn': CN_TEXT_3,
-    'en': EN_TEXT_3,
-}, _checker_3, level=4)
+register_question(
+    {
+        'cn': CN_TEXT_3,
+        'en': EN_TEXT_3,
+    },
+    checkers=_checker_3,
+    name={'cn': '4-3 自然之密', 'en': '4-3'},
+    level=4,
+)

llmriddles/questions/level5.py

@@ -1,5 +1,6 @@
 from .question import register_question
 
+
 def count_words(text: str, contain_punctuation: bool = False):
     chinese_words = []
     english_words = []
@@ -20,9 +21,10 @@ def count_words(text: str, contain_punctuation: bool = False):
                     temp_english_words = []
                 other_words.append(char)
     if contain_punctuation:
-        return len(chinese_words)+len(english_words)+len(other_words)
+        return len(chinese_words) + len(english_words) + len(other_words)
     else:
-        return len(chinese_words)+len(english_words)
+        return len(chinese_words) + len(english_words)
+
 
 CN_TEXT_1 = """
 第五章第一题（口是心非）请构造一个不少于十个字的问题，使得回答中不包含问题中的任意字符。
@@ -36,21 +38,27 @@ For the first question in chapter 5, Please construct a question of no less than
 Please enter your query below and click the submit button
 """
 
+
 def _checker_1(question_text: str, user_text: str, answer_text: str, lang: str):
     _ = question_text, lang
     answer_text = answer_text.strip()
     user_text = user_text.strip()
 
     if count_words(user_text) < 10:
-        return False, "用户的问题长度应该至少10个字" if lang=='cn' else 'Question should be no less than 10 words.'
+        return False, "用户的问题长度应该至少10个字" if lang == 'cn' else 'Question should be no less than 10 words.'
 
     if answer_text == user_text:
         return True, None
     else:
         return False, None
 
-register_question({
-    'cn': CN_TEXT_1,
-    'en': EN_TEXT_1,
-}, _checker_1, level=5)
 
+register_question(
+    {
+        'cn': CN_TEXT_1,
+        'en': EN_TEXT_1,
+    },
+    checkers=_checker_1,
+    name={'cn': '5-1 口是心非', 'en': '5-1'},
+    level=5,
+)

llmriddles/questions/question.py

@@ -3,14 +3,15 @@ from dataclasses import dataclass
 from typing import Union, Mapping, Literal, Callable, Tuple, List, Optional
 
 LangTyping = Literal['en', 'cn']
-MultiLangCheckerTyping = Callable[[str, str, str], Tuple[bool, Optional[str]]]
-SingleLangCheckerTyping = Callable[[str, str], Tuple[bool, Optional[str]]]
+MultiLangCheckerTyping = Callable[[str, str, str, str], Tuple[bool, Optional[str]]]
+SingleLangCheckerTyping = Callable[[str, str, str], Tuple[bool, Optional[str]]]
 
 
 @dataclass
 class Question:
     texts: Mapping[str, str]
     checker: MultiLangCheckerTyping
+    names: Mapping[str, str]
     level: int
 
 
@@ -19,6 +20,7 @@ _KNOWN_PROBLEMS = []
 
 def register_question(text: Union[Mapping[str, str], str],
                       checkers: Union[Mapping[str, SingleLangCheckerTyping], MultiLangCheckerTyping],
+                      name=Union[Mapping[str, str], str],
                       level: int = 1, default_lang='cn'):
     if isinstance(checkers, collections.abc.Mapping):
         _origin_checkers = checkers
@@ -35,7 +37,12 @@ def register_question(text: Union[Mapping[str, str], str],
     else:
         texts = text
 
-    _KNOWN_PROBLEMS.append(Question(texts, checker, level))
+    if isinstance(name, str):
+        names = {default_lang: name}
+    else:
+        names = name
+
+    _KNOWN_PROBLEMS.append(Question(texts, checker, names, level))
 
 
 def list_ordered_questions() -> List[Question]: