commit files to HF hub

config.json

@@ -18,7 +18,7 @@
     "neutral": 1
   },
   "max_length": 128,
-  "model_type": "fasttext_jp",
+  "model_type": "fasttext_classification",
   "ngram": 2,
   "tokenizerI_class": "FastTextJpTokenizer",
   "tokenizer_class": "FastTextJpTokenizer",

fasttext_fsc.py

@@ -1,6 +1,4 @@
 from __future__ import annotations
-from transformers import PretrainedConfig
-from torch import nn
 import torch
 from torchtyping import TensorType
 from .fasttext_jp_embedding import FastTextJpModel, FastTextJpConfig
@@ -10,7 +8,7 @@ from transformers.modeling_outputs import SequenceClassifierOutput
 class FastTextForSeuqenceClassificationConfig(FastTextJpConfig):
     """FastTextJpModelのConfig
     """
-    model_type = "fasttext_jp"
+    model_type = "fasttext_classification"
 
     def __init__(self,
                  ngram: int = 2,

mecab_tokenizer.py

@@ -14,9 +14,9 @@ class MeCabResult(NamedTuple):
     hinshi_saibunrui_3: str
     katsuyokei_1: str
     katsuyokei_2: str
-    genkei: str
-    yomi: str
-    hatsuon: str
+    genkei: str = ""
+    yomi: str = ""
+    hatsuon: str = ""
 
 
 class MeCabTokenizer(PreTrainedTokenizer):
@@ -34,9 +34,9 @@ class MeCabTokenizer(PreTrainedTokenizer):
 
         self.target_hinshi = hinshi
         if mecab_dicdir is not None:
-            self.mecab = MeCab.Tagger(f"-d {mecab_dicdir}")
+            self.mecab = MeCab.Tagger(f"-d {mecab_dicdir} -O '' -F '%m,%H\n'")
         else:
-            self.mecab = MeCab.Tagger()
+            self.mecab = MeCab.Tagger("-O '' -F '%m,%H\n'")
 
         super().__init__(**kwargs)
 
@@ -76,17 +76,30 @@ class MeCabTokenizer(PreTrainedTokenizer):
         Returns:
             list[MeCabResult]: MeCabの解析結果
         """
-        node = self.mecab.parseToNode(text)
+        nodes = self.mecab.parse(text).split("\n")
         #形態素1つ1つを処理
         out = []
-        while node:
-            args = []
-            args.append(node.surface)
-            feature = node.feature.split(",")
-            args.extend(feature)
-            mecab_result = MeCabResult(args[0], args[1], args[2], args[3],
-                                       args[4], args[5], args[6], args[7],
-                                       args[8], args[9])
+        for node in nodes:
+            args = node.split(",")
+            if args[0] in ["EOS", ""]:
+                continue
+            # 辞書によって異なる
+            if len(args) == 10:
+                mecab_result = MeCabResult(args[0], args[1], args[2], args[3],
+                                           args[4], args[5], args[6], args[7],
+                                           args[8], args[9])
+            elif len(args) == 7:
+                # 英語
+                mecab_result = MeCabResult(args[0], args[1], args[2], args[3],
+                                           args[4], args[5], args[6], "*", "*",
+                                           "*")
+            elif len(args) == 27:
+                # 補助記号
+                mecab_result = MeCabResult(args[0], args[1], args[2], args[3],
+                                           args[4], args[5], args[6], "*", "*",
+                                           "*")
+            else:
+                raise NotImplementedError(
+                    f"unsupposed parse args_length: {len(args)}, args: {args}")
             out.append(mecab_result)
-            node = node.next  # 最後のEOSを省く
         return out