Update zeroshot.py

zeroshot.py

@@ -93,16 +93,14 @@ def filter_lexicon(lexicon, word_counts):
 
 def load_words(filepath):
     words = {}
-    num_sentences = 0
     with open(filepath) as f:
-        for line in f:
-            line = line.strip().lower()
-            num_sentences += 1
-            line = text_normalize(line, iso_code="xxx")
-            # ignore invalid words.
-            for w in line.split():
-                words.setdefault(w, 0)
-                words[w] += 1
+        lines = f.readlines()
+    num_sentences = len(lines)
+    all_sentences = " ".join([l.strip() for l in lines])
+    norm_all_sentences = text_normalize(all_sentences)
+    for w in norm_all_sentences.split():
+        words.setdefault(w, 0)
+        words[w] += 1
     return words, num_sentences