Update modules/chatbot.py
Browse files- modules/chatbot.py +3 -0
modules/chatbot.py
CHANGED
|
@@ -5,6 +5,9 @@ from torch.utils.data import DataLoader, Dataset
|
|
| 5 |
import json
|
| 6 |
import tqdm
|
| 7 |
|
|
|
|
|
|
|
|
|
|
| 8 |
class MultilingualChatData(Dataset):
|
| 9 |
def __init__(self, file_path, tokenizer, max_length=512):
|
| 10 |
with open(file_path, 'r', encoding='utf-8') as f:
|
|
|
|
| 5 |
import json
|
| 6 |
import tqdm
|
| 7 |
|
| 8 |
+
tokenizer = GPT2Tokenizer.from_pretrained("openai-community/gpt2")
|
| 9 |
+
model = GPT2LMHeadModel.from_pretrained("openai-community/gpt2")
|
| 10 |
+
|
| 11 |
class MultilingualChatData(Dataset):
|
| 12 |
def __init__(self, file_path, tokenizer, max_length=512):
|
| 13 |
with open(file_path, 'r', encoding='utf-8') as f:
|