Spaces:
Running
Running
raushanrajcareer
commited on
Commit
•
07a4da5
1
Parent(s):
bc467c2
Update app.py
Browse files
app.py
CHANGED
@@ -18,6 +18,8 @@ dataset = Dataset.from_list(data)
|
|
18 |
|
19 |
# Load the tokenizer
|
20 |
tokenizer = AutoTokenizer.from_pretrained("microsoft/DialoGPT-medium")
|
|
|
|
|
21 |
|
22 |
def preprocess_function(examples):
|
23 |
inputs = [f"Question: {q}" for q in examples["question"]]
|
@@ -36,6 +38,8 @@ tokenized_dataset = dataset.map(preprocess_function, batched=True)
|
|
36 |
|
37 |
|
38 |
model = AutoModelForCausalLM.from_pretrained("microsoft/DialoGPT-medium")
|
|
|
|
|
39 |
|
40 |
# Define training arguments
|
41 |
training_args = TrainingArguments(
|
|
|
18 |
|
19 |
# Load the tokenizer
|
20 |
tokenizer = AutoTokenizer.from_pretrained("microsoft/DialoGPT-medium")
|
21 |
+
if tokenizer.pad_token is None:
|
22 |
+
tokenizer.pad_token = tokenizer.eos_token
|
23 |
|
24 |
def preprocess_function(examples):
|
25 |
inputs = [f"Question: {q}" for q in examples["question"]]
|
|
|
38 |
|
39 |
|
40 |
model = AutoModelForCausalLM.from_pretrained("microsoft/DialoGPT-medium")
|
41 |
+
model.resize_token_embeddings(len(tokenizer))
|
42 |
+
|
43 |
|
44 |
# Define training arguments
|
45 |
training_args = TrainingArguments(
|