raushanrajcareer commited on
Commit
07a4da5
1 Parent(s): bc467c2

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +4 -0
app.py CHANGED
@@ -18,6 +18,8 @@ dataset = Dataset.from_list(data)
18
 
19
  # Load the tokenizer
20
  tokenizer = AutoTokenizer.from_pretrained("microsoft/DialoGPT-medium")
 
 
21
 
22
  def preprocess_function(examples):
23
  inputs = [f"Question: {q}" for q in examples["question"]]
@@ -36,6 +38,8 @@ tokenized_dataset = dataset.map(preprocess_function, batched=True)
36
 
37
 
38
  model = AutoModelForCausalLM.from_pretrained("microsoft/DialoGPT-medium")
 
 
39
 
40
  # Define training arguments
41
  training_args = TrainingArguments(
 
18
 
19
  # Load the tokenizer
20
  tokenizer = AutoTokenizer.from_pretrained("microsoft/DialoGPT-medium")
21
+ if tokenizer.pad_token is None:
22
+ tokenizer.pad_token = tokenizer.eos_token
23
 
24
  def preprocess_function(examples):
25
  inputs = [f"Question: {q}" for q in examples["question"]]
 
38
 
39
 
40
  model = AutoModelForCausalLM.from_pretrained("microsoft/DialoGPT-medium")
41
+ model.resize_token_embeddings(len(tokenizer))
42
+
43
 
44
  # Define training arguments
45
  training_args = TrainingArguments(