feat: Upgrade model from t5-small to t5-base

Upgraded the model from t5-small to t5-base for improved performance and accuracy. Additionally, increased the maximum generation length to 300 tokens in text generation, enhancing the capacity for more detailed outputs. Removed dataset truncation to utilize the entire dataset, helping in better model training and evaluation.

app.py

@@ -1,13 +1,13 @@
 import gradio as gr
 from transformers import AutoTokenizer, AutoModelForSeq2SeqLM
 
-model_name = "t5-small"
+model_name = "t5-base"
 tokenizer = AutoTokenizer.from_pretrained(model_name)
 model = AutoModelForSeq2SeqLM.from_pretrained(model_name)
 
 def generate_text(input_text):
     input_ids = tokenizer("summarize: " + input_text, return_tensors="pt", max_length=512, truncation=True).input_ids
-    outputs = model.generate(input_ids, max_length=150, num_return_sequences=1, no_repeat_ngram_size=2)
+    outputs = model.generate(input_ids, max_length=300, num_return_sequences=1, no_repeat_ngram_size=2)
     return tokenizer.decode(outputs[0], skip_special_tokens=True)
 
 iface = gr.Interface(

humanizer.py

@@ -13,7 +13,6 @@ logger = logging.getLogger(__name__)
 logger.info("Loading dataset...")
 dataset = load_dataset("LucasChu/reddit_comments")
 dataset = dataset.shuffle(seed=42)
-dataset["train"] = dataset["train"].select(range(10000))
 logger.info("Dataset loaded, shuffled, and truncated to 10,000 samples.")
 
 # Split the train dataset into train and test
@@ -41,7 +40,7 @@ processed_dataset = {split: data.map(prepare_data) for split, data in dataset.it
 logger.info("Dataset prepared.")
 
 # Tokenize the dataset
-model_name = "t5-small"
+model_name = "t5-base"
 tokenizer = AutoTokenizer.from_pretrained(model_name)
 
 def tokenize_function(examples):