Spaces:
Runtime error
Runtime error
File size: 6,322 Bytes
a671d2f 88e5a9f 4a941bb a671d2f 7c18751 4a941bb c9e7b3d 7c18751 4a941bb 7c18751 c9e7b3d b52c6e2 c9e7b3d b52c6e2 c9e7b3d 88e5a9f 4a941bb 88e5a9f 4a941bb a671d2f |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 |
import tweepy as tw
import streamlit as st
import pandas as pd
import torch
import numpy as np
import re
from torch.utils.data import TensorDataset, DataLoader, RandomSampler, SequentialSampler
from transformers import AutoTokenizer, AutoModelForSequenceClassification,AdamW
tokenizer = AutoTokenizer.from_pretrained('hackathon-pln-es/twitter_sexismo-finetuned-exist2021-metwo')
model = AutoModelForSequenceClassification.from_pretrained("hackathon-pln-es/twitter_sexismo-finetuned-exist2021-metwo")
import torch
if torch.cuda.is_available():
device = torch.device("cuda")
print('I will use the GPU:', torch.cuda.get_device_name(0))
else:
print('No GPU available, using the CPU instead.')
device = torch.device("cpu")
consumer_key = st.secrets["consumer_key"]
consumer_secret = st.secrets["consumer_secret"]
access_token = st.secrets["access_token"]
access_token_secret = st.secrets["access_token_secret"]
auth = tw.OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_token, access_token_secret)
api = tw.API(auth, wait_on_rate_limit=True)
def preprocess(text):
text=text.lower()
# remove hyperlinks
text = re.sub(r'https?:\/\/.*[\r\n]*', '', text)
text = re.sub(r'http?:\/\/.*[\r\n]*', '', text)
#Replace &, <, > with &,<,> respectively
text=text.replace(r'&?',r'and')
text=text.replace(r'<',r'<')
text=text.replace(r'>',r'>')
#remove hashtag sign
#text=re.sub(r"#","",text)
#remove mentions
text = re.sub(r"(?:\@)\w+", '', text)
#text=re.sub(r"@","",text)
#remove non ascii chars
text=text.encode("ascii",errors="ignore").decode()
#remove some puncts (except . ! ?)
text=re.sub(r'[:"#$%&\*+,-/:;<=>@\\^_`{|}~]+','',text)
text=re.sub(r'[!]+','!',text)
text=re.sub(r'[?]+','?',text)
text=re.sub(r'[.]+','.',text)
text=re.sub(r"'","",text)
text=re.sub(r"\(","",text)
text=re.sub(r"\)","",text)
text=" ".join(text.split())
return text
st.title('Analisis de comentarios sexistas en Twitter con Tweepy and HuggingFace Transformers')
st.markdown('Esta app utiliza tweepy para descargar tweets de twitter en base a la información de entrada y procesa los tweets usando transformers de HuggingFace para detectar comentarios sexistas. El resultado y los tweets correspondientes se almacenan en un dataframe para mostrarlo que es lo que se ve como resultado')
def run():
with st.form(key='Introduzca nombre'):
search_words = st.text_input('Introduzca el termino para analizar')
search_words1 = st.text_input('Introduzca el usuario para analizar')
number_of_tweets = st.number_input('Introduzca número de twweets a analizar. Máximo 50', 0,50,10)
submit_button = st.form_submit_button(label='Submit')
if submit_button:
if (search_words != ' '):
new_search = search_words + " -filter:retweets"
tweets =tw.Cursor(api.search_tweets,q=new_search,lang="es",since=date_since).items(number_of_tweets)
if (search_words1 != ' '):
tweets = api.user_timeline(screen_name = search_words1,count=50)
date_since = "2020-09-14"
#new_search = search_words + " -filter:retweets"
#tweets = tweepy.Cursor(api.search,q=new_search,lang="es",since=date_since).items(number_of_tweets)
#tweets =tw.Cursor(api.search_tweets,q=search_words).items(number_of_tweets)
#tweets =tw.Cursor(api.search_tweets,q=new_search,lang="es",since=date_since).items(number_of_tweets)
tweet_list = [i.text for i in tweets]
#tweet_list = [strip_undesired_chars(i.text) for i in tweets]
text= pd.DataFrame(tweet_list)
text[0] = text[0].apply(preprocess)
text1=text[0].values
indices1=tokenizer.batch_encode_plus(text1.tolist(),
max_length=128,
add_special_tokens=True,
return_attention_mask=True,
pad_to_max_length=True,
truncation=True)
input_ids1=indices1["input_ids"]
attention_masks1=indices1["attention_mask"]
prediction_inputs1= torch.tensor(input_ids1)
prediction_masks1 = torch.tensor(attention_masks1)
# Set the batch size.
batch_size = 25
# Create the DataLoader.
prediction_data1 = TensorDataset(prediction_inputs1, prediction_masks1)
prediction_sampler1 = SequentialSampler(prediction_data1)
prediction_dataloader1 = DataLoader(prediction_data1, sampler=prediction_sampler1, batch_size=batch_size)
print('Predicting labels for {:,} test sentences...'.format(len(prediction_inputs1)))
# Put model in evaluation mode
model.eval()
# Tracking variables
predictions = []
# Predict
for batch in prediction_dataloader1:
batch = tuple(t.to(device) for t in batch)
# Unpack the inputs from our dataloader
b_input_ids1, b_input_mask1 = batch
# Telling the model not to compute or store gradients, saving memory and # speeding up prediction
with torch.no_grad():
# Forward pass, calculate logit predictions
outputs1 = model(b_input_ids1, token_type_ids=None,attention_mask=b_input_mask1)
logits1 = outputs1[0]
# Move logits and labels to CPU
logits1 = logits1.detach().cpu().numpy()
# Store predictions and true labels
predictions.append(logits1)
flat_predictions = [item for sublist in predictions for item in sublist]
flat_predictions = np.argmax(flat_predictions, axis=1).flatten()#p = [i for i in classifier(tweet_list)]
df = pd.DataFrame(list(zip(tweet_list, flat_predictions)),columns =['Latest'+str(number_of_tweets)+'Tweets'+' on '+search_words, 'Sexista'])
df['Sexista']= np.where(df['Sexista']== 0, 'No Sexista', 'Sexista')
st.table(df)
#st.write(df)
run() |