Update app.py

app.py

@@ -27,7 +27,13 @@ auth = tw.OAuthHandler(consumer_key, consumer_secret)
 auth.set_access_token(access_token, access_token_secret)
 api = tw.API(auth, wait_on_rate_limit=True)
 
-
+def strip_undesired_chars(tweet):
+    stripped_tweet = tweet.replace('\n', ' ').replace('\r', '')
+    char_list = [stripped_tweet[j] for j in range(len(stripped_tweet)) if ord(stripped_tweet[j]) in range(65536)]
+    stripped_tweet=''
+    for j in char_list:
+        stripped_tweet=stripped_tweet+j
+    return stripped_tweet
 
 
 st.title('Analisis de comentarios sexistas en Twitter con Tweepy and HuggingFace Transformers')
@@ -44,7 +50,8 @@ def run():
             #tweets = tweepy.Cursor(api.search,q=new_search,lang="es",since=date_since).items(number_of_tweets)
             #tweets =tw.Cursor(api.search_tweets,q=search_words).items(number_of_tweets)
             tweets =tw.Cursor(api.search_tweets,q=new_search,lang="es",since=date_since).items(number_of_tweets)
-            tweet_list = [i.text for i in tweets]
+            #tweet_list = [i.text for i in tweets]
+            tweet_list = [strip_undesired_chars(i.text) for i in tweets]
             text= pd.DataFrame(tweet_list)
             text1=text[0].values
             indices1=tokenizer.batch_encode_plus(text1.tolist(),