Commit
•
2ee1fd2
1
Parent(s):
62f8d3d
first commit
Browse filesThis view is limited to 50 files because it contains too many changes.
See raw diff
- .gitattributes +1 -0
- .gitignore +1 -0
- app.py +77 -0
- plots/ar_character_repetition_ratio.png +3 -0
- plots/ar_closed_class_word_ratio.png +3 -0
- plots/ar_flagged_word_ratio.png +3 -0
- plots/ar_number_of_words.png +3 -0
- plots/ar_perplexity_score.png +3 -0
- plots/ar_special_character_ratio.png +3 -0
- plots/ar_word_repetition_ratio.png +3 -0
- plots/bn_character_repetition_ratio.png +3 -0
- plots/bn_closed_class_word_ratio.png +3 -0
- plots/bn_flagged_word_ratio.png +3 -0
- plots/bn_number_of_words.png +3 -0
- plots/bn_perplexity_score.png +3 -0
- plots/bn_special_character_ratio.png +3 -0
- plots/bn_word_repetition_ratio.png +3 -0
- plots/ca_character_repetition_ratio.png +3 -0
- plots/ca_closed_class_word_ratio.png +3 -0
- plots/ca_flagged_word_ratio.png +3 -0
- plots/ca_number_of_words.png +3 -0
- plots/ca_perplexity_score.png +3 -0
- plots/ca_special_character_ratio.png +3 -0
- plots/ca_word_repetition_ratio.png +3 -0
- plots/en_character_repetition_ratio.png +3 -0
- plots/en_closed_class_word_ratio.png +3 -0
- plots/en_flagged_word_ratio.png +3 -0
- plots/en_number_of_words.png +3 -0
- plots/en_perplexity_score.png +3 -0
- plots/en_special_character_ratio.png +3 -0
- plots/en_word_repetition_ratio.png +3 -0
- plots/es_character_repetition_ratio.png +3 -0
- plots/es_closed_class_word_ratio.png +3 -0
- plots/es_flagged_word_ratio.png +3 -0
- plots/es_number_of_words.png +3 -0
- plots/es_perplexity_score.png +3 -0
- plots/es_special_character_ratio.png +3 -0
- plots/es_word_repetition_ratio.png +3 -0
- plots/eu_character_repetition_ratio.png +3 -0
- plots/eu_closed_class_word_ratio.png +3 -0
- plots/eu_flagged_word_ratio.png +3 -0
- plots/eu_number_of_words.png +3 -0
- plots/eu_perplexity_score.png +3 -0
- plots/eu_special_character_ratio.png +3 -0
- plots/eu_word_repetition_ratio.png +3 -0
- plots/fr_character_repetition_ratio.png +3 -0
- plots/fr_closed_class_word_ratio.png +3 -0
- plots/fr_flagged_word_ratio.png +3 -0
- plots/fr_number_of_words.png +3 -0
- plots/fr_perplexity_score.png +3 -0
.gitattributes
CHANGED
@@ -25,3 +25,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
|
25 |
*.zip filter=lfs diff=lfs merge=lfs -text
|
26 |
*.zstandard filter=lfs diff=lfs merge=lfs -text
|
27 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
|
|
|
25 |
*.zip filter=lfs diff=lfs merge=lfs -text
|
26 |
*.zstandard filter=lfs diff=lfs merge=lfs -text
|
27 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
28 |
+
*.png filter=lfs diff=lfs merge=lfs -text
|
.gitignore
ADDED
@@ -0,0 +1 @@
|
|
|
|
|
1 |
+
*DS_Store
|
app.py
ADDED
@@ -0,0 +1,77 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
import streamlit as st
|
2 |
+
|
3 |
+
|
4 |
+
PATH_PLOTS = "./plots"
|
5 |
+
|
6 |
+
LANGUAGES = {
|
7 |
+
"Arabic": "ar",
|
8 |
+
"Basque": "eu",
|
9 |
+
"Bengali": "bn",
|
10 |
+
"Catalan": "ca",
|
11 |
+
"Chinese": "zh",
|
12 |
+
"English": "en",
|
13 |
+
"French": "fr",
|
14 |
+
"Hindi": "hi",
|
15 |
+
"Indonesian": "id",
|
16 |
+
"Portuguese": "pt",
|
17 |
+
"Spanish": "es",
|
18 |
+
"Urdu": "ur",
|
19 |
+
"Vietnamese": "vi",
|
20 |
+
}
|
21 |
+
|
22 |
+
FILTERS = [
|
23 |
+
"number of words",
|
24 |
+
"character repetition ratio",
|
25 |
+
"word repetition ratio",
|
26 |
+
"special character ratio",
|
27 |
+
"closed class word ratio",
|
28 |
+
"flagged word ratio",
|
29 |
+
"perplexity score",
|
30 |
+
]
|
31 |
+
|
32 |
+
|
33 |
+
class Visualization:
|
34 |
+
def __init__(self):
|
35 |
+
pass
|
36 |
+
|
37 |
+
def set_title(self):
|
38 |
+
st.title("Visualization of the distributions of the filter values for the BigScience Corpus")
|
39 |
+
|
40 |
+
def choose_language(self):
|
41 |
+
chosen_language = st.sidebar.selectbox(
|
42 |
+
"Language",
|
43 |
+
options=list(LANGUAGES.keys()),
|
44 |
+
index=5 # English
|
45 |
+
)
|
46 |
+
self.chosen_language = LANGUAGES[chosen_language]
|
47 |
+
|
48 |
+
def choose_filter(self):
|
49 |
+
chosen_filter = st.sidebar.selectbox(
|
50 |
+
"Filter on the",
|
51 |
+
options=FILTERS,
|
52 |
+
index=0
|
53 |
+
)
|
54 |
+
self.chosen_filter = chosen_filter.replace(" ", "_")
|
55 |
+
|
56 |
+
def display_plot(self):
|
57 |
+
path_image = f"{PATH_PLOTS}/{self.chosen_language}_{self.chosen_filter}.png"
|
58 |
+
|
59 |
+
col1, col2, col3 = st.columns([1,6,1])
|
60 |
+
with col1:
|
61 |
+
st.write("")
|
62 |
+
with col2:
|
63 |
+
st.image(path_image)
|
64 |
+
with col3:
|
65 |
+
st.write("")
|
66 |
+
|
67 |
+
def visualization(self):
|
68 |
+
self.set_title()
|
69 |
+
self.choose_language()
|
70 |
+
self.choose_filter()
|
71 |
+
self.display_plot()
|
72 |
+
|
73 |
+
|
74 |
+
if __name__ == "__main__":
|
75 |
+
st.set_page_config(layout="wide")
|
76 |
+
visualization = Visualization()
|
77 |
+
visualization.visualization()
|
plots/ar_character_repetition_ratio.png
ADDED
![]() |
Git LFS Details
|
plots/ar_closed_class_word_ratio.png
ADDED
![]() |
Git LFS Details
|
plots/ar_flagged_word_ratio.png
ADDED
![]() |
Git LFS Details
|
plots/ar_number_of_words.png
ADDED
![]() |
Git LFS Details
|
plots/ar_perplexity_score.png
ADDED
![]() |
Git LFS Details
|
plots/ar_special_character_ratio.png
ADDED
![]() |
Git LFS Details
|
plots/ar_word_repetition_ratio.png
ADDED
![]() |
Git LFS Details
|
plots/bn_character_repetition_ratio.png
ADDED
![]() |
Git LFS Details
|
plots/bn_closed_class_word_ratio.png
ADDED
![]() |
Git LFS Details
|
plots/bn_flagged_word_ratio.png
ADDED
![]() |
Git LFS Details
|
plots/bn_number_of_words.png
ADDED
![]() |
Git LFS Details
|
plots/bn_perplexity_score.png
ADDED
![]() |
Git LFS Details
|
plots/bn_special_character_ratio.png
ADDED
![]() |
Git LFS Details
|
plots/bn_word_repetition_ratio.png
ADDED
![]() |
Git LFS Details
|
plots/ca_character_repetition_ratio.png
ADDED
![]() |
Git LFS Details
|
plots/ca_closed_class_word_ratio.png
ADDED
![]() |
Git LFS Details
|
plots/ca_flagged_word_ratio.png
ADDED
![]() |
Git LFS Details
|
plots/ca_number_of_words.png
ADDED
![]() |
Git LFS Details
|
plots/ca_perplexity_score.png
ADDED
![]() |
Git LFS Details
|
plots/ca_special_character_ratio.png
ADDED
![]() |
Git LFS Details
|
plots/ca_word_repetition_ratio.png
ADDED
![]() |
Git LFS Details
|
plots/en_character_repetition_ratio.png
ADDED
![]() |
Git LFS Details
|
plots/en_closed_class_word_ratio.png
ADDED
![]() |
Git LFS Details
|
plots/en_flagged_word_ratio.png
ADDED
![]() |
Git LFS Details
|
plots/en_number_of_words.png
ADDED
![]() |
Git LFS Details
|
plots/en_perplexity_score.png
ADDED
![]() |
Git LFS Details
|
plots/en_special_character_ratio.png
ADDED
![]() |
Git LFS Details
|
plots/en_word_repetition_ratio.png
ADDED
![]() |
Git LFS Details
|
plots/es_character_repetition_ratio.png
ADDED
![]() |
Git LFS Details
|
plots/es_closed_class_word_ratio.png
ADDED
![]() |
Git LFS Details
|
plots/es_flagged_word_ratio.png
ADDED
![]() |
Git LFS Details
|
plots/es_number_of_words.png
ADDED
![]() |
Git LFS Details
|
plots/es_perplexity_score.png
ADDED
![]() |
Git LFS Details
|
plots/es_special_character_ratio.png
ADDED
![]() |
Git LFS Details
|
plots/es_word_repetition_ratio.png
ADDED
![]() |
Git LFS Details
|
plots/eu_character_repetition_ratio.png
ADDED
![]() |
Git LFS Details
|
plots/eu_closed_class_word_ratio.png
ADDED
![]() |
Git LFS Details
|
plots/eu_flagged_word_ratio.png
ADDED
![]() |
Git LFS Details
|
plots/eu_number_of_words.png
ADDED
![]() |
Git LFS Details
|
plots/eu_perplexity_score.png
ADDED
![]() |
Git LFS Details
|
plots/eu_special_character_ratio.png
ADDED
![]() |
Git LFS Details
|
plots/eu_word_repetition_ratio.png
ADDED
![]() |
Git LFS Details
|
plots/fr_character_repetition_ratio.png
ADDED
![]() |
Git LFS Details
|
plots/fr_closed_class_word_ratio.png
ADDED
![]() |
Git LFS Details
|
plots/fr_flagged_word_ratio.png
ADDED
![]() |
Git LFS Details
|
plots/fr_number_of_words.png
ADDED
![]() |
Git LFS Details
|
plots/fr_perplexity_score.png
ADDED
![]() |
Git LFS Details
|