Merge conflicts

curated.py

@@ -131,18 +131,16 @@ wikipedia_filter = pd.DataFrame(
             "0.00%",
         ],
         "Percent Removed After Local Dedup": [
-            "",
+            "0.31%",
         ],
         "Total Percentage Remaining": [
-            "",
+            "97.84%",
         ],
     }
 )
 
 table_html_wikipedia = wikipedia_filter.to_html(index=False, border=0)
-table_div_wikipedia = Div(
-    NotStr(table_html_wikipedia), style="margin-left: auto; width: 80%; align: center;"
-)
+table_div_wikipedia = Div(NotStr(table_html_wikipedia))
 
 freelaw_filter = pd.DataFrame(
     {
@@ -171,9 +169,7 @@ freelaw_filter = pd.DataFrame(
 )
 
 table_html_freelaw = freelaw_filter.to_html(index=False, border=0)
-table_div_freelaw = Div(
-    NotStr(table_html_freelaw), style="margin-left: auto; width: 80%; align: center;"
-)
+table_div_freelaw = Div(NotStr(table_html_freelaw))
 
 dmm_filter = pd.DataFrame(
     {
@@ -193,18 +189,16 @@ dmm_filter = pd.DataFrame(
             "0.00%",
         ],
         "Percent Removed After Local Dedup": [
-            "",
+            "0.00%",
         ],
         "Total Percentage Remaining": [
-            "%",
+            "100.00%",
         ],
     }
 )
 
 table_html_dmm = dmm_filter.to_html(index=False, border=0)
-table_div_dmm = Div(
-    NotStr(table_html_dmm), style="margin-left: auto; width: 80%; align: center;"
-)
+table_div_dmm = Div(NotStr(table_html_dmm))
 
 
 uspto_filter = pd.DataFrame(
@@ -225,18 +219,16 @@ uspto_filter = pd.DataFrame(
             "0.01%",
         ],
         "Percent Removed After Local Dedup": [
-            "",
+            "22.94%",
         ],
         "Total Percentage Remaining": [
-            "%",
+            "75.60%",
         ],
     }
 )
 
 table_html_uspto = uspto_filter.to_html(index=False, border=0)
-table_div_uspto = Div(
-    NotStr(table_html_uspto), style="margin-left: auto; width: 80%; align: center;"
-)
+table_div_uspto = Div(NotStr(table_html_uspto))
 
 pg19_filter = pd.DataFrame(
     {
@@ -256,18 +248,16 @@ pg19_filter = pd.DataFrame(
             "0.17%",
         ],
         "Percent Removed After Local Dedup": [
-            "",
+            "0.80%",
         ],
         "Total Percentage Remaining": [
-            "%",
+            "98.78%",
         ],
     }
 )
 
 table_html_pg19 = pg19_filter.to_html(index=False, border=0)
-table_div_pg19 = Div(
-    NotStr(table_html_pg19), style="margin-left: auto; width: 80%; align: center;"
-)
+table_div_pg19 = Div(NotStr(table_html_pg19))
 
 
 hn_filter = pd.DataFrame(
@@ -288,18 +278,16 @@ hn_filter = pd.DataFrame(
             "0.34%",
         ],
         "Percent Removed After Local Dedup": [
-            "",
+            "61.84%",
         ],
         "Total Percentage Remaining": [
-            "%",
+            "37.03%",
         ],
     }
 )
 
 table_html_hn = hn_filter.to_html(index=False, border=0)
-table_div_hn = Div(
-    NotStr(table_html_hn), style="margin-left: auto; width: 80%; align: center;"
-)
+table_div_hn = Div(NotStr(table_html_hn))
 
 
 uirc_filter = pd.DataFrame(
@@ -320,18 +308,16 @@ uirc_filter = pd.DataFrame(
             "1.12%",
         ],
         "Percent Removed After Local Dedup": [
-            "",
+            "0.66%",
         ],
         "Total Percentage Remaining": [
-            "%",
+            "60.72%",
         ],
     }
 )
 
 table_html_uirc = uirc_filter.to_html(index=False, border=0)
-table_div_uirc = Div(
-    NotStr(table_html_uirc), style="margin-left: auto; width: 80%; align: center;"
-)
+table_div_uirc = Div(NotStr(table_html_uirc))
 
 up_filter = pd.DataFrame(
     {
@@ -351,18 +337,16 @@ up_filter = pd.DataFrame(
             "0.00%",
         ],
         "Percent Removed After Local Dedup": [
-            "",
+            "1.00%",
         ],
         "Total Percentage Remaining": [
-            "%",
+            "99.00%",
         ],
     }
 )
 
 table_html_up = up_filter.to_html(index=False, border=0)
-table_div_up = Div(
-    NotStr(table_html_up), style="margin-left: auto; width: 80%; align: center;"
-)
+table_div_up = Div(NotStr(table_html_up))
 
 se_filter = pd.DataFrame(
     {
@@ -382,18 +366,16 @@ se_filter = pd.DataFrame(
             "0.00%",
         ],
         "Percent Removed After Local Dedup": [
-            "",
+            "0.00%",
         ],
         "Total Percentage Remaining": [
-            "%",
+            "100.00%",
         ],
     }
 )
 
 table_html_se = se_filter.to_html(index=False, border=0)
-table_div_se = Div(
-    NotStr(table_html_se), style="margin-left: auto; width: 80%; align: center;"
-)
+table_div_se = Div(NotStr(table_html_se))
 
 arx_filter = pd.DataFrame(
     {
@@ -413,18 +395,16 @@ arx_filter = pd.DataFrame(
             "0.07%",
         ],
         "Percent Removed After Local Dedup": [
-            "",
+            "0.00%",
         ],
         "Total Percentage Remaining": [
-            "%",
+            "92.20%",
         ],
     }
 )
 
 table_html_arx = arx_filter.to_html(index=False, border=0)
-table_div_arx = Div(
-    NotStr(table_html_arx), style="margin-left: auto; width: 80%; align: center;"
-)
+table_div_arx = Div(NotStr(table_html_arx))
 
 s2o_filter = pd.DataFrame(
     {
@@ -444,18 +424,16 @@ s2o_filter = pd.DataFrame(
             "0.00%",
         ],
         "Percent Removed After Local Dedup": [
-            "",
+            "0.00%",
         ],
         "Total Percentage Remaining": [
-            "%",
+            "100.00%",
         ],
     }
 )
 
 table_html_s2o = s2o_filter.to_html(index=False, border=0)
-table_div_s2o = Div(
-    NotStr(table_html_s2o), style="margin-left: auto; width: 80%; align: center;"
-)
+table_div_s2o = Div(NotStr(table_html_s2o))
 
 med_filter = pd.DataFrame(
     {
@@ -475,18 +453,16 @@ med_filter = pd.DataFrame(
             "0.02%",
         ],
         "Percent Removed After Local Dedup": [
-            "",
+            "0.00%",
         ],
         "Total Percentage Remaining": [
-            "%",
+            "91.14%",
         ],
     }
 )
 
 table_html_med = med_filter.to_html(index=False, border=0)
-table_div_med = Div(
-    NotStr(table_html_med), style="margin-left: auto; width: 80%; align: center;"
-)
+table_div_med = Div(NotStr(table_html_med))
 
 phil_filter = pd.DataFrame(
     {
@@ -506,18 +482,16 @@ phil_filter = pd.DataFrame(
             "0.12%",
         ],
         "Percent Removed After Local Dedup": [
-            "",
+            "0.00%",
         ],
         "Total Percentage Remaining": [
-            "%",
+            "79.22%",
         ],
     }
 )
 
 table_html_phil = phil_filter.to_html(index=False, border=0)
-table_div_phil = Div(
-    NotStr(table_html_phil), style="margin-left: auto; width: 80%; align: center;"
-)
+table_div_phil = Div(NotStr(table_html_phil))
 ## end individual tables showing filterin
 
 

main.py

@@ -757,7 +757,7 @@ dataset_sources = pd.DataFrame(
             "StackExchange",
         ],
         "Raw Data Size": [
-            "11 TB",
+            "9.2 TB",
             "712 GB",
             "210 GB",
             "23 GB",
@@ -770,7 +770,7 @@ dataset_sources = pd.DataFrame(
             "45 GB",
         ],
         "Token Count": [
-            "5.71T",
+            "4.83T",
             "154.96B",
             "4.75B",
             "7.34B",

web.py

@@ -376,6 +376,7 @@ def web_data():
     return Div(
         Section(
         Div(
+        H1("Web Data Processing"),
         H2("Common Crawl Snapshot Processing"),
         H3("What This Section Contains"),
         P("This section provides a complete discussion on the filtering applied to the 99 Common Crawl snapshots that comprise the web data section of TxT360. The section is split into the following topic areas: "),