Non-web revision

curated.py

@@ -33,7 +33,9 @@ curated_sources_intro = Div(
     P(
         "Curated sources comprise high-quality datasets that contain domain-specificity.",
         B(
-            " TxT360 was strongly influenced by The Pile",  D_cite(bibtex_key="thepile"), " regarding both inclusion of the dataset and filtering techniques."
+            " TxT360 was strongly influenced by The Pile",
+            D_cite(bibtex_key="thepile"),
+            " regarding both inclusion of the dataset and filtering techniques.",
         ),
         " These sources, such as Arxiv, Wikipedia, and Stack Exchange, provide valuable data that is excluded from the web dataset mentioned above. Analyzing and processing non-web data can yield insights and opportunities for various applications. Details about each of the sources are provided below. ",
     ),
@@ -129,16 +131,16 @@ wikipedia_filter = pd.DataFrame(
             "0.00%",
         ],
         "Percent Removed After Local Dedup": [
-            "",
+            "0.31%",
         ],
         "Total Percentage Remaining": [
-            "",
+            "97.84%",
         ],
     }
 )
 
 table_html_wikipedia = wikipedia_filter.to_html(index=False, border=0)
-table_div_wikipedia = Div(NotStr(table_html_wikipedia), style="margin-left: auto; width: 80%; align: center;")
+table_div_wikipedia = Div(NotStr(table_html_wikipedia))
 
 freelaw_filter = pd.DataFrame(
     {
@@ -167,7 +169,7 @@ freelaw_filter = pd.DataFrame(
 )
 
 table_html_freelaw = freelaw_filter.to_html(index=False, border=0)
-table_div_freelaw = Div(NotStr(table_html_freelaw), style="margin-left: auto; width: 80%; align: center;")
+table_div_freelaw = Div(NotStr(table_html_freelaw))
 
 dmm_filter = pd.DataFrame(
     {
@@ -187,16 +189,16 @@ dmm_filter = pd.DataFrame(
             "0.00%",
         ],
         "Percent Removed After Local Dedup": [
-            "",
+            "0.00%",
         ],
         "Total Percentage Remaining": [
-            "%",
+            "100.00%",
         ],
     }
 )
 
 table_html_dmm = dmm_filter.to_html(index=False, border=0)
-table_div_dmm = Div(NotStr(table_html_dmm), style="margin-left: auto; width: 80%; align: center;")
+table_div_dmm = Div(NotStr(table_html_dmm))
 
 
 uspto_filter = pd.DataFrame(
@@ -217,16 +219,16 @@ uspto_filter = pd.DataFrame(
             "0.01%",
         ],
         "Percent Removed After Local Dedup": [
-            "",
+            "22.94%",
         ],
         "Total Percentage Remaining": [
-            "%",
+            "75.60%",
         ],
     }
 )
 
 table_html_uspto = uspto_filter.to_html(index=False, border=0)
-table_div_uspto = Div(NotStr(table_html_uspto), style="margin-left: auto; width: 80%; align: center;")
+table_div_uspto = Div(NotStr(table_html_uspto))
 
 pg19_filter = pd.DataFrame(
     {
@@ -246,16 +248,16 @@ pg19_filter = pd.DataFrame(
             "0.17%",
         ],
         "Percent Removed After Local Dedup": [
-            "",
+            "0.80%",
         ],
         "Total Percentage Remaining": [
-            "%",
+            "98.78%",
         ],
     }
 )
 
 table_html_pg19 = pg19_filter.to_html(index=False, border=0)
-table_div_pg19 = Div(NotStr(table_html_pg19), style="margin-left: auto; width: 80%; align: center;")
+table_div_pg19 = Div(NotStr(table_html_pg19))
 
 
 hn_filter = pd.DataFrame(
@@ -276,16 +278,16 @@ hn_filter = pd.DataFrame(
             "0.34%",
         ],
         "Percent Removed After Local Dedup": [
-            "",
+            "61.84%",
         ],
         "Total Percentage Remaining": [
-            "%",
+            "37.03%",
         ],
     }
 )
 
 table_html_hn = hn_filter.to_html(index=False, border=0)
-table_div_hn = Div(NotStr(table_html_hn), style="margin-left: auto; width: 80%; align: center;")
+table_div_hn = Div(NotStr(table_html_hn))
 
 
 uirc_filter = pd.DataFrame(
@@ -306,16 +308,16 @@ uirc_filter = pd.DataFrame(
             "1.12%",
         ],
         "Percent Removed After Local Dedup": [
-            "",
+            "0.66%",
         ],
         "Total Percentage Remaining": [
-            "%",
+            "60.72%",
         ],
     }
 )
 
 table_html_uirc = uirc_filter.to_html(index=False, border=0)
-table_div_uirc = Div(NotStr(table_html_uirc), style="margin-left: auto; width: 80%; align: center;")
+table_div_uirc = Div(NotStr(table_html_uirc))
 
 up_filter = pd.DataFrame(
     {
@@ -335,16 +337,16 @@ up_filter = pd.DataFrame(
             "0.00%",
         ],
         "Percent Removed After Local Dedup": [
-            "",
+            "1.00%",
         ],
         "Total Percentage Remaining": [
-            "%",
+            "99.00%",
         ],
     }
 )
 
 table_html_up = up_filter.to_html(index=False, border=0)
-table_div_up = Div(NotStr(table_html_up), style="margin-left: auto; width: 80%; align: center;")
+table_div_up = Div(NotStr(table_html_up))
 
 se_filter = pd.DataFrame(
     {
@@ -364,16 +366,16 @@ se_filter = pd.DataFrame(
             "0.00%",
         ],
         "Percent Removed After Local Dedup": [
-            "",
+            "0.00%",
         ],
         "Total Percentage Remaining": [
-            "%",
+            "100.00%",
         ],
     }
 )
 
 table_html_se = se_filter.to_html(index=False, border=0)
-table_div_se = Div(NotStr(table_html_se), style="margin-left: auto; width: 80%; align: center;")
+table_div_se = Div(NotStr(table_html_se))
 
 arx_filter = pd.DataFrame(
     {
@@ -393,16 +395,16 @@ arx_filter = pd.DataFrame(
             "0.07%",
         ],
         "Percent Removed After Local Dedup": [
-            "",
+            "0.00%",
         ],
         "Total Percentage Remaining": [
-            "%",
+            "92.20%",
         ],
     }
 )
 
 table_html_arx = arx_filter.to_html(index=False, border=0)
-table_div_arx = Div(NotStr(table_html_arx), style="margin-left: auto; width: 80%; align: center;")
+table_div_arx = Div(NotStr(table_html_arx))
 
 s2o_filter = pd.DataFrame(
     {
@@ -422,16 +424,16 @@ s2o_filter = pd.DataFrame(
             "0.00%",
         ],
         "Percent Removed After Local Dedup": [
-            "",
+            "0.00%",
         ],
         "Total Percentage Remaining": [
-            "%",
+            "100.00%",
         ],
     }
 )
 
 table_html_s2o = s2o_filter.to_html(index=False, border=0)
-table_div_s2o = Div(NotStr(table_html_s2o), style="margin-left: auto; width: 80%; align: center;")
+table_div_s2o = Div(NotStr(table_html_s2o))
 
 med_filter = pd.DataFrame(
     {
@@ -451,16 +453,16 @@ med_filter = pd.DataFrame(
             "0.02%",
         ],
         "Percent Removed After Local Dedup": [
-            "",
+            "0.00%",
         ],
         "Total Percentage Remaining": [
-            "%",
+            "91.14%",
         ],
     }
 )
 
 table_html_med = med_filter.to_html(index=False, border=0)
-table_div_med = Div(NotStr(table_html_med), style="margin-left: auto; width: 80%; align: center;")
+table_div_med = Div(NotStr(table_html_med))
 
 phil_filter = pd.DataFrame(
     {
@@ -480,16 +482,16 @@ phil_filter = pd.DataFrame(
             "0.12%",
         ],
         "Percent Removed After Local Dedup": [
-            "",
+            "0.00%",
         ],
         "Total Percentage Remaining": [
-            "%",
+            "79.22%",
         ],
     }
 )
 
 table_html_phil = phil_filter.to_html(index=False, border=0)
-table_div_phil = Div(NotStr(table_html_phil), style="margin-left: auto; width: 80%; align: center;")
+table_div_phil = Div(NotStr(table_html_phil))
 ## end individual tables showing filterin
 
 
@@ -681,24 +683,51 @@ filtering_process = Div(
             P(
                 B("Download and Extraction: "),
                 "All the data was downloaded in original latex format from ArXiv official S3 repo: ",
-                A("s3://arxic/src", href="s3://arxic/src"),
-                ". We try to encode the downloaded data into utf-8 or guess encoding using chardet library. After that pandoc was used to extract information from the latex files and saved as markdown format",
+                A("s3://arxiv/src", href="s3://arxiv/src"),
+                ". We aim to encode the downloaded data in UTF-8 format, and when necessary, utilize the chardet library to infer the appropriate encoding. After that, we use ",
+                A("Pandoc", href="https://pandoc.org/"),
+                " to extract information from the latex files into markdown format. The command we use is",
                 D_code(
-                    "pandoc -s {tex} -o out/{out_name}.md --wrap=none",
-                    language="python",
+                    "pandoc <raw_tex_path> -s -o <output_markdown_path> -f latex+raw_tex -t markdown_mmd [--lua-filter <lua_filter_path>]",
+                    language="bash",
                 ),
-                ". All markdowns were combined to create jsonl files.",
+                ". Finally, all markdowns were combined to create jsonl files.",
             ),
             P(B("Unique Data Preparation Challenges: ")),
+            P(
+                "When converting LaTeX files into Markdown using Pandoc, it is crucial to account for different data formats to minimize information loss while also filtering out noisy content in LaTeX. Below, we outline our considerations and methods for handling various data types during this conversion process:"
+            ),
             Ul(
                 Li(
-                    "Due to large amounts of meaningful data being contained in table formats, special consideration was taken to extract the data and proper metadata.",
+                    B("Tables: "),
+                    "The process for handling tables follows three main approaches. First, tables compatible with Pandoc’s built-in formats are directly converted into standard Markdown tables. Notably, LaTeX’s '\\multicolumn' and '\\multirow' commands can be successfully translated into valid Markdown tables. Second, tables unsupported by Pandoc’s native functionality, such as deluxetable or other complex LaTeX types, are preserved in their original LaTeX format to maintain the integrity of complex structures. Third, only a few remaining tables have been converted to HTML web tables.",
+                    style="margin-bottom: -3px",
+                ),
+                Li(
+                    B("Mathematical Expressions: "),
+                    "Inline mathematical expressions are rendered in Markdown. More complex equations remain unchanged, e.g., presented as '\\begin{aligned}' blocks, to ensure accuracy and readability.",
+                    style="margin-bottom: -3px",
+                ),
+                Li(
+                    B("Figures: "),
+                    "All figures are removed during the conversion process. Placeholder figures might not contribute to the paper’s data quality and, as such, have been omitted to streamline the output.",
+                    style="margin-bottom: -3px",
+                ),
+                Li(
+                    B("Section Headers: "),
+                    "Section headers are converted into markdown format, using leading '#' symbols to represent the heading levels.",
+                    style="margin-bottom: -3px",
+                ),
+                Li(
+                    B("References: "),
+                    "References are removed. Although they may be informative, references often introduce formatting inconsistencies or add little value compared to the core content of the paper.",
                     style="margin-bottom: -3px",
                 ),
             ),
             P(
                 B(" Filters Applied: "),
-                "multiple filters are used here after manually verifying output of all the filters as suggested by peS2o dataset",  D_cite(bibtex_key="peS2o"),
+                "multiple filters are used here after manually verifying output of all the filters as suggested by peS2o dataset",
+                D_cite(bibtex_key="peS2o"),
             ),
             Ul(
                 Li(
@@ -851,13 +880,16 @@ filtering_process = Div(
                     href="ttps://ftp.ncbi.nlm.nih.gov/pub/pmc/oa_package/",
                 ),
                 ". PubMed Central (PMC) files are downloaded in an xml.tar format. The tar files are opened and converted to markdown format using pandoc",
-                D_code("pandoc -f jats {nxml} -o {pmcid}.md", language="bash"),
-                ". The markdown files are combined to create jsonl files. PubMed Abstract (PMA) files  were downloaded in xml. The BeautifulSoup library was used to extract the abstract, title, and PMID. All files were stored in jsonl format.",
+                D_code(
+                    "pandoc <raw_xml_path> -s -o <output_markdown_path> -f jats -t markdown_mmd [--lua-filter <lua_filter_path>]",
+                    language="bash",
+                ),
+                ". The markdown files are combined to create jsonl files. PubMed Abstract (PMA) files were downloaded in xml. The BeautifulSoup library was used to extract the abstract, title, and PMID. All files were stored in jsonl format.",
             ),
             P(B("Unique Data Preparation Challenges: ")),
             Ul(
                 Li(
-                    "Due to large amounts of meaningful data being contained in table formats, speical consideration was taken to extract the data and proper metadata.",
+                    "We tried similar attempts on PMC as we did on ArXiv. The resulted markdown might have slight difference due to the different structure of the XML files.",
                     style="margin-bottom: -3px",
                 ),
             ),
@@ -1584,7 +1616,8 @@ def curated():
     table_html = data_preparation_steps.to_html(index=False, border=0)
     table_div = Div(NotStr(table_html), style="margin: 40px;")
 
-    text = P("""This initial stage serves as the foundation for the entire
+    text = P(
+        """This initial stage serves as the foundation for the entire
     process. Here, we focus on acquiring and extracting the raw data, which can
     come from various sources such as crawling websites, using HTTP/FTP dumps,
     or working with archive dumps.  For instance, to download and prepare a
@@ -1594,7 +1627,8 @@ def curated():
     preparation process: It is worth noting that some pipelines might require
     invoking additional functions or scripts to handle specific data sources or
     formats.  These helper scripts can be located within specific directories
-    or modules dedicated to the dataset.""")
+    or modules dedicated to the dataset."""
+    )
 
     return Div(
         Section(

main.py

@@ -175,7 +175,7 @@ def main():
                     Div(
                         A(
                             "TxT360",
-                            href="#section1",
+                            href="#section11",
                         )
                     ),
                     Div(
@@ -359,8 +359,8 @@ def main():
                 ),
             ),
             intro(),
-            curated.curated(),
             web.web_data(),
+            curated.curated(),
             common.common_steps(),
             results.results(),
         ),
@@ -757,7 +757,7 @@ dataset_sources = pd.DataFrame(
             "StackExchange",
         ],
         "Raw Data Size": [
-            "11 TB",
+            "9.2 TB",
             "712 GB",
             "210 GB",
             "23 GB",
@@ -770,7 +770,7 @@ dataset_sources = pd.DataFrame(
             "45 GB",
         ],
         "Token Count": [
-            "5.71T",
+            "4.83T",
             "154.96B",
             "4.75B",
             "7.34B",

web.py

@@ -376,6 +376,7 @@ def web_data():
     return Div(
         Section(
         Div(
+        H1("Web Data Processing"),
         H2("Common Crawl Snapshot Processing"),
         H3("What This Section Contains"),
         P("This section provides a complete discussion on the filtering applied to the 99 Common Crawl snapshots that comprise the web data section of TxT360. The section is split into the following topic areas: "),