Spaces:

LLM360
/

TxT360

Running

File size: 26,006 Bytes

from fasthtml.common import *
from fasthtml.components import *
import json
from fh_plotly import plotly2fasthtml
from plotly import graph_objects as go
import pandas as pd
import plotly.express as px

#Perplexity Across Different Buckets (global)
# The data you provided
DATA = [
    ["2014", [["1-1", "2-5", "6-10", "11-100", "101-1000", "1001-30000000"], [17.410227605477868, 16.11176217183986, 15.632757662414805, 15.446116676532212, 16.716943171826703, 18.156821563322765]]],
    ["2015", [["1-1", "2-5", "6-10", "11-100", "101-1000", "1001-30000000"], [17.446573602753478, 16.14852530113782, 15.627408549576069, 15.0055028132117, 15.565430373421485, 17.314701050452452]]],
    ["2016", [["1-1", "2-5", "6-10", "11-100", "101-1000", "1001-30000000"], [17.307221780905284, 16.297702171159543, 15.948641884223639, 14.799690714225637, 14.935989931859659, 16.09585768919658]]],
    ["2017", [["1-1", "2-5", "6-10", "11-100", "101-1000", "1001-30000000"], [17.338525603992114, 15.960924352297502, 15.912187993988933, 14.822102470001267, 14.778913482337416, 15.428145290012955]]],
    ["2018", [["1-1", "2-5", "6-10", "11-100", "101-1000", "1001-30000000"], [17.08551151136689, 16.187802102106698, 14.935072408852303, 14.832038213200583, 14.508674264491997, 14.800605964649103]]],
    ["2019", [["1-1", "2-5", "6-10", "11-100", "101-1000", "1001-30000000"], [16.818363305107052, 16.474269837858706, 14.944741674400241, 14.568394784374943, 14.690158822673334, 15.990949424635108]]],
    ["2020", [["1-1", "2-5", "6-10", "11-100", "101-1000", "1001-30000000"], [16.98821894111693, 15.936494557783181, 14.79960386342691, 14.435682562274105, 14.58651834886038, 15.869365567783806]]],
    ["2021", [["1-1", "2-5", "6-10", "11-100", "101-1000", "1001-30000000"], [17.125795647512877, 15.780419457145868, 14.631430892394002, 14.276477514399625, 14.337146941773641, 15.872474774329305]]],
    ["2022", [["1-1", "2-5", "6-10", "11-100", "101-1000", "1001-30000000"], [16.573462144306383, 15.283018703313582, 14.378277745163881, 14.0611924390084, 13.9886330091318, 15.769421394877273]]],
    ["2023", [["1-1", "2-5", "6-10", "11-100", "101-1000", "1001-30000000"], [15.4293630385597, 14.608379914730168, 14.118271697056592, 13.880215644749589, 13.767106666731275, 15.05749135510839]]]
]

# Extract ranges (buckets) and years
ranges = DATA[0][1][0]
years = [year_data[0] for year_data in DATA]
all_values = [year_data[1][1] for year_data in DATA]

# Create the figure
fig = go.Figure()

# Add a trace for each year
for i, year in enumerate(years):
    values = all_values[i]
    fig.add_trace(go.Scatter(x=ranges, y=values, mode='lines+markers', name=year))

# Update layout
fig.update_layout(
    title="Perplexity Versus Buckets for Different Years",
    xaxis_title="Buckets",
    yaxis_title="Perplexity",
    legend_title="Years",
    hovermode="x unified"
)

Perplexity_Across_Different_Buckets_global_graph = fig

import plotly.graph_objects as go

# The data you provided
DATA = [
    ["2014", [["1-1", "2-5", "6-10", "11-100", "101-1000", "1001-30000000"], [17.410227605477868, 16.11176217183986, 15.632757662414805, 15.446116676532212, 16.716943171826703, 18.156821563322765]]],
    ["2015", [["1-1", "2-5", "6-10", "11-100", "101-1000", "1001-30000000"], [17.446573602753478, 16.14852530113782, 15.627408549576069, 15.0055028132117, 15.565430373421485, 17.314701050452452]]],
    ["2016", [["1-1", "2-5", "6-10", "11-100", "101-1000", "1001-30000000"], [17.307221780905284, 16.297702171159543, 15.948641884223639, 14.799690714225637, 14.935989931859659, 16.09585768919658]]],
    ["2017", [["1-1", "2-5", "6-10", "11-100", "101-1000", "1001-30000000"], [17.338525603992114, 15.960924352297502, 15.912187993988933, 14.822102470001267, 14.778913482337416, 15.428145290012955]]],
    ["2018", [["1-1", "2-5", "6-10", "11-100", "101-1000", "1001-30000000"], [17.08551151136689, 16.187802102106698, 14.935072408852303, 14.832038213200583, 14.508674264491997, 14.800605964649103]]],
    ["2019", [["1-1", "2-5", "6-10", "11-100", "101-1000", "1001-30000000"], [16.818363305107052, 16.474269837858706, 14.944741674400241, 14.568394784374943, 14.690158822673334, 15.990949424635108]]],
    ["2020", [["1-1", "2-5", "6-10", "11-100", "101-1000", "1001-30000000"], [16.98821894111693, 15.936494557783181, 14.79960386342691, 14.435682562274105, 14.58651834886038, 15.869365567783806]]],
    ["2021", [["1-1", "2-5", "6-10", "11-100", "101-1000", "1001-30000000"], [17.125795647512877, 15.780419457145868, 14.631430892394002, 14.276477514399625, 14.337146941773641, 15.872474774329305]]],
    ["2022", [["1-1", "2-5", "6-10", "11-100", "101-1000", "1001-30000000"], [16.573462144306383, 15.283018703313582, 14.378277745163881, 14.0611924390084, 13.9886330091318, 15.769421394877273]]],
    ["2023", [["1-1", "2-5", "6-10", "11-100", "101-1000", "1001-30000000"], [15.4293630385597, 14.608379914730168, 14.118271697056592, 13.880215644749589, 13.767106666731275, 15.05749135510839]]]
]

# Extract years and ranges (buckets)
years = [year_data[0] for year_data in DATA]
ranges = DATA[0][1][0]
all_values = [year_data[1][1] for year_data in DATA]

# Create the figure
fig = go.Figure()

# Add a trace for each range (bucket)
for i, range_label in enumerate(ranges):
    values = [year_values[i] for year_values in all_values]
    fig.add_trace(go.Scatter(x=years, y=values, mode='lines+markers', name=range_label))

# Update layout
fig.update_layout(
    title="Perplexity over Time by Buckets",
    xaxis_title="Year",
    yaxis_title="Perplexity",
    legend_title="Buckets",
    hovermode="x unified"
)

# Show the plot

Perplexity_Across_Different_years_graph = fig

#graph 3 tbd


##graph 4

# Data
data = {
    "2014": [[1, 2, 3, 4, 5, 6, 7], [16.82572561037536, 15.568688184472002, 15.81492345088457, 15.206355227978303, 14.957766124315622, 15.0626618458479, 14.868694970954875]],
    "2015": [[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14], [16.89591942700286, 16.065453332424525, 16.27127382267357, 15.98729389349345, 15.820859416672139, 15.607600911192797, 15.110138119917048, 14.775887397593593, 14.660655022362318, 14.588547400309, 14.55020529201701, 14.60127048285861, 14.629467286160864, 14.724440862889]],
    "2016": [[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23], [16.937063110995666, 16.159401045667217, 16.24178377816029, 16.073494730595225, 15.774737597940442, 15.621947075605162, 15.652955728810767, 15.490807185265608, 15.107547930881259, 14.976802818244652, 14.84517846815987, 14.839768640933237, 14.794471716989547, 14.63826024444852, 14.57725826735028, 14.446983381957654, 14.357863327746747, 14.370146480850329, 14.207796791759758, 14.067617540013867, 13.968037234662484, 14.015386030957798, 14.240341030736081]],
    "2017": [[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34], [16.731010662045737, 16.534529481773635, 16.624818021435388, 16.6027968727952, 16.309053610063657, 15.897351021398148, 15.28125612085656, 15.247650153402024, 15.254523671401472, 14.914419299947483, 14.734687782223004, 14.78413990847839, 14.632065773173599, 14.553246204352645, 14.566566717229817, 14.598997883895558, 14.630837564582592, 14.577208610859856, 14.519413074334954, 14.378498447103155, 14.230943065441672, 14.21486380216727, 14.267422050116286, 14.281606945252474, 14.285535565035358, 14.267266452303197, 14.183136371592417, 14.148268300539986, 14.21036055571916, 14.403658436211263, 14.07740471177247, 13.978242496152737, 14.002633023111342, 13.921442192327655]],
    "2018": [[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46], [16.36546565307699, 16.40198820766892, 16.223802437472045, 15.876190858008776, 15.67866795074444, 15.06202559354935, 14.90395971116697, 14.678203166785968, 14.584307457124408, 14.566603376547835, 14.716527825876883, 15.325114798824192, 15.153635738494962, 15.02749650458142, 15.001501640976104, 14.928437622685198, 14.82066263298361, 14.821925418558614, 14.766538823073754, 14.742836908980854, 14.56365471694568, 14.676889493065413, 14.554327268897188, 14.532997117657752, 14.424665661602356, 14.498427111442304, 14.503622356394782, 14.465294436509943, 14.238209588803269, 14.138531026657803, 14.11633073136879, 14.139167312709546, 14.018054330979824, 13.99555243574117, 13.944770385859002, 13.893353445566834, 14.062344563726617, 13.93602183204734, 14.021518977400955, 14.205273479299835, 14.476114932821364, 14.707612238290547, 14.403724822460271, 14.134464626849528, 14.444143421676507, 14.372513155481114]], 
    "2019": [[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58], [16.430122616908218, 16.959894024725404, 17.530494343918114, 16.868158683774006, 15.881925908038214, 15.534071764082064, 15.131599277920825, 14.912091189771328, 14.670101527122705, 14.591504400695127, 14.405999306200146, 14.445437111619501, 14.460150974610382, 14.484377024024335, 14.454923933668082, 14.419367200283267, 14.413528998792588, 14.362242243379736, 14.326113076490996, 14.296366861932851, 14.241137636937816, 14.199018730937293, 14.229939247452181, 14.136900232594229, 14.118747592622253, 14.084639441555339, 14.215359965064522, 14.321335291582407, 14.324483466296755, 14.22250351319463, 14.086153725088337, 13.835546707594524, 13.861133395338289, 13.943395906095482, 13.924621051048792, 13.978711084778347, 13.905837106100599, 13.895058375110484, 13.965442215811176, 13.960110799963521, 13.99641930380663, 14.036529641608142, 14.138294825947872, 14.3360579115842, 14.381214824082278, 14.389155929858507, 14.645175007732961, 14.477402464547152, 14.51444505845359, 14.452153618691407, 14.369430039316658, 14.777857326947053, 14.919588044831785, 14.44053007434645, 15.18596756335163, 15.362545102466955, 14.19095014510728, 14.974116009038374]],
    "2020": [[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67], [16.370879996210345, 15.8912555858172, 15.30157381561316, 15.461362783916728, 15.499602726814674, 15.329200065422526, 15.15238010628145, 14.763158730707703, 14.554444724050978, 14.391542531017906, 14.36062417910266, 14.38408773153761, 14.326398992080678, 14.30559398460406, 14.296620966953196, 14.257641523080219, 14.239785079623704, 14.257963344315925, 14.242187947481078, 14.227247960684823, 14.246528806504662, 14.218691858857001, 14.119027315863725, 14.16049126937658, 14.1393355330565, 14.045446781499932, 14.157717833319223, 14.118844952908322, 14.196031529646438, 14.213151244097208, 13.961708881578405, 13.933434907040512, 13.851967361230258, 13.837032186918211, 13.83609308408103, 13.74084041793037, 13.725470779946555, 13.828238575588477, 13.821753201986834, 13.81152543072784, 13.848104780631868, 13.867032605563987, 13.91567734841802, 13.951714207206178, 14.049401288158215, 14.000020142532282, 14.097054408814992, 13.973915015915077, 14.001305792897133, 14.144447551433583, 14.259794771973013, 14.451580305620148, 14.51523695971965, 14.491781843544896, 14.282130970472108, 15.107471010795585, 14.507809743534436, 14.521069179252333, 14.434674191986367, 14.608533166251355, 14.638444470251414, 14.533496524040576, 14.798278071146985, 14.426231926251468, 14.697215649677831, 16.000970311873413, 15.962112261396376]], 
    "2021": [[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76], [16.40998076014569, 16.214064988829055, 16.102423371338, 15.61960664445389, 15.365806380257993, 15.258505702187641, 14.987691202287733, 14.702934989974425, 14.47618699872759, 14.27294780183996, 14.149902488372312, 14.205153888507418, 14.180406523176302, 14.179162306409303, 14.148172641684653, 14.146789562089962, 14.13207546325995, 14.095045629570848, 14.11059784727951, 14.099774964675055, 14.123404475264994, 14.04205645064162, 14.071957340879374, 14.076002230293538, 13.997579723886533, 14.055982816834012, 13.969566421026498, 14.044197137615983, 13.970800321234117, 13.991254850488748, 13.882773884993162, 13.878676595695538, 13.738534890032913, 13.731264526843741, 13.743931319054433, 13.744385448034985, 13.654269807446708, 13.649793785589242, 13.659669286701138, 13.70981039304929, 13.706017327034093, 13.793638036031938, 13.820253019972634, 13.895844222751752, 13.805532005211788, 13.879969410141696, 13.918667434527839, 13.905080860878972, 13.928649229766597, 13.92971106121914, 14.027211058825408, 13.869387194129342, 14.002439547556614, 14.197189244986761, 14.171599593405029, 14.189026952927275, 14.317088413682809, 14.266884102594418, 14.419819049388389, 14.394102801112203, 14.28486572885178, 14.463889605992794, 14.33813105749612, 14.49991185507524, 14.143521529230846, 14.020378845184027, 13.929518285869944, 14.785566241242854, 14.273112202537183, 14.214773231874327, 15.148250280334482, 14.563912283938823, 14.552655401047055, 15.103655560583926, 15.10169368308145, 15.51226670578053]], 
    "2022": [[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82], [15.816438075969113, 16.09580666529207, 15.996333190110686, 15.943848815283268, 15.656093894572393, 15.117394092950262, 14.78287160043243, 14.543141634926663, 14.305058018637094, 14.125611753367533, 14.020067247129745, 14.009478965058692, 14.001669618595649, 13.985248005578073, 13.973728782674742, 13.977117237954532, 13.937202607945661, 13.935625722728988, 13.910945770197134, 13.893835251097759, 13.89679037463924, 13.881064193374637, 13.904299030382344, 13.904090626213797, 13.888526913768459, 13.838568567514612, 13.829178463025315, 13.835043064902585, 13.81757570730985, 13.776370281729609, 13.762638676519028, 13.755074572847866, 13.70045870380896, 13.636805800875507, 13.621671132216942, 13.633589926779942, 13.602080944483301, 13.555848610415586, 13.558904660155596, 13.515784046498322, 13.609062507319349, 13.583137427458137, 13.483909540769114, 13.552547586090368, 13.598431457952788, 13.553603210491028, 13.57678073518515, 13.638974685844074, 13.536340332636364, 13.665029112653803, 13.792584632059397, 13.694618901117751, 13.76826442085512, 13.712141215124436, 13.606927085734855, 13.715251358141709, 13.832372298562165, 13.833057817510603, 14.158282794215733, 13.720855464980609, 13.803417413649743, 13.969854384335488, 13.892171370159991, 13.778269401162506, 14.338216122354686, 13.954993084366778, 14.124621205791534, 14.23737489353975, 13.735373769168353, 14.680642990647039, 14.955805632680471, 14.657580957949671, 14.515390831490835, 14.664661248141265, 14.497750142907295, 14.662641057721332, 14.889760996859867, 14.044358215057608, 14.201835355776206, 15.739648390380614, 16.209835613545337, 16.152241961523004]], 
    "2023": [[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87], [14.789164834058647, 14.816940361326127, 14.859128791010018, 14.874353138456074, 14.799558338831533, 14.521955875294703, 14.344439116861626, 14.188768591243738, 14.068214649484007, 13.975247139727212, 13.892361806680233, 13.873812682757855, 13.855623423924795, 13.834034581531931, 13.820704041290954, 13.809345726828388, 13.799809305012891, 13.783733425983497, 13.777650159371621, 13.769295261894937, 13.759922637457876, 13.763157283741176, 13.746892077566258, 13.76043892787152, 13.780799936451544, 13.769219249583701, 13.776107653632701, 13.791395904208231, 13.796017873163873, 13.793511195440795, 13.752832872061946, 13.744140909741626, 13.708291014183953, 13.71791539682566, 13.687888739966342, 13.639779981608498, 13.60820604942688, 13.647619221630219, 13.586732110007599, 13.578824230397977, 13.56286775189147, 13.54887132723798, 13.528005768240323, 13.562278263274335, 13.50423334091739, 13.534814940452101, 13.500524731734556, 13.50415611423685, 13.539792624454682, 13.524280636462276, 13.508635254011224, 13.559898859515048, 13.52340866000508, 13.580987242413414, 13.55406078682323, 13.51949982056399, 13.554946096777044, 13.582384634669271, 13.596405485832927, 13.56397157046417, 13.50076890156882, 13.5996552161972, 13.640177272640555, 13.488306430506505, 13.561144538501607, 13.538229936728902, 13.523274401746107, 13.549803559768709, 13.473822808409837, 13.604196078834873, 13.495970010370977, 13.511897788189435, 13.480753572940852, 13.790758773339501, 13.507154643917646, 13.451282820788672, 13.690887252346345, 13.752272760476266, 13.844969190718858, 13.547043033265561, 14.011397551896307, 13.686917203544905, 13.87527368747695, 13.86828531076653, 14.200910595590095, 14.228639023300655, 14.830844952562549]], 
}

# Create figure
fig = go.Figure()

# Add a line for each year
for year, values in data.items():
    x_values = values[0]  # Dumps duplication count
    y_values = values[1]  # Average perplexity
    
    fig.add_trace(go.Scatter(
        x=x_values, 
        y=y_values, 
        mode='lines+markers',
        name=year
    ))

# Update layout
fig.update_layout(
    title="Perplexity Across Different Dump Duplication Counts (global)",
    xaxis_title="Number of Dumps Duplication",
    yaxis_title="Average Perplexity",
    legend_title="Year"
)

# Show figure
graph4 = fig

#graph 5

# Data
data = {
    "2014": [["1-1", "2-5", "6-10", "11-100", "101-1000", "1001-30000000"], [14.993513389984585, 14.011578085529056, 13.85873136108517, 14.36377987125267, 15.699475274542996, 17.613998844460845]],
    "2015": [["1-1", "2-5", "6-10", "11-100", "101-1000", "1001-30000000"], [14.471687658163091, 13.778144432139904, 14.021706888859466, 14.434268636964038, 15.738483406741972, 17.840661656870072]],
    "2016": [["1-1", "2-5", "6-10", "11-100", "101-1000", "1001-30000000"], [14.438546004899326, 13.785269267918505, 14.181493168285378, 14.616607805125264, 16.2678738383595, 18.109357061988614]],
    "2017": [["1-1", "2-5", "6-10", "11-100", "101-1000", "1001-30000000"], [14.567324655164636, 13.93914285288248, 14.443292405833132, 14.749691886458645, 17.212768319133406, 18.09921482533539]],
    "2018": [["1-1", "2-5", "6-10", "11-100", "101-1000", "1001-30000000"], [14.79413309579284, 14.047920703385312, 14.732393204862687, 15.245556746489475, 17.523092460922793, 17.979494192145445]],
    "2019": [["1-1", "2-5", "6-10", "11-100", "101-1000", "1001-30000000"], [14.755894620552562, 14.061328527771183, 14.921709511248036, 15.746644701243795, 17.6501867538675, 18.65346532837317]],
    "2020": [["1-1", "2-5", "6-10", "11-100", "101-1000", "1001-30000000"], [14.808975955128705, 14.075864850752456, 14.857958945409392, 15.491042607097183, 17.423084409503307, 17.519155049505677]],
    "2021": [["1-1", "2-5", "6-10", "11-100", "101-1000", "1001-30000000"], [14.97373049489925, 14.044277002343486, 14.943719820749207, 15.730257843247038, 17.289265921502192, 17.00252056705296]],
    "2022": [["1-1", "2-5", "6-10", "11-100", "101-1000", "1001-30000000"], [14.865869368871065, 14.028544413850119, 14.920117888974543, 15.693608290158098, 17.33525765189499, 17.724085896005107]],
    "2023": [["1-1", "2-5", "6-10", "11-100", "101-1000", "1001-30000000"], [14.499660832882716, 13.98118612654411, 14.86417535071986, 15.440738449977292, 16.94447963146979, 16.40768745134217]]
}

# Create figure
fig = go.Figure()

# Add a line for each year
for year, values in data.items():
    x_values = values[0]  # Buckets (duplicate count range)
    y_values = values[1]  # Average perplexity
    
    fig.add_trace(go.Scatter(
        x=x_values, 
        y=y_values, 
        mode='lines+markers',
        name=year
    ))

# Update layout
fig.update_layout(
    title="Perplexity Across Different Buckets (local)",
    xaxis_title="Bucket (Duplicate Count Range)",
    yaxis_title="Average Perplexity",
    legend_title="Year"
)

# Show figure
graph5 = fig


intro_div = Div(
        H2("Perplexity Evaluation on Duplicate Data"),
        H3("Model based Quality Estimation"),
        P("We took one of the model-based data quality evaluation strategies adopted by [DataComp-LM](https://arxiv.org/abs/2406.11794), which used perplexity filtering as a candidate for quality filtering. DataComp-LM followed [CCNet’s](https://arxiv.org/abs/1911.00359) practice to use a 5-gram Kneser-Ney model as implemented in the [KenLM](https://github.com/kpu/kenlm) library for efficient perplexity calculation. Following this practice, we estimated data quality by taking a KenLM model (from [edugp/kenlm](https://huggingface.co/edugp/kenlm)) trained on English Wikipedia data to compute perplexity on data with different duplication patterns. Lower perplexity is regarded as a signal of higher quality."),
        H3("Sampling Strategy"),
        P("We started from a processed Common Crawl (CC) ablation dataset divided by the number of duplicates of each document. For each CC dump, we have different buckets each holding chunks of document with different duplicate count ranges (1-1, 2-5, 6-10, 11-100, 101-1000, 1001-30000000). We sampled the first 10k documents from each chunk with their meta data."),

)

perp1_div = Div(
        Section(
            H3("Perplexity vs Buckets"),
            P("For each bucket, we aggregated all the chunks that belong to a single year and calculated the average perplexity for each (bucket, year) data point."),
            Img(src="images/prep-diff-buckets-global.png", height = "300", width = "600" ),
            plotly2fasthtml(Perplexity_Across_Different_Buckets_global_graph),
        ),
        Section(
            H3("Perplexity vs Years"),
            P("Taking the same data, we can convert it into a graph indicating the yearly trend. For most buckets, the average perplexity of dumps from more recent years seem to be lower than that of former years."),
            Img(src="images/prep-across-diff-year-global-dup-buckets.png", height = "300", width = "600" ),
            plotly2fasthtml(Perplexity_Across_Different_years_graph),
        ),
    Section(
            H3("Perplexity vs Document Duplication"),
            P("We can also break each bucket into distinct document counts. The graph becomes a bit noisy at the end because of insufficient samples with larger duplication counts."),
            Img(src="images/prep-across-diff-docs-dup-count-global.png", height = "300", width = "600" ),
           # plotly2fasthtml(different_document_dup_counts_global),
        ),
    Section(
            H3("Perplexity vs Dump Duplication"),
            P("We are also interested in how the number of dumps a document is in affect data quality. From the graph below we can see that documents that are duplicated across around 40 - 60 dumps usually have lower perplexity."),
            Img(src="images/prep-across-diff-dump-dup-counts-global.png", height = "300", width = "600" ),
            plotly2fasthtml(graph4),
        ),
    Section(
            H3("Perplexity vs Local Buckets"),
            P("Previously we have seen that documents in recent dumps tend to have lower perplexity. This might be related to the way how global deduplication was implemented. During global deduplication, we only keep copy in the latest dump. Hence documents that are duplicated across multiple dumps only appear in the latest one. To avoid bias brought by this strategy, we tried to recover the states before the global deduplication by reading the metadata attached with each document."),
            Img(src="images/prep-across-diff-buckets-local.png", height = "300", width = "600" ),
            plotly2fasthtml(graph5),
        ),
    Section(
            H3("Perplexity vs Local Dump Duplication"),
            P("Following the same practice, we can plot the local version of the graph of average perplexity with respect to dump duplication."),
            Img(src="images/prep-diff-dump-dump-counts-local.png", height = "300", width = "600" ),
        ),
)


llama_div = Div(
        Section(
            H2("Llama 3.1 8B"),
            P("For comparison purpose, we run the same perplexity evaluation with llama 3.1 8B model.")
        ),
        Section(
            H3("Perplexity vs Buckets"),
            Img(src="images/perp-across-diff-buckets-global.png", height = "300", width = "600" ),
        ),
        Section(
            H3("Perplexity vs Years"),
            Img(src="images/prep-across-diff-years-global.png", height = "300", width = "600" ),
        ),
    Section(
            H3("Perplexity vs Dump Duplication"),
            Img(src="images/prep-vs-dump-dup-global.png", height = "300", width = "600" ),
        ),
    Section(
            H3("Perplexity vs Local Buckets"),
            Img(src="images/prep-diff-buckets-local.png", height = "300", width = "600" ),
        ),
    Section(
            H3("Perplexity vs Local Dump Duplication"),
            Img(src="images/prep-vs-dump-dup-global.png", height = "300", width = "600" ),
        ),
)


def results():
    return Div(
                Section(
                    intro_div,
                    perp1_div,
                    llama_div,
                    P("test plotly"),
                    id="inner-text"
                )
    )