{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.9995419147961521,
  "eval_steps": 100,
  "global_step": 1091,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.0009161704076958314,
      "grad_norm": 0.8362945914268494,
      "learning_rate": 9.09090909090909e-09,
      "logits/chosen": -0.33099836111068726,
      "logits/rejected": -0.6684360504150391,
      "logps/chosen": -201.79928588867188,
      "logps/rejected": -183.10525512695312,
      "loss": 0.6931,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "epoch": 0.009161704076958314,
      "grad_norm": 0.7959509491920471,
      "learning_rate": 9.09090909090909e-08,
      "logits/chosen": -0.8238513469696045,
      "logits/rejected": -0.6410912275314331,
      "logps/chosen": -101.17930603027344,
      "logps/rejected": -201.2146759033203,
      "loss": 0.693,
      "rewards/accuracies": 0.5833333134651184,
      "rewards/chosen": 0.0013685817830264568,
      "rewards/margins": 0.003857892006635666,
      "rewards/rejected": -0.002489310223609209,
      "step": 10
    },
    {
      "epoch": 0.01832340815391663,
      "grad_norm": 0.791206419467926,
      "learning_rate": 1.818181818181818e-07,
      "logits/chosen": -0.696135401725769,
      "logits/rejected": -0.6421085000038147,
      "logps/chosen": -103.54312896728516,
      "logps/rejected": -195.77432250976562,
      "loss": 0.6932,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.0055698854848742485,
      "rewards/margins": -0.009762462228536606,
      "rewards/rejected": 0.004192576743662357,
      "step": 20
    },
    {
      "epoch": 0.027485112230874943,
      "grad_norm": 0.8699631690979004,
      "learning_rate": 2.727272727272727e-07,
      "logits/chosen": -0.7419141530990601,
      "logits/rejected": -0.6500910520553589,
      "logps/chosen": -137.67446899414062,
      "logps/rejected": -262.87994384765625,
      "loss": 0.6912,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": 0.00041480286745354533,
      "rewards/margins": 0.004314146004617214,
      "rewards/rejected": -0.0038993440102785826,
      "step": 30
    },
    {
      "epoch": 0.03664681630783326,
      "grad_norm": 0.8166934847831726,
      "learning_rate": 3.636363636363636e-07,
      "logits/chosen": -0.7547220587730408,
      "logits/rejected": -0.5034279823303223,
      "logps/chosen": -94.3946762084961,
      "logps/rejected": -208.92074584960938,
      "loss": 0.6937,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.002515423344448209,
      "rewards/margins": -0.0004801487666554749,
      "rewards/rejected": -0.002035274403169751,
      "step": 40
    },
    {
      "epoch": 0.04580852038479157,
      "grad_norm": 0.8770909309387207,
      "learning_rate": 4.545454545454545e-07,
      "logits/chosen": -0.7312489151954651,
      "logits/rejected": -0.521498441696167,
      "logps/chosen": -132.73512268066406,
      "logps/rejected": -239.8402862548828,
      "loss": 0.6924,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -7.614419155288488e-05,
      "rewards/margins": 0.0043979184702038765,
      "rewards/rejected": -0.004474063403904438,
      "step": 50
    },
    {
      "epoch": 0.054970224461749886,
      "grad_norm": 0.8896933794021606,
      "learning_rate": 5.454545454545454e-07,
      "logits/chosen": -0.7862143516540527,
      "logits/rejected": -0.5695743560791016,
      "logps/chosen": -166.35006713867188,
      "logps/rejected": -254.9455108642578,
      "loss": 0.6945,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.003317029681056738,
      "rewards/margins": 0.011281955055892467,
      "rewards/rejected": -0.007964925840497017,
      "step": 60
    },
    {
      "epoch": 0.0641319285387082,
      "grad_norm": 1.035460114479065,
      "learning_rate": 6.363636363636363e-07,
      "logits/chosen": -0.8119982481002808,
      "logits/rejected": -0.6650353670120239,
      "logps/chosen": -91.4764404296875,
      "logps/rejected": -195.1119842529297,
      "loss": 0.6915,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.005646915640681982,
      "rewards/margins": 0.004837614018470049,
      "rewards/rejected": 0.000809302378911525,
      "step": 70
    },
    {
      "epoch": 0.07329363261566652,
      "grad_norm": 1.1378659009933472,
      "learning_rate": 7.272727272727272e-07,
      "logits/chosen": -0.8396841287612915,
      "logits/rejected": -0.739294707775116,
      "logps/chosen": -128.26870727539062,
      "logps/rejected": -237.46743774414062,
      "loss": 0.688,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.007331106811761856,
      "rewards/margins": 0.0071782395243644714,
      "rewards/rejected": 0.0001528669090475887,
      "step": 80
    },
    {
      "epoch": 0.08245533669262482,
      "grad_norm": 0.8795006275177002,
      "learning_rate": 8.181818181818182e-07,
      "logits/chosen": -0.7932317852973938,
      "logits/rejected": -0.634912371635437,
      "logps/chosen": -111.9941177368164,
      "logps/rejected": -242.5846710205078,
      "loss": 0.6868,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.021863078698515892,
      "rewards/margins": 0.009898947551846504,
      "rewards/rejected": 0.011964133009314537,
      "step": 90
    },
    {
      "epoch": 0.09161704076958314,
      "grad_norm": 0.9012777209281921,
      "learning_rate": 9.09090909090909e-07,
      "logits/chosen": -0.7805367708206177,
      "logits/rejected": -0.49868839979171753,
      "logps/chosen": -140.55088806152344,
      "logps/rejected": -290.98150634765625,
      "loss": 0.6806,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": 0.02105652168393135,
      "rewards/margins": 0.013599460944533348,
      "rewards/rejected": 0.00745706120505929,
      "step": 100
    },
    {
      "epoch": 0.09161704076958314,
      "eval_logits/chosen": -0.7638165950775146,
      "eval_logits/rejected": -0.5878660082817078,
      "eval_logps/chosen": -139.51101684570312,
      "eval_logps/rejected": -264.5260009765625,
      "eval_loss": 0.6816309690475464,
      "eval_rewards/accuracies": 0.6445086598396301,
      "eval_rewards/chosen": 0.030344096943736076,
      "eval_rewards/margins": 0.02048710361123085,
      "eval_rewards/rejected": 0.009856995195150375,
      "eval_runtime": 253.9092,
      "eval_samples_per_second": 10.882,
      "eval_steps_per_second": 1.363,
      "step": 100
    },
    {
      "epoch": 0.10077874484654145,
      "grad_norm": 1.0326433181762695,
      "learning_rate": 1e-06,
      "logits/chosen": -0.8220171928405762,
      "logits/rejected": -0.720903754234314,
      "logps/chosen": -186.0214385986328,
      "logps/rejected": -243.47702026367188,
      "loss": 0.6773,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.05213131755590439,
      "rewards/margins": 0.03893345221877098,
      "rewards/rejected": 0.013197864405810833,
      "step": 110
    },
    {
      "epoch": 0.10994044892349977,
      "grad_norm": 1.0610991716384888,
      "learning_rate": 9.997436315234263e-07,
      "logits/chosen": -0.687503457069397,
      "logits/rejected": -0.6214867830276489,
      "logps/chosen": -119.43013000488281,
      "logps/rejected": -170.53787231445312,
      "loss": 0.6699,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": 0.09322745352983475,
      "rewards/margins": 0.058328378945589066,
      "rewards/rejected": 0.034899067133665085,
      "step": 120
    },
    {
      "epoch": 0.11910215300045808,
      "grad_norm": 0.9824616312980652,
      "learning_rate": 9.989747889928883e-07,
      "logits/chosen": -0.7877914905548096,
      "logits/rejected": -0.6358338594436646,
      "logps/chosen": -128.0840606689453,
      "logps/rejected": -254.0060272216797,
      "loss": 0.656,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.102009616792202,
      "rewards/margins": 0.07630841434001923,
      "rewards/rejected": 0.02570119872689247,
      "step": 130
    },
    {
      "epoch": 0.1282638570774164,
      "grad_norm": 1.0044574737548828,
      "learning_rate": 9.976942608363393e-07,
      "logits/chosen": -0.8087643384933472,
      "logits/rejected": -0.5704258680343628,
      "logps/chosen": -112.33183288574219,
      "logps/rejected": -218.6199951171875,
      "loss": 0.6455,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": 0.18401500582695007,
      "rewards/margins": 0.12449517101049423,
      "rewards/rejected": 0.05951983854174614,
      "step": 140
    },
    {
      "epoch": 0.1374255611543747,
      "grad_norm": 1.028503179550171,
      "learning_rate": 9.9590336020199e-07,
      "logits/chosen": -0.7007342576980591,
      "logits/rejected": -0.5621221661567688,
      "logps/chosen": -114.86798095703125,
      "logps/rejected": -248.943603515625,
      "loss": 0.6348,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 0.24283933639526367,
      "rewards/margins": 0.14451688528060913,
      "rewards/rejected": 0.09832244366407394,
      "step": 150
    },
    {
      "epoch": 0.14658726523133303,
      "grad_norm": 0.8501743674278259,
      "learning_rate": 9.936039236117095e-07,
      "logits/chosen": -0.7554874420166016,
      "logits/rejected": -0.7505234479904175,
      "logps/chosen": -147.8970184326172,
      "logps/rejected": -204.843505859375,
      "loss": 0.6207,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": 0.26698964834213257,
      "rewards/margins": 0.1363282948732376,
      "rewards/rejected": 0.13066136837005615,
      "step": 160
    },
    {
      "epoch": 0.15574896930829135,
      "grad_norm": 0.8963977098464966,
      "learning_rate": 9.907983090777206e-07,
      "logits/chosen": -0.7809524536132812,
      "logits/rejected": -0.6654596924781799,
      "logps/chosen": -131.55320739746094,
      "logps/rejected": -207.1016082763672,
      "loss": 0.6215,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": 0.30690616369247437,
      "rewards/margins": 0.15369756519794464,
      "rewards/rejected": 0.15320861339569092,
      "step": 170
    },
    {
      "epoch": 0.16491067338524965,
      "grad_norm": 0.9032484292984009,
      "learning_rate": 9.874893936845187e-07,
      "logits/chosen": -0.6498181819915771,
      "logits/rejected": -0.5487117767333984,
      "logps/chosen": -152.4180908203125,
      "logps/rejected": -292.68438720703125,
      "loss": 0.6068,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": 0.3112695813179016,
      "rewards/margins": 0.18446235358715057,
      "rewards/rejected": 0.12680718302726746,
      "step": 180
    },
    {
      "epoch": 0.17407237746220797,
      "grad_norm": 0.7833600044250488,
      "learning_rate": 9.836805706384983e-07,
      "logits/chosen": -0.8308868408203125,
      "logits/rejected": -0.66489177942276,
      "logps/chosen": -97.64247131347656,
      "logps/rejected": -185.3607635498047,
      "loss": 0.5958,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": 0.47603702545166016,
      "rewards/margins": 0.26404884457588196,
      "rewards/rejected": 0.21198812127113342,
      "step": 190
    },
    {
      "epoch": 0.1832340815391663,
      "grad_norm": 0.9265146255493164,
      "learning_rate": 9.793757457883061e-07,
      "logits/chosen": -0.8464080691337585,
      "logits/rejected": -0.6489285826683044,
      "logps/chosen": -81.71176147460938,
      "logps/rejected": -186.20758056640625,
      "loss": 0.5704,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.5865594148635864,
      "rewards/margins": 0.3980158567428589,
      "rewards/rejected": 0.18854358792304993,
      "step": 200
    },
    {
      "epoch": 0.1832340815391663,
      "eval_logits/chosen": -0.7638057470321655,
      "eval_logits/rejected": -0.5881015062332153,
      "eval_logps/chosen": -133.12771606445312,
      "eval_logps/rejected": -261.77801513671875,
      "eval_loss": 0.5992826819419861,
      "eval_rewards/accuracies": 0.823699414730072,
      "eval_rewards/chosen": 0.34951016306877136,
      "eval_rewards/margins": 0.20225529372692108,
      "eval_rewards/rejected": 0.14725488424301147,
      "eval_runtime": 252.1498,
      "eval_samples_per_second": 10.958,
      "eval_steps_per_second": 1.372,
      "step": 200
    },
    {
      "epoch": 0.1923957856161246,
      "grad_norm": 0.8853036165237427,
      "learning_rate": 9.745793336194975e-07,
      "logits/chosen": -0.674169659614563,
      "logits/rejected": -0.5945926904678345,
      "logps/chosen": -106.3974380493164,
      "logps/rejected": -215.2452850341797,
      "loss": 0.574,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 0.6075299978256226,
      "rewards/margins": 0.40666985511779785,
      "rewards/rejected": 0.20086011290550232,
      "step": 210
    },
    {
      "epoch": 0.2015574896930829,
      "grad_norm": 0.8835374116897583,
      "learning_rate": 9.69296252727595e-07,
      "logits/chosen": -0.6831656098365784,
      "logits/rejected": -0.6844693422317505,
      "logps/chosen": -99.75518035888672,
      "logps/rejected": -196.4925994873047,
      "loss": 0.5604,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": 0.5753337740898132,
      "rewards/margins": 0.31991419196128845,
      "rewards/rejected": 0.25541964173316956,
      "step": 220
    },
    {
      "epoch": 0.21071919377004122,
      "grad_norm": 0.8146816492080688,
      "learning_rate": 9.63531920774199e-07,
      "logits/chosen": -0.8509872555732727,
      "logits/rejected": -0.7722374796867371,
      "logps/chosen": -84.59940338134766,
      "logps/rejected": -179.6902618408203,
      "loss": 0.5335,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.8087629079818726,
      "rewards/margins": 0.43924540281295776,
      "rewards/rejected": 0.36951744556427,
      "step": 230
    },
    {
      "epoch": 0.21988089784699955,
      "grad_norm": 0.7882716059684753,
      "learning_rate": 9.572922489313142e-07,
      "logits/chosen": -0.7912776470184326,
      "logits/rejected": -0.7742729187011719,
      "logps/chosen": -122.55216217041016,
      "logps/rejected": -203.78738403320312,
      "loss": 0.5186,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": 0.6593917608261108,
      "rewards/margins": 0.20427684485912323,
      "rewards/rejected": 0.4551149010658264,
      "step": 240
    },
    {
      "epoch": 0.22904260192395787,
      "grad_norm": 0.778360903263092,
      "learning_rate": 9.505836358195993e-07,
      "logits/chosen": -0.8406602144241333,
      "logits/rejected": -0.6831453442573547,
      "logps/chosen": -95.79948425292969,
      "logps/rejected": -225.69931030273438,
      "loss": 0.5362,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 0.8291288614273071,
      "rewards/margins": 0.5095328092575073,
      "rewards/rejected": 0.3195960223674774,
      "step": 250
    },
    {
      "epoch": 0.23820430600091616,
      "grad_norm": 0.8873792290687561,
      "learning_rate": 9.434129609467483e-07,
      "logits/chosen": -0.6178407669067383,
      "logits/rejected": -0.6204096674919128,
      "logps/chosen": -194.87338256835938,
      "logps/rejected": -260.9487609863281,
      "loss": 0.5372,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.5945974588394165,
      "rewards/margins": 0.42120856046676636,
      "rewards/rejected": 0.17338886857032776,
      "step": 260
    },
    {
      "epoch": 0.24736601007787448,
      "grad_norm": 0.833759069442749,
      "learning_rate": 9.357875776527333e-07,
      "logits/chosen": -0.7258036732673645,
      "logits/rejected": -0.6290136575698853,
      "logps/chosen": -110.7699966430664,
      "logps/rejected": -214.27734375,
      "loss": 0.5062,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 0.6229942440986633,
      "rewards/margins": 0.48382797837257385,
      "rewards/rejected": 0.13916628062725067,
      "step": 270
    },
    {
      "epoch": 0.2565277141548328,
      "grad_norm": 0.7017952799797058,
      "learning_rate": 9.27715305569148e-07,
      "logits/chosen": -0.7022525072097778,
      "logits/rejected": -0.5847650766372681,
      "logps/chosen": -103.59033203125,
      "logps/rejected": -211.4336700439453,
      "loss": 0.5042,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 0.7977592349052429,
      "rewards/margins": 0.5717019438743591,
      "rewards/rejected": 0.2260572463274002,
      "step": 280
    },
    {
      "epoch": 0.2656894182317911,
      "grad_norm": 0.8911390900611877,
      "learning_rate": 9.192044226003788e-07,
      "logits/chosen": -0.7471307516098022,
      "logits/rejected": -0.6816040277481079,
      "logps/chosen": -121.2009048461914,
      "logps/rejected": -197.65383911132812,
      "loss": 0.5119,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.9325343370437622,
      "rewards/margins": 0.6139361262321472,
      "rewards/rejected": 0.3185981810092926,
      "step": 290
    },
    {
      "epoch": 0.2748511223087494,
      "grad_norm": 0.7208541631698608,
      "learning_rate": 9.102636564348294e-07,
      "logits/chosen": -0.7008062601089478,
      "logits/rejected": -0.6801473498344421,
      "logps/chosen": -114.77880859375,
      "logps/rejected": -210.2012176513672,
      "loss": 0.5032,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 0.8345085978507996,
      "rewards/margins": 0.5057986378669739,
      "rewards/rejected": 0.3287099599838257,
      "step": 300
    },
    {
      "epoch": 0.2748511223087494,
      "eval_logits/chosen": -0.7651150226593018,
      "eval_logits/rejected": -0.5892514586448669,
      "eval_logps/chosen": -128.52711486816406,
      "eval_logps/rejected": -261.1383056640625,
      "eval_loss": 0.5312727093696594,
      "eval_rewards/accuracies": 0.852601170539856,
      "eval_rewards/chosen": 0.5795396566390991,
      "eval_rewards/margins": 0.40029817819595337,
      "eval_rewards/rejected": 0.17924147844314575,
      "eval_runtime": 252.4578,
      "eval_samples_per_second": 10.944,
      "eval_steps_per_second": 1.371,
      "step": 300
    },
    {
      "epoch": 0.28401282638570774,
      "grad_norm": 0.8867717385292053,
      "learning_rate": 9.009021755949051e-07,
      "logits/chosen": -0.7086865901947021,
      "logits/rejected": -0.7210379242897034,
      "logps/chosen": -109.81558990478516,
      "logps/rejected": -152.71084594726562,
      "loss": 0.5059,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.8528501391410828,
      "rewards/margins": 0.4039060175418854,
      "rewards/rejected": 0.4489441514015198,
      "step": 310
    },
    {
      "epoch": 0.29317453046266606,
      "grad_norm": 0.9950509071350098,
      "learning_rate": 8.911295800349314e-07,
      "logits/chosen": -0.6988198757171631,
      "logits/rejected": -0.6140611171722412,
      "logps/chosen": -160.05844116210938,
      "logps/rejected": -236.6884307861328,
      "loss": 0.4987,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 0.7523847818374634,
      "rewards/margins": 0.4543587267398834,
      "rewards/rejected": 0.2980259656906128,
      "step": 320
    },
    {
      "epoch": 0.3023362345396244,
      "grad_norm": 0.9498128294944763,
      "learning_rate": 8.809558912966519e-07,
      "logits/chosen": -0.6651901006698608,
      "logits/rejected": -0.6055413484573364,
      "logps/chosen": -84.35199737548828,
      "logps/rejected": -208.5164794921875,
      "loss": 0.4501,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 0.9873196482658386,
      "rewards/margins": 0.7461606860160828,
      "rewards/rejected": 0.24115900695323944,
      "step": 330
    },
    {
      "epoch": 0.3114979386165827,
      "grad_norm": 0.7728273272514343,
      "learning_rate": 8.703915422323984e-07,
      "logits/chosen": -0.5462226867675781,
      "logits/rejected": -0.46084174513816833,
      "logps/chosen": -129.86944580078125,
      "logps/rejected": -210.6329345703125,
      "loss": 0.4827,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": 0.8226272463798523,
      "rewards/margins": 0.4101462960243225,
      "rewards/rejected": 0.4124809801578522,
      "step": 340
    },
    {
      "epoch": 0.320659642693541,
      "grad_norm": 0.6749230623245239,
      "learning_rate": 8.594473663064734e-07,
      "logits/chosen": -0.8189064264297485,
      "logits/rejected": -0.6820628046989441,
      "logps/chosen": -83.3464126586914,
      "logps/rejected": -180.37059020996094,
      "loss": 0.4865,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 1.0249218940734863,
      "rewards/margins": 0.6868400573730469,
      "rewards/rejected": 0.3380817770957947,
      "step": 350
    },
    {
      "epoch": 0.3298213467704993,
      "grad_norm": 0.917298436164856,
      "learning_rate": 8.481345864857146e-07,
      "logits/chosen": -0.7375888824462891,
      "logits/rejected": -0.5009106993675232,
      "logps/chosen": -119.41560363769531,
      "logps/rejected": -241.78866577148438,
      "loss": 0.4839,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": 0.7711573839187622,
      "rewards/margins": 0.5962369441986084,
      "rewards/rejected": 0.1749204695224762,
      "step": 360
    },
    {
      "epoch": 0.3389830508474576,
      "grad_norm": 0.6750978827476501,
      "learning_rate": 8.36464803730636e-07,
      "logits/chosen": -0.8278859853744507,
      "logits/rejected": -0.761436402797699,
      "logps/chosen": -101.12157440185547,
      "logps/rejected": -170.44569396972656,
      "loss": 0.4589,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 1.0102306604385376,
      "rewards/margins": 0.6717247366905212,
      "rewards/rejected": 0.33850589394569397,
      "step": 370
    },
    {
      "epoch": 0.34814475492441593,
      "grad_norm": 0.8837040066719055,
      "learning_rate": 8.244499850989451e-07,
      "logits/chosen": -0.8623167276382446,
      "logits/rejected": -0.699422299861908,
      "logps/chosen": -77.54528045654297,
      "logps/rejected": -172.49044799804688,
      "loss": 0.456,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 1.0262943506240845,
      "rewards/margins": 0.7692972421646118,
      "rewards/rejected": 0.25699710845947266,
      "step": 380
    },
    {
      "epoch": 0.35730645900137425,
      "grad_norm": 0.6525785326957703,
      "learning_rate": 8.121024514736377e-07,
      "logits/chosen": -0.7333749532699585,
      "logits/rejected": -0.5934922099113464,
      "logps/chosen": -71.39646911621094,
      "logps/rejected": -198.8276824951172,
      "loss": 0.425,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 0.9156519770622253,
      "rewards/margins": 0.7522441148757935,
      "rewards/rejected": 0.16340789198875427,
      "step": 390
    },
    {
      "epoch": 0.3664681630783326,
      "grad_norm": 0.795269250869751,
      "learning_rate": 7.994348649282532e-07,
      "logits/chosen": -0.7343798279762268,
      "logits/rejected": -0.5222686529159546,
      "logps/chosen": -116.50679779052734,
      "logps/rejected": -223.85409545898438,
      "loss": 0.4548,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.9559024572372437,
      "rewards/margins": 0.7852972745895386,
      "rewards/rejected": 0.17060522735118866,
      "step": 400
    },
    {
      "epoch": 0.3664681630783326,
      "eval_logits/chosen": -0.7659581303596497,
      "eval_logits/rejected": -0.5900666117668152,
      "eval_logps/chosen": -127.30506896972656,
      "eval_logps/rejected": -263.6780090332031,
      "eval_loss": 0.47267913818359375,
      "eval_rewards/accuracies": 0.884393036365509,
      "eval_rewards/chosen": 0.6406417489051819,
      "eval_rewards/margins": 0.5883842706680298,
      "eval_rewards/rejected": 0.05225752666592598,
      "eval_runtime": 252.1281,
      "eval_samples_per_second": 10.959,
      "eval_steps_per_second": 1.372,
      "step": 400
    },
    {
      "epoch": 0.3756298671552909,
      "grad_norm": 0.6601365208625793,
      "learning_rate": 7.8646021574225e-07,
      "logits/chosen": -0.6952235102653503,
      "logits/rejected": -0.46782374382019043,
      "logps/chosen": -104.0670394897461,
      "logps/rejected": -239.7484130859375,
      "loss": 0.4482,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.9586408734321594,
      "rewards/margins": 0.7240998148918152,
      "rewards/rejected": 0.23454102873802185,
      "step": 410
    },
    {
      "epoch": 0.3847915712322492,
      "grad_norm": 0.7361147999763489,
      "learning_rate": 7.731918090798113e-07,
      "logits/chosen": -0.8017680048942566,
      "logits/rejected": -0.6514943838119507,
      "logps/chosen": -97.84058380126953,
      "logps/rejected": -181.79403686523438,
      "loss": 0.4194,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 1.1092697381973267,
      "rewards/margins": 0.7227686047554016,
      "rewards/rejected": 0.3865012228488922,
      "step": 420
    },
    {
      "epoch": 0.39395327530920754,
      "grad_norm": 0.8337231278419495,
      "learning_rate": 7.596432513457482e-07,
      "logits/chosen": -0.7028661966323853,
      "logits/rejected": -0.6256899833679199,
      "logps/chosen": -96.13346862792969,
      "logps/rejected": -165.75747680664062,
      "loss": 0.426,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 1.0767663717269897,
      "rewards/margins": 0.7708722949028015,
      "rewards/rejected": 0.30589404702186584,
      "step": 430
    },
    {
      "epoch": 0.4031149793861658,
      "grad_norm": 0.826393723487854,
      "learning_rate": 7.458284362324842e-07,
      "logits/chosen": -0.7026184797286987,
      "logits/rejected": -0.5175060033798218,
      "logps/chosen": -73.34812927246094,
      "logps/rejected": -228.7108917236328,
      "loss": 0.4312,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 1.0083404779434204,
      "rewards/margins": 0.9987894296646118,
      "rewards/rejected": 0.009551113471388817,
      "step": 440
    },
    {
      "epoch": 0.4122766834631241,
      "grad_norm": 0.7044612765312195,
      "learning_rate": 7.317615304724387e-07,
      "logits/chosen": -0.6982114911079407,
      "logits/rejected": -0.6542345881462097,
      "logps/chosen": -97.9085922241211,
      "logps/rejected": -188.80284118652344,
      "loss": 0.4191,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 1.0013755559921265,
      "rewards/margins": 0.9130918383598328,
      "rewards/rejected": 0.08828376233577728,
      "step": 450
    },
    {
      "epoch": 0.42143838754008245,
      "grad_norm": 0.8462466597557068,
      "learning_rate": 7.174569593104108e-07,
      "logits/chosen": -0.8011992573738098,
      "logits/rejected": -0.7858909368515015,
      "logps/chosen": -111.00840759277344,
      "logps/rejected": -198.2548370361328,
      "loss": 0.4307,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 1.0858404636383057,
      "rewards/margins": 0.7915989756584167,
      "rewards/rejected": 0.29424166679382324,
      "step": 460
    },
    {
      "epoch": 0.43060009161704077,
      "grad_norm": 1.0214639902114868,
      "learning_rate": 7.029293917108677e-07,
      "logits/chosen": -0.6210706233978271,
      "logits/rejected": -0.5300668478012085,
      "logps/chosen": -198.18032836914062,
      "logps/rejected": -255.27877807617188,
      "loss": 0.4096,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.7058089971542358,
      "rewards/margins": 0.7782974243164062,
      "rewards/rejected": -0.0724884420633316,
      "step": 470
    },
    {
      "epoch": 0.4397617956939991,
      "grad_norm": 0.8063747882843018,
      "learning_rate": 6.881937253153051e-07,
      "logits/chosen": -0.8666943311691284,
      "logits/rejected": -0.8363106846809387,
      "logps/chosen": -119.9945068359375,
      "logps/rejected": -200.77627563476562,
      "loss": 0.4004,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 1.2585500478744507,
      "rewards/margins": 1.000775694847107,
      "rewards/rejected": 0.25777435302734375,
      "step": 480
    },
    {
      "epoch": 0.4489234997709574,
      "grad_norm": 0.7443576455116272,
      "learning_rate": 6.732650711651031e-07,
      "logits/chosen": -0.7361257076263428,
      "logits/rejected": -0.5734608769416809,
      "logps/chosen": -127.10151672363281,
      "logps/rejected": -238.904296875,
      "loss": 0.4098,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 0.6828312873840332,
      "rewards/margins": 0.7480920553207397,
      "rewards/rejected": -0.06526081264019012,
      "step": 490
    },
    {
      "epoch": 0.45808520384791573,
      "grad_norm": 0.6595803499221802,
      "learning_rate": 6.581587382055491e-07,
      "logits/chosen": -0.8137266039848328,
      "logits/rejected": -0.7571356296539307,
      "logps/chosen": -104.3070297241211,
      "logps/rejected": -201.35647583007812,
      "loss": 0.3823,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 1.131914496421814,
      "rewards/margins": 1.0377442836761475,
      "rewards/rejected": 0.09417015314102173,
      "step": 500
    },
    {
      "epoch": 0.45808520384791573,
      "eval_logits/chosen": -0.7672035098075867,
      "eval_logits/rejected": -0.591365396976471,
      "eval_logps/chosen": -127.29341125488281,
      "eval_logps/rejected": -267.3507080078125,
      "eval_loss": 0.4234526753425598,
      "eval_rewards/accuracies": 0.8930636048316956,
      "eval_rewards/chosen": 0.6412254571914673,
      "eval_rewards/margins": 0.7726045250892639,
      "eval_rewards/rejected": -0.13137909770011902,
      "eval_runtime": 252.2594,
      "eval_samples_per_second": 10.953,
      "eval_steps_per_second": 1.372,
      "step": 500
    },
    {
      "epoch": 0.467246907924874,
      "grad_norm": 0.7072412371635437,
      "learning_rate": 6.428902175869126e-07,
      "logits/chosen": -0.7340083122253418,
      "logits/rejected": -0.6606656908988953,
      "logps/chosen": -126.58953857421875,
      "logps/rejected": -193.69363403320312,
      "loss": 0.3779,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 1.1136077642440796,
      "rewards/margins": 1.0119339227676392,
      "rewards/rejected": 0.10167402029037476,
      "step": 510
    },
    {
      "epoch": 0.4764086120018323,
      "grad_norm": 0.7274028658866882,
      "learning_rate": 6.274751667786761e-07,
      "logits/chosen": -0.6646409630775452,
      "logits/rejected": -0.5588616728782654,
      "logps/chosen": -177.66925048828125,
      "logps/rejected": -301.41876220703125,
      "loss": 0.4,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.8406426310539246,
      "rewards/margins": 0.9421254992485046,
      "rewards/rejected": -0.10148291289806366,
      "step": 520
    },
    {
      "epoch": 0.48557031607879064,
      "grad_norm": 0.8565487861633301,
      "learning_rate": 6.119293935132075e-07,
      "logits/chosen": -0.7664127349853516,
      "logits/rejected": -0.6106132864952087,
      "logps/chosen": -105.25531005859375,
      "logps/rejected": -184.4231414794922,
      "loss": 0.3778,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 0.9561585187911987,
      "rewards/margins": 0.908736526966095,
      "rewards/rejected": 0.04742204025387764,
      "step": 530
    },
    {
      "epoch": 0.49473202015574896,
      "grad_norm": 0.8583614826202393,
      "learning_rate": 5.962688395753437e-07,
      "logits/chosen": -0.9779294729232788,
      "logits/rejected": -0.892353355884552,
      "logps/chosen": -84.86984252929688,
      "logps/rejected": -183.84295654296875,
      "loss": 0.3823,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.9522911310195923,
      "rewards/margins": 0.9800866842269897,
      "rewards/rejected": -0.027795499190688133,
      "step": 540
    },
    {
      "epoch": 0.5038937242327073,
      "grad_norm": 0.9459156394004822,
      "learning_rate": 5.80509564454506e-07,
      "logits/chosen": -0.7560229897499084,
      "logits/rejected": -0.6336179971694946,
      "logps/chosen": -60.79301834106445,
      "logps/rejected": -198.7034912109375,
      "loss": 0.3682,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 1.1121090650558472,
      "rewards/margins": 0.9977458715438843,
      "rewards/rejected": 0.11436320841312408,
      "step": 550
    },
    {
      "epoch": 0.5130554283096656,
      "grad_norm": 0.7807260155677795,
      "learning_rate": 5.646677288761132e-07,
      "logits/chosen": -0.7362383604049683,
      "logits/rejected": -0.6392463445663452,
      "logps/chosen": -94.71185302734375,
      "logps/rejected": -229.9622802734375,
      "loss": 0.3765,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.8846451640129089,
      "rewards/margins": 1.0037345886230469,
      "rewards/rejected": -0.11908946186304092,
      "step": 560
    },
    {
      "epoch": 0.5222171323866239,
      "grad_norm": 0.9470974802970886,
      "learning_rate": 5.487595782291784e-07,
      "logits/chosen": -0.807681679725647,
      "logits/rejected": -0.6749477982521057,
      "logps/chosen": -118.90516662597656,
      "logps/rejected": -185.31967163085938,
      "loss": 0.3591,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 1.0300986766815186,
      "rewards/margins": 1.0025510787963867,
      "rewards/rejected": 0.02754763886332512,
      "step": 570
    },
    {
      "epoch": 0.5313788364635822,
      "grad_norm": 0.776768684387207,
      "learning_rate": 5.328014259070878e-07,
      "logits/chosen": -0.7403959035873413,
      "logits/rejected": -0.5926883816719055,
      "logps/chosen": -135.42613220214844,
      "logps/rejected": -229.9550323486328,
      "loss": 0.3776,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 0.7646117806434631,
      "rewards/margins": 0.9938881993293762,
      "rewards/rejected": -0.22927632927894592,
      "step": 580
    },
    {
      "epoch": 0.5405405405405406,
      "grad_norm": 0.890230119228363,
      "learning_rate": 5.168096365786402e-07,
      "logits/chosen": -0.8721317052841187,
      "logits/rejected": -0.6174881458282471,
      "logps/chosen": -113.02799987792969,
      "logps/rejected": -223.82821655273438,
      "loss": 0.3664,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 0.9226465225219727,
      "rewards/margins": 1.041733980178833,
      "rewards/rejected": -0.11908741295337677,
      "step": 590
    },
    {
      "epoch": 0.5497022446174988,
      "grad_norm": 0.7550140023231506,
      "learning_rate": 5.008006094065069e-07,
      "logits/chosen": -0.6947864294052124,
      "logits/rejected": -0.6369723081588745,
      "logps/chosen": -108.70173645019531,
      "logps/rejected": -202.474365234375,
      "loss": 0.3513,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 0.9735838174819946,
      "rewards/margins": 0.9605382680892944,
      "rewards/rejected": 0.013045459985733032,
      "step": 600
    },
    {
      "epoch": 0.5497022446174988,
      "eval_logits/chosen": -0.7693285346031189,
      "eval_logits/rejected": -0.5936357975006104,
      "eval_logps/chosen": -127.94479370117188,
      "eval_logps/rejected": -271.55316162109375,
      "eval_loss": 0.3842833936214447,
      "eval_rewards/accuracies": 0.913294792175293,
      "eval_rewards/chosen": 0.608656644821167,
      "eval_rewards/margins": 0.9501577615737915,
      "eval_rewards/rejected": -0.3415011465549469,
      "eval_runtime": 252.1964,
      "eval_samples_per_second": 10.956,
      "eval_steps_per_second": 1.372,
      "step": 600
    },
    {
      "epoch": 0.5588639486944572,
      "grad_norm": 0.7903901934623718,
      "learning_rate": 4.847907612303182e-07,
      "logits/chosen": -0.7707523107528687,
      "logits/rejected": -0.6204859614372253,
      "logps/chosen": -131.9110870361328,
      "logps/rejected": -249.21353149414062,
      "loss": 0.3699,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 0.5509034991264343,
      "rewards/margins": 0.738714337348938,
      "rewards/rejected": -0.18781085312366486,
      "step": 610
    },
    {
      "epoch": 0.5680256527714155,
      "grad_norm": 0.7700456380844116,
      "learning_rate": 4.687965097316223e-07,
      "logits/chosen": -0.8288325071334839,
      "logits/rejected": -0.6177199482917786,
      "logps/chosen": -75.93597412109375,
      "logps/rejected": -225.97067260742188,
      "loss": 0.3304,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 1.001242756843567,
      "rewards/margins": 1.3316209316253662,
      "rewards/rejected": -0.33037835359573364,
      "step": 620
    },
    {
      "epoch": 0.5771873568483737,
      "grad_norm": 0.9662141799926758,
      "learning_rate": 4.5283425659798175e-07,
      "logits/chosen": -0.8291882276535034,
      "logits/rejected": -0.8248087167739868,
      "logps/chosen": -143.07180786132812,
      "logps/rejected": -264.81463623046875,
      "loss": 0.3757,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 0.9661464691162109,
      "rewards/margins": 1.1662724018096924,
      "rewards/rejected": -0.20012597739696503,
      "step": 630
    },
    {
      "epoch": 0.5863490609253321,
      "grad_norm": 0.7089081406593323,
      "learning_rate": 4.3692037070347123e-07,
      "logits/chosen": -0.8086493611335754,
      "logits/rejected": -0.6815121173858643,
      "logps/chosen": -89.39459228515625,
      "logps/rejected": -196.5904541015625,
      "loss": 0.3598,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 0.9766321182250977,
      "rewards/margins": 1.233741044998169,
      "rewards/rejected": -0.2571088671684265,
      "step": 640
    },
    {
      "epoch": 0.5955107650022904,
      "grad_norm": 0.6838498115539551,
      "learning_rate": 4.21071171322823e-07,
      "logits/chosen": -0.6819313764572144,
      "logits/rejected": -0.649137556552887,
      "logps/chosen": -202.49642944335938,
      "logps/rejected": -305.983154296875,
      "loss": 0.358,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.5383878946304321,
      "rewards/margins": 0.8751532435417175,
      "rewards/rejected": -0.3367653489112854,
      "step": 650
    },
    {
      "epoch": 0.6046724690792488,
      "grad_norm": 0.8088337779045105,
      "learning_rate": 4.0530291139643755e-07,
      "logits/chosen": -0.8444887399673462,
      "logits/rejected": -0.7773497700691223,
      "logps/chosen": -92.40638732910156,
      "logps/rejected": -210.07962036132812,
      "loss": 0.3479,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 1.1181201934814453,
      "rewards/margins": 1.1484359502792358,
      "rewards/rejected": -0.030315812677145004,
      "step": 660
    },
    {
      "epoch": 0.613834173156207,
      "grad_norm": 0.743868350982666,
      "learning_rate": 3.8963176086341727e-07,
      "logits/chosen": -0.7523199319839478,
      "logits/rejected": -0.545455813407898,
      "logps/chosen": -94.30660247802734,
      "logps/rejected": -229.30667114257812,
      "loss": 0.3394,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 0.5909605026245117,
      "rewards/margins": 1.0719151496887207,
      "rewards/rejected": -0.4809547960758209,
      "step": 670
    },
    {
      "epoch": 0.6229958772331654,
      "grad_norm": 0.7818168997764587,
      "learning_rate": 3.7407379007971506e-07,
      "logits/chosen": -0.6859625577926636,
      "logits/rejected": -0.5579516887664795,
      "logps/chosen": -143.53274536132812,
      "logps/rejected": -291.33453369140625,
      "loss": 0.3549,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.8891765475273132,
      "rewards/margins": 1.2517534494400024,
      "rewards/rejected": -0.36257684230804443,
      "step": 680
    },
    {
      "epoch": 0.6321575813101237,
      "grad_norm": 0.8612805604934692,
      "learning_rate": 3.586449533384048e-07,
      "logits/chosen": -0.6445218324661255,
      "logits/rejected": -0.5638427138328552,
      "logps/chosen": -84.69303894042969,
      "logps/rejected": -175.0728302001953,
      "loss": 0.3413,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 0.8365532159805298,
      "rewards/margins": 1.054081916809082,
      "rewards/rejected": -0.21752884984016418,
      "step": 690
    },
    {
      "epoch": 0.641319285387082,
      "grad_norm": 0.6171064972877502,
      "learning_rate": 3.433610725089692e-07,
      "logits/chosen": -0.7035859823226929,
      "logits/rejected": -0.5256271362304688,
      "logps/chosen": -138.380615234375,
      "logps/rejected": -248.87451171875,
      "loss": 0.3444,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.8877826929092407,
      "rewards/margins": 1.0520169734954834,
      "rewards/rejected": -0.16423439979553223,
      "step": 700
    },
    {
      "epoch": 0.641319285387082,
      "eval_logits/chosen": -0.7720704078674316,
      "eval_logits/rejected": -0.5964747667312622,
      "eval_logps/chosen": -128.3763427734375,
      "eval_logps/rejected": -274.77838134765625,
      "eval_loss": 0.35713571310043335,
      "eval_rewards/accuracies": 0.910404622554779,
      "eval_rewards/chosen": 0.5870785117149353,
      "eval_rewards/margins": 1.0898393392562866,
      "eval_rewards/rejected": -0.5027608871459961,
      "eval_runtime": 252.0905,
      "eval_samples_per_second": 10.96,
      "eval_steps_per_second": 1.373,
      "step": 700
    },
    {
      "epoch": 0.6504809894640403,
      "grad_norm": 0.8962298631668091,
      "learning_rate": 3.2823782081238555e-07,
      "logits/chosen": -0.7447826266288757,
      "logits/rejected": -0.7338870167732239,
      "logps/chosen": -95.43080139160156,
      "logps/rejected": -192.2973175048828,
      "loss": 0.3405,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 1.0037095546722412,
      "rewards/margins": 1.1646350622177124,
      "rewards/rejected": -0.16092538833618164,
      "step": 710
    },
    {
      "epoch": 0.6596426935409986,
      "grad_norm": 0.7918722033500671,
      "learning_rate": 3.132907067486471e-07,
      "logits/chosen": -0.7636629343032837,
      "logits/rejected": -0.6649468541145325,
      "logps/chosen": -114.23177337646484,
      "logps/rejected": -215.4693145751953,
      "loss": 0.3439,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 0.6278459429740906,
      "rewards/margins": 0.8962922096252441,
      "rewards/rejected": -0.26844626665115356,
      "step": 720
    },
    {
      "epoch": 0.668804397617957,
      "grad_norm": 0.8416666388511658,
      "learning_rate": 2.985350581932005e-07,
      "logits/chosen": -0.776689350605011,
      "logits/rejected": -0.6382275819778442,
      "logps/chosen": -142.0974884033203,
      "logps/rejected": -211.6706085205078,
      "loss": 0.3482,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": 1.0184391736984253,
      "rewards/margins": 1.137699842453003,
      "rewards/rejected": -0.11926088482141495,
      "step": 730
    },
    {
      "epoch": 0.6779661016949152,
      "grad_norm": 0.8943964242935181,
      "learning_rate": 2.839860066786103e-07,
      "logits/chosen": -0.7382106781005859,
      "logits/rejected": -0.6326214075088501,
      "logps/chosen": -84.48265075683594,
      "logps/rejected": -175.4539031982422,
      "loss": 0.3261,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 1.1386158466339111,
      "rewards/margins": 1.2899872064590454,
      "rewards/rejected": -0.1513712853193283,
      "step": 740
    },
    {
      "epoch": 0.6871278057718736,
      "grad_norm": 0.8377578258514404,
      "learning_rate": 2.6965847187756553e-07,
      "logits/chosen": -0.7496690154075623,
      "logits/rejected": -0.6893197298049927,
      "logps/chosen": -96.63633728027344,
      "logps/rejected": -162.9729461669922,
      "loss": 0.3157,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 1.0216400623321533,
      "rewards/margins": 1.3754501342773438,
      "rewards/rejected": -0.3538101315498352,
      "step": 750
    },
    {
      "epoch": 0.6962895098488319,
      "grad_norm": 0.8681157827377319,
      "learning_rate": 2.5556714630314613e-07,
      "logits/chosen": -0.8700932264328003,
      "logits/rejected": -0.7129749059677124,
      "logps/chosen": -68.58832550048828,
      "logps/rejected": -177.4335479736328,
      "loss": 0.3173,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 1.1690860986709595,
      "rewards/margins": 1.593080759048462,
      "rewards/rejected": -0.42399463057518005,
      "step": 760
    },
    {
      "epoch": 0.7054512139257902,
      "grad_norm": 0.7554333209991455,
      "learning_rate": 2.417264802420343e-07,
      "logits/chosen": -0.8102506399154663,
      "logits/rejected": -0.6963295936584473,
      "logps/chosen": -94.93528747558594,
      "logps/rejected": -227.2413787841797,
      "loss": 0.3225,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 0.9498472213745117,
      "rewards/margins": 1.3431434631347656,
      "rewards/rejected": -0.39329633116722107,
      "step": 770
    },
    {
      "epoch": 0.7146129180027485,
      "grad_norm": 0.9492266178131104,
      "learning_rate": 2.2815066693612117e-07,
      "logits/chosen": -0.7795412540435791,
      "logits/rejected": -0.6747149229049683,
      "logps/chosen": -101.27651977539062,
      "logps/rejected": -204.02151489257812,
      "loss": 0.321,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 1.01050865650177,
      "rewards/margins": 1.2811126708984375,
      "rewards/rejected": -0.2706039547920227,
      "step": 780
    },
    {
      "epoch": 0.7237746220797068,
      "grad_norm": 0.8731204867362976,
      "learning_rate": 2.1485362802770862e-07,
      "logits/chosen": -0.7779867053031921,
      "logits/rejected": -0.6321139335632324,
      "logps/chosen": -139.72247314453125,
      "logps/rejected": -309.1371154785156,
      "loss": 0.3098,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.7068808078765869,
      "rewards/margins": 1.2700955867767334,
      "rewards/rejected": -0.563214898109436,
      "step": 790
    },
    {
      "epoch": 0.7329363261566652,
      "grad_norm": 0.9384542107582092,
      "learning_rate": 2.018489992832283e-07,
      "logits/chosen": -0.7126377820968628,
      "logits/rejected": -0.5895640850067139,
      "logps/chosen": -134.41358947753906,
      "logps/rejected": -231.8523712158203,
      "loss": 0.3486,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": 0.8817362785339355,
      "rewards/margins": 1.268080234527588,
      "rewards/rejected": -0.3863441050052643,
      "step": 800
    },
    {
      "epoch": 0.7329363261566652,
      "eval_logits/chosen": -0.7724946141242981,
      "eval_logits/rejected": -0.5970612168312073,
      "eval_logps/chosen": -128.75588989257812,
      "eval_logps/rejected": -277.0340576171875,
      "eval_loss": 0.3426840603351593,
      "eval_rewards/accuracies": 0.910404622554779,
      "eval_rewards/chosen": 0.5681010484695435,
      "eval_rewards/margins": 1.1836459636688232,
      "eval_rewards/rejected": -0.6155449151992798,
      "eval_runtime": 251.9975,
      "eval_samples_per_second": 10.964,
      "eval_steps_per_second": 1.373,
      "step": 800
    },
    {
      "epoch": 0.7420980302336234,
      "grad_norm": 0.9169598817825317,
      "learning_rate": 1.891501166101187e-07,
      "logits/chosen": -0.7657705545425415,
      "logits/rejected": -0.6736264228820801,
      "logps/chosen": -82.07408905029297,
      "logps/rejected": -185.07350158691406,
      "loss": 0.3199,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.8503938913345337,
      "rewards/margins": 1.3659098148345947,
      "rewards/rejected": -0.515515923500061,
      "step": 810
    },
    {
      "epoch": 0.7512597343105818,
      "grad_norm": 0.8475902080535889,
      "learning_rate": 1.767700023812e-07,
      "logits/chosen": -0.8032963871955872,
      "logits/rejected": -0.5944005846977234,
      "logps/chosen": -111.13800048828125,
      "logps/rejected": -270.47247314453125,
      "loss": 0.3154,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.8725293278694153,
      "rewards/margins": 1.7224490642547607,
      "rewards/rejected": -0.8499197959899902,
      "step": 820
    },
    {
      "epoch": 0.7604214383875401,
      "grad_norm": 0.9381293058395386,
      "learning_rate": 1.6472135208057125e-07,
      "logits/chosen": -0.78323894739151,
      "logits/rejected": -0.6326896548271179,
      "logps/chosen": -106.88468170166016,
      "logps/rejected": -183.2010498046875,
      "loss": 0.3454,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.8930851221084595,
      "rewards/margins": 1.1470720767974854,
      "rewards/rejected": -0.25398701429367065,
      "step": 830
    },
    {
      "epoch": 0.7695831424644984,
      "grad_norm": 0.8112012147903442,
      "learning_rate": 1.530165212847217e-07,
      "logits/chosen": -0.7700347900390625,
      "logits/rejected": -0.7001415491104126,
      "logps/chosen": -93.79769134521484,
      "logps/rejected": -191.95217895507812,
      "loss": 0.3337,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 1.0522786378860474,
      "rewards/margins": 1.627384901046753,
      "rewards/rejected": -0.5751065015792847,
      "step": 840
    },
    {
      "epoch": 0.7787448465414567,
      "grad_norm": 0.674938976764679,
      "learning_rate": 1.4166751299221003e-07,
      "logits/chosen": -0.70830899477005,
      "logits/rejected": -0.6749675273895264,
      "logps/chosen": -106.66004943847656,
      "logps/rejected": -215.84115600585938,
      "loss": 0.3163,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 1.1951297521591187,
      "rewards/margins": 1.7206119298934937,
      "rewards/rejected": -0.5254822969436646,
      "step": 850
    },
    {
      "epoch": 0.7879065506184151,
      "grad_norm": 0.7585360407829285,
      "learning_rate": 1.306859653149025e-07,
      "logits/chosen": -0.7158334851264954,
      "logits/rejected": -0.6363730430603027,
      "logps/chosen": -111.70841979980469,
      "logps/rejected": -218.56442260742188,
      "loss": 0.3239,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.8049896359443665,
      "rewards/margins": 1.498777985572815,
      "rewards/rejected": -0.6937884092330933,
      "step": 860
    },
    {
      "epoch": 0.7970682546953733,
      "grad_norm": 0.8166142106056213,
      "learning_rate": 1.2008313954339305e-07,
      "logits/chosen": -0.5638805627822876,
      "logits/rejected": -0.42692404985427856,
      "logps/chosen": -148.00503540039062,
      "logps/rejected": -250.53756713867188,
      "loss": 0.3133,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 0.6730099320411682,
      "rewards/margins": 1.2878690958023071,
      "rewards/rejected": -0.6148591637611389,
      "step": 870
    },
    {
      "epoch": 0.8062299587723316,
      "grad_norm": 0.6894632577896118,
      "learning_rate": 1.098699085988432e-07,
      "logits/chosen": -0.6931108236312866,
      "logits/rejected": -0.6952135562896729,
      "logps/chosen": -102.37767791748047,
      "logps/rejected": -263.8074035644531,
      "loss": 0.3437,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.7925483584403992,
      "rewards/margins": 1.331838607788086,
      "rewards/rejected": -0.5392903089523315,
      "step": 880
    },
    {
      "epoch": 0.81539166284929,
      "grad_norm": 0.8443871736526489,
      "learning_rate": 1.0005674588308566e-07,
      "logits/chosen": -0.8288379907608032,
      "logits/rejected": -0.7619950175285339,
      "logps/chosen": -87.10929870605469,
      "logps/rejected": -192.11410522460938,
      "loss": 0.2805,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 1.0762431621551514,
      "rewards/margins": 1.3870527744293213,
      "rewards/rejected": -0.31080955266952515,
      "step": 890
    },
    {
      "epoch": 0.8245533669262483,
      "grad_norm": 1.089480996131897,
      "learning_rate": 9.065371453842358e-08,
      "logits/chosen": -0.7295564413070679,
      "logits/rejected": -0.66633141040802,
      "logps/chosen": -80.64302062988281,
      "logps/rejected": -198.11285400390625,
      "loss": 0.3317,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.9881649017333984,
      "rewards/margins": 1.6359241008758545,
      "rewards/rejected": -0.6477591395378113,
      "step": 900
    },
    {
      "epoch": 0.8245533669262483,
      "eval_logits/chosen": -0.7748228907585144,
      "eval_logits/rejected": -0.5993096232414246,
      "eval_logps/chosen": -128.94505310058594,
      "eval_logps/rejected": -278.2012939453125,
      "eval_loss": 0.33487406373023987,
      "eval_rewards/accuracies": 0.913294792175293,
      "eval_rewards/chosen": 0.5586429238319397,
      "eval_rewards/margins": 1.2325509786605835,
      "eval_rewards/rejected": -0.6739080548286438,
      "eval_runtime": 251.978,
      "eval_samples_per_second": 10.965,
      "eval_steps_per_second": 1.373,
      "step": 900
    },
    {
      "epoch": 0.8337150710032066,
      "grad_norm": 0.7584892511367798,
      "learning_rate": 8.167045712814108e-08,
      "logits/chosen": -0.6891669034957886,
      "logits/rejected": -0.509884774684906,
      "logps/chosen": -120.35499572753906,
      "logps/rejected": -280.7765197753906,
      "loss": 0.3101,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.7489343881607056,
      "rewards/margins": 1.3420490026474,
      "rewards/rejected": -0.5931144952774048,
      "step": 910
    },
    {
      "epoch": 0.8428767750801649,
      "grad_norm": 0.6903491020202637,
      "learning_rate": 7.311618574830569e-08,
      "logits/chosen": -0.6809369921684265,
      "logits/rejected": -0.5410372614860535,
      "logps/chosen": -115.5818862915039,
      "logps/rejected": -269.8819885253906,
      "loss": 0.2988,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 1.0188477039337158,
      "rewards/margins": 1.640873908996582,
      "rewards/rejected": -0.6220260262489319,
      "step": 920
    },
    {
      "epoch": 0.8520384791571233,
      "grad_norm": 0.837735116481781,
      "learning_rate": 6.499967258100514e-08,
      "logits/chosen": -0.7608433365821838,
      "logits/rejected": -0.6938787698745728,
      "logps/chosen": -108.01826477050781,
      "logps/rejected": -226.8501434326172,
      "loss": 0.3083,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 0.8835248947143555,
      "rewards/margins": 1.5439478158950806,
      "rewards/rejected": -0.6604230403900146,
      "step": 930
    },
    {
      "epoch": 0.8612001832340815,
      "grad_norm": 0.5987643599510193,
      "learning_rate": 5.732924089870245e-08,
      "logits/chosen": -0.6554332971572876,
      "logits/rejected": -0.46006011962890625,
      "logps/chosen": -141.659423828125,
      "logps/rejected": -257.38958740234375,
      "loss": 0.2997,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.7329229712486267,
      "rewards/margins": 1.5281693935394287,
      "rewards/rejected": -0.795246422290802,
      "step": 940
    },
    {
      "epoch": 0.8703618873110398,
      "grad_norm": 0.7886483073234558,
      "learning_rate": 5.011275652893782e-08,
      "logits/chosen": -0.696278989315033,
      "logits/rejected": -0.6031322479248047,
      "logps/chosen": -92.44393157958984,
      "logps/rejected": -203.4006805419922,
      "loss": 0.2837,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 0.9351547360420227,
      "rewards/margins": 1.3968000411987305,
      "rewards/rejected": -0.46164530515670776,
      "step": 950
    },
    {
      "epoch": 0.8795235913879982,
      "grad_norm": 0.8714445233345032,
      "learning_rate": 4.3357619788127634e-08,
      "logits/chosen": -0.7752223014831543,
      "logits/rejected": -0.7101688981056213,
      "logps/chosen": -135.9874725341797,
      "logps/rejected": -251.438232421875,
      "loss": 0.3198,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.7239378094673157,
      "rewards/margins": 1.1686517000198364,
      "rewards/rejected": -0.444713830947876,
      "step": 960
    },
    {
      "epoch": 0.8886852954649564,
      "grad_norm": 0.7179840207099915,
      "learning_rate": 3.707075789273306e-08,
      "logits/chosen": -0.7246268391609192,
      "logits/rejected": -0.726759135723114,
      "logps/chosen": -98.63682556152344,
      "logps/rejected": -221.41909790039062,
      "loss": 0.3284,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 1.0240366458892822,
      "rewards/margins": 1.735687255859375,
      "rewards/rejected": -0.711650550365448,
      "step": 970
    },
    {
      "epoch": 0.8978469995419148,
      "grad_norm": 0.8668420314788818,
      "learning_rate": 3.125861785558015e-08,
      "logits/chosen": -0.8971655964851379,
      "logits/rejected": -0.7343107461929321,
      "logps/chosen": -121.60533142089844,
      "logps/rejected": -274.42510986328125,
      "loss": 0.3273,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.9639977216720581,
      "rewards/margins": 1.4876940250396729,
      "rewards/rejected": -0.52369624376297,
      "step": 980
    },
    {
      "epoch": 0.9070087036188731,
      "grad_norm": 0.8045462965965271,
      "learning_rate": 2.592715987461702e-08,
      "logits/chosen": -0.707330584526062,
      "logits/rejected": -0.6783408522605896,
      "logps/chosen": -161.0232391357422,
      "logps/rejected": -251.8162384033203,
      "loss": 0.3285,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 0.64153653383255,
      "rewards/margins": 1.0937219858169556,
      "rewards/rejected": -0.45218539237976074,
      "step": 990
    },
    {
      "epoch": 0.9161704076958315,
      "grad_norm": 0.7876152992248535,
      "learning_rate": 2.108185122088546e-08,
      "logits/chosen": -0.8192548751831055,
      "logits/rejected": -0.7370203733444214,
      "logps/chosen": -109.0053939819336,
      "logps/rejected": -241.18704223632812,
      "loss": 0.3077,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 0.7523542642593384,
      "rewards/margins": 1.3361942768096924,
      "rewards/rejected": -0.5838399529457092,
      "step": 1000
    },
    {
      "epoch": 0.9161704076958315,
      "eval_logits/chosen": -0.7753556370735168,
      "eval_logits/rejected": -0.5998021960258484,
      "eval_logps/chosen": -129.05845642089844,
      "eval_logps/rejected": -278.6715087890625,
      "eval_loss": 0.33283570408821106,
      "eval_rewards/accuracies": 0.9075144529342651,
      "eval_rewards/chosen": 0.5529729723930359,
      "eval_rewards/margins": 1.2503920793533325,
      "eval_rewards/rejected": -0.697419285774231,
      "eval_runtime": 411.7768,
      "eval_samples_per_second": 6.71,
      "eval_steps_per_second": 0.84,
      "step": 1000
    },
    {
      "epoch": 0.9253321117727897,
      "grad_norm": 0.9411417245864868,
      "learning_rate": 1.672766063197789e-08,
      "logits/chosen": -0.6959950923919678,
      "logits/rejected": -0.671855092048645,
      "logps/chosen": -124.23388671875,
      "logps/rejected": -235.0277862548828,
      "loss": 0.3114,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.9731966257095337,
      "rewards/margins": 1.4448806047439575,
      "rewards/rejected": -0.4716837406158447,
      "step": 1010
    },
    {
      "epoch": 0.934493815849748,
      "grad_norm": 0.7529547214508057,
      "learning_rate": 1.286905321672621e-08,
      "logits/chosen": -0.7256031632423401,
      "logits/rejected": -0.6639115214347839,
      "logps/chosen": -74.08079528808594,
      "logps/rejected": -190.6580047607422,
      "loss": 0.2967,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 1.259799838066101,
      "rewards/margins": 1.864816665649414,
      "rewards/rejected": -0.605016827583313,
      "step": 1020
    },
    {
      "epoch": 0.9436555199267064,
      "grad_norm": 0.8753419518470764,
      "learning_rate": 9.509985876349491e-09,
      "logits/chosen": -0.6738966703414917,
      "logits/rejected": -0.5317025184631348,
      "logps/chosen": -84.36373138427734,
      "logps/rejected": -239.25881958007812,
      "loss": 0.2962,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.9429308772087097,
      "rewards/margins": 1.350412368774414,
      "rewards/rejected": -0.40748143196105957,
      "step": 1030
    },
    {
      "epoch": 0.9528172240036646,
      "grad_norm": 0.8753317594528198,
      "learning_rate": 6.6539032467546885e-09,
      "logits/chosen": -0.7684177756309509,
      "logits/rejected": -0.7221516370773315,
      "logps/chosen": -195.75209045410156,
      "logps/rejected": -291.89117431640625,
      "loss": 0.2878,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 0.786228597164154,
      "rewards/margins": 1.300559163093567,
      "rewards/rejected": -0.5143305063247681,
      "step": 1040
    },
    {
      "epoch": 0.961978928080623,
      "grad_norm": 0.7298064827919006,
      "learning_rate": 4.303734166152706e-09,
      "logits/chosen": -0.6869091987609863,
      "logits/rejected": -0.7067757844924927,
      "logps/chosen": -120.041748046875,
      "logps/rejected": -194.99925231933594,
      "loss": 0.3014,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 1.13614022731781,
      "rewards/margins": 1.6430591344833374,
      "rewards/rejected": -0.5069188475608826,
      "step": 1050
    },
    {
      "epoch": 0.9711406321575813,
      "grad_norm": 0.6257432103157043,
      "learning_rate": 2.4618886716110676e-09,
      "logits/chosen": -0.7349929213523865,
      "logits/rejected": -0.6031204462051392,
      "logps/chosen": -102.07683563232422,
      "logps/rejected": -239.6611785888672,
      "loss": 0.3347,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.5327378511428833,
      "rewards/margins": 1.1345620155334473,
      "rewards/rejected": -0.6018242835998535,
      "step": 1060
    },
    {
      "epoch": 0.9803023362345397,
      "grad_norm": 0.6151149868965149,
      "learning_rate": 1.1302555276238579e-09,
      "logits/chosen": -0.8125348091125488,
      "logits/rejected": -0.6750482320785522,
      "logps/chosen": -64.71864318847656,
      "logps/rejected": -191.86572265625,
      "loss": 0.2674,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 1.005023717880249,
      "rewards/margins": 1.796847939491272,
      "rewards/rejected": -0.7918239831924438,
      "step": 1070
    },
    {
      "epoch": 0.9894640403114979,
      "grad_norm": 0.7649210691452026,
      "learning_rate": 3.102002892329536e-10,
      "logits/chosen": -0.6314386129379272,
      "logits/rejected": -0.5421333909034729,
      "logps/chosen": -96.51834869384766,
      "logps/rejected": -234.3397979736328,
      "loss": 0.2975,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 1.1401491165161133,
      "rewards/margins": 1.9231317043304443,
      "rewards/rejected": -0.7829826474189758,
      "step": 1080
    },
    {
      "epoch": 0.9986257443884563,
      "grad_norm": 0.7103332877159119,
      "learning_rate": 2.5639016871248366e-12,
      "logits/chosen": -0.7133598923683167,
      "logits/rejected": -0.5197003483772278,
      "logps/chosen": -163.91204833984375,
      "logps/rejected": -249.02682495117188,
      "loss": 0.3111,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 0.7740644216537476,
      "rewards/margins": 1.3091305494308472,
      "rewards/rejected": -0.5350660681724548,
      "step": 1090
    },
    {
      "epoch": 0.9995419147961521,
      "step": 1091,
      "total_flos": 0.0,
      "train_loss": 0.4306906612229719,
      "train_runtime": 8689.0202,
      "train_samples_per_second": 4.019,
      "train_steps_per_second": 0.126
    }
  ],
  "logging_steps": 10,
  "max_steps": 1091,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 100,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 1,
  "trial_name": null,
  "trial_params": null
}