Model save

README.md

@@ -2,14 +2,10 @@
 license: apache-2.0
 library_name: peft
 tags:
-- alignment-handbook
-- generated_from_trainer
 - trl
 - dpo
+- generated_from_trainer
 base_model: alignment-handbook/zephyr-7b-sft-full
-datasets:
-- updated
-- original
 model-index:
 - name: nash_dpo_doff_no_golden_iter_1
   results: []
@@ -20,17 +16,17 @@ should probably proofread and complete it, then remove this comment. -->
 
 # nash_dpo_doff_no_golden_iter_1
 
-This model is a fine-tuned version of [alignment-handbook/zephyr-7b-sft-full](https://huggingface.co/alignment-handbook/zephyr-7b-sft-full) on the updated and the original datasets.
+This model is a fine-tuned version of [alignment-handbook/zephyr-7b-sft-full](https://huggingface.co/alignment-handbook/zephyr-7b-sft-full) on the None dataset.
 It achieves the following results on the evaluation set:
+- Logits/chosen: -2.7444
+- Logits/rejected: -2.6646
+- Logps/chosen: -288.1525
+- Logps/rejected: -266.5563
 - Loss: 0.6749
-- Rewards/chosen: -0.0411
-- Rewards/rejected: -0.0904
 - Rewards/accuracies: 0.6100
+- Rewards/chosen: -0.0411
 - Rewards/margins: 0.0493
-- Logps/rejected: -266.5563
-- Logps/chosen: -288.1525
-- Logits/rejected: -2.6646
-- Logits/chosen: -2.7444
+- Rewards/rejected: -0.0904
 
 ## Model description
 
@@ -65,9 +61,9 @@ The following hyperparameters were used during training:
 
 ### Training results
 
-| Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
-|:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
-| 0.6794        | 0.65  | 100  | 0.6749          | -0.0411        | -0.0904          | 0.6100             | 0.0493          | -266.5563      | -288.1525    | -2.6646         | -2.7444       |
+| Training Loss | Epoch | Step | Logits/chosen | Logits/rejected | Logps/chosen | Logps/rejected | Validation Loss | Rewards/accuracies | Rewards/chosen | Rewards/margins | Rewards/rejected |
+|:-------------:|:-----:|:----:|:-------------:|:---------------:|:------------:|:--------------:|:---------------:|:------------------:|:--------------:|:---------------:|:----------------:|
+| 0.6794        | 0.65  | 100  | -2.7444       | -2.6646         | -288.1525    | -266.5563      | 0.6749          | 0.6100             | -0.0411        | 0.0493          | -0.0904          |
 
 
 ### Framework versions

adapter_config.json

@@ -20,12 +20,12 @@
   "revision": null,
   "target_modules": [
     "gate_proj",
-    "up_proj",
-    "down_proj",
+    "o_proj",
     "k_proj",
     "q_proj",
     "v_proj",
-    "o_proj"
+    "down_proj",
+    "up_proj"
   ],
   "task_type": "CAUSAL_LM"
 }

all_results.json

@@ -1,8 +1,21 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.682184520889731,
-    "train_runtime": 6533.4659,
+    "eval_logits/chosen": -2.7313623428344727,
+    "eval_logits/rejected": -2.6492152214050293,
+    "eval_logps/chosen": -291.2025146484375,
+    "eval_logps/rejected": -270.01739501953125,
+    "eval_loss": 0.6726499795913696,
+    "eval_rewards/accuracies": 0.6100000143051147,
+    "eval_rewards/chosen": -0.07162749022245407,
+    "eval_rewards/margins": 0.05338801071047783,
+    "eval_rewards/rejected": -0.1250154972076416,
+    "eval_runtime": 393.8336,
+    "eval_samples": 2000,
+    "eval_samples_per_second": 5.078,
+    "eval_steps_per_second": 0.635,
+    "train_loss": 0.0,
+    "train_runtime": 0.012,
     "train_samples": 19600,
-    "train_samples_per_second": 3.0,
-    "train_steps_per_second": 0.023
+    "train_samples_per_second": 1639445.565,
+    "train_steps_per_second": 12797.713
 }

eval_results.json

@@ -0,0 +1,16 @@
+{
+    "epoch": 1.0,
+    "eval_logits/chosen": -2.7313623428344727,
+    "eval_logits/rejected": -2.6492152214050293,
+    "eval_logps/chosen": -291.2025146484375,
+    "eval_logps/rejected": -270.01739501953125,
+    "eval_loss": 0.6726499795913696,
+    "eval_rewards/accuracies": 0.6100000143051147,
+    "eval_rewards/chosen": -0.07162749022245407,
+    "eval_rewards/margins": 0.05338801071047783,
+    "eval_rewards/rejected": -0.1250154972076416,
+    "eval_runtime": 393.8336,
+    "eval_samples": 2000,
+    "eval_samples_per_second": 5.078,
+    "eval_steps_per_second": 0.635
+}

train_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.682184520889731,
-    "train_runtime": 6533.4659,
+    "train_loss": 0.0,
+    "train_runtime": 0.012,
     "train_samples": 19600,
-    "train_samples_per_second": 3.0,
-    "train_steps_per_second": 0.023
+    "train_samples_per_second": 1639445.565,
+    "train_steps_per_second": 12797.713
 }

trainer_state.json

@@ -252,10 +252,10 @@
       "epoch": 1.0,
       "step": 153,
       "total_flos": 0.0,
-      "train_loss": 0.682184520889731,
-      "train_runtime": 6533.4659,
-      "train_samples_per_second": 3.0,
-      "train_steps_per_second": 0.023
+      "train_loss": 0.0,
+      "train_runtime": 0.012,
+      "train_samples_per_second": 1639445.565,
+      "train_steps_per_second": 12797.713
     }
   ],
   "logging_steps": 10,

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8b97ce1508a83f54f4a01efc85e5b209fe67bdb7f873ee5e2fe6aca2d8f9b412
+oid sha256:9b0bfa9e38082baebb0956946fc19884374c70a778f9edf69afbe947876ecc28
 size 4920