Training in progress, step 100

adapter_config.json

@@ -15,17 +15,17 @@
   "megatron_core": "megatron.core",
   "modules_to_save": null,
   "peft_type": "LORA",
-  "r": 64,
+  "r": 128,
   "rank_pattern": {},
   "revision": null,
   "target_modules": [
     "up_proj",
-    "down_proj",
-    "o_proj",
-    "k_proj",
+    "q_proj",
     "v_proj",
+    "k_proj",
     "gate_proj",
-    "q_proj"
+    "down_proj",
+    "o_proj"
   ],
   "task_type": "CAUSAL_LM",
   "use_rslora": false

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:535e080e3a598aa5b85dadb5c4fd070e75f03dc507f5fef18b6ba8c29b2834c4
-size 639691872
+oid sha256:50aefb9eeac7ccef6381ee4dc413f2db08d745faf0f6f0b7772260c4fff4416d
+size 1279323952

angle.config

@@ -5,18 +5,18 @@
   "pooling_strategy": "cls",
   "lora_config_kwargs": {
     "task_type": "CAUSAL_LM",
-    "r": 64,
+    "r": 128,
     "lora_alpha": 8,
     "lora_dropout": 0.05,
     "bias": "none",
     "target_modules": [
-      "down_proj",
-      "o_proj",
-      "v_proj",
       "k_proj",
-      "up_proj",
+      "q_proj",
+      "v_proj",
+      "o_proj",
       "gate_proj",
-      "q_proj"
+      "down_proj",
+      "up_proj"
     ]
   },
   "apply_lora": 1

runs/Mar26_14-12-55_instance-20240226-074643/events.out.tfevents.1711462382.instance-20240226-074643.1699661.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:26faf6d6cdb71cd3dca5706f56fa8c590500cc4b8562416bef8959505c4f5621
+size 5413

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0fb9527e14cb8b3ca4d8e0759e3c1bc0a71a0a26b94f93e690e0b6463819110d
+oid sha256:b8c41d6589f3f961edd3468c3af13c3c5dd8f49872630e845f622fd35b2314b2
 size 5048