update weights

config.json

@@ -8,7 +8,7 @@
   "hidden_act": "silu",
   "hidden_size": 5120,
   "initializer_range": 0.02,
-  "intermediate_size": 13696,
+  "intermediate_size": 14436,
   "max_position_embeddings": 32768,
   "max_window_layers": 35,
   "model_type": "qwen2",

model-00001-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d5c7d6bdded3e62e7786ec46f79bdaf04605cc6cd174cd5678068249c950d524
-size 3981821832
+oid sha256:6dae54bb1ace1fa5fbf4381acf3b151f88da07e556419d9ab1dc2be718f8bbcd
+size 3980417584

model-00002-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a5f84db35ce352f487c8a6c640358e0ded3d22dcd197118f8a14b18bcf9cc935
-size 3988050688
+oid sha256:658b8b195751b07b866a3bebaa56ec9f0f74d392173eb56a69f147bff72f0ed8
+size 3967789304

model-00003-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e8f5d5772ee940d9631f6f0350af2d1300f146b76c8555114376d5cb6aac9a00
-size 1707574440
+oid sha256:cebc8cf4f30c67a07134c5b6f1394d38acf985cbd819ab5ddab8baca5f9afe33
+size 1933732848

model.safetensors.index.json

@@ -1,6 +1,6 @@
 {
   "metadata": {
-    "total_size": 9677281280
+    "total_size": 9881774080
   },
   "weight_map": {
     "lm_head.weight": "model-00003-of-00003.safetensors",
@@ -228,16 +228,16 @@
     "model.layers.13.self_attn.v_proj.qzeros": "model-00001-of-00003.safetensors",
     "model.layers.13.self_attn.v_proj.scales": "model-00001-of-00003.safetensors",
     "model.layers.14.input_layernorm.weight": "model-00002-of-00003.safetensors",
-    "model.layers.14.mlp.down_proj.bias": "model-00001-of-00003.safetensors",
-    "model.layers.14.mlp.down_proj.g_idx": "model-00001-of-00003.safetensors",
-    "model.layers.14.mlp.down_proj.qweight": "model-00001-of-00003.safetensors",
-    "model.layers.14.mlp.down_proj.qzeros": "model-00001-of-00003.safetensors",
-    "model.layers.14.mlp.down_proj.scales": "model-00001-of-00003.safetensors",
-    "model.layers.14.mlp.gate_proj.bias": "model-00001-of-00003.safetensors",
-    "model.layers.14.mlp.gate_proj.g_idx": "model-00001-of-00003.safetensors",
-    "model.layers.14.mlp.gate_proj.qweight": "model-00001-of-00003.safetensors",
-    "model.layers.14.mlp.gate_proj.qzeros": "model-00001-of-00003.safetensors",
-    "model.layers.14.mlp.gate_proj.scales": "model-00001-of-00003.safetensors",
+    "model.layers.14.mlp.down_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.down_proj.g_idx": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.down_proj.qweight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.down_proj.qzeros": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.down_proj.scales": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.gate_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.gate_proj.g_idx": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.gate_proj.qweight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.gate_proj.qzeros": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.gate_proj.scales": "model-00002-of-00003.safetensors",
     "model.layers.14.mlp.up_proj.bias": "model-00002-of-00003.safetensors",
     "model.layers.14.mlp.up_proj.g_idx": "model-00002-of-00003.safetensors",
     "model.layers.14.mlp.up_proj.qweight": "model-00002-of-00003.safetensors",
@@ -1152,7 +1152,7 @@
     "model.layers.36.self_attn.v_proj.qweight": "model-00002-of-00003.safetensors",
     "model.layers.36.self_attn.v_proj.qzeros": "model-00002-of-00003.safetensors",
     "model.layers.36.self_attn.v_proj.scales": "model-00002-of-00003.safetensors",
-    "model.layers.37.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.37.input_layernorm.weight": "model-00003-of-00003.safetensors",
     "model.layers.37.mlp.down_proj.bias": "model-00002-of-00003.safetensors",
     "model.layers.37.mlp.down_proj.g_idx": "model-00002-of-00003.safetensors",
     "model.layers.37.mlp.down_proj.qweight": "model-00002-of-00003.safetensors",
@@ -1163,12 +1163,12 @@
     "model.layers.37.mlp.gate_proj.qweight": "model-00002-of-00003.safetensors",
     "model.layers.37.mlp.gate_proj.qzeros": "model-00002-of-00003.safetensors",
     "model.layers.37.mlp.gate_proj.scales": "model-00002-of-00003.safetensors",
-    "model.layers.37.mlp.up_proj.bias": "model-00002-of-00003.safetensors",
-    "model.layers.37.mlp.up_proj.g_idx": "model-00002-of-00003.safetensors",
-    "model.layers.37.mlp.up_proj.qweight": "model-00002-of-00003.safetensors",
-    "model.layers.37.mlp.up_proj.qzeros": "model-00002-of-00003.safetensors",
-    "model.layers.37.mlp.up_proj.scales": "model-00002-of-00003.safetensors",
-    "model.layers.37.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.37.mlp.up_proj.bias": "model-00003-of-00003.safetensors",
+    "model.layers.37.mlp.up_proj.g_idx": "model-00003-of-00003.safetensors",
+    "model.layers.37.mlp.up_proj.qweight": "model-00003-of-00003.safetensors",
+    "model.layers.37.mlp.up_proj.qzeros": "model-00003-of-00003.safetensors",
+    "model.layers.37.mlp.up_proj.scales": "model-00003-of-00003.safetensors",
+    "model.layers.37.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
     "model.layers.37.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
     "model.layers.37.self_attn.k_proj.g_idx": "model-00002-of-00003.safetensors",
     "model.layers.37.self_attn.k_proj.qweight": "model-00002-of-00003.safetensors",
@@ -1189,43 +1189,43 @@
     "model.layers.37.self_attn.v_proj.qweight": "model-00002-of-00003.safetensors",
     "model.layers.37.self_attn.v_proj.qzeros": "model-00002-of-00003.safetensors",
     "model.layers.37.self_attn.v_proj.scales": "model-00002-of-00003.safetensors",
-    "model.layers.38.input_layernorm.weight": "model-00002-of-00003.safetensors",
-    "model.layers.38.mlp.down_proj.bias": "model-00002-of-00003.safetensors",
-    "model.layers.38.mlp.down_proj.g_idx": "model-00002-of-00003.safetensors",
-    "model.layers.38.mlp.down_proj.qweight": "model-00002-of-00003.safetensors",
-    "model.layers.38.mlp.down_proj.qzeros": "model-00002-of-00003.safetensors",
-    "model.layers.38.mlp.down_proj.scales": "model-00002-of-00003.safetensors",
-    "model.layers.38.mlp.gate_proj.bias": "model-00002-of-00003.safetensors",
-    "model.layers.38.mlp.gate_proj.g_idx": "model-00002-of-00003.safetensors",
-    "model.layers.38.mlp.gate_proj.qweight": "model-00002-of-00003.safetensors",
-    "model.layers.38.mlp.gate_proj.qzeros": "model-00002-of-00003.safetensors",
-    "model.layers.38.mlp.gate_proj.scales": "model-00002-of-00003.safetensors",
-    "model.layers.38.mlp.up_proj.bias": "model-00002-of-00003.safetensors",
-    "model.layers.38.mlp.up_proj.g_idx": "model-00002-of-00003.safetensors",
-    "model.layers.38.mlp.up_proj.qweight": "model-00002-of-00003.safetensors",
-    "model.layers.38.mlp.up_proj.qzeros": "model-00002-of-00003.safetensors",
-    "model.layers.38.mlp.up_proj.scales": "model-00002-of-00003.safetensors",
-    "model.layers.38.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
-    "model.layers.38.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
-    "model.layers.38.self_attn.k_proj.g_idx": "model-00002-of-00003.safetensors",
-    "model.layers.38.self_attn.k_proj.qweight": "model-00002-of-00003.safetensors",
-    "model.layers.38.self_attn.k_proj.qzeros": "model-00002-of-00003.safetensors",
-    "model.layers.38.self_attn.k_proj.scales": "model-00002-of-00003.safetensors",
-    "model.layers.38.self_attn.o_proj.bias": "model-00002-of-00003.safetensors",
-    "model.layers.38.self_attn.o_proj.g_idx": "model-00002-of-00003.safetensors",
-    "model.layers.38.self_attn.o_proj.qweight": "model-00002-of-00003.safetensors",
-    "model.layers.38.self_attn.o_proj.qzeros": "model-00002-of-00003.safetensors",
-    "model.layers.38.self_attn.o_proj.scales": "model-00002-of-00003.safetensors",
-    "model.layers.38.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
-    "model.layers.38.self_attn.q_proj.g_idx": "model-00002-of-00003.safetensors",
-    "model.layers.38.self_attn.q_proj.qweight": "model-00002-of-00003.safetensors",
-    "model.layers.38.self_attn.q_proj.qzeros": "model-00002-of-00003.safetensors",
-    "model.layers.38.self_attn.q_proj.scales": "model-00002-of-00003.safetensors",
-    "model.layers.38.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
-    "model.layers.38.self_attn.v_proj.g_idx": "model-00002-of-00003.safetensors",
-    "model.layers.38.self_attn.v_proj.qweight": "model-00002-of-00003.safetensors",
-    "model.layers.38.self_attn.v_proj.qzeros": "model-00002-of-00003.safetensors",
-    "model.layers.38.self_attn.v_proj.scales": "model-00002-of-00003.safetensors",
+    "model.layers.38.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.38.mlp.down_proj.bias": "model-00003-of-00003.safetensors",
+    "model.layers.38.mlp.down_proj.g_idx": "model-00003-of-00003.safetensors",
+    "model.layers.38.mlp.down_proj.qweight": "model-00003-of-00003.safetensors",
+    "model.layers.38.mlp.down_proj.qzeros": "model-00003-of-00003.safetensors",
+    "model.layers.38.mlp.down_proj.scales": "model-00003-of-00003.safetensors",
+    "model.layers.38.mlp.gate_proj.bias": "model-00003-of-00003.safetensors",
+    "model.layers.38.mlp.gate_proj.g_idx": "model-00003-of-00003.safetensors",
+    "model.layers.38.mlp.gate_proj.qweight": "model-00003-of-00003.safetensors",
+    "model.layers.38.mlp.gate_proj.qzeros": "model-00003-of-00003.safetensors",
+    "model.layers.38.mlp.gate_proj.scales": "model-00003-of-00003.safetensors",
+    "model.layers.38.mlp.up_proj.bias": "model-00003-of-00003.safetensors",
+    "model.layers.38.mlp.up_proj.g_idx": "model-00003-of-00003.safetensors",
+    "model.layers.38.mlp.up_proj.qweight": "model-00003-of-00003.safetensors",
+    "model.layers.38.mlp.up_proj.qzeros": "model-00003-of-00003.safetensors",
+    "model.layers.38.mlp.up_proj.scales": "model-00003-of-00003.safetensors",
+    "model.layers.38.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.38.self_attn.k_proj.bias": "model-00003-of-00003.safetensors",
+    "model.layers.38.self_attn.k_proj.g_idx": "model-00003-of-00003.safetensors",
+    "model.layers.38.self_attn.k_proj.qweight": "model-00003-of-00003.safetensors",
+    "model.layers.38.self_attn.k_proj.qzeros": "model-00003-of-00003.safetensors",
+    "model.layers.38.self_attn.k_proj.scales": "model-00003-of-00003.safetensors",
+    "model.layers.38.self_attn.o_proj.bias": "model-00003-of-00003.safetensors",
+    "model.layers.38.self_attn.o_proj.g_idx": "model-00003-of-00003.safetensors",
+    "model.layers.38.self_attn.o_proj.qweight": "model-00003-of-00003.safetensors",
+    "model.layers.38.self_attn.o_proj.qzeros": "model-00003-of-00003.safetensors",
+    "model.layers.38.self_attn.o_proj.scales": "model-00003-of-00003.safetensors",
+    "model.layers.38.self_attn.q_proj.bias": "model-00003-of-00003.safetensors",
+    "model.layers.38.self_attn.q_proj.g_idx": "model-00003-of-00003.safetensors",
+    "model.layers.38.self_attn.q_proj.qweight": "model-00003-of-00003.safetensors",
+    "model.layers.38.self_attn.q_proj.qzeros": "model-00003-of-00003.safetensors",
+    "model.layers.38.self_attn.q_proj.scales": "model-00003-of-00003.safetensors",
+    "model.layers.38.self_attn.v_proj.bias": "model-00003-of-00003.safetensors",
+    "model.layers.38.self_attn.v_proj.g_idx": "model-00003-of-00003.safetensors",
+    "model.layers.38.self_attn.v_proj.qweight": "model-00003-of-00003.safetensors",
+    "model.layers.38.self_attn.v_proj.qzeros": "model-00003-of-00003.safetensors",
+    "model.layers.38.self_attn.v_proj.scales": "model-00003-of-00003.safetensors",
     "model.layers.39.input_layernorm.weight": "model-00003-of-00003.safetensors",
     "model.layers.39.mlp.down_proj.bias": "model-00003-of-00003.safetensors",
     "model.layers.39.mlp.down_proj.g_idx": "model-00003-of-00003.safetensors",
@@ -1243,11 +1243,11 @@
     "model.layers.39.mlp.up_proj.qzeros": "model-00003-of-00003.safetensors",
     "model.layers.39.mlp.up_proj.scales": "model-00003-of-00003.safetensors",
     "model.layers.39.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
-    "model.layers.39.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
-    "model.layers.39.self_attn.k_proj.g_idx": "model-00002-of-00003.safetensors",
-    "model.layers.39.self_attn.k_proj.qweight": "model-00002-of-00003.safetensors",
-    "model.layers.39.self_attn.k_proj.qzeros": "model-00002-of-00003.safetensors",
-    "model.layers.39.self_attn.k_proj.scales": "model-00002-of-00003.safetensors",
+    "model.layers.39.self_attn.k_proj.bias": "model-00003-of-00003.safetensors",
+    "model.layers.39.self_attn.k_proj.g_idx": "model-00003-of-00003.safetensors",
+    "model.layers.39.self_attn.k_proj.qweight": "model-00003-of-00003.safetensors",
+    "model.layers.39.self_attn.k_proj.qzeros": "model-00003-of-00003.safetensors",
+    "model.layers.39.self_attn.k_proj.scales": "model-00003-of-00003.safetensors",
     "model.layers.39.self_attn.o_proj.bias": "model-00003-of-00003.safetensors",
     "model.layers.39.self_attn.o_proj.g_idx": "model-00003-of-00003.safetensors",
     "model.layers.39.self_attn.o_proj.qweight": "model-00003-of-00003.safetensors",