Training in progress, step 500

config.json

@@ -1,5 +1,5 @@
 {
-  "_name_or_path": "ai21labs/Jamba-v0.1",
+  "_name_or_path": "PyJamba1B/checkpoint-265000",
   "architectures": [
     "JambaForCausalLM"
   ],
@@ -7,9 +7,9 @@
   "attn_layer_offset": 4,
   "attn_layer_period": 8,
   "auto_map": {
-    "AutoConfig": "ai21labs/Jamba-v0.1--configuration_jamba.JambaConfig",
+    "AutoConfig": "configuration_jamba.JambaConfig",
     "AutoModel": "ai21labs/Jamba-v0.1--modeling_jamba.JambaModel",
-    "AutoModelForCausalLM": "ai21labs/Jamba-v0.1--modeling_jamba.JambaForCausalLM",
+    "AutoModelForCausalLM": "modeling_jamba.JambaForCausalLM",
     "AutoModelForSequenceClassification": "ai21labs/Jamba-v0.1--model.JambaForSequenceClassification"
   },
   "bos_token_id": 1,

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5d8222927fc1b08840d8537b70188d9b75bd8a49b1d2b639d6eae1c8285d668e
+oid sha256:5ef35299fdad7c9943ff2813779e8c46ea8e50805c3c469ef39758059b404624
 size 2039780704

tokenizer_config.json

@@ -38,10 +38,17 @@
   "bos_token": "<|startoftext|>",
   "clean_up_tokenization_spaces": false,
   "eos_token": "<|endoftext|>",
+  "max_length": 8192,
   "model_max_length": 1000000000000000019884624838656,
+  "pad_to_multiple_of": null,
   "pad_token": "<|pad|>",
+  "pad_token_type_id": 0,
+  "padding_side": "left",
   "spaces_between_special_tokens": false,
+  "stride": 0,
   "tokenizer_class": "LlamaTokenizer",
+  "truncation_side": "right",
+  "truncation_strategy": "longest_first",
   "unk_token": "<|unk|>",
   "use_default_system_prompt": false
 }

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5f8e26607c647da36be57ccc25c5bc6bf9f9ad2ed48631a1620e3dafcd826f6e
+oid sha256:aab7473090ad215f01ae27b884f529a64b7e25ed3088c0e3bc5b9257406e4e92
 size 4920