Upload folder using huggingface_hub

README.md

@@ -0,0 +1,44 @@
+---
+license: apache-2.0
+base_model: alpindale/WizardLM-2-8x22B
+---
+
+# This repo contains the copy of the original quantized to FP8. Original: [rAIfle/SorcererLM-8x22b-bf16](https://huggingface.co/rAIfle/SorcererLM-8x22b-bf16)
+
+# SorcererLM-8x22b-bf16
+
+<img src="https://files.catbox.moe/1kohx8.png" width="400"/>
+
+<audio controls src="https://cdn-uploads.huggingface.co/production/uploads/6569a4ed2419be6072890cf8/L_uGojVkNUsK6QHvWgs9o.mpga"></audio>
+
+Oh boy, here we go. Low-rank (`r=16, alpha=32`) 16bit-LoRA on top of [WizardLM-2-8x22B](https://huggingface.co/alpindale/WizardLM-2-8x22B), trained on 2 epochs of (cleaned & deduped) c2-logs. As far as I can tell, this is an upgrade from `WizardLM-2-8x22B` for RP purposes.
+
+Alongside this ready-to-use release I'm also releasing [the LoRA itself](https://huggingface.co/rAIfle/SorcererLM-8x22b-epoch2-LoRA) as well as [the earlier `epoch1`-checkpoint of the LoRA](https://huggingface.co/rAIfle/SorcererLM-8x22b-epoch1-LoRA).
+
+## Why A LoRA?
+
+The choice was fully intentional. I briefly considered a FFT but for this particular use-case a LoRA seemed a better fit. `WizardLM-2-8x22B` is smart by itself but its used vocabulary leaves much to be desired when it comes to RP. By training a low-rank LoRA on top of it to teach it some of Claude's writing style, we remedy that.
+
+## Prompting
+
+- Use the templates in [Quant-Cartel/Recommended-Settings](https://huggingface.co/Quant-Cartel/Recommended-Settings) under the `SorcererLM`-folder.
+- Or Vicuna 1.1 and a sane context template. It's somewhat sensitive to samplers, I'd recommend Temperature 1, MinP 0.05 and a dash of DRY but YMMV. Shorter prompts seem to work better, too.
+
+## Quantized Versions
+
+- [iMat GGUFs](https://huggingface.co/Quant-Cartel/SorcererLM-8x22b-iMat-GGUF)
+- [longcal exl2s](https://huggingface.co/Quant-Cartel/SorcererLM-8x22b-exl2-longcal)
+
+## Acknowledgments
+
+The main shoutout I want to make is to my [Cartel](https://huggingface.co/Quant-Cartel) bros, [Envoid](https://huggingface.co/Envoid) and particularly [I^2](https://huggingface.co/InferenceIllusionist), for being amazing. I count this as a team effort, so they deserve kudos too if you like this.
+
+
+## Training
+
+Trained using [qlora-pipe](https://github.com/tdrussell/qlora-pipe). Configs included in the `train`-subfolder.
+
+## Safety
+
+... n/a
+

config.json

@@ -0,0 +1,38 @@
+{
+  "_name_or_path": "/workspace/SorcererLM-8x22b-bf16",
+  "architectures": [
+    "MixtralForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "hidden_act": "silu",
+  "hidden_size": 6144,
+  "initializer_range": 0.02,
+  "intermediate_size": 16384,
+  "max_position_embeddings": 65536,
+  "model_type": "mixtral",
+  "num_attention_heads": 48,
+  "num_experts_per_tok": 2,
+  "num_hidden_layers": 56,
+  "num_key_value_heads": 8,
+  "num_local_experts": 8,
+  "output_router_logits": false,
+  "quantization_config": {
+    "activation_scheme": "dynamic",
+    "ignored_layers": [
+      "lm_head"
+    ],
+    "quant_method": "fp8"
+  },
+  "rms_norm_eps": 1e-05,
+  "rope_theta": 1000000,
+  "router_aux_loss_coef": 0.001,
+  "router_jitter_noise": 0.0,
+  "sliding_window": null,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.45.1",
+  "use_cache": false,
+  "vocab_size": 32000
+}

generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "transformers_version": "4.45.1"
+}

model-00001-of-00029.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:587c2f3a0f78ef7637e8887e91c157290791e03327de7b2ff006917739055f35
+size 4998698216

model-00002-of-00029.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:460b9fb867247bd7f41fafbd607f9d0e24246fdbabb45f3f30e758eb4031c24d
+size 4907497516

model-00003-of-00029.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e60f29512c385981bebe1505ddf048ba289760a52787c5bf9d89eb35c2a5133d
+size 4907497516

model-00004-of-00029.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8adbf88761a0a1545a569ca58ffd06f966756e33bb28db74199869eac24345b5
+size 4907497516

model-00005-of-00029.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0504adcba55dbe8b02ab3797f5e4903004160746f35360ad0d508efc4201c1dc
+size 4907497516

model-00006-of-00029.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a4609cf8fd0ef9b717a4ed0897352b69725ce9d377921775964e432126aa14df
+size 4907497620

model-00007-of-00029.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3be349ad5002b18e5bbf94c30ba84a0045fe284f3705add969d62bb18234fa8c
+size 4907497636

model-00008-of-00029.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f503b7c4c4616ee9c09ed73e70d38a87bbdab22eb66a4e59ac17a3dcb61064d4
+size 4907497636

model-00009-of-00029.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a3d395f22be572393cc9d66313a89ca522908f7041d654a1a6bc774c9b70a823
+size 4907497636

model-00010-of-00029.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:98e58c87b1247b7844e9c2beacc77dae244daa4ce05350deaa40e5b0396b4dac
+size 4907497636

model-00011-of-00029.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:72ad58a2691724f177fd72f0b7bda8f7ab0dfdacae607526854577969eef3954
+size 4907497636

model-00012-of-00029.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5a7f368d22983b8f2fd3db61a0155a4f57630a36c8d0eb52876539da28d66596
+size 4907497636

model-00013-of-00029.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9177a02563fca43527128251ccdc5330821bd50b682f447dc8684c651fdf921c
+size 4907497636

model-00014-of-00029.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6459d0412be024efde6407ff49cdea991bbd4511e7b108fa9302f01ae36af767
+size 4907497636

model-00015-of-00029.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d244322cd3af6e6045bd8933bd38175d87f821a745e3eedeae986ca0465895d4
+size 4907497636

model-00016-of-00029.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fd962dc0f066bc27caebe936fe26e126c93633c3037aa2a6b15597ca729b7040
+size 4907497636

model-00017-of-00029.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:06f215e34ae2955c248ed4dbb51020849eb0d59304a4e78b87045532ae8ae17b
+size 4907497636

model-00018-of-00029.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f705b2406ce8169e39a46045429c35f3a54d0126431f4bf7f6aeedba727ea7e9
+size 4907497636

model-00019-of-00029.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e22b599eff7ec65e8bdee295de65fd2d18ffb96f7a018de9157838db7240b78c
+size 4907497636

model-00020-of-00029.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:26334d2161cd504676f652645a9968a9f5df5be016e1de6993893059392a35a9
+size 4907497636

model-00021-of-00029.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a0e250dac8b8d2b3cd293ff27ab8aac70858ff8d97f0050542e64049ce96a20a
+size 4907497636

model-00022-of-00029.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8e838a3a0e94b43b64d05e6dc63f778916832b39d9cb76f2a76142917e574a97
+size 4970362840

model-00023-of-00029.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b98332bf509bd1dcc1d4d7921e30952b52cf43911a5f7f1f4865d4910b07de5a
+size 4995577824

model-00024-of-00029.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2ba09b35c1381d04ee7e550d62fea7638cebd863f82ae0099c17216c0e7fe47d
+size 4970412220

model-00025-of-00029.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ea449237779634c54581673ed4d020f4dad04ab3778bd83f17b89a2949d160ba
+size 4907472844

model-00026-of-00029.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6056699e0593fa1a3b4a432b34c3abd9863df7380077f7d4719bb8660a5f9186
+size 4907497636

model-00027-of-00029.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d25efecad9156b4f60f8fe69b69fecfe95aaece84d8ea38221a09dc40bd7bf17
+size 4907497636

model-00028-of-00029.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7f5dc70c6993f2d0a882826f81a3a429374eb826c28fd67e585e9697523f389a
+size 4907497636

model-00029-of-00029.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e7430dc5d079f75d2532d29191d1ff0eb82d9c7ef6553a702f0cd588e53bb173
+size 3299988060

special_tokens_map.json

@@ -0,0 +1,30 @@
+{
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dadfd56d766715c61d2ef780a525ab43b8e6da4de6865bda3d95fdef5e134055
+size 493443

tokenizer_config.json

@@ -0,0 +1,45 @@
+{
+  "add_bos_token": true,
+  "add_eos_token": false,
+  "add_prefix_space": true,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [],
+  "bos_token": "<s>",
+  "chat_template": "{% if messages[0]['role'] == 'system' %}{% set loop_messages = messages[1:] %}{{ messages[0]['content'].strip() }}{% else %}{% set loop_messages = messages %}{{ 'A chat between a curious user and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the user\\'s questions.' }}{% endif %}{% for message in loop_messages %}{% if loop.index0 == 0 %}{% if message['role'] == 'system' or message['role'] == 'user' %}{{ ' USER: ' + message['content'].strip() }}{% else %}{{ ' ASSISTANT: ' + message['content'].strip() + eos_token }}{% endif %}{% else %}{% if message['role'] == 'system' or message['role'] == 'user' %}{{ '\nUSER: ' + message['content'].strip() }}{% else %}{{ ' ASSISTANT: ' + message['content'].strip() + eos_token }}{% endif %}{% endif %}{% endfor %}{% if add_generation_prompt %}{{ ' ASSISTANT:' }}{% endif %}",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "legacy": true,
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<unk>",
+  "padding_side": "right",
+  "sp_model_kwargs": {},
+  "spaces_between_special_tokens": false,
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": true
+}