Kukedlc

/

NeuralSirKrishna-7b

like 4

Hi Thejasxl, the context windows is the same among the others mistral models, wich is 4096. If you want to see the context windows of any model, you can check de config.json file.
It´s look like this:
{
"_name_or_path": "Kukedlc/NeuralSirKrishna-7b",
"architectures": [
"MistralForCausalLM"
],
"attention_dropout": 0.0,
"bos_token_id": 1,
"eos_token_id": 2,
"hidden_act": "silu",
"hidden_size": 4096,
"initializer_range": 0.02,
"intermediate_size": 14336,
"max_position_embeddings": 32768,
"model_type": "mistral",
"num_attention_heads": 32,
"num_hidden_layers": 32,
"num_key_value_heads": 8,
"rms_norm_eps": 1e-05,
"rope_theta": 10000.0,
"sliding_window": 4096, # This is the context window
"tie_word_embeddings": false,
"torch_dtype": "bfloat16",
"transformers_version": "4.38.2",
"use_cache": true,
"vocab_size": 32000
}