|
main: build = 3037 (cce3dcff) |
|
main: built with cc (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0 for x86_64-linux-gnu |
|
main: seed = 1717008037 |
|
llama_model_loader: loaded meta data with 25 key-value pairs and 507 tensors from Codestral-22B-v0.1-hf-IMat-GGUF/Codestral-22B-v0.1-hf.gguf (version GGUF V3 (latest)) |
|
llama_model_loader: Dumping metadata keys/values. Note: KV overrides do not apply in this output. |
|
llama_model_loader: - kv 0: general.architecture str = llama |
|
llama_model_loader: - kv 1: general.name str = Codestral-22B-v0.1-hf |
|
llama_model_loader: - kv 2: llama.block_count u32 = 56 |
|
llama_model_loader: - kv 3: llama.context_length u32 = 32768 |
|
llama_model_loader: - kv 4: llama.embedding_length u32 = 6144 |
|
llama_model_loader: - kv 5: llama.feed_forward_length u32 = 16384 |
|
llama_model_loader: - kv 6: llama.attention.head_count u32 = 48 |
|
llama_model_loader: - kv 7: llama.attention.head_count_kv u32 = 8 |
|
llama_model_loader: - kv 8: llama.rope.freq_base f32 = 1000000.000000 |
|
llama_model_loader: - kv 9: llama.attention.layer_norm_rms_epsilon f32 = 0.000010 |
|
llama_model_loader: - kv 10: general.file_type u32 = 0 |
|
llama_model_loader: - kv 11: llama.vocab_size u32 = 32768 |
|
llama_model_loader: - kv 12: llama.rope.dimension_count u32 = 128 |
|
llama_model_loader: - kv 13: tokenizer.ggml.add_space_prefix bool = true |
|
llama_model_loader: - kv 14: tokenizer.ggml.model str = llama |
|
llama_model_loader: - kv 15: tokenizer.ggml.pre str = default |
|
llama_model_loader: - kv 16: tokenizer.ggml.tokens arr[str,32768] = ["<unk>", "<s>", "</s>", "[INST]", "[... |
|
llama_model_loader: - kv 17: tokenizer.ggml.scores arr[f32,32768] = [0.000000, 0.000000, 0.000000, 0.0000... |
|
llama_model_loader: - kv 18: tokenizer.ggml.token_type arr[i32,32768] = [2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, ... |
|
llama_model_loader: - kv 19: tokenizer.ggml.bos_token_id u32 = 1 |
|
llama_model_loader: - kv 20: tokenizer.ggml.eos_token_id u32 = 2 |
|
llama_model_loader: - kv 21: tokenizer.ggml.unknown_token_id u32 = 0 |
|
llama_model_loader: - kv 22: tokenizer.ggml.add_bos_token bool = true |
|
llama_model_loader: - kv 23: tokenizer.ggml.add_eos_token bool = false |
|
llama_model_loader: - kv 24: general.quantization_version u32 = 2 |
|
llama_model_loader: - type f32: 507 tensors |
|
llm_load_vocab: special tokens cache size = 1027. |
|
llm_load_print_meta: format = GGUF V3 (latest) |
|
llm_load_print_meta: arch = llama |
|
llm_load_print_meta: vocab type = SPM |
|
llm_load_print_meta: n_vocab = 32768 |
|
llm_load_print_meta: n_merges = 0 |
|
llm_load_print_meta: n_ctx_train = 32768 |
|
llm_load_print_meta: n_embd = 6144 |
|
llm_load_print_meta: n_head = 48 |
|
llm_load_print_meta: n_head_kv = 8 |
|
llm_load_print_meta: n_layer = 56 |
|
llm_load_print_meta: n_rot = 128 |
|
llm_load_print_meta: n_embd_head_k = 128 |
|
llm_load_print_meta: n_embd_head_v = 128 |
|
llm_load_print_meta: n_gqa = 6 |
|
llm_load_print_meta: n_embd_k_gqa = 1024 |
|
llm_load_print_meta: n_embd_v_gqa = 1024 |
|
llm_load_print_meta: f_norm_eps = 0.0e+00 |
|
llm_load_print_meta: f_norm_rms_eps = 1.0e-05 |
|
llm_load_print_meta: f_clamp_kqv = 0.0e+00 |
|
llm_load_print_meta: f_max_alibi_bias = 0.0e+00 |
|
llm_load_print_meta: f_logit_scale = 0.0e+00 |
|
llm_load_print_meta: n_ff = 16384 |
|
llm_load_print_meta: n_expert = 0 |
|
llm_load_print_meta: n_expert_used = 0 |
|
llm_load_print_meta: causal attn = 1 |
|
llm_load_print_meta: pooling type = 0 |
|
llm_load_print_meta: rope type = 0 |
|
llm_load_print_meta: rope scaling = linear |
|
llm_load_print_meta: freq_base_train = 1000000.0 |
|
llm_load_print_meta: freq_scale_train = 1 |
|
llm_load_print_meta: n_yarn_orig_ctx = 32768 |
|
llm_load_print_meta: rope_finetuned = unknown |
|
llm_load_print_meta: ssm_d_conv = 0 |
|
llm_load_print_meta: ssm_d_inner = 0 |
|
llm_load_print_meta: ssm_d_state = 0 |
|
llm_load_print_meta: ssm_dt_rank = 0 |
|
llm_load_print_meta: model type = ?B |
|
llm_load_print_meta: model ftype = all F32 |
|
llm_load_print_meta: model params = 22.25 B |
|
llm_load_print_meta: model size = 82.88 GiB (32.00 BPW) |
|
llm_load_print_meta: general.name = Codestral-22B-v0.1-hf |
|
llm_load_print_meta: BOS token = 1 '<s>' |
|
llm_load_print_meta: EOS token = 2 '</s>' |
|
llm_load_print_meta: UNK token = 0 '<unk>' |
|
llm_load_print_meta: LF token = 781 '<0x0A>' |
|
llm_load_print_meta: PRE token = 32007 '材' |
|
llm_load_print_meta: SUF token = 32008 'ホ' |
|
llm_load_print_meta: MID token = 32009 '張' |
|
llm_load_print_meta: EOT token = 32010 '洞' |
|
ggml_cuda_init: GGML_CUDA_FORCE_MMQ: no |
|
ggml_cuda_init: CUDA_USE_TENSOR_CORES: yes |
|
ggml_cuda_init: found 1 CUDA devices: |
|
Device 0: NVIDIA GeForce RTX 4090, compute capability 8.9, VMM: yes |
|
llm_load_tensors: ggml ctx size = 0.52 MiB |
|
llm_load_tensors: offloading 15 repeating layers to GPU |
|
llm_load_tensors: offloaded 15/57 layers to GPU |
|
llm_load_tensors: CPU buffer size = 84866.65 MiB |
|
llm_load_tensors: CUDA0 buffer size = 22320.70 MiB |
|
.................................................................................................... |
|
llama_new_context_with_model: n_ctx = 512 |
|
llama_new_context_with_model: n_batch = 512 |
|
llama_new_context_with_model: n_ubatch = 512 |
|
llama_new_context_with_model: flash_attn = 0 |
|
llama_new_context_with_model: freq_base = 1000000.0 |
|
llama_new_context_with_model: freq_scale = 1 |
|
llama_kv_cache_init: CUDA_Host KV buffer size = 82.00 MiB |
|
llama_kv_cache_init: CUDA0 KV buffer size = 30.00 MiB |
|
llama_new_context_with_model: KV self size = 112.00 MiB, K (f16): 56.00 MiB, V (f16): 56.00 MiB |
|
llama_new_context_with_model: CUDA_Host output buffer size = 0.12 MiB |
|
llama_new_context_with_model: CUDA0 compute buffer size = 844.00 MiB |
|
llama_new_context_with_model: CUDA_Host compute buffer size = 13.01 MiB |
|
llama_new_context_with_model: graph nodes = 1798 |
|
llama_new_context_with_model: graph splits = 455 |
|
|
|
system_info: n_threads = 32 / 32 | AVX = 1 | AVX_VNNI = 0 | AVX2 = 1 | AVX512 = 1 | AVX512_VBMI = 1 | AVX512_VNNI = 1 | AVX512_BF16 = 1 | FMA = 1 | NEON = 0 | SVE = 0 | ARM_FMA = 0 | F16C = 1 | FP16_VA = 0 | WASM_SIMD = 0 | BLAS = 1 | SSE3 = 1 | SSSE3 = 1 | VSX = 0 | MATMUL_INT8 = 0 | LLAMAFILE = 1 | |
|
compute_imatrix: tokenizing the input .. |
|
compute_imatrix: tokenization took 138.82 ms |
|
compute_imatrix: computing over 228 chunks with batch_size 512 |
|
compute_imatrix: 4.79 seconds per pass - ETA 18.22 minutes |
|
[1]3.6206,[2]2.8149,[3]2.8979,[4]3.1307,[5]3.5843,[6]3.5066,[7]3.1873,[8]3.6962,[9]3.7453, |
|
save_imatrix: stored collected data after 10 chunks in Codestral-22B-v0.1-hf-IMat-GGUF/imatrix.dat |
|
[10]4.1889,[11]4.3081,[12]4.1932,[13]4.4807,[14]4.2222,[15]4.6157,[16]4.7609,[17]5.0368,[18]5.1680,[19]5.3220, |
|
save_imatrix: stored collected data after 20 chunks in Codestral-22B-v0.1-hf-IMat-GGUF/imatrix.dat |
|
[20]5.4484,[21]5.6082,[22]5.4802,[23]5.2685,[24]5.3531,[25]5.0754,[26]4.8463,[27]4.6727,[28]4.5832,[29]4.5770, |
|
save_imatrix: stored collected data after 30 chunks in Codestral-22B-v0.1-hf-IMat-GGUF/imatrix.dat |
|
[30]4.6711,[31]4.7958,[32]4.9050,[33]4.9496,[34]5.0253,[35]4.8420,[36]4.7165,[37]4.6649,[38]4.6706,[39]4.6698, |
|
save_imatrix: stored collected data after 40 chunks in Codestral-22B-v0.1-hf-IMat-GGUF/imatrix.dat |
|
[40]4.6425,[41]4.6986,[42]4.6637,[43]4.7284,[44]4.8258,[45]4.8480,[46]4.9396,[47]5.0852,[48]5.1953,[49]5.3392, |
|
save_imatrix: stored collected data after 50 chunks in Codestral-22B-v0.1-hf-IMat-GGUF/imatrix.dat |
|
[50]5.4228,[51]5.4439,[52]5.3860,[53]5.3287,[54]5.2154,[55]5.2875,[56]5.3471,[57]5.4037,[58]5.4583,[59]5.4802, |
|
save_imatrix: stored collected data after 60 chunks in Codestral-22B-v0.1-hf-IMat-GGUF/imatrix.dat |
|
[60]5.5583,[61]5.6004,[62]5.6432,[63]5.6571,[64]5.6904,[65]5.7266,[66]5.7612,[67]5.8229,[68]5.8764,[69]5.9056, |
|
save_imatrix: stored collected data after 70 chunks in Codestral-22B-v0.1-hf-IMat-GGUF/imatrix.dat |
|
[70]5.9264,[71]5.8846,[72]5.8326,[73]5.7879,[74]5.7538,[75]5.7250,[76]5.6958,[77]5.6570,[78]5.5948,[79]5.5695, |
|
save_imatrix: stored collected data after 80 chunks in Codestral-22B-v0.1-hf-IMat-GGUF/imatrix.dat |
|
[80]5.5568,[81]5.5119,[82]5.5662,[83]5.6069,[84]5.6251,[85]5.5685,[86]5.5872,[87]5.5409,[88]5.4422,[89]5.3923, |
|
save_imatrix: stored collected data after 90 chunks in Codestral-22B-v0.1-hf-IMat-GGUF/imatrix.dat |
|
[90]5.3469,[91]5.3411,[92]5.3173,[93]5.3136,[94]5.3077,[95]5.2651,[96]5.2372,[97]5.2329,[98]5.2656,[99]5.2864, |
|
save_imatrix: stored collected data after 100 chunks in Codestral-22B-v0.1-hf-IMat-GGUF/imatrix.dat |
|
[100]5.2841,[101]5.2798,[102]5.2652,[103]5.2753,[104]5.2718,[105]5.2586,[106]5.2419,[107]5.2376,[108]5.2387,[109]5.2534, |
|
save_imatrix: stored collected data after 110 chunks in Codestral-22B-v0.1-hf-IMat-GGUF/imatrix.dat |
|
[110]5.2363,[111]5.2469,[112]5.2396,[113]5.2359,[114]5.2267,[115]5.2328,[116]5.2306,[117]5.2235,[118]5.1921,[119]5.1974, |
|
save_imatrix: stored collected data after 120 chunks in Codestral-22B-v0.1-hf-IMat-GGUF/imatrix.dat |
|
[120]5.2192,[121]5.2305,[122]5.2273,[123]5.2396,[124]5.2493,[125]5.2733,[126]5.2258,[127]5.2232,[128]5.2110,[129]5.1853, |
|
save_imatrix: stored collected data after 130 chunks in Codestral-22B-v0.1-hf-IMat-GGUF/imatrix.dat |
|
[130]5.1992,[131]5.1702,[132]5.1418,[133]5.1129,[134]5.0822,[135]5.0549,[136]5.0276,[137]5.0028,[138]4.9769,[139]4.9620, |
|
save_imatrix: stored collected data after 140 chunks in Codestral-22B-v0.1-hf-IMat-GGUF/imatrix.dat |
|
[140]4.9497,[141]4.9379,[142]4.9125,[143]4.9016,[144]4.8828,[145]4.8682,[146]4.8482,[147]4.8343,[148]4.8259,[149]4.8058, |
|
save_imatrix: stored collected data after 150 chunks in Codestral-22B-v0.1-hf-IMat-GGUF/imatrix.dat |
|
[150]4.7923,[151]4.8015,[152]4.7726,[153]4.7718,[154]4.7926,[155]4.8079,[156]4.8176,[157]4.8289,[158]4.8414,[159]4.8695, |
|
save_imatrix: stored collected data after 160 chunks in Codestral-22B-v0.1-hf-IMat-GGUF/imatrix.dat |
|
[160]4.8807,[161]4.8925,[162]4.8983,[163]4.9078,[164]4.9196,[165]4.9140,[166]4.9203,[167]4.9314,[168]4.9385,[169]4.9510, |
|
save_imatrix: stored collected data after 170 chunks in Codestral-22B-v0.1-hf-IMat-GGUF/imatrix.dat |
|
[170]4.9479,[171]4.9549,[172]4.9685,[173]4.9724,[174]4.9820,[175]4.9670,[176]4.9837,[177]4.9955,[178]5.0128,[179]5.0121, |
|
save_imatrix: stored collected data after 180 chunks in Codestral-22B-v0.1-hf-IMat-GGUF/imatrix.dat |
|
[180]5.0300,[181]5.0326,[182]5.0338,[183]5.0372,[184]5.0388,[185]5.0501,[186]5.0595,[187]5.0836,[188]5.0919,[189]5.0794, |
|
save_imatrix: stored collected data after 190 chunks in Codestral-22B-v0.1-hf-IMat-GGUF/imatrix.dat |
|
[190]5.1144,[191]5.1499,[192]5.1812,[193]5.2296,[194]5.2654,[195]5.2764,[196]5.2873,[197]5.2721,[198]5.2801,[199]5.3001, |
|
save_imatrix: stored collected data after 200 chunks in Codestral-22B-v0.1-hf-IMat-GGUF/imatrix.dat |
|
[200]5.3280,[201]5.3264,[202]5.3275,[203]5.3350,[204]5.3510,[205]5.3574,[206]5.3676,[207]5.3763,[208]5.3902,[209]5.4076, |
|
save_imatrix: stored collected data after 210 chunks in Codestral-22B-v0.1-hf-IMat-GGUF/imatrix.dat |
|
[210]5.4320,[211]5.4264,[212]5.4334,[213]5.4305,[214]5.4307,[215]5.4283,[216]5.4246,[217]5.4195,[218]5.4396,[219]5.4232, |
|
save_imatrix: stored collected data after 220 chunks in Codestral-22B-v0.1-hf-IMat-GGUF/imatrix.dat |
|
[220]5.4398,[221]5.4743,[222]5.5035,[223]5.5333,[224]5.5496,[225]5.5515,[226]5.5226,[227]5.4973,[228]5.4685, |
|
save_imatrix: stored collected data after 228 chunks in Codestral-22B-v0.1-hf-IMat-GGUF/imatrix.dat |
|
|
|
llama_print_timings: load time = 8573.25 ms |
|
llama_print_timings: sample time = 0.00 ms / 1 runs ( 0.00 ms per token, inf tokens per second) |
|
llama_print_timings: prompt eval time = 1070343.24 ms / 116736 tokens ( 9.17 ms per token, 109.06 tokens per second) |
|
llama_print_timings: eval time = 0.00 ms / 1 runs ( 0.00 ms per token, inf tokens per second) |
|
llama_print_timings: total time = 1074991.99 ms / 116737 tokens |
|
|
|
Final estimate: PPL = 5.4685 +/- 0.05302 |
