piotr25691
commited on
Commit
•
050948f
1
Parent(s):
0af877a
conversion with imatrix
Browse files
conv.sh
CHANGED
@@ -2,7 +2,7 @@
|
|
2 |
|
3 |
# convert PyTorch model
|
4 |
# LLaMA 3
|
5 |
-
convert --vocab-type bpe --outtype f16 --outfile conv.bin .
|
6 |
|
7 |
# LLaMA 2
|
8 |
# convert --outtype f16 --outfile conv.bin .
|
@@ -10,8 +10,12 @@ convert --vocab-type bpe --outtype f16 --outfile conv.bin .
|
|
10 |
# define TheBloke quants and create them
|
11 |
declare -a quants=(Q2_K Q3_K_S Q3_K_M Q3_K_L Q4_0 Q4_K_S Q4_K_M Q5_0 Q5_K_S Q5_K_M Q6_K Q8_0)
|
12 |
currpath=${PWD##*/}
|
|
|
13 |
|
14 |
for quant in "${quants[@]}";
|
15 |
do
|
16 |
-
|
17 |
-
|
|
|
|
|
|
|
|
2 |
|
3 |
# convert PyTorch model
|
4 |
# LLaMA 3
|
5 |
+
# convert --vocab-type bpe --outtype f16 --outfile conv.bin .
|
6 |
|
7 |
# LLaMA 2
|
8 |
# convert --outtype f16 --outfile conv.bin .
|
|
|
10 |
# define TheBloke quants and create them
|
11 |
declare -a quants=(Q2_K Q3_K_S Q3_K_M Q3_K_L Q4_0 Q4_K_S Q4_K_M Q5_0 Q5_K_S Q5_K_M Q6_K Q8_0)
|
12 |
currpath=${PWD##*/}
|
13 |
+
basefile=$(ls -Art *.F16.gguf)
|
14 |
|
15 |
for quant in "${quants[@]}";
|
16 |
do
|
17 |
+
# basic GGUF
|
18 |
+
quantize $basefile ${currpath::-5}.$quant.gguf $quant
|
19 |
+
# imatrix GGUF
|
20 |
+
# quantize --imatrix model.imatrix $basefile ${currpath::-5}.$quant.gguf $quant
|
21 |
+
done
|