baalajimaestro/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

examples: fix quantize example (#1387)

Browse source 
@iyubondyrev thank you!
This commit is contained in:
iyubondyrev 2024-04-28 02:48:47 +02:00 committed by GitHub
parent c58b56123d
commit e6bbfb863c
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
1 changed files with 8 additions and 5 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

13
examples/low_level_api/quantize.py
View file

@ -4,14 +4,16 @@ import llama_cpp
def main(args):
fname_inp = args.fname_inp.encode("utf-8")
fname_out = args.fname_out.encode("utf-8")
if not os.path.exists(fname_inp):
raise RuntimeError(f"Input file does not exist ({fname_inp})")
if os.path.exists(fname_out):
raise RuntimeError(f"Output file already exists ({fname_out})")
fname_inp = args.fname_inp.encode("utf-8")
fname_out = args.fname_out.encode("utf-8")
itype = args.itype
return_code = llama_cpp.llama_model_quantize(fname_inp, fname_out, itype)
ftype = args.type
args = llama_cpp.llama_model_quantize_default_params()
args.ftype = ftype
return_code = llama_cpp.llama_model_quantize(fname_inp, fname_out, args)
if return_code != 0:
raise RuntimeError("Failed to quantize model")
@ -20,6 +22,7 @@ if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument("fname_inp", type=str, help="Path to input model")
parser.add_argument("fname_out", type=str, help="Path to output model")
parser.add_argument("type", type=int, help="Type of quantization (2: q4_0, 3: q4_1)")
parser.add_argument("type", type=int, help="Type of quantization (2: q4_0, 3: q4_1), see llama_cpp.py for enum")
args = parser.parse_args()
main(args)