docs(examples): Add huggingface pull example
This commit is contained in:
parent
bd4ec2e612
commit
252e1ff2b4
1 changed files with 39 additions and 0 deletions
39
examples/hf_pull/main.py
Normal file
39
examples/hf_pull/main.py
Normal file
|
@ -0,0 +1,39 @@
|
|||
import llama_cpp
|
||||
import llama_cpp.llama_tokenizer
|
||||
|
||||
|
||||
llama = llama_cpp.Llama.from_pretrained(
|
||||
repo_id="Qwen/Qwen1.5-0.5B-Chat-GGUF",
|
||||
filename="*q8_0.gguf",
|
||||
tokenizer=llama_cpp.llama_tokenizer.LlamaHFTokenizer.from_pretrained("Qwen/Qwen1.5-0.5B"),
|
||||
verbose=False
|
||||
)
|
||||
|
||||
response = llama.create_chat_completion(
|
||||
messages=[
|
||||
{
|
||||
"role": "user",
|
||||
"content": "What is the capital of France?"
|
||||
}
|
||||
],
|
||||
response_format={
|
||||
"type": "json_object",
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"country": {"type": "string"},
|
||||
"capital": {"type": "string"}
|
||||
},
|
||||
"required": ["country", "capital"],
|
||||
}
|
||||
},
|
||||
stream=True
|
||||
)
|
||||
|
||||
for chunk in response:
|
||||
delta = chunk["choices"][0]["delta"]
|
||||
if "content" not in delta:
|
||||
continue
|
||||
print(delta["content"], end="", flush=True)
|
||||
|
||||
print()
|
Loading…
Reference in a new issue