baalajimaestro/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Bugfix: enable embeddings for fastapi server

Browse source
This commit is contained in:
Andrei Betlen 2023-04-01 15:12:25 -04:00
parent c25b7dfc86
commit f28bf3f13d
1 changed files with 4 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
examples/fastapi_server.py
View file

@ -31,6 +31,7 @@ llama = llama_cpp.Llama(
settings.model, settings.model,
f16_kv=True, f16_kv=True,
use_mlock=True, use_mlock=True,
embedding=True,
n_threads=6, n_threads=6,
n_batch=2048, n_batch=2048,
) )
@ -93,4 +94,6 @@ CreateEmbeddingResponse = create_model_from_typeddict(llama_cpp.Embedding)
response_model=CreateEmbeddingResponse, response_model=CreateEmbeddingResponse,
) )
def create_embedding(request: CreateEmbeddingRequest): def create_embedding(request: CreateEmbeddingRequest):
return llama.create_embedding(**request.dict()) # print(request)
# return llama.create_embedding(**request.dict(exclude={"model", "user"}))
return llama.create_embedding(request.input)