baalajimaestro/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Adds openai-processing-ms response header (#748)

Browse source
This commit is contained in:
Viacheslav/Slava Tradunsky 2023-09-25 13:55:58 -04:00 committed by GitHub
parent dbca136fea
commit 3d5e5b1c04
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
1 changed files with 6 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
llama_cpp/server/app.py
View file

@ -1,5 +1,6 @@
import json
import multiprocessing
import time
from re import compile, Match, Pattern
from threading import Lock
from functools import partial
@ -271,7 +272,11 @@ class RouteErrorHandler(APIRoute):
async def custom_route_handler(request: Request) -> Response:
try:
return await original_route_handler(request)
start_sec = time.perf_counter()
response = await original_route_handler(request)
elapsed_time_ms = int((time.perf_counter() - start_sec) * 1000)
response.headers["openai-processing-ms"] = f"{elapsed_time_ms}"
return response
except Exception as exc:
json_body = await request.json()
try: