baalajimaestro/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Merge branch 'main' of github.com:abetlen/llama_cpp_python into main

Browse source
This commit is contained in:
Andrei Betlen 2023-09-25 13:57:00 -04:00
commit cf8ae5a69c
1 changed files with 6 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
llama_cpp/server/app.py
View file

@ -1,5 +1,6 @@
import json import json
import multiprocessing import multiprocessing
import time
from re import compile, Match, Pattern from re import compile, Match, Pattern
from threading import Lock from threading import Lock
from functools import partial from functools import partial
@ -271,7 +272,11 @@ class RouteErrorHandler(APIRoute):
async def custom_route_handler(request: Request) -> Response: async def custom_route_handler(request: Request) -> Response:
try: try:
return await original_route_handler(request) start_sec = time.perf_counter()
response = await original_route_handler(request)
elapsed_time_ms = int((time.perf_counter() - start_sec) * 1000)
response.headers["openai-processing-ms"] = f"{elapsed_time_ms}"
return response
except Exception as exc: except Exception as exc:
json_body = await request.json() json_body = await request.json()
try: try: