From 3d5e5b1c0453345b771847539306dfef135062be Mon Sep 17 00:00:00 2001 From: Viacheslav/Slava Tradunsky Date: Mon, 25 Sep 2023 13:55:58 -0400 Subject: [PATCH] Adds openai-processing-ms response header (#748) --- llama_cpp/server/app.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/llama_cpp/server/app.py b/llama_cpp/server/app.py index 053c308..535952c 100644 --- a/llama_cpp/server/app.py +++ b/llama_cpp/server/app.py @@ -1,5 +1,6 @@ import json import multiprocessing +import time from re import compile, Match, Pattern from threading import Lock from functools import partial @@ -271,7 +272,11 @@ class RouteErrorHandler(APIRoute): async def custom_route_handler(request: Request) -> Response: try: - return await original_route_handler(request) + start_sec = time.perf_counter() + response = await original_route_handler(request) + elapsed_time_ms = int((time.perf_counter() - start_sec) * 1000) + response.headers["openai-processing-ms"] = f"{elapsed_time_ms}" + return response except Exception as exc: json_body = await request.json() try: