baalajimaestro/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Support Accept text/event-stream in chat and completion endpoints, resolves #1083 (#1088)

Browse source 
Co-authored-by: Anil Pathak <anil@heyday.com>
Co-authored-by: Andrei Betlen <abetlen@gmail.com>
This commit is contained in:
anil 2024-01-16 11:52:52 -06:00 committed by GitHub
parent e39778f8eb
commit cfb7da98ed
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
1 changed files with 57 additions and 2 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

59
llama_cpp/server/app.py
View file

@ -197,7 +197,36 @@ async def authenticate(
@router.post( @router.post(
"/v1/completions", summary="Completion", dependencies=[Depends(authenticate)] "/v1/completions",
summary="Completion",
dependencies=[Depends(authenticate)],
response_model= Union[
llama_cpp.CreateCompletionResponse,
str,
],
responses={
"200": {
"description": "Successful Response",
"content": {
"application/json": {
"schema": {
"anyOf": [
{"$ref": "#/components/schemas/CreateCompletionResponse"}
],
"title": "Completion response, when stream=False",
}
},
"text/event-stream":{
"schema": {
"type": "string",
"title": "Server Side Streaming response, when stream=True. " +
"See SSE format: https://developer.mozilla.org/en-US/docs/Web/API/Server-sent_events/Using_server-sent_events#Event_stream_format", # noqa: E501
"example": """data: {... see CreateCompletionResponse ...} \\n\\n data: ... \\n\\n ... data: [DONE]"""
}
}
},
}
},
) )
@router.post( @router.post(
"/v1/engines/copilot-codex/completions", "/v1/engines/copilot-codex/completions",
@ -280,7 +309,33 @@ async def create_embedding(
@router.post( @router.post(
"/v1/chat/completions", summary="Chat", dependencies=[Depends(authenticate)] "/v1/chat/completions", summary="Chat", dependencies=[Depends(authenticate)],
response_model= Union[
llama_cpp.ChatCompletion, str
],
responses={
"200": {
"description": "Successful Response",
"content": {
"application/json": {
"schema": {
"anyOf": [
{"$ref": "#/components/schemas/CreateChatCompletionResponse"}
],
"title": "Completion response, when stream=False",
}
},
"text/event-stream":{
"schema": {
"type": "string",
"title": "Server Side Streaming response, when stream=True" +
"See SSE format: https://developer.mozilla.org/en-US/docs/Web/API/Server-sent_events/Using_server-sent_events#Event_stream_format", # noqa: E501
"example": """data: {... see CreateChatCompletionResponse ...} \\n\\n data: ... \\n\\n ... data: [DONE]"""
}
}
},
}
},
) )
async def create_chat_completion( async def create_chat_completion(
request: Request, request: Request,