From 15e3dc78975f70164819482b4563b0ca6cdd72d0 Mon Sep 17 00:00:00 2001
From: Andrei Betlen <abetlen@gmail.com>
Date: Fri, 24 Mar 2023 01:41:24 -0400
Subject: [PATCH] Add fastapi example

---
 examples/fastapi_server.py | 42 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 42 insertions(+)
 create mode 100644 examples/fastapi_server.py

diff --git a/examples/fastapi_server.py b/examples/fastapi_server.py
new file mode 100644
index 0000000..7ea804c
--- /dev/null
+++ b/examples/fastapi_server.py
@@ -0,0 +1,42 @@
+from typing import List, Optional
+
+from llama_cpp import Llama
+
+from fastapi import FastAPI
+from pydantic import BaseModel, BaseSettings, Field
+
+class Settings(BaseSettings):
+    model: str
+
+app = FastAPI(
+    title="🦙 llama.cpp Python API",
+    version="0.0.1",
+)
+settings = Settings()
+llama = Llama(settings.model)
+
+class CompletionRequest(BaseModel):
+    prompt: str
+    suffix: Optional[str] = Field(None)
+    max_tokens: int = 16
+    temperature: float = 0.8
+    top_p: float = 0.95
+    logprobs: Optional[int] = Field(None)
+    echo: bool = False
+    stop: List[str] = []
+    repeat_penalty: float = 1.1
+    top_k: int = 40
+
+    class Config:
+        schema_extra = {
+            "example": {
+                "prompt": "Question: What is the capital of France? Answer: ",
+                "stop": ["\n", " Question:"]
+            }
+        }
+
+
+
+@app.post("/v1/completions")
+def completions(request: CompletionRequest):
+    return llama(**request.dict())
\ No newline at end of file