"""Example FastAPI server for llama.cpp. To run this example: ```bash pip install fastapi uvicorn sse-starlette pydantic-settings export MODEL=../models/7B/... ``` Then run: ``` uvicorn llama_cpp.server.app:app --reload ``` or ``` python3 -m llama_cpp.server ``` Then visit http://localhost:8000/docs to see the interactive API docs. """ import os import argparse from typing import List, Literal, Union import uvicorn from llama_cpp.server.app import create_app, Settings def get_base_type(annotation): if getattr(annotation, '__origin__', None) is Literal: return type(annotation.__args__[0]) elif getattr(annotation, '__origin__', None) is Union: non_optional_args = [arg for arg in annotation.__args__ if arg is not type(None)] if non_optional_args: return get_base_type(non_optional_args[0]) elif getattr(annotation, '__origin__', None) is list or getattr(annotation, '__origin__', None) is List: return get_base_type(annotation.__args__[0]) else: return annotation def contains_list_type(annotation) -> bool: origin = getattr(annotation, '__origin__', None) if origin is list or origin is List: return True elif origin in (Literal, Union): return any(contains_list_type(arg) for arg in annotation.__args__) else: return False def parse_bool_arg(arg): if isinstance(arg, bytes): arg = arg.decode('utf-8') true_values = {'1', 'on', 't', 'true', 'y', 'yes'} false_values = {'0', 'off', 'f', 'false', 'n', 'no'} arg_str = str(arg).lower().strip() if arg_str in true_values: return True elif arg_str in false_values: return False else: raise ValueError(f'Invalid boolean argument: {arg}') if __name__ == "__main__": parser = argparse.ArgumentParser() for name, field in Settings.model_fields.items(): description = field.description if field.default is not None and description is not None: description += f" (default: {field.default})" base_type = get_base_type(field.annotation) if field.annotation is not None else str list_type = contains_list_type(field.annotation) if base_type is not bool: parser.add_argument( f"--{name}", dest=name, nargs="*" if list_type else None, type=base_type, help=description, ) if base_type is bool: parser.add_argument( f"--{name}", dest=name, type=parse_bool_arg, help=f"{description}", ) args = parser.parse_args() settings = Settings(**{k: v for k, v in vars(args).items() if v is not None}) app = create_app(settings=settings) uvicorn.run( app, host=os.getenv("HOST", settings.host), port=int(os.getenv("PORT", settings.port)) )