2023-11-29 19:00:37 +00:00
|
|
|
//go:build !darwin
|
|
|
|
|
|
|
|
package llm
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
|
|
|
#include <stdlib.h>
|
2023-12-20 18:36:01 +00:00
|
|
|
#include "dynamic_shim.h"
|
2023-11-29 19:00:37 +00:00
|
|
|
|
|
|
|
*/
|
|
|
|
import "C"
|
|
|
|
import (
|
|
|
|
"context"
|
|
|
|
"errors"
|
|
|
|
"fmt"
|
2023-12-23 19:35:44 +00:00
|
|
|
"io"
|
2023-11-29 19:00:37 +00:00
|
|
|
"io/fs"
|
|
|
|
"log"
|
|
|
|
"os"
|
|
|
|
"path/filepath"
|
2023-12-20 18:36:01 +00:00
|
|
|
"strings"
|
2023-11-29 19:00:37 +00:00
|
|
|
"sync"
|
|
|
|
"unsafe"
|
|
|
|
|
|
|
|
"github.com/jmorganca/ollama/api"
|
|
|
|
)
|
|
|
|
|
|
|
|
type shimExtServer struct {
|
2023-12-20 18:36:01 +00:00
|
|
|
s C.struct_dynamic_llama_server
|
2023-11-29 19:00:37 +00:00
|
|
|
options api.Options
|
|
|
|
}
|
|
|
|
|
|
|
|
// Note: current implementation does not support concurrent instantiations
|
|
|
|
var shimMutex sync.Mutex
|
|
|
|
var llm *shimExtServer
|
|
|
|
|
2024-01-04 17:40:15 +00:00
|
|
|
const pathComponentCount = 6
|
|
|
|
|
2023-11-29 19:00:37 +00:00
|
|
|
func (llm *shimExtServer) llama_server_init(sparams *C.ext_server_params_t, err *C.ext_server_resp_t) {
|
2023-12-20 18:36:01 +00:00
|
|
|
C.dynamic_shim_llama_server_init(llm.s, sparams, err)
|
2023-11-29 19:00:37 +00:00
|
|
|
}
|
|
|
|
func (llm *shimExtServer) llama_server_start() {
|
2023-12-20 18:36:01 +00:00
|
|
|
C.dynamic_shim_llama_server_start(llm.s)
|
2023-11-29 19:00:37 +00:00
|
|
|
}
|
|
|
|
func (llm *shimExtServer) llama_server_stop() {
|
2023-12-20 18:36:01 +00:00
|
|
|
C.dynamic_shim_llama_server_stop(llm.s)
|
2023-11-29 19:00:37 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
func (llm *shimExtServer) llama_server_completion(json_req *C.char, resp *C.ext_server_resp_t) {
|
2023-12-20 18:36:01 +00:00
|
|
|
C.dynamic_shim_llama_server_completion(llm.s, json_req, resp)
|
2023-11-29 19:00:37 +00:00
|
|
|
}
|
|
|
|
func (llm *shimExtServer) llama_server_completion_next_result(task_id C.int, resp *C.ext_server_task_result_t) {
|
2023-12-20 18:36:01 +00:00
|
|
|
C.dynamic_shim_llama_server_completion_next_result(llm.s, task_id, resp)
|
2023-11-29 19:00:37 +00:00
|
|
|
}
|
|
|
|
func (llm *shimExtServer) llama_server_completion_cancel(task_id C.int, err *C.ext_server_resp_t) {
|
2023-12-20 18:36:01 +00:00
|
|
|
C.dynamic_shim_llama_server_completion_cancel(llm.s, task_id, err)
|
2023-11-29 19:00:37 +00:00
|
|
|
}
|
|
|
|
func (llm *shimExtServer) llama_server_release_task_result(result *C.ext_server_task_result_t) {
|
2023-12-20 18:36:01 +00:00
|
|
|
C.dynamic_shim_llama_server_release_task_result(llm.s, result)
|
2023-11-29 19:00:37 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
func (llm *shimExtServer) llama_server_tokenize(json_req *C.char, json_resp **C.char, err *C.ext_server_resp_t) {
|
2023-12-20 18:36:01 +00:00
|
|
|
C.dynamic_shim_llama_server_tokenize(llm.s, json_req, json_resp, err)
|
2023-11-29 19:00:37 +00:00
|
|
|
}
|
|
|
|
func (llm *shimExtServer) llama_server_detokenize(json_req *C.char, json_resp **C.char, err *C.ext_server_resp_t) {
|
2023-12-20 18:36:01 +00:00
|
|
|
C.dynamic_shim_llama_server_detokenize(llm.s, json_req, json_resp, err)
|
2023-11-29 19:00:37 +00:00
|
|
|
}
|
|
|
|
func (llm *shimExtServer) llama_server_embedding(json_req *C.char, json_resp **C.char, err *C.ext_server_resp_t) {
|
2023-12-20 18:36:01 +00:00
|
|
|
C.dynamic_shim_llama_server_embedding(llm.s, json_req, json_resp, err)
|
2023-11-29 19:00:37 +00:00
|
|
|
}
|
|
|
|
func (llm *shimExtServer) llama_server_release_json_resp(json_resp **C.char) {
|
2023-12-20 18:36:01 +00:00
|
|
|
C.dynamic_shim_llama_server_release_json_resp(llm.s, json_resp)
|
2023-11-29 19:00:37 +00:00
|
|
|
}
|
|
|
|
|
2024-01-08 21:42:00 +00:00
|
|
|
func newDynamicShimExtServer(library, model string, adapters, projectors []string, opts api.Options) (extServer, error) {
|
2023-12-20 18:36:01 +00:00
|
|
|
shimMutex.Lock()
|
|
|
|
defer shimMutex.Unlock()
|
2023-12-23 19:35:44 +00:00
|
|
|
updatePath(filepath.Dir(library))
|
2023-12-20 18:36:01 +00:00
|
|
|
libPath := C.CString(library)
|
|
|
|
defer C.free(unsafe.Pointer(libPath))
|
|
|
|
resp := newExtServerResp(128)
|
|
|
|
defer freeExtServerResp(resp)
|
|
|
|
var srv C.struct_dynamic_llama_server
|
|
|
|
C.dynamic_shim_init(libPath, &srv, &resp)
|
|
|
|
if resp.id < 0 {
|
|
|
|
return nil, fmt.Errorf("Unable to load dynamic library: %s", C.GoString(resp.msg))
|
2023-11-29 19:00:37 +00:00
|
|
|
}
|
2023-12-20 18:36:01 +00:00
|
|
|
llm = &shimExtServer{
|
|
|
|
s: srv,
|
|
|
|
options: opts,
|
2023-11-29 19:00:37 +00:00
|
|
|
}
|
2023-12-20 18:36:01 +00:00
|
|
|
log.Printf("Loading Dynamic Shim llm server: %s", library)
|
2024-01-08 21:42:00 +00:00
|
|
|
return newExtServer(llm, model, adapters, projectors, opts)
|
2023-11-29 19:00:37 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
func (llm *shimExtServer) Predict(ctx context.Context, pred PredictOpts, fn func(PredictResult)) error {
|
2024-01-03 17:01:42 +00:00
|
|
|
return predict(ctx, llm, pred, fn)
|
2023-11-29 19:00:37 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
func (llm *shimExtServer) Encode(ctx context.Context, prompt string) ([]int, error) {
|
|
|
|
return encode(llm, ctx, prompt)
|
|
|
|
}
|
|
|
|
|
|
|
|
func (llm *shimExtServer) Decode(ctx context.Context, tokens []int) (string, error) {
|
|
|
|
return decode(llm, ctx, tokens)
|
|
|
|
}
|
|
|
|
|
|
|
|
func (llm *shimExtServer) Embedding(ctx context.Context, input string) ([]float64, error) {
|
|
|
|
return embedding(llm, ctx, input)
|
|
|
|
}
|
|
|
|
|
|
|
|
func (llm *shimExtServer) Close() {
|
|
|
|
close(llm)
|
|
|
|
}
|
|
|
|
|
|
|
|
func nativeInit(workdir string) error {
|
2024-01-04 17:40:15 +00:00
|
|
|
libs, err := extractDynamicLibs(workdir, "llama.cpp/build/*/*/lib/*")
|
2023-11-29 19:00:37 +00:00
|
|
|
if err != nil {
|
2023-12-19 02:32:04 +00:00
|
|
|
if err == payloadMissing {
|
2023-12-20 18:36:01 +00:00
|
|
|
log.Printf("%s", payloadMissing)
|
2023-11-29 19:00:37 +00:00
|
|
|
return nil
|
|
|
|
}
|
|
|
|
return err
|
2023-12-20 18:36:01 +00:00
|
|
|
}
|
|
|
|
for _, lib := range libs {
|
2023-12-23 19:35:44 +00:00
|
|
|
// The last dir component is the variant name
|
|
|
|
variant := filepath.Base(filepath.Dir(lib))
|
|
|
|
AvailableShims[variant] = lib
|
|
|
|
}
|
|
|
|
|
|
|
|
if err := verifyDriverAccess(); err != nil {
|
|
|
|
return err
|
2023-11-29 19:00:37 +00:00
|
|
|
}
|
|
|
|
|
2023-12-23 19:35:44 +00:00
|
|
|
// Report which dynamic libraries we have loaded to assist troubleshooting
|
|
|
|
variants := make([]string, len(AvailableShims))
|
|
|
|
i := 0
|
|
|
|
for variant := range AvailableShims {
|
|
|
|
variants[i] = variant
|
|
|
|
i++
|
|
|
|
}
|
|
|
|
log.Printf("Dynamic LLM variants %v", variants)
|
|
|
|
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
func extractDynamicLibs(workDir, glob string) ([]string, error) {
|
|
|
|
files, err := fs.Glob(libEmbed, glob)
|
|
|
|
if err != nil || len(files) == 0 {
|
|
|
|
return nil, payloadMissing
|
|
|
|
}
|
2024-01-04 00:08:34 +00:00
|
|
|
libs := []string{}
|
2023-12-23 19:35:44 +00:00
|
|
|
|
2024-01-04 00:08:34 +00:00
|
|
|
for _, file := range files {
|
2023-12-23 19:35:44 +00:00
|
|
|
pathComps := strings.Split(file, "/")
|
2024-01-04 17:40:15 +00:00
|
|
|
if len(pathComps) != pathComponentCount {
|
2023-12-23 19:35:44 +00:00
|
|
|
log.Printf("unexpected payload components: %v", pathComps)
|
|
|
|
continue
|
|
|
|
}
|
2024-01-04 17:40:15 +00:00
|
|
|
// llama.cpp/build/$OS/$VARIANT/lib/$LIBRARY
|
2023-12-23 19:35:44 +00:00
|
|
|
// Include the variant in the path to avoid conflicts between multiple server libs
|
2024-01-04 17:40:15 +00:00
|
|
|
targetDir := filepath.Join(workDir, pathComps[pathComponentCount-3])
|
2023-12-23 19:35:44 +00:00
|
|
|
srcFile, err := libEmbed.Open(file)
|
2023-12-20 18:36:01 +00:00
|
|
|
if err != nil {
|
2023-12-23 19:35:44 +00:00
|
|
|
return nil, fmt.Errorf("read payload %s: %v", file, err)
|
|
|
|
}
|
|
|
|
defer srcFile.Close()
|
|
|
|
if err := os.MkdirAll(targetDir, 0o755); err != nil {
|
|
|
|
return nil, fmt.Errorf("create payload temp dir %s: %v", workDir, err)
|
|
|
|
}
|
2023-12-20 18:36:01 +00:00
|
|
|
|
2023-12-23 19:35:44 +00:00
|
|
|
destFile := filepath.Join(targetDir, filepath.Base(file))
|
|
|
|
if strings.Contains(destFile, "server") {
|
2024-01-04 00:08:34 +00:00
|
|
|
libs = append(libs, destFile)
|
2023-11-29 19:00:37 +00:00
|
|
|
}
|
|
|
|
|
2023-12-23 19:35:44 +00:00
|
|
|
_, err = os.Stat(destFile)
|
|
|
|
switch {
|
|
|
|
case errors.Is(err, os.ErrNotExist):
|
|
|
|
destFile, err := os.OpenFile(destFile, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o755)
|
|
|
|
if err != nil {
|
|
|
|
return nil, fmt.Errorf("write payload %s: %v", file, err)
|
|
|
|
}
|
|
|
|
defer destFile.Close()
|
|
|
|
if _, err := io.Copy(destFile, srcFile); err != nil {
|
|
|
|
return nil, fmt.Errorf("copy payload %s: %v", file, err)
|
|
|
|
}
|
|
|
|
case err != nil:
|
|
|
|
return nil, fmt.Errorf("stat payload %s: %v", file, err)
|
|
|
|
}
|
2023-11-29 19:00:37 +00:00
|
|
|
}
|
2023-12-23 19:35:44 +00:00
|
|
|
return libs, nil
|
2023-11-29 19:00:37 +00:00
|
|
|
}
|