baalajimaestro/ollama
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

dyn global

Browse source
This commit is contained in:
Michael Yang 2024-03-15 16:34:38 -07:00
parent 22f326464e
commit 3c4ad0ecab
1 changed files with 2 additions and 5 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
llm/dyn_ext_server.go
View file

@ -64,9 +64,6 @@ func extServerResponseToErr(resp C.ext_server_resp_t) error {
return fmt.Errorf(C.GoString(resp.msg)) return fmt.Errorf(C.GoString(resp.msg))
} }
// Note: current implementation does not support concurrent instantiations
var llm *dynExtServer
func newDynExtServer(library, model string, adapters, projectors []string, opts api.Options) (LLM, error) { func newDynExtServer(library, model string, adapters, projectors []string, opts api.Options) (LLM, error) {
if !mutex.TryLock() { if !mutex.TryLock() {
slog.Info("concurrent llm servers not yet supported, waiting for prior server to complete") slog.Info("concurrent llm servers not yet supported, waiting for prior server to complete")
@ -83,7 +80,7 @@ func newDynExtServer(library, model string, adapters, projectors []string, opts
mutex.Unlock() mutex.Unlock()
return nil, fmt.Errorf("Unable to load dynamic library: %s", C.GoString(resp.msg)) return nil, fmt.Errorf("Unable to load dynamic library: %s", C.GoString(resp.msg))
} }
llm = &dynExtServer{ llm := dynExtServer{
s: srv, s: srv,
options: opts, options: opts,
} }
@ -161,7 +158,7 @@ func newDynExtServer(library, model string, adapters, projectors []string, opts
slog.Info("Starting llama main loop") slog.Info("Starting llama main loop")
C.dyn_llama_server_start(llm.s) C.dyn_llama_server_start(llm.s)
return llm, nil return &llm, nil
} }
func (llm *dynExtServer) Predict(ctx context.Context, predict PredictOpts, fn func(PredictResult)) error { func (llm *dynExtServer) Predict(ctx context.Context, predict PredictOpts, fn func(PredictResult)) error {