baalajimaestro/ollama
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

dyn global

Browse source
This commit is contained in:
Michael Yang 2024-03-15 16:34:38 -07:00
parent 22f326464e
commit 3c4ad0ecab
1 changed files with 2 additions and 5 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
llm/dyn_ext_server.go
View file

@ -64,9 +64,6 @@ func extServerResponseToErr(resp C.ext_server_resp_t) error {
return fmt.Errorf(C.GoString(resp.msg))
}
// Note: current implementation does not support concurrent instantiations
var llm *dynExtServer
func newDynExtServer(library, model string, adapters, projectors []string, opts api.Options) (LLM, error) {
if !mutex.TryLock() {
slog.Info("concurrent llm servers not yet supported, waiting for prior server to complete")
@ -83,7 +80,7 @@ func newDynExtServer(library, model string, adapters, projectors []string, opts
mutex.Unlock()
return nil, fmt.Errorf("Unable to load dynamic library: %s", C.GoString(resp.msg))
}
llm = &dynExtServer{
llm := dynExtServer{
s: srv,
options: opts,
}
@ -161,7 +158,7 @@ func newDynExtServer(library, model string, adapters, projectors []string, opts
slog.Info("Starting llama main loop")
C.dyn_llama_server_start(llm.s)
return llm, nil
return &llm, nil
}
func (llm *dynExtServer) Predict(ctx context.Context, predict PredictOpts, fn func(PredictResult)) error {