From 386169205c94aef035456ecfea8047f9cb76cf0a Mon Sep 17 00:00:00 2001 From: Michael Yang Date: Fri, 20 Oct 2023 18:17:14 -0700 Subject: [PATCH] update runtime options (#864) --- server/routes.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/server/routes.go b/server/routes.go index cdd3caf5..5c52dbfe 100644 --- a/server/routes.go +++ b/server/routes.go @@ -112,6 +112,10 @@ func load(ctx context.Context, workDir string, model *Model, reqOpts map[string] loaded.Options = &opts } + // update options for the loaded llm + // TODO(mxyng): this isn't thread safe, but it should be fine for now + loaded.runner.SetOptions(opts) + loaded.expireAt = time.Now().Add(sessionDuration) if loaded.expireTimer == nil {