baalajimaestro/ollama
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

exponential back-off (#1484)

Browse source
This commit is contained in:
Bruce MacDonald 2023-12-12 12:33:02 -05:00 • committed by GitHub
parent c0960e29b5
commit 3144e2a439
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
1 changed files with 3 additions and 2 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
llm/llama.go
View file

@ -545,8 +545,7 @@ type prediction struct {
}
const maxBufferSize = 512 * format.KiloByte
const maxRetries = 3
const retryDelay = 1 * time.Second
const maxRetries = 6
type PredictOpts struct {
Prompt string
@ -610,9 +609,11 @@ func (llm *llama) Predict(ctx context.Context, predict PredictOpts, fn func(Pred
request["grammar"] = jsonGrammar
}
retryDelay := 100 * time.Microsecond
for retries := 0; retries < maxRetries; retries++ {
if retries > 0 {
time.Sleep(retryDelay) // wait before retrying
retryDelay *= 2 // exponential backoff
}
// Handling JSON marshaling with special characters unescaped.