34b9db5afc
This change adds support for multiple concurrent requests, as well as loading multiple models by spawning multiple runners. The default settings are currently set at 1 concurrent request per model and only 1 loaded model at a time, but these can be adjusted by setting OLLAMA_NUM_PARALLEL and OLLAMA_MAX_LOADED_MODELS.
47 lines
1 KiB
Go
47 lines
1 KiB
Go
//go:build integration
|
|
|
|
package integration
|
|
|
|
import (
|
|
"context"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/ollama/ollama/api"
|
|
)
|
|
|
|
// TODO - this would ideally be in the llm package, but that would require some refactoring of interfaces in the server
|
|
// package to avoid circular dependencies
|
|
|
|
var (
|
|
stream = false
|
|
req = [2]api.GenerateRequest{
|
|
{
|
|
Model: "orca-mini",
|
|
Prompt: "why is the ocean blue?",
|
|
Stream: &stream,
|
|
Options: map[string]interface{}{
|
|
"seed": 42,
|
|
"temperature": 0.0,
|
|
},
|
|
}, {
|
|
Model: "orca-mini",
|
|
Prompt: "what is the origin of the us thanksgiving holiday?",
|
|
Stream: &stream,
|
|
Options: map[string]interface{}{
|
|
"seed": 42,
|
|
"temperature": 0.0,
|
|
},
|
|
},
|
|
}
|
|
resp = [2][]string{
|
|
[]string{"sunlight"},
|
|
[]string{"england", "english", "massachusetts", "pilgrims"},
|
|
}
|
|
)
|
|
|
|
func TestIntegrationSimpleOrcaMini(t *testing.T) {
|
|
ctx, cancel := context.WithTimeout(context.Background(), time.Second*120)
|
|
defer cancel()
|
|
GenerateTestHelper(ctx, t, req[0], resp[0])
|
|
}
|