baalajimaestro/ollama
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

only package 11.8 runner

Browse source
This commit is contained in:
Bruce MacDonald 2023-09-20 20:00:41 +01:00
parent b9bb5ca288
commit 1255bc9b45
4 changed files with 17 additions and 69 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
docs/development.md
View file

@ -35,5 +35,5 @@ Now you can run `ollama`:
## Building on Linux with GPU support
- Install cmake and nvidia-cuda-toolkit
- run `CUDA_VERSION=11 CUDA_PATH=/path/to/libcuda.so CUBLAS_PATH=/path/to/libcublas.so CUDART_PATH=/path/to/libcudart.so CUBLASLT_PATH=/path/to/libcublasLt.so go generate ./...`
- run `CUDA_PATH=/path/to/libcuda.so CUBLAS_PATH=/path/to/libcublas.so CUDART_PATH=/path/to/libcudart.so CUBLASLT_PATH=/path/to/libcublasLt.so go generate ./...`
- run `go build .`

24
llm/llama.cpp/generate_linux.go
View file

@ -15,15 +15,15 @@ package llm
//go:generate cmake -S gguf -B gguf/build/cpu -DLLAMA_K_QUANTS=on
//go:generate cmake --build gguf/build/cpu --target server --config Release
//go:generate cmake -S ggml -B ggml/build/cuda-${CUDA_VERSION} -DLLAMA_CUBLAS=on -DLLAMA_ACCELERATE=on -DLLAMA_K_QUANTS=on
//go:generate cmake --build ggml/build/cuda-${CUDA_VERSION} --target server --config Release
//go:generate cmake -S gguf -B gguf/build/cuda-${CUDA_VERSION} -DLLAMA_CUBLAS=on -DLLAMA_ACCELERATE=on -DLLAMA_K_QUANTS=on
//go:generate cmake --build gguf/build/cuda-${CUDA_VERSION} --target server --config Release
//go:generate cp --dereference ${CUDA_PATH} ggml/build/cuda-${CUDA_VERSION}/bin/libcuda.so
//go:generate cp --dereference ${CUDA_PATH} gguf/build/cuda-${CUDA_VERSION}/bin/libcuda.so
//go:generate cp --dereference ${CUBLAS_PATH} ggml/build/cuda-${CUDA_VERSION}/bin/libcublas.so.${CUDA_VERSION}
//go:generate cp --dereference ${CUBLAS_PATH} gguf/build/cuda-${CUDA_VERSION}/bin/libcublas.so.${CUDA_VERSION}
//go:generate cp --dereference ${CUDART_PATH} ggml/build/cuda-${CUDA_VERSION}/bin/libcudart.so.${CUDA_VERSION}.0
//go:generate cp --dereference ${CUDART_PATH} gguf/build/cuda-${CUDA_VERSION}/bin/libcudart.so.${CUDA_VERSION}.0
//go:generate cp --dereference ${CUBLASLT_PATH} ggml/build/cuda-${CUDA_VERSION}/bin/libcublasLt.so.${CUDA_VERSION}
//go:generate cp --dereference ${CUBLASLT_PATH} gguf/build/cuda-${CUDA_VERSION}/bin/libcublasLt.so.${CUDA_VERSION}
//go:generate cmake -S ggml -B ggml/build/cuda -DLLAMA_CUBLAS=on -DLLAMA_ACCELERATE=on -DLLAMA_K_QUANTS=on
//go:generate cmake --build ggml/build/cuda --target server --config Release
//go:generate cmake -S gguf -B gguf/build/cuda -DLLAMA_CUBLAS=on -DLLAMA_ACCELERATE=on -DLLAMA_K_QUANTS=on
//go:generate cmake --build gguf/build/cuda --target server --config Release
//go:generate cp --dereference ${CUDA_PATH} ggml/build/cuda/bin/libcuda.so
//go:generate cp --dereference ${CUDA_PATH} gguf/build/cuda/bin/libcuda.so
//go:generate cp --dereference ${CUBLAS_PATH} ggml/build/cuda/bin/libcublas.so.11
//go:generate cp --dereference ${CUBLAS_PATH} gguf/build/cuda/bin/libcublas.so.11
//go:generate cp --dereference ${CUDART_PATH} ggml/build/cuda/bin/libcudart.so.11.0
//go:generate cp --dereference ${CUDART_PATH} gguf/build/cuda/bin/libcudart.so.11.0
//go:generate cp --dereference ${CUBLASLT_PATH} ggml/build/cuda/bin/libcublasLt.so.11
//go:generate cp --dereference ${CUBLASLT_PATH} gguf/build/cuda/bin/libcublasLt.so.11

54
llm/llama.go
View file

@ -17,7 +17,6 @@ import (
"os/exec"
"path"
"path/filepath"
"regexp"
"runtime"
"strconv"
"strings"
@ -29,46 +28,6 @@ import (
//go:embed llama.cpp/*/build/*/bin/*
var llamaCppEmbed embed.FS
func cudaVersion() int {
// first try nvcc, it gives the most accurate version if available
cmd := exec.Command("nvcc", "--version")
output, err := cmd.CombinedOutput()
if err == nil {
// regex to match the CUDA version line in nvcc --version output
re := regexp.MustCompile(`release (\d+\.\d+),`)
matches := re.FindStringSubmatch(string(output))
if len(matches) >= 2 {
cudaVersion := matches[1]
cudaVersionParts := strings.Split(cudaVersion, ".")
cudaMajorVersion, err := strconv.Atoi(cudaVersionParts[0])
if err == nil {
return cudaMajorVersion
}
}
}
// fallback to nvidia-smi
cmd = exec.Command("nvidia-smi")
output, err = cmd.CombinedOutput()
if err != nil {
return -1
}
re := regexp.MustCompile(`CUDA Version: (\d+\.\d+)`)
matches := re.FindStringSubmatch(string(output))
if len(matches) < 2 {
return -1
}
cudaVersion := matches[1]
cudaVersionParts := strings.Split(cudaVersion, ".")
cudaMajorVersion, err := strconv.Atoi(cudaVersionParts[0])
if err != nil {
return -1
}
return cudaMajorVersion
}
type ModelRunner struct {
Path string // path to the model runner executable
}
@ -86,21 +45,10 @@ func chooseRunners(runnerType string) []ModelRunner {
path.Join(buildPath, "cpu", "bin", "server"),
}
case "linux":
cuda := cudaVersion()
if cuda == 11 {
// prioritize CUDA 11 runner
runners = []string{
path.Join(buildPath, "cuda-11", "bin", "server"),
path.Join(buildPath, "cuda-12", "bin", "server"),
path.Join(buildPath, "cuda", "bin", "server"),
path.Join(buildPath, "cpu", "bin", "server"),
}
} else {
runners = []string{
path.Join(buildPath, "cuda-12", "bin", "server"),
path.Join(buildPath, "cuda-11", "bin", "server"),
path.Join(buildPath, "cpu", "bin", "server"),
}
}
case "windows":
// TODO: select windows GPU runner here when available
runners = []string{

2
server/routes.go
View file

@ -556,7 +556,7 @@ func Serve(ln net.Listener, origins []string) error {
if runtime.GOOS == "linux" {
// check compatibility to log warnings
if _, err := llm.CheckVRAM(); err != nil {
log.Printf("Warning: GPU support not enabled, you may need to install GPU drivers: %v", err)
log.Printf("Warning: GPU support may not enabled, check you have installed install GPU drivers: %v", err)
}
}