From d966b730ac34631cc2251e8bf14c62f558a745fe Mon Sep 17 00:00:00 2001 From: Daniel Hiltgen Date: Sat, 23 Dec 2023 11:35:44 -0800 Subject: [PATCH] Switch windows build to fully dynamic Refactor where we store build outputs, and support a fully dynamic loading model on windows so the base executable has no special dependencies thus doesn't require a special PATH. --- gpu/gpu.go | 24 ++-- gpu/gpu_darwin.go | 1 - gpu/gpu_test.go | 2 +- gpu/types.go | 1 - llm/dynamic_shim.c | 27 +++-- llm/{ext_server.go => ext_server_common.go} | 120 +++++--------------- llm/ext_server_default.go | 80 +++++++++++++ llm/ext_server_windows.go | 15 +++ llm/llama.cpp/gen_common.sh | 9 ++ llm/llama.cpp/gen_darwin.sh | 3 +- llm/llama.cpp/gen_linux.sh | 30 ++--- llm/llama.cpp/gen_windows.ps1 | 47 +++----- llm/llama.go | 41 ------- llm/shim_darwin.go | 41 ++++++- llm/shim_ext_server.go | 91 ++++++++++----- llm/shim_ext_server_linux.go | 46 ++++++++ llm/shim_ext_server_windows.go | 29 +++++ 17 files changed, 379 insertions(+), 228 deletions(-) rename llm/{ext_server.go => ext_server_common.go} (77%) create mode 100644 llm/ext_server_default.go create mode 100644 llm/ext_server_windows.go create mode 100644 llm/shim_ext_server_linux.go create mode 100644 llm/shim_ext_server_windows.go diff --git a/gpu/gpu.go b/gpu/gpu.go index 91ced3a8..52da39ec 100644 --- a/gpu/gpu.go +++ b/gpu/gpu.go @@ -13,6 +13,7 @@ import "C" import ( "fmt" "log" + "runtime" "sync" "unsafe" @@ -65,15 +66,14 @@ func GetGPUInfo() GpuInfo { } var memInfo C.mem_info_t - resp := GpuInfo{"", "", 0, 0} + resp := GpuInfo{"", 0, 0} if gpuHandles.cuda != nil { C.cuda_check_vram(*gpuHandles.cuda, &memInfo) if memInfo.err != nil { log.Printf("error looking up CUDA GPU memory: %s", C.GoString(memInfo.err)) C.free(unsafe.Pointer(memInfo.err)) } else { - resp.Driver = "CUDA" - resp.Library = "cuda_server" + resp.Library = "cuda" } } else if gpuHandles.rocm != nil { C.rocm_check_vram(*gpuHandles.rocm, &memInfo) @@ -81,15 +81,17 @@ func GetGPUInfo() GpuInfo { log.Printf("error looking up ROCm GPU memory: %s", C.GoString(memInfo.err)) C.free(unsafe.Pointer(memInfo.err)) } else { - resp.Driver = "ROCM" - resp.Library = "rocm_server" + resp.Library = "rocm" } } - if resp.Driver == "" { + if resp.Library == "" { C.cpu_check_ram(&memInfo) - resp.Driver = "CPU" // In the future we may offer multiple CPU variants to tune CPU features - resp.Library = "default" + if runtime.GOOS == "windows" { + resp.Library = "cpu" + } else { + resp.Library = "default" + } } if memInfo.err != nil { log.Printf("error looking up CPU memory: %s", C.GoString(memInfo.err)) @@ -103,7 +105,7 @@ func GetGPUInfo() GpuInfo { func CheckVRAM() (int64, error) { gpuInfo := GetGPUInfo() - if gpuInfo.FreeMemory > 0 && gpuInfo.Driver != "CPU" { + if gpuInfo.FreeMemory > 0 && (gpuInfo.Library == "cuda" || gpuInfo.Library == "rocm") { return int64(gpuInfo.FreeMemory), nil } return 0, fmt.Errorf("no GPU detected") // TODO - better handling of CPU based memory determiniation @@ -114,7 +116,7 @@ func NumGPU(numLayer, fileSizeBytes int64, opts api.Options) int { return opts.NumGPU } info := GetGPUInfo() - if info.Driver == "CPU" { + if info.Library == "cpu" || info.Library == "default" { return 0 } @@ -128,7 +130,7 @@ func NumGPU(numLayer, fileSizeBytes int64, opts api.Options) int { // 75% of the absolute max number of layers we can fit in available VRAM, off-loading too many layers to the GPU can cause OOM errors layers := int(info.FreeMemory/bytesPerLayer) * 3 / 4 - log.Printf("%d MB VRAM available, loading up to %d %s GPU layers out of %d", info.FreeMemory/(1024*1024), layers, info.Driver, numLayer) + log.Printf("%d MB VRAM available, loading up to %d %s GPU layers out of %d", info.FreeMemory/(1024*1024), layers, info.Library, numLayer) return layers } diff --git a/gpu/gpu_darwin.go b/gpu/gpu_darwin.go index ccf67b51..3a7319b4 100644 --- a/gpu/gpu_darwin.go +++ b/gpu/gpu_darwin.go @@ -20,7 +20,6 @@ func GetGPUInfo() GpuInfo { // TODO - Metal vs. x86 macs... return GpuInfo{ - Driver: "METAL", Library: "default", TotalMemory: 0, FreeMemory: 0, diff --git a/gpu/gpu_test.go b/gpu/gpu_test.go index cbdcf3ec..a5b5c892 100644 --- a/gpu/gpu_test.go +++ b/gpu/gpu_test.go @@ -9,7 +9,7 @@ import ( func TestBasicGetGPUInfo(t *testing.T) { info := GetGPUInfo() - assert.Contains(t, "CUDA ROCM CPU METAL", info.Driver) + assert.Contains(t, "cuda rocm cpu default", info.Library) switch runtime.GOOS { case "darwin": diff --git a/gpu/types.go b/gpu/types.go index a56da45e..637e32e6 100644 --- a/gpu/types.go +++ b/gpu/types.go @@ -2,7 +2,6 @@ package gpu // Beginning of an `ollama info` command type GpuInfo struct { - Driver string `json:"driver,omitempty"` Library string `json:"library,omitempty"` TotalMemory uint64 `json:"total_memory,omitempty"` FreeMemory uint64 `json:"free_memory,omitempty"` diff --git a/llm/dynamic_shim.c b/llm/dynamic_shim.c index 8b5d67c9..c3e74d4a 100644 --- a/llm/dynamic_shim.c +++ b/llm/dynamic_shim.c @@ -7,24 +7,29 @@ #include #define LOAD_LIBRARY(lib, flags) dlopen(lib, flags | RTLD_DEEPBIND) #define LOAD_SYMBOL(handle, sym) dlsym(handle, sym) -#define LOAD_ERR() dlerror() +#define LOAD_ERR() strdup(dlerror()) #define UNLOAD_LIBRARY(handle) dlclose(handle) #elif _WIN32 #include #define LOAD_LIBRARY(lib, flags) LoadLibrary(lib) #define LOAD_SYMBOL(handle, sym) GetProcAddress(handle, sym) #define UNLOAD_LIBRARY(handle) FreeLibrary(handle) -// TODO - refactor this with proper error message handling on windows -inline static char *LOAD_ERR() { - static char errbuf[8]; - snprintf(errbuf, 8, "0x%lx", GetLastError()); - return errbuf; +inline char *LOAD_ERR() { + LPSTR messageBuffer = NULL; + size_t size = FormatMessageA( + FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM | + FORMAT_MESSAGE_IGNORE_INSERTS, + NULL, GetLastError(), MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), + (LPSTR)&messageBuffer, 0, NULL); + char *resp = strdup(messageBuffer); + LocalFree(messageBuffer); + return resp; } #else #include #define LOAD_LIBRARY(lib, flags) dlopen(lib, flags) #define LOAD_SYMBOL(handle, sym) dlsym(handle, sym) -#define LOAD_ERR() dlerror() +#define LOAD_ERR() strdup(dlerror()) #define UNLOAD_LIBRARY(handle) dlclose(handle) #endif @@ -57,8 +62,10 @@ void dynamic_shim_init(const char *libPath, struct dynamic_llama_server *s, s->handle = LOAD_LIBRARY(libPath, RTLD_NOW); if (!s->handle) { err->id = -1; + char *msg = LOAD_ERR(); snprintf(err->msg, err->msg_len, - "Unable to load dynamic server library: %s", LOAD_ERR()); + "Unable to load dynamic server library: %s", msg); + free(msg); return; } @@ -67,8 +74,10 @@ void dynamic_shim_init(const char *libPath, struct dynamic_llama_server *s, if (!l[i].p) { UNLOAD_LIBRARY(s->handle); err->id = -1; + char *msg = LOAD_ERR(); snprintf(err->msg, err->msg_len, "symbol lookup for %s failed: %s", - l[i].s, LOAD_ERR()); + l[i].s, msg); + free(msg); return; } } diff --git a/llm/ext_server.go b/llm/ext_server_common.go similarity index 77% rename from llm/ext_server.go rename to llm/ext_server_common.go index 87cd6600..8e5f34e3 100644 --- a/llm/ext_server.go +++ b/llm/ext_server_common.go @@ -10,20 +10,19 @@ package llm #cgo darwin CPPFLAGS: -DGGML_USE_METAL -DGGML_METAL_NDEBUG #cgo darwin LDFLAGS: -lc++ -framework Accelerate #cgo darwin LDFLAGS: -framework Foundation -framework Metal -framework MetalKit -framework MetalPerformanceShaders -#cgo darwin LDFLAGS: ${SRCDIR}/llama.cpp/gguf/build/metal/common/libcommon.a -#cgo darwin LDFLAGS: ${SRCDIR}/llama.cpp/gguf/build/metal/examples/server/libext_server.a -#cgo darwin LDFLAGS: ${SRCDIR}/llama.cpp/gguf/build/metal/libllama.a -#cgo darwin LDFLAGS: ${SRCDIR}/llama.cpp/gguf/build/metal/libggml_static.a +#cgo darwin LDFLAGS: ${SRCDIR}/llama.cpp/gguf/build/darwin/metal/lib/libcommon.a +#cgo darwin LDFLAGS: ${SRCDIR}/llama.cpp/gguf/build/darwin/metal/lib/libext_server.a +#cgo darwin LDFLAGS: ${SRCDIR}/llama.cpp/gguf/build/darwin/metal/lib/libllama.a +#cgo darwin LDFLAGS: ${SRCDIR}/llama.cpp/gguf/build/darwin/metal/lib/libggml_static.a #cgo linux CFLAGS: -D_GNU_SOURCE #cgo linux windows CFLAGS: -DGGML_CUDA_DMMV_X=32 -DGGML_CUDA_MMV_Y=1 -DGGML_CUDA_PEER_MAX_BATCH_SIZE=128 -DGGML_USE_CUBLAS #cgo linux LDFLAGS: -L/usr/local/cuda/targets/x86_64-linux/lib -L/usr/local/cuda/lib64 -L/usr/local/cuda/targets/x86_64-linux/lib/stubs -#cgo linux LDFLAGS: ${SRCDIR}/llama.cpp/gguf/build/cpu/examples/server/libext_server.a -#cgo linux LDFLAGS: ${SRCDIR}/llama.cpp/gguf/build/cpu/common/libcommon.a -#cgo linux LDFLAGS: ${SRCDIR}/llama.cpp/gguf/build/cpu/libllama.a -#cgo linux LDFLAGS: ${SRCDIR}/llama.cpp/gguf/build/cpu/libggml_static.a -#cgo linux LDFLAGS: -lrt -lpthread -ldl -lstdc++ -lm -#cgo windows LDFLAGS: -L${SRCDIR}/llama.cpp/gguf/build/lib -#cgo windows LDFLAGS: -lext_server_shared -lpthread +#cgo linux LDFLAGS: ${SRCDIR}/llama.cpp/gguf/build/linux/cpu/lib/libext_server.a +#cgo linux LDFLAGS: ${SRCDIR}/llama.cpp/gguf/build/linux/cpu/lib/libcommon.a +#cgo linux LDFLAGS: ${SRCDIR}/llama.cpp/gguf/build/linux/cpu/lib/libllama.a +#cgo linux LDFLAGS: ${SRCDIR}/llama.cpp/gguf/build/linux/cpu/lib/libggml_static.a +#cgo linux LDFLAGS: -lrt -ldl -lstdc++ -lm +#cgo linux windows LDFLAGS: -lpthread #include #include "ext_server.h" @@ -46,6 +45,24 @@ import ( "github.com/jmorganca/ollama/gpu" ) +type extServer interface { + LLM + llama_server_init(sparams *C.ext_server_params_t, err *C.ext_server_resp_t) + llama_server_start() + llama_server_stop() + llama_server_completion(json_req *C.char, resp *C.ext_server_resp_t) + llama_server_completion_next_result(task_id C.int, resp *C.ext_server_task_result_t) + llama_server_completion_cancel(task_id C.int, err *C.ext_server_resp_t) + llama_server_release_task_result(result *C.ext_server_task_result_t) + llama_server_tokenize(json_req *C.char, json_resp **C.char, err *C.ext_server_resp_t) + llama_server_detokenize(json_req *C.char, json_resp **C.char, err *C.ext_server_resp_t) + llama_server_embedding(json_req *C.char, json_resp **C.char, err *C.ext_server_resp_t) + llama_server_release_json_resp(json_resp **C.char) +} + +// Note: current implementation does not support concurrent instantiations +var mutex sync.Mutex + func newExtServerResp(len C.size_t) C.ext_server_resp_t { var resp C.ext_server_resp_t resp.msg_len = len @@ -65,69 +82,6 @@ func extServerResponseToErr(resp C.ext_server_resp_t) error { return fmt.Errorf(C.GoString(resp.msg)) } -type extServer interface { - LLM - llama_server_init(sparams *C.ext_server_params_t, err *C.ext_server_resp_t) - llama_server_start() - llama_server_stop() - llama_server_completion(json_req *C.char, resp *C.ext_server_resp_t) - llama_server_completion_next_result(task_id C.int, resp *C.ext_server_task_result_t) - llama_server_completion_cancel(task_id C.int, err *C.ext_server_resp_t) - llama_server_release_task_result(result *C.ext_server_task_result_t) - llama_server_tokenize(json_req *C.char, json_resp **C.char, err *C.ext_server_resp_t) - llama_server_detokenize(json_req *C.char, json_resp **C.char, err *C.ext_server_resp_t) - llama_server_embedding(json_req *C.char, json_resp **C.char, err *C.ext_server_resp_t) - llama_server_release_json_resp(json_resp **C.char) -} - -type llamaExtServer struct { - api.Options -} - -// Note: current implementation does not support concurrent instantiations -var mutex sync.Mutex - -func (llm *llamaExtServer) llama_server_init(sparams *C.ext_server_params_t, err *C.ext_server_resp_t) { - C.llama_server_init(sparams, err) -} -func (llm *llamaExtServer) llama_server_start() { - C.llama_server_start() -} -func (llm *llamaExtServer) llama_server_stop() { - C.llama_server_stop() -} - -func (llm *llamaExtServer) llama_server_completion(json_req *C.char, resp *C.ext_server_resp_t) { - C.llama_server_completion(json_req, resp) -} -func (llm *llamaExtServer) llama_server_completion_next_result(task_id C.int, resp *C.ext_server_task_result_t) { - C.llama_server_completion_next_result(task_id, resp) -} -func (llm *llamaExtServer) llama_server_completion_cancel(task_id C.int, err *C.ext_server_resp_t) { - C.llama_server_completion_cancel(task_id, err) -} -func (llm *llamaExtServer) llama_server_release_task_result(result *C.ext_server_task_result_t) { - C.llama_server_release_task_result(result) -} - -func (llm *llamaExtServer) llama_server_tokenize(json_req *C.char, json_resp **C.char, err *C.ext_server_resp_t) { - C.llama_server_tokenize(json_req, json_resp, err) -} -func (llm *llamaExtServer) llama_server_detokenize(json_req *C.char, json_resp **C.char, err *C.ext_server_resp_t) { - C.llama_server_detokenize(json_req, json_resp, err) -} -func (llm *llamaExtServer) llama_server_embedding(json_req *C.char, json_resp **C.char, err *C.ext_server_resp_t) { - C.llama_server_embedding(json_req, json_resp, err) -} -func (llm *llamaExtServer) llama_server_release_json_resp(json_resp **C.char) { - C.llama_server_release_json_resp(json_resp) -} - -func newDefaultExtServer(model string, adapters, projectors []string, numLayers int64, opts api.Options) (extServer, error) { - server := &llamaExtServer{opts} - return newExtServer(server, model, adapters, projectors, numLayers, opts) -} - func newExtServer(server extServer, model string, adapters, projectors []string, numLayers int64, opts api.Options) (extServer, error) { if !mutex.TryLock() { log.Printf("concurrent llm servers not yet supported, waiting for prior server to complete") @@ -199,10 +153,6 @@ func newExtServer(server extServer, model string, adapters, projectors []string, return server, nil } -func (llm *llamaExtServer) Predict(ctx context.Context, pred PredictOpts, fn func(PredictResult)) error { - return predict(llm, llm.Options, ctx, pred, fn) -} - func predict(llm extServer, opts api.Options, ctx context.Context, predict PredictOpts, fn func(PredictResult)) error { resp := newExtServerResp(128) defer freeExtServerResp(resp) @@ -326,9 +276,6 @@ func predict(llm extServer, opts api.Options, ctx context.Context, predict Predi // should never reach here ideally return fmt.Errorf("max retries exceeded") } -func (llm *llamaExtServer) Encode(ctx context.Context, prompt string) ([]int, error) { - return encode(llm, ctx, prompt) -} func encode(llm extServer, ctx context.Context, prompt string) ([]int, error) { data, err := json.Marshal(TokenizeRequest{Content: prompt}) @@ -354,10 +301,6 @@ func encode(llm extServer, ctx context.Context, prompt string) ([]int, error) { return encoded.Tokens, err } -func (llm *llamaExtServer) Decode(ctx context.Context, tokens []int) (string, error) { - return decode(llm, ctx, tokens) -} - func decode(llm extServer, ctx context.Context, tokens []int) (string, error) { if len(tokens) == 0 { return "", nil @@ -386,9 +329,6 @@ func decode(llm extServer, ctx context.Context, tokens []int) (string, error) { return decoded.Content, err } -func (llm *llamaExtServer) Embedding(ctx context.Context, input string) ([]float64, error) { - return embedding(llm, ctx, input) -} func embedding(llm extServer, ctx context.Context, input string) ([]float64, error) { data, err := json.Marshal(TokenizeRequest{Content: input}) if err != nil { @@ -414,10 +354,6 @@ func embedding(llm extServer, ctx context.Context, input string) ([]float64, err return embedding.Embedding, nil } -func (llm *llamaExtServer) Close() { - close(llm) -} - func close(llm extServer) { llm.llama_server_stop() mutex.Unlock() diff --git a/llm/ext_server_default.go b/llm/ext_server_default.go new file mode 100644 index 00000000..797eb851 --- /dev/null +++ b/llm/ext_server_default.go @@ -0,0 +1,80 @@ +//go:build !windows + +package llm + +/* +#include +#include "ext_server.h" + +*/ +import "C" +import ( + "context" + + "github.com/jmorganca/ollama/api" +) + +type llamaExtServer struct { + api.Options +} + +func (llm *llamaExtServer) llama_server_init(sparams *C.ext_server_params_t, err *C.ext_server_resp_t) { + C.llama_server_init(sparams, err) +} +func (llm *llamaExtServer) llama_server_start() { + C.llama_server_start() +} +func (llm *llamaExtServer) llama_server_stop() { + C.llama_server_stop() +} + +func (llm *llamaExtServer) llama_server_completion(json_req *C.char, resp *C.ext_server_resp_t) { + C.llama_server_completion(json_req, resp) +} +func (llm *llamaExtServer) llama_server_completion_next_result(task_id C.int, resp *C.ext_server_task_result_t) { + C.llama_server_completion_next_result(task_id, resp) +} +func (llm *llamaExtServer) llama_server_completion_cancel(task_id C.int, err *C.ext_server_resp_t) { + C.llama_server_completion_cancel(task_id, err) +} +func (llm *llamaExtServer) llama_server_release_task_result(result *C.ext_server_task_result_t) { + C.llama_server_release_task_result(result) +} + +func (llm *llamaExtServer) llama_server_tokenize(json_req *C.char, json_resp **C.char, err *C.ext_server_resp_t) { + C.llama_server_tokenize(json_req, json_resp, err) +} +func (llm *llamaExtServer) llama_server_detokenize(json_req *C.char, json_resp **C.char, err *C.ext_server_resp_t) { + C.llama_server_detokenize(json_req, json_resp, err) +} +func (llm *llamaExtServer) llama_server_embedding(json_req *C.char, json_resp **C.char, err *C.ext_server_resp_t) { + C.llama_server_embedding(json_req, json_resp, err) +} +func (llm *llamaExtServer) llama_server_release_json_resp(json_resp **C.char) { + C.llama_server_release_json_resp(json_resp) +} + +func newDefaultExtServer(model string, adapters, projectors []string, numLayers int64, opts api.Options) (extServer, error) { + server := &llamaExtServer{opts} + return newExtServer(server, model, adapters, projectors, numLayers, opts) +} + +func (llm *llamaExtServer) Predict(ctx context.Context, pred PredictOpts, fn func(PredictResult)) error { + return predict(llm, llm.Options, ctx, pred, fn) +} + +func (llm *llamaExtServer) Encode(ctx context.Context, prompt string) ([]int, error) { + return encode(llm, ctx, prompt) +} + +func (llm *llamaExtServer) Decode(ctx context.Context, tokens []int) (string, error) { + return decode(llm, ctx, tokens) +} + +func (llm *llamaExtServer) Embedding(ctx context.Context, input string) ([]float64, error) { + return embedding(llm, ctx, input) +} + +func (llm *llamaExtServer) Close() { + close(llm) +} diff --git a/llm/ext_server_windows.go b/llm/ext_server_windows.go new file mode 100644 index 00000000..24b57b6d --- /dev/null +++ b/llm/ext_server_windows.go @@ -0,0 +1,15 @@ +package llm + +import ( + "fmt" + + "github.com/jmorganca/ollama/api" +) + +func newDefaultExtServer(model string, adapters, projectors []string, numLayers int64, opts api.Options) (extServer, error) { + // On windows we always load the llama.cpp libraries dynamically to avoid startup DLL dependencies + // This ensures we can update the PATH at runtime to get everything loaded + + // Should not happen + return nil, fmt.Errorf("no default impl on windows - all dynamic") +} diff --git a/llm/llama.cpp/gen_common.sh b/llm/llama.cpp/gen_common.sh index d0795981..25b98f2f 100644 --- a/llm/llama.cpp/gen_common.sh +++ b/llm/llama.cpp/gen_common.sh @@ -39,6 +39,15 @@ build() { cmake --build ${BUILD_DIR} ${CMAKE_TARGETS} -j8 } +install() { + rm -rf ${BUILD_DIR}/lib + mkdir -p ${BUILD_DIR}/lib + cp ${BUILD_DIR}/examples/server/libext_server.a ${BUILD_DIR}/lib + cp ${BUILD_DIR}/common/libcommon.a ${BUILD_DIR}/lib + cp ${BUILD_DIR}/libllama.a ${BUILD_DIR}/lib + cp ${BUILD_DIR}/libggml_static.a ${BUILD_DIR}/lib +} + # Keep the local tree clean after we're done with the build cleanup() { (cd gguf/examples/server/ && git checkout CMakeLists.txt server.cpp) diff --git a/llm/llama.cpp/gen_darwin.sh b/llm/llama.cpp/gen_darwin.sh index 1eb5b074..e9b9f038 100755 --- a/llm/llama.cpp/gen_darwin.sh +++ b/llm/llama.cpp/gen_darwin.sh @@ -10,7 +10,7 @@ echo "Starting darwin generate script" source $(dirname $0)/gen_common.sh init_vars CMAKE_DEFS="-DCMAKE_OSX_DEPLOYMENT_TARGET=11.0 -DLLAMA_METAL=on ${CMAKE_DEFS}" -BUILD_DIR="gguf/build/metal" +BUILD_DIR="gguf/build/darwin/metal" case "${GOARCH}" in "amd64") CMAKE_DEFS="-DCMAKE_SYSTEM_PROCESSOR=x86_64 -DCMAKE_OSX_ARCHITECTURES=x86_64 ${CMAKE_DEFS}" @@ -28,4 +28,5 @@ esac git_module_setup apply_patches build +install cleanup diff --git a/llm/llama.cpp/gen_linux.sh b/llm/llama.cpp/gen_linux.sh index ab0229db..721e20a9 100755 --- a/llm/llama.cpp/gen_linux.sh +++ b/llm/llama.cpp/gen_linux.sh @@ -21,34 +21,33 @@ if [ -z "${CUDACXX}" -a -x /usr/local/cuda/bin/nvcc ]; then export CUDACXX=/usr/local/cuda/bin/nvcc fi COMMON_CMAKE_DEFS="-DCMAKE_POSITION_INDEPENDENT_CODE=on -DLLAMA_ACCELERATE=on -DLLAMA_NATIVE=off -DLLAMA_AVX=on -DLLAMA_AVX2=off -DLLAMA_AVX512=off -DLLAMA_FMA=off -DLLAMA_F16C=off" -OLLAMA_DYN_LIB_DIR="gguf/build/lib" source $(dirname $0)/gen_common.sh init_vars git_module_setup apply_patches -mkdir -p ${OLLAMA_DYN_LIB_DIR} -touch ${OLLAMA_DYN_LIB_DIR}/.generated - # # CPU first for the default library # CMAKE_DEFS="${COMMON_CMAKE_DEFS} ${CMAKE_DEFS}" -BUILD_DIR="gguf/build/cpu" +BUILD_DIR="gguf/build/linux/cpu" + build +install if [ -d /usr/local/cuda/lib64/ ]; then echo "CUDA libraries detected - building dynamic CUDA library" init_vars CMAKE_DEFS="-DLLAMA_CUBLAS=on ${COMMON_CMAKE_DEFS} ${CMAKE_DEFS}" - BUILD_DIR="gguf/build/cuda" + BUILD_DIR="gguf/build/linux/cuda" CUDA_LIB_DIR=/usr/local/cuda/lib64 build - gcc -fPIC -g -shared -o ${OLLAMA_DYN_LIB_DIR}/libcuda_server.so \ + install + gcc -fPIC -g -shared -o ${BUILD_DIR}/lib/libext_server.so \ -Wl,--whole-archive \ - ${BUILD_DIR}/examples/server/libext_server.a \ - ${BUILD_DIR}/common/libcommon.a \ - ${BUILD_DIR}/libllama.a \ + ${BUILD_DIR}/lib/libext_server.a \ + ${BUILD_DIR}/lib/libcommon.a \ + ${BUILD_DIR}/lib/libllama.a \ -Wl,--no-whole-archive \ ${CUDA_LIB_DIR}/libcudart_static.a \ ${CUDA_LIB_DIR}/libcublas_static.a \ @@ -74,13 +73,14 @@ if [ -d "${ROCM_PATH}" ]; then echo "ROCm libraries detected - building dynamic ROCm library" init_vars CMAKE_DEFS="${COMMON_CMAKE_DEFS} ${CMAKE_DEFS} -DLLAMA_HIPBLAS=on -DCMAKE_C_COMPILER=$ROCM_PATH/llvm/bin/clang -DCMAKE_CXX_COMPILER=$ROCM_PATH/llvm/bin/clang++ -DAMDGPU_TARGETS='gfx803;gfx900;gfx906:xnack-;gfx908:xnack-;gfx90a:xnack+;gfx90a:xnack-;gfx1010;gfx1012;gfx1030;gfx1100;gfx1101;gfx1102' -DGPU_TARGETS='gfx803;gfx900;gfx906:xnack-;gfx908:xnack-;gfx90a:xnack+;gfx90a:xnack-;gfx1010;gfx1012;gfx1030;gfx1100;gfx1101;gfx1102'" - BUILD_DIR="gguf/build/rocm" + BUILD_DIR="gguf/build/linux/rocm" build - gcc -fPIC -g -shared -o ${OLLAMA_DYN_LIB_DIR}/librocm_server.so \ + install + gcc -fPIC -g -shared -o ${BUILD_DIR}/lib/libext_server.so \ -Wl,--whole-archive \ - ${BUILD_DIR}/examples/server/libext_server.a \ - ${BUILD_DIR}/common/libcommon.a \ - ${BUILD_DIR}/libllama.a \ + ${BUILD_DIR}/lib/libext_server.a \ + ${BUILD_DIR}/lib/libcommon.a \ + ${BUILD_DIR}/lib/libllama.a \ -Wl,--no-whole-archive \ -lrt -lpthread -ldl -lstdc++ -lm \ -L/opt/rocm/lib -L/opt/amdgpu/lib/x86_64-linux-gnu/ \ diff --git a/llm/llama.cpp/gen_windows.ps1 b/llm/llama.cpp/gen_windows.ps1 index 02ff2ffe..3f85cbb2 100644 --- a/llm/llama.cpp/gen_windows.ps1 +++ b/llm/llama.cpp/gen_windows.ps1 @@ -44,6 +44,13 @@ function build { if ($LASTEXITCODE -ne 0) { exit($LASTEXITCODE)} } +function install { + rm -ea 0 -recurse -force -path "${script:buildDir}/lib" + md "${script:buildDir}/lib" -ea 0 > $null + cp "${script:buildDir}/bin/${script:config}/ext_server_shared.dll" "${script:buildDir}/lib" + cp "${script:buildDir}/bin/${script:config}/llama.dll" "${script:buildDir}/lib" +} + function cleanup { Set-Location "gguf/examples/server" git checkout CMakeLists.txt server.cpp @@ -54,42 +61,24 @@ git_module_setup apply_patches # first build CPU based -$script:buildDir="gguf/build/wincpu" +$script:buildDir="gguf/build/windows/cpu" build -# install - -md gguf/build/lib -ea 0 -md gguf/build/wincpu/dist/lib -ea 0 -cp -force gguf/build/wincpu/bin/$script:config/ext_server_shared.dll gguf/build/lib/ext_server_shared.dll -cp -force gguf/build/wincpu/bin/$script:config/llama.dll gguf/build/lib/llama.dll - -# Nope, this barfs on lots of symbol problems -#mv gguf/build/wincpu/examples/server/$script:config/ext_server_shared.dll gguf/build/wincpu/dist/lib/cpu_server.lib -# Nope: this needs lots of include paths to pull in things like msvcprt.lib and other deps -# & cl.exe ` -# gguf/build/wincpu/examples/server/$script:config/ext_server.lib ` -# gguf/build/wincpu/common/$script:config/common.lib ` -# gguf/build/wincpu/$script:config/llama.lib ` -# gguf/build/wincpu/$script:config/ggml_static.lib ` -# /link /DLL /DEF:cpu_server.def /NOENTRY /MACHINE:X64 /OUT:gguf/build/wincpu/dist/lib/cpu_server.dll -# if ($LASTEXITCODE -ne 0) { exit($LASTEXITCODE)} +install # Then build cuda as a dynamically loaded library init_vars -$script:buildDir="gguf/build/wincuda" -$script:cmakeDefs += @("-DLLAMA_CUBLAS=ON", "-DBUILD_SHARED_LIBS=on") +$script:buildDir="gguf/build/windows/cuda" +$script:cmakeDefs += @("-DLLAMA_CUBLAS=ON") build -# install -cp -force gguf/build/wincuda/bin/$script:config/ext_server_shared.dll gguf/build/lib/cuda_server.dll +install -# TODO - more to do here to create a usable dll +# TODO - actually implement ROCm support on windows +$script:buildDir="gguf/build/windows/rocm" - -# TODO - implement ROCm support on windows -md gguf/build/winrocm/lib -ea 0 -echo $null >> gguf/build/winrocm/lib/.generated +rm -ea 0 -recurse -force -path "${script:buildDir}/lib" +md "${script:buildDir}/lib" -ea 0 > $null +echo $null >> "${script:buildDir}/lib/.generated" cleanup - -write-host "go generate completed" \ No newline at end of file +write-host "`ngo generate completed" \ No newline at end of file diff --git a/llm/llama.go b/llm/llama.go index 6278abc3..f8e22960 100644 --- a/llm/llama.go +++ b/llm/llama.go @@ -6,11 +6,8 @@ import ( _ "embed" "errors" "fmt" - "io" - "io/fs" "os" "os/exec" - "path/filepath" "sync" "time" @@ -206,41 +203,3 @@ type EmbeddingRequest struct { type EmbeddingResponse struct { Embedding []float64 `json:"embedding"` } - -func extractDynamicLibs(workDir, glob string) ([]string, error) { - files, err := fs.Glob(libEmbed, glob) - if err != nil || len(files) == 0 { - return nil, payloadMissing - } - libs := make([]string, len(files)) - - for i, file := range files { - srcFile, err := libEmbed.Open(file) - if err != nil { - return nil, fmt.Errorf("read payload %s: %v", file, err) - } - defer srcFile.Close() - if err := os.MkdirAll(workDir, 0o755); err != nil { - return nil, fmt.Errorf("create payload temp dir %s: %v", workDir, err) - } - - destFile := filepath.Join(workDir, filepath.Base(file)) - libs[i] = destFile - - _, err = os.Stat(destFile) - switch { - case errors.Is(err, os.ErrNotExist): - destFile, err := os.OpenFile(destFile, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o755) - if err != nil { - return nil, fmt.Errorf("write payload %s: %v", file, err) - } - defer destFile.Close() - if _, err := io.Copy(destFile, srcFile); err != nil { - return nil, fmt.Errorf("copy payload %s: %v", file, err) - } - case err != nil: - return nil, fmt.Errorf("stat payload %s: %v", file, err) - } - } - return libs, nil -} diff --git a/llm/shim_darwin.go b/llm/shim_darwin.go index 98e7a7d5..f7427b96 100644 --- a/llm/shim_darwin.go +++ b/llm/shim_darwin.go @@ -2,9 +2,13 @@ package llm import ( "embed" + "errors" "fmt" + "io" + "io/fs" "log" "os" + "path/filepath" "github.com/jmorganca/ollama/api" ) @@ -18,7 +22,7 @@ func newDynamicShimExtServer(library, model string, adapters, projectors []strin } func nativeInit(workdir string) error { - _, err := extractDynamicLibs(workdir, "llama.cpp/gguf/ggml-metal.metal") + err := extractPayloadFiles(workdir, "llama.cpp/gguf/ggml-metal.metal") if err != nil { if err == payloadMissing { // TODO perhaps consider this a hard failure on arm macs? @@ -30,3 +34,38 @@ func nativeInit(workdir string) error { os.Setenv("GGML_METAL_PATH_RESOURCES", workdir) return nil } + +func extractPayloadFiles(workDir, glob string) error { + files, err := fs.Glob(libEmbed, glob) + if err != nil || len(files) == 0 { + return payloadMissing + } + + for _, file := range files { + srcFile, err := libEmbed.Open(file) + if err != nil { + return fmt.Errorf("read payload %s: %v", file, err) + } + defer srcFile.Close() + if err := os.MkdirAll(workDir, 0o755); err != nil { + return fmt.Errorf("create payload temp dir %s: %v", workDir, err) + } + + destFile := filepath.Join(workDir, filepath.Base(file)) + _, err = os.Stat(destFile) + switch { + case errors.Is(err, os.ErrNotExist): + destFile, err := os.OpenFile(destFile, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o755) + if err != nil { + return fmt.Errorf("write payload %s: %v", file, err) + } + defer destFile.Close() + if _, err := io.Copy(destFile, srcFile); err != nil { + return fmt.Errorf("copy payload %s: %v", file, err) + } + case err != nil: + return fmt.Errorf("stat payload %s: %v", file, err) + } + } + return nil +} diff --git a/llm/shim_ext_server.go b/llm/shim_ext_server.go index d9c2df46..146456d3 100644 --- a/llm/shim_ext_server.go +++ b/llm/shim_ext_server.go @@ -11,9 +11,9 @@ package llm import "C" import ( "context" - "embed" "errors" "fmt" + "io" "io/fs" "log" "os" @@ -25,11 +25,6 @@ import ( "github.com/jmorganca/ollama/api" ) -//go:embed llama.cpp/gguf/build/lib/* -var libEmbed embed.FS - -var RocmShimMissing = fmt.Errorf("ROCm shim library not included in this build of ollama. Radeon GPUs are not supported") - type shimExtServer struct { s C.struct_dynamic_llama_server options api.Options @@ -78,6 +73,7 @@ func (llm *shimExtServer) llama_server_release_json_resp(json_resp **C.char) { func newDynamicShimExtServer(library, model string, adapters, projectors []string, numLayers int64, opts api.Options) (extServer, error) { shimMutex.Lock() defer shimMutex.Unlock() + updatePath(filepath.Dir(library)) libPath := C.CString(library) defer C.free(unsafe.Pointer(libPath)) resp := newExtServerResp(128) @@ -116,7 +112,7 @@ func (llm *shimExtServer) Close() { } func nativeInit(workdir string) error { - libs, err := extractDynamicLibs(workdir, "llama.cpp/gguf/build/lib/*server*") + libs, err := extractDynamicLibs(workdir, "llama.cpp/gguf/build/*/*/lib/*") if err != nil { if err == payloadMissing { log.Printf("%s", payloadMissing) @@ -125,28 +121,71 @@ func nativeInit(workdir string) error { return err } for _, lib := range libs { - libName := strings.Split(strings.TrimPrefix(filepath.Base(lib), "lib"), ".")[0] - AvailableShims[libName] = lib + // The last dir component is the variant name + variant := filepath.Base(filepath.Dir(lib)) + AvailableShims[variant] = lib } - // Only check ROCm access if we have the dynamic lib loaded - if _, rocmPresent := AvailableShims["rocm_server"]; rocmPresent { - // Verify we have permissions - either running as root, or we have group access to the driver - fd, err := os.OpenFile("/dev/kfd", os.O_RDWR, 0666) - if err != nil { - if errors.Is(err, fs.ErrPermission) { - log.Fatalf("Radeon card detected, but permissions not set up properly. Either run ollama as root, or add you user account to the render group.") - return err - } else if errors.Is(err, fs.ErrNotExist) { - // expected behavior without a radeon card - return nil - } - - return fmt.Errorf("failed to check permission on /dev/kfd: %w", err) - } - fd.Close() - + if err := verifyDriverAccess(); err != nil { + return err } + // Report which dynamic libraries we have loaded to assist troubleshooting + variants := make([]string, len(AvailableShims)) + i := 0 + for variant := range AvailableShims { + variants[i] = variant + i++ + } + log.Printf("Dynamic LLM variants %v", variants) + return nil } + +func extractDynamicLibs(workDir, glob string) ([]string, error) { + files, err := fs.Glob(libEmbed, glob) + if err != nil || len(files) == 0 { + return nil, payloadMissing + } + libs := make([]string, len(files)) + + for i, file := range files { + pathComps := strings.Split(file, "/") + if len(pathComps) != 7 { + log.Printf("unexpected payload components: %v", pathComps) + continue + } + // llama.cpp/gguf/build/$OS/$VARIANT/lib/$LIBRARY + // Include the variant in the path to avoid conflicts between multiple server libs + targetDir := filepath.Join(workDir, pathComps[4]) + srcFile, err := libEmbed.Open(file) + if err != nil { + return nil, fmt.Errorf("read payload %s: %v", file, err) + } + defer srcFile.Close() + if err := os.MkdirAll(targetDir, 0o755); err != nil { + return nil, fmt.Errorf("create payload temp dir %s: %v", workDir, err) + } + + destFile := filepath.Join(targetDir, filepath.Base(file)) + if strings.Contains(destFile, "server") { + libs[i] = destFile + } + + _, err = os.Stat(destFile) + switch { + case errors.Is(err, os.ErrNotExist): + destFile, err := os.OpenFile(destFile, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o755) + if err != nil { + return nil, fmt.Errorf("write payload %s: %v", file, err) + } + defer destFile.Close() + if _, err := io.Copy(destFile, srcFile); err != nil { + return nil, fmt.Errorf("copy payload %s: %v", file, err) + } + case err != nil: + return nil, fmt.Errorf("stat payload %s: %v", file, err) + } + } + return libs, nil +} diff --git a/llm/shim_ext_server_linux.go b/llm/shim_ext_server_linux.go new file mode 100644 index 00000000..83409fc8 --- /dev/null +++ b/llm/shim_ext_server_linux.go @@ -0,0 +1,46 @@ +package llm + +import ( + "embed" + "errors" + "fmt" + "io/fs" + "log" + "os" + "strings" +) + +//go:embed llama.cpp/gguf/build/*/*/lib/*.so +var libEmbed embed.FS + +func updatePath(dir string) { + pathComponents := strings.Split(os.Getenv("PATH"), ":") + for _, comp := range pathComponents { + if comp == dir { + return + } + } + newPath := strings.Join(append(pathComponents, dir), ":") + log.Printf("Updating PATH to %s", newPath) + os.Setenv("PATH", newPath) +} + +func verifyDriverAccess() error { + // Only check ROCm access if we have the dynamic lib loaded + if _, rocmPresent := AvailableShims["rocm"]; rocmPresent { + // Verify we have permissions - either running as root, or we have group access to the driver + fd, err := os.OpenFile("/dev/kfd", os.O_RDWR, 0666) + if err != nil { + if errors.Is(err, fs.ErrPermission) { + return fmt.Errorf("Radeon card detected, but permissions not set up properly. Either run ollama as root, or add you user account to the render group.") + } else if errors.Is(err, fs.ErrNotExist) { + // expected behavior without a radeon card + return nil + } + + return fmt.Errorf("failed to check permission on /dev/kfd: %w", err) + } + fd.Close() + } + return nil +} diff --git a/llm/shim_ext_server_windows.go b/llm/shim_ext_server_windows.go new file mode 100644 index 00000000..6a12b61d --- /dev/null +++ b/llm/shim_ext_server_windows.go @@ -0,0 +1,29 @@ +package llm + +import ( + "embed" + "log" + "os" + "strings" +) + +//go:embed llama.cpp/gguf/build/windows/*/lib/*.dll +var libEmbed embed.FS + +func updatePath(dir string) { + pathComponents := strings.Split(os.Getenv("PATH"), ";") + for _, comp := range pathComponents { + // Case incensitive + if strings.ToLower(comp) == strings.ToLower(dir) { + return + } + } + newPath := strings.Join(append(pathComponents, dir), ";") + log.Printf("Updating PATH to %s", newPath) + os.Setenv("PATH", newPath) +} + +func verifyDriverAccess() error { + // TODO if applicable + return nil +}