diff --git a/docs/development.md b/docs/development.md index e1b00bac..85cf34c6 100644 --- a/docs/development.md +++ b/docs/development.md @@ -35,5 +35,5 @@ Now you can run `ollama`: ## Building on Linux with GPU support - Install cmake and nvidia-cuda-toolkit -- run `CUBLAS_PATH=/path/to/libcublas.so CUDART_PATH=/path/to/libcudart.so CUBLASLT_PATH=/path/to/libcublasLt.so go generate ./...` +- run `go generate ./...` - run `go build .` diff --git a/llm/llama.cpp/generate.go b/llm/llama.cpp/generate.go index 40a42708..19179ac2 100644 --- a/llm/llama.cpp/generate.go +++ b/llm/llama.cpp/generate.go @@ -6,9 +6,8 @@ package llm //go:generate git submodule init //go:generate git submodule update --force ggml -//go:generate -command git-apply git -C ggml apply -//go:generate git-apply ../ggml_patch/0001-add-detokenize-endpoint.patch -//go:generate git-apply ../ggml_patch/0002-34B-model-support.patch +//go:generate git -C ggml apply ../patches/0001-add-detokenize-endpoint.patch +//go:generate git -C ggml apply ../patches/0002-34B-model-support.patch //go:generate cmake -S ggml -B ggml/build/cpu -DLLAMA_K_QUANTS=on //go:generate cmake --build ggml/build/cpu --target server --config Release diff --git a/llm/llama.cpp/generate_darwin_amd64.go b/llm/llama.cpp/generate_darwin_amd64.go index a8b4f0ad..9b782db3 100644 --- a/llm/llama.cpp/generate_darwin_amd64.go +++ b/llm/llama.cpp/generate_darwin_amd64.go @@ -3,11 +3,10 @@ package llm //go:generate git submodule init //go:generate git submodule update --force ggml -//go:generate -command git-apply git -C ggml apply -//go:generate git-apply ../ggml_patch/0001-add-detokenize-endpoint.patch -//go:generate git-apply ../ggml_patch/0002-34B-model-support.patch -//go:generate git-apply ../ggml_patch/0003-metal-fix-synchronization-in-new-matrix-multiplicati.patch -//go:generate git-apply ../ggml_patch/0004-metal-add-missing-barriers-for-mul-mat-2699.patch +//go:generate git -C ggml apply ../patches/0001-add-detokenize-endpoint.patch +//go:generate git -C ggml apply ../patches/0002-34B-model-support.patch +//go:generate git -C ggml apply ../patches/0003-metal-fix-synchronization-in-new-matrix-multiplicati.patch +//go:generate git -C ggml apply ../patches/0004-metal-add-missing-barriers-for-mul-mat-2699.patch //go:generate cmake -S ggml -B ggml/build/cpu -DLLAMA_ACCELERATE=on -DLLAMA_K_QUANTS=on -DCMAKE_SYSTEM_PROCESSOR=x86_64 -DCMAKE_OSX_ARCHITECTURES=x86_64 -DCMAKE_OSX_DEPLOYMENT_TARGET=11.0 //go:generate cmake --build ggml/build/cpu --target server --config Release diff --git a/llm/llama.cpp/generate_darwin_arm64.go b/llm/llama.cpp/generate_darwin_arm64.go index 4923fefb..72d175ef 100644 --- a/llm/llama.cpp/generate_darwin_arm64.go +++ b/llm/llama.cpp/generate_darwin_arm64.go @@ -3,11 +3,10 @@ package llm //go:generate git submodule init //go:generate git submodule update --force ggml -//go:generate -command git-apply git -C ggml apply -//go:generate git-apply ../ggml_patch/0001-add-detokenize-endpoint.patch -//go:generate git-apply ../ggml_patch/0002-34B-model-support.patch -//go:generate git-apply ../ggml_patch/0003-metal-fix-synchronization-in-new-matrix-multiplicati.patch -//go:generate git-apply ../ggml_patch/0004-metal-add-missing-barriers-for-mul-mat-2699.patch +//go:generate git -C ggml apply ../patches/0001-add-detokenize-endpoint.patch +//go:generate git -C ggml apply ../patches/0002-34B-model-support.patch +//go:generate git -C ggml apply ../patches/0003-metal-fix-synchronization-in-new-matrix-multiplicati.patch +//go:generate git -C ggml apply ../patches/0004-metal-add-missing-barriers-for-mul-mat-2699.patch //go:generate cmake -S ggml -B ggml/build/metal -DLLAMA_METAL=on -DLLAMA_ACCELERATE=on -DLLAMA_K_QUANTS=on -DCMAKE_SYSTEM_PROCESSOR=arm64 -DCMAKE_OSX_ARCHITECTURES=arm64 -DCMAKE_OSX_DEPLOYMENT_TARGET=11.0 //go:generate cmake --build ggml/build/metal --target server --config Release diff --git a/llm/llama.cpp/generate_linux.go b/llm/llama.cpp/generate_linux.go index a3ee703d..76be15d5 100644 --- a/llm/llama.cpp/generate_linux.go +++ b/llm/llama.cpp/generate_linux.go @@ -3,15 +3,15 @@ package llm //go:generate git submodule init //go:generate git submodule update --force ggml -//go:generate -command git-apply git -C ggml apply -//go:generate git-apply ../ggml_patch/0001-add-detokenize-endpoint.patch -//go:generate git-apply ../ggml_patch/0002-34B-model-support.patch -//go:generate git-apply ../ggml_patch/0005-ggml-support-CUDA-s-half-type-for-aarch64-1455-2670.patch - +//go:generate git -C ggml apply ../patches/0001-add-detokenize-endpoint.patch +//go:generate git -C ggml apply ../patches/0002-34B-model-support.patch +//go:generate git -C ggml apply ../patches/0005-ggml-support-CUDA-s-half-type-for-aarch64-1455-2670.patch +//go:generate git -C ggml apply ../patches/0001-copy-cuda-runtime-libraries.patch //go:generate cmake -S ggml -B ggml/build/cpu -DLLAMA_K_QUANTS=on //go:generate cmake --build ggml/build/cpu --target server --config Release //go:generate git submodule update --force gguf +//go:generate git -C gguf apply ../patches/0001-copy-cuda-runtime-libraries.patch //go:generate cmake -S gguf -B gguf/build/cpu -DLLAMA_K_QUANTS=on //go:generate cmake --build gguf/build/cpu --target server --config Release @@ -19,10 +19,3 @@ package llm //go:generate cmake --build ggml/build/cuda --target server --config Release //go:generate cmake -S gguf -B gguf/build/cuda -DLLAMA_CUBLAS=on -DLLAMA_ACCELERATE=on -DLLAMA_K_QUANTS=on //go:generate cmake --build gguf/build/cuda --target server --config Release - -//go:generate cp --dereference ${CUBLAS_PATH} ggml/build/cuda/bin/libcublas.so.11 -//go:generate cp --dereference ${CUBLAS_PATH} gguf/build/cuda/bin/libcublas.so.11 -//go:generate cp --dereference ${CUDART_PATH} ggml/build/cuda/bin/libcudart.so.11.0 -//go:generate cp --dereference ${CUDART_PATH} gguf/build/cuda/bin/libcudart.so.11.0 -//go:generate cp --dereference ${CUBLASLT_PATH} ggml/build/cuda/bin/libcublasLt.so.11 -//go:generate cp --dereference ${CUBLASLT_PATH} gguf/build/cuda/bin/libcublasLt.so.11 diff --git a/llm/llama.cpp/ggml_patch/0003-metal-add-missing-barriers-for-mul-mat-2699.patch b/llm/llama.cpp/ggml_patch/0003-metal-add-missing-barriers-for-mul-mat-2699.patch deleted file mode 100644 index 870e982a..00000000 --- a/llm/llama.cpp/ggml_patch/0003-metal-add-missing-barriers-for-mul-mat-2699.patch +++ /dev/null @@ -1,32 +0,0 @@ -From 8c0ea847ac1460bca534d92266e3471cb31471be Mon Sep 17 00:00:00 2001 -From: Bruce MacDonald -Date: Tue, 5 Sep 2023 16:05:08 -0400 -Subject: [PATCH] metal: add missing barriers for mul-mat #2699 - ---- - ggml-metal.metal | 2 ++ - 1 file changed, 2 insertions(+) - -diff --git a/ggml-metal.metal b/ggml-metal.metal -index 3f31252..ce3541f 100644 ---- a/ggml-metal.metal -+++ b/ggml-metal.metal -@@ -1850,6 +1850,7 @@ kernel void kernel_mul_mm(device const uchar * src0, - //load data and store to threadgroup memory - half4x4 temp_a; - dequantize_func(x, il, temp_a); -+ threadgroup_barrier(mem_flags::mem_threadgroup); - #pragma unroll(16) - for (int i = 0; i < 16; i++) { - *(sa + SG_MAT_SIZE * ((tiitg / THREAD_PER_ROW / 8) \ -@@ -1895,6 +1896,7 @@ kernel void kernel_mul_mm(device const uchar * src0, - } - } else { - // block is smaller than 64x32, we should avoid writing data outside of the matrix -+ threadgroup_barrier(mem_flags::mem_threadgroup); - threadgroup float *temp_str = ((threadgroup float *)shared_memory) \ - + 32 * (sgitg&1) + (16 * (sgitg>>1)) * BLOCK_SIZE_M; - for (int i = 0; i < 8; i++) { --- -2.39.2 (Apple Git-143) - diff --git a/llm/llama.cpp/ggml_patch/0001-add-detokenize-endpoint.patch b/llm/llama.cpp/patches/0001-add-detokenize-endpoint.patch similarity index 100% rename from llm/llama.cpp/ggml_patch/0001-add-detokenize-endpoint.patch rename to llm/llama.cpp/patches/0001-add-detokenize-endpoint.patch diff --git a/llm/llama.cpp/patches/0001-copy-cuda-runtime-libraries.patch b/llm/llama.cpp/patches/0001-copy-cuda-runtime-libraries.patch new file mode 100644 index 00000000..1fd07973 --- /dev/null +++ b/llm/llama.cpp/patches/0001-copy-cuda-runtime-libraries.patch @@ -0,0 +1,27 @@ +From 5dd02993e8cc2ce309157736b95bb572f274a3fd Mon Sep 17 00:00:00 2001 +From: Michael Yang +Date: Wed, 20 Sep 2023 14:19:52 -0700 +Subject: [PATCH] copy cuda runtime libraries + +--- + CMakeLists.txt | 4 ++++ + 1 file changed, 4 insertions(+) + +diff --git a/CMakeLists.txt b/CMakeLists.txt +index 824d9f2..dd24137 100644 +--- a/CMakeLists.txt ++++ b/CMakeLists.txt +@@ -274,6 +274,10 @@ if (LLAMA_CUBLAS) + set(LLAMA_EXTRA_LIBS ${LLAMA_EXTRA_LIBS} CUDA::cudart CUDA::cublas CUDA::cublasLt) + endif() + ++ configure_file(${CUDAToolkit_LIBRARY_DIR}/libcudart.so ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/libcudart.so.${CUDAToolkit_VERSION_MAJOR}.0 COPYONLY) ++ configure_file(${CUDAToolkit_LIBRARY_DIR}/libcublas.so ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/libcublas.so.${CUDAToolkit_VERSION_MAJOR} COPYONLY) ++ configure_file(${CUDAToolkit_LIBRARY_DIR}/libcublasLt.so ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/libcublasLt.so.${CUDAToolkit_VERSION_MAJOR} COPYONLY) ++ + if (NOT DEFINED CMAKE_CUDA_ARCHITECTURES) + # 52 == lowest CUDA 12 standard + # 60 == f16 CUDA intrinsics +-- +2.42.0 + diff --git a/llm/llama.cpp/ggml_patch/0002-34B-model-support.patch b/llm/llama.cpp/patches/0002-34B-model-support.patch similarity index 100% rename from llm/llama.cpp/ggml_patch/0002-34B-model-support.patch rename to llm/llama.cpp/patches/0002-34B-model-support.patch diff --git a/llm/llama.cpp/ggml_patch/0003-metal-fix-synchronization-in-new-matrix-multiplicati.patch b/llm/llama.cpp/patches/0003-metal-fix-synchronization-in-new-matrix-multiplicati.patch similarity index 100% rename from llm/llama.cpp/ggml_patch/0003-metal-fix-synchronization-in-new-matrix-multiplicati.patch rename to llm/llama.cpp/patches/0003-metal-fix-synchronization-in-new-matrix-multiplicati.patch diff --git a/llm/llama.cpp/ggml_patch/0004-metal-add-missing-barriers-for-mul-mat-2699.patch b/llm/llama.cpp/patches/0004-metal-add-missing-barriers-for-mul-mat-2699.patch similarity index 100% rename from llm/llama.cpp/ggml_patch/0004-metal-add-missing-barriers-for-mul-mat-2699.patch rename to llm/llama.cpp/patches/0004-metal-add-missing-barriers-for-mul-mat-2699.patch diff --git a/llm/llama.cpp/ggml_patch/0005-ggml-support-CUDA-s-half-type-for-aarch64-1455-2670.patch b/llm/llama.cpp/patches/0005-ggml-support-CUDA-s-half-type-for-aarch64-1455-2670.patch similarity index 100% rename from llm/llama.cpp/ggml_patch/0005-ggml-support-CUDA-s-half-type-for-aarch64-1455-2670.patch rename to llm/llama.cpp/patches/0005-ggml-support-CUDA-s-half-type-for-aarch64-1455-2670.patch