ollama/llm/patches/02-cudaleaks.diff

diff --git a/examples/server/server.cpp b/examples/server/server.cpp
index b14cca61..02bfd4b1 100644
--- a/examples/server/server.cpp
+++ b/examples/server/server.cpp
@@ -29,6 +29,10 @@
 #include <signal.h>
 #include <memory>
 
+#ifdef GGML_USE_CUBLAS
+extern "C" GGML_CALL void ggml_free_cublas(void);
+#endif
+
 using json = nlohmann::json;
 
 bool server_verbose = false;
@@ -664,6 +668,10 @@ struct server_context {
             llama_free_model(model);
             model = nullptr;
         }
+
+#ifdef GGML_USE_CUBLAS
+        ggml_free_cublas();
+#endif
     }
 
     bool load_model(const gpt_params & params_) {
@@ -3499,6 +3507,7 @@ int main(int argc, char ** argv) {
     sigemptyset (&sigint_action.sa_mask);
     sigint_action.sa_flags = 0;
     sigaction(SIGINT, &sigint_action, NULL);
+    sigaction(SIGUSR1, &sigint_action, NULL);
 #elif defined (_WIN32)
     auto console_ctrl_handler = +[](DWORD ctrl_type) -> BOOL {
         return (ctrl_type == CTRL_C_EVENT) ? (signal_handler(SIGINT), true) : false;
diff --git a/ggml-cuda.cu b/ggml-cuda.cu
index c207ff87..945708a4 100644
--- a/ggml-cuda.cu
+++ b/ggml-cuda.cu
@@ -46,6 +46,7 @@
 #define __shfl_xor_sync(mask, var, laneMask, width) __shfl_xor(var, laneMask, width)
 #define cublasComputeType_t hipblasDatatype_t //deprecated, new hipblasComputeType_t not in 5.6
 #define cublasCreate hipblasCreate
+#define cublasDestroy hipblasDestroy
 #define cublasGemmEx hipblasGemmEx
 #define cublasGemmBatchedEx hipblasGemmBatchedEx
 #define cublasGemmStridedBatchedEx hipblasGemmStridedBatchedEx
@@ -8014,10 +8015,10 @@ GGML_CALL bool ggml_cublas_loaded(void) {
     return g_cublas_loaded;
 }
 
-GGML_CALL void ggml_init_cublas() {
-    static bool initialized = false;
+static bool g_cublas_initialized = false;
 
-    if (!initialized) {
+GGML_CALL void ggml_init_cublas() {
+    if (!g_cublas_initialized) {
 
 #ifdef __HIP_PLATFORM_AMD__
         // Workaround for a rocBLAS bug when using multiple graphics cards:
@@ -8027,7 +8028,7 @@ GGML_CALL void ggml_init_cublas() {
 #endif
 
         if (cudaGetDeviceCount(&g_device_count) != cudaSuccess) {
-            initialized = true;
+            g_cublas_initialized = true;
             g_cublas_loaded = false;
             fprintf(stderr, "%s: no " GGML_CUDA_NAME " devices found, " GGML_CUDA_NAME " will be disabled\n", __func__);
             return;
@@ -8098,7 +8099,7 @@ GGML_CALL void ggml_init_cublas() {
         // configure logging to stdout
         // CUBLAS_CHECK(cublasLoggerConfigure(1, 1, 0, nullptr));
 
-        initialized = true;
+        g_cublas_initialized = true;
         g_cublas_loaded = true;
     }
 }
@@ -11753,3 +11754,23 @@ GGML_CALL int ggml_backend_cuda_reg_devices() {
     }
     return device_count;
 }
+
+
+extern "C" GGML_CALL void ggml_free_cublas(void);
+GGML_CALL void ggml_free_cublas(void) {
+    for (int id = 0; id < g_device_count; ++id) {
+#if !(defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__))
+        if (g_device_caps[id].vmm) {
+            CU_CHECK(cuMemUnmap(g_cuda_pool_addr[id], g_cuda_pool_size[id]));
+            g_cuda_pool_size[id] = 0;
+            g_cuda_pool_addr[id] = 0;
+        }
+#endif
+        // TODO: free legacy non-vmm memory
+        // destroy cublas handle
+        CUBLAS_CHECK(cublasDestroy(g_cublas_handles[id]));
+        g_cublas_handles[id] = nullptr;
+    }
+
+    g_cublas_initialized = false;
+}
diff --git a/ggml-cuda.h b/ggml-cuda.h
index b1ebd61d..6dd58ddf 100644
--- a/ggml-cuda.h
+++ b/ggml-cuda.h
@@ -23,6 +23,9 @@ GGML_API GGML_CALL void   ggml_init_cublas(void);
 // Returns `true` if there are available CUDA devices and cublas loads successfully; otherwise, it returns `false`.
 GGML_API GGML_CALL bool   ggml_cublas_loaded(void);
 
+// Release CUDA resources
+GGML_API GGML_CALL void   ggml_free_cublas(void);
+
 GGML_API GGML_CALL void * ggml_cuda_host_malloc(size_t size);
 GGML_API GGML_CALL void   ggml_cuda_host_free(void * ptr);