From 8d75016549e2ff62a511b1119d966ffc0df5c77b Mon Sep 17 00:00:00 2001
From: Andrei Betlen <abetlen@gmail.com>
Date: Sat, 16 Sep 2023 14:57:49 -0400
Subject: [PATCH] Install required runtime dlls to package directory on windows

---
 CMakeLists.txt         | 11 ++++++++++-
 llama_cpp/llama_cpp.py |  2 +-
 pyproject.toml         |  2 +-
 3 files changed, 12 insertions(+), 3 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index c503d48..de6e850 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -1,4 +1,4 @@
-cmake_minimum_required(VERSION 3.4...3.22)
+cmake_minimum_required(VERSION 3.21)
 
 project(llama_cpp)
 
@@ -33,4 +33,13 @@ if (LLAMA_BUILD)
         FRAMEWORK DESTINATION ${CMAKE_CURRENT_SOURCE_DIR}/llama_cpp
         RESOURCE DESTINATION ${CMAKE_CURRENT_SOURCE_DIR}/llama_cpp
     )
+    # Workaround for Windows + CUDA https://github.com/abetlen/llama-cpp-python/issues/563
+    install(
+        FILES $<TARGET_RUNTIME_DLLS:llama>
+        DESTINATION ${SKBUILD_PLATLIB_DIR}/llama_cpp
+    )
+    install(
+        FILES $<TARGET_RUNTIME_DLLS:llama>
+        DESTINATION ${CMAKE_CURRENT_SOURCE_DIR}/llama_cpp
+    )
 endif()
diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py
index 9f0dfd2..f7e562d 100644
--- a/llama_cpp/llama_cpp.py
+++ b/llama_cpp/llama_cpp.py
@@ -58,7 +58,7 @@ def _load_shared_library(lib_base_name: str):
         if "CUDA_PATH" in os.environ:
             os.add_dll_directory(os.path.join(os.environ["CUDA_PATH"], "bin"))
             os.add_dll_directory(os.path.join(os.environ["CUDA_PATH"], "lib"))
-        cdll_args["winmode"] = 0
+        cdll_args["winmode"] = ctypes.RTLD_GLOBAL
 
     # Try to load the shared library, handling potential errors
     for _lib_path in _lib_paths:
diff --git a/pyproject.toml b/pyproject.toml
index e2ba21a..152fbc0 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -54,7 +54,7 @@ all = [
 [tool.scikit-build]
 wheel.packages = ["llama_cpp"]
 cmake.verbose = true
-cmake.minimum-version = "3.12"
+cmake.minimum-version = "3.21"
 minimum-version = "0.5"
 sdist.exclude = [".git", "vendor/llama.cpp/.git"]