baalajimaestro/ollama
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Merge pull request #3308 from dhiltgen/bump_more

Browse source 
Bump llama.cpp to b2510
This commit is contained in:
Daniel Hiltgen 2024-03-25 12:56:12 -07:00 committed by GitHub
commit 560be5e0b6
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
5 changed files with 7 additions and 184 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
llm/ext_server/server.cpp vendored
View file

@ -343,6 +343,12 @@ struct llama_server_context
~llama_server_context()
{
if (clp_ctx)
{
LOG_INFO("freeing clip model", {});
clip_free(clp_ctx);
clp_ctx = nullptr;
}
if (ctx)
{
llama_free(ctx);

2
llm/llama.cpp

@ -1 +1 @@
Subproject commit ceca1aef0738b57951cd12c603c3477e75312dec
Subproject commit 1b26aebe4de4f048ac99996efd8a2c9af150904d

21
llm/patches/01-cache.diff
View file

@ -1,21 +0,0 @@
diff --git a/examples/server/server.cpp b/examples/server/server.cpp
index 8fe5e0b1..3e82acb9 100644
--- a/examples/server/server.cpp
+++ b/examples/server/server.cpp
@@ -997,13 +997,15 @@ struct llama_server_context
slot.n_sent_text += result.text_to_send.size();
// add the token to slot queue and cache
}
- slot.add_token_string(result);
+
if (slot.params.stream)
{
send_partial_response(slot, result);
}
}
+ slot.add_token_string(result);
+
if (incomplete)
{
slot.has_next_token = true;

117
llm/patches/02-cudaleaks.diff
View file

@ -1,117 +0,0 @@
diff --git a/examples/server/server.cpp b/examples/server/server.cpp
index 8fe5e0b1..53bf39c1 100644
--- a/examples/server/server.cpp
+++ b/examples/server/server.cpp
@@ -31,6 +31,10 @@
#include <atomic>
#include <signal.h>
+#ifdef GGML_USE_CUBLAS
+extern "C" GGML_CALL void ggml_free_cublas(void);
+#endif
+
using json = nlohmann::json;
struct server_params {
@@ -363,6 +367,10 @@ struct llama_server_context
llama_free_model(model);
model = nullptr;
}
+
+#ifdef GGML_USE_CUBLAS
+ ggml_free_cublas();
+#endif
}
bool load_model(const gpt_params &params_)
@@ -3543,6 +3551,7 @@ int main(int argc, char **argv)
sigemptyset (&sigint_action.sa_mask);
sigint_action.sa_flags = 0;
sigaction(SIGINT, &sigint_action, NULL);
+ sigaction(SIGUSR1, &sigint_action, NULL);
#elif defined (_WIN32)
auto console_ctrl_handler = +[](DWORD ctrl_type) -> BOOL {
return (ctrl_type == CTRL_C_EVENT) ? (signal_handler(SIGINT), true) : false;
diff --git a/ggml-cuda.cu b/ggml-cuda.cu
index 72bcec8c..6c934e8c 100644
--- a/ggml-cuda.cu
+++ b/ggml-cuda.cu
@@ -43,6 +43,7 @@
#define __shfl_xor_sync(mask, var, laneMask, width) __shfl_xor(var, laneMask, width)
#define cublasComputeType_t hipblasDatatype_t //deprecated, new hipblasComputeType_t not in 5.6
#define cublasCreate hipblasCreate
+#define cublasDestroy hipblasDestroy
#define cublasGemmEx hipblasGemmEx
#define cublasGemmBatchedEx hipblasGemmBatchedEx
#define cublasGemmStridedBatchedEx hipblasGemmStridedBatchedEx
@@ -8751,10 +8752,10 @@ GGML_CALL bool ggml_cublas_loaded(void) {
return g_cublas_loaded;
}
-GGML_CALL void ggml_init_cublas() {
- static bool initialized = false;
+static bool g_cublas_initialized = false;
- if (!initialized) {
+GGML_CALL void ggml_init_cublas() {
+ if (!g_cublas_initialized) {
#ifdef __HIP_PLATFORM_AMD__
// Workaround for a rocBLAS bug when using multiple graphics cards:
@@ -8764,7 +8765,7 @@ GGML_CALL void ggml_init_cublas() {
#endif
if (cudaGetDeviceCount(&g_device_count) != cudaSuccess) {
- initialized = true;
+ g_cublas_initialized = true;
g_cublas_loaded = false;
fprintf(stderr, "%s: no " GGML_CUDA_NAME " devices found, " GGML_CUDA_NAME " will be disabled\n", __func__);
return;
@@ -8835,7 +8836,7 @@ GGML_CALL void ggml_init_cublas() {
// configure logging to stdout
// CUBLAS_CHECK(cublasLoggerConfigure(1, 1, 0, nullptr));
- initialized = true;
+ g_cublas_initialized = true;
g_cublas_loaded = true;
}
}
@@ -12490,3 +12491,23 @@ GGML_CALL int ggml_backend_cuda_reg_devices() {
}
return device_count;
}
+
+
+extern "C" GGML_CALL void ggml_free_cublas(void);
+GGML_CALL void ggml_free_cublas(void) {
+ for (int id = 0; id < g_device_count; ++id) {
+#if !(defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__))
+ if (g_device_caps[id].vmm) {
+ CU_CHECK(cuMemUnmap(g_cuda_pool_addr[id], g_cuda_pool_size[id]));
+ g_cuda_pool_size[id] = 0;
+ g_cuda_pool_addr[id] = 0;
+ }
+#endif
+ // TODO: free legacy non-vmm memory
+ // destroy cublas handle
+ CUBLAS_CHECK(cublasDestroy(g_cublas_handles[id]));
+ g_cublas_handles[id] = nullptr;
+ }
+
+ g_cublas_initialized = false;
+}
\ No newline at end of file
diff --git a/ggml-cuda.h b/ggml-cuda.h
index b1ebd61d..6dd58ddf 100644
--- a/ggml-cuda.h
+++ b/ggml-cuda.h
@@ -23,6 +23,9 @@ GGML_API GGML_CALL void ggml_init_cublas(void);
// Returns `true` if there are available CUDA devices and cublas loads successfully; otherwise, it returns `false`.
GGML_API GGML_CALL bool ggml_cublas_loaded(void);
+// Release CUDA resources
+GGML_API GGML_CALL void ggml_free_cublas(void);
+
GGML_API GGML_CALL void * ggml_cuda_host_malloc(size_t size);
GGML_API GGML_CALL void ggml_cuda_host_free(void * ptr);

45
llm/patches/05-fix-clip-free.diff
View file

@ -1,45 +0,0 @@
From 9192432daf90b1bfec75577434a99b4ea70d54c8 Mon Sep 17 00:00:00 2001
From: Michael Yang <mxyng@pm.me>
Date: Thu, 14 Mar 2024 12:09:50 -0700
Subject: [PATCH] fix clip free
---
examples/llava/clip.cpp | 4 ++++
examples/server/server.cpp | 6 ++++++
2 files changed, 10 insertions(+)
diff --git a/examples/llava/clip.cpp b/examples/llava/clip.cpp
index ef9e4ba7..b4ddfe6b 100644
--- a/examples/llava/clip.cpp
+++ b/examples/llava/clip.cpp
@@ -1673,6 +1673,10 @@ void clip_free(clip_ctx * ctx) {
ggml_free(ctx->ctx_data);
gguf_free(ctx->ctx_gguf);
+ ggml_backend_buffer_free(ctx->params_buffer);
+ ggml_backend_buffer_free(ctx->compute_buffer);
+ ggml_backend_free(ctx->backend);
+ ggml_gallocr_free(ctx->compute_alloc);
delete ctx;
}
diff --git a/examples/server/server.cpp b/examples/server/server.cpp
index 8fe5e0b1..f927336b 100644
--- a/examples/server/server.cpp
+++ b/examples/server/server.cpp
@@ -353,6 +353,12 @@ struct llama_server_context
~llama_server_context()
{
+ if (clp_ctx)
+ {
+ LOG_INFO("freeing clip model", {});
+ clip_free(clp_ctx);
+ clp_ctx = nullptr;
+ }
if (ctx)
{
llama_free(ctx);
--
2.43.2