2023-12-20 18:36:01 +00:00
|
|
|
#include "dynamic_shim.h"
|
2023-11-29 19:00:37 +00:00
|
|
|
|
|
|
|
#include <stdio.h>
|
|
|
|
#include <string.h>
|
|
|
|
|
2023-12-18 20:05:59 +00:00
|
|
|
#ifdef __linux__
|
2023-11-29 19:00:37 +00:00
|
|
|
#include <dlfcn.h>
|
2023-12-18 20:05:59 +00:00
|
|
|
#define LOAD_LIBRARY(lib, flags) dlopen(lib, flags | RTLD_DEEPBIND)
|
2023-11-29 19:00:37 +00:00
|
|
|
#define LOAD_SYMBOL(handle, sym) dlsym(handle, sym)
|
|
|
|
#define LOAD_ERR() dlerror()
|
|
|
|
#define UNLOAD_LIBRARY(handle) dlclose(handle)
|
2023-12-18 20:05:59 +00:00
|
|
|
#elif _WIN32
|
2023-11-29 19:00:37 +00:00
|
|
|
#include <windows.h>
|
|
|
|
#define LOAD_LIBRARY(lib, flags) LoadLibrary(lib)
|
|
|
|
#define LOAD_SYMBOL(handle, sym) GetProcAddress(handle, sym)
|
|
|
|
#define UNLOAD_LIBRARY(handle) FreeLibrary(handle)
|
|
|
|
// TODO - refactor this with proper error message handling on windows
|
|
|
|
inline static char *LOAD_ERR() {
|
|
|
|
static char errbuf[8];
|
|
|
|
snprintf(errbuf, 8, "0x%lx", GetLastError());
|
|
|
|
return errbuf;
|
|
|
|
}
|
2023-12-18 20:05:59 +00:00
|
|
|
#else
|
|
|
|
#include <dlfcn.h>
|
|
|
|
#define LOAD_LIBRARY(lib, flags) dlopen(lib, flags)
|
|
|
|
#define LOAD_SYMBOL(handle, sym) dlsym(handle, sym)
|
|
|
|
#define LOAD_ERR() dlerror()
|
|
|
|
#define UNLOAD_LIBRARY(handle) dlclose(handle)
|
2023-11-29 19:00:37 +00:00
|
|
|
#endif
|
|
|
|
|
2023-12-20 18:36:01 +00:00
|
|
|
void dynamic_shim_init(const char *libPath, struct dynamic_llama_server *s,
|
|
|
|
ext_server_resp_t *err) {
|
2023-11-29 19:00:37 +00:00
|
|
|
int i = 0;
|
|
|
|
struct lookup {
|
|
|
|
char *s;
|
|
|
|
void **p;
|
|
|
|
} l[] = {
|
|
|
|
{"llama_server_init", (void *)&s->llama_server_init},
|
|
|
|
{"llama_server_start", (void *)&s->llama_server_start},
|
|
|
|
{"llama_server_stop", (void *)&s->llama_server_stop},
|
|
|
|
{"llama_server_completion", (void *)&s->llama_server_completion},
|
|
|
|
{"llama_server_completion_next_result",
|
|
|
|
(void *)&s->llama_server_completion_next_result},
|
|
|
|
{"llama_server_completion_cancel",
|
|
|
|
(void *)&s->llama_server_completion_cancel},
|
|
|
|
{"llama_server_release_task_result",
|
|
|
|
(void *)&s->llama_server_release_task_result},
|
|
|
|
{"llama_server_tokenize", (void *)&s->llama_server_tokenize},
|
|
|
|
{"llama_server_detokenize", (void *)&s->llama_server_detokenize},
|
|
|
|
{"llama_server_embedding", (void *)&s->llama_server_embedding},
|
|
|
|
{"llama_server_release_json_resp",
|
|
|
|
(void *)&s->llama_server_release_json_resp},
|
|
|
|
{"", NULL},
|
|
|
|
};
|
|
|
|
|
|
|
|
printf("Lazy loading %s library\n", libPath);
|
2023-12-18 20:05:59 +00:00
|
|
|
s->handle = LOAD_LIBRARY(libPath, RTLD_NOW);
|
2023-11-29 19:00:37 +00:00
|
|
|
if (!s->handle) {
|
|
|
|
err->id = -1;
|
2023-12-20 18:36:01 +00:00
|
|
|
snprintf(err->msg, err->msg_len,
|
|
|
|
"Unable to load dynamic server library: %s", LOAD_ERR());
|
2023-11-29 19:00:37 +00:00
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
for (i = 0; l[i].p != NULL; i++) {
|
|
|
|
*l[i].p = LOAD_SYMBOL(s->handle, l[i].s);
|
|
|
|
if (!l[i].p) {
|
|
|
|
UNLOAD_LIBRARY(s->handle);
|
|
|
|
err->id = -1;
|
|
|
|
snprintf(err->msg, err->msg_len, "symbol lookup for %s failed: %s",
|
|
|
|
l[i].s, LOAD_ERR());
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2023-12-20 18:36:01 +00:00
|
|
|
inline void dynamic_shim_llama_server_init(struct dynamic_llama_server s,
|
|
|
|
ext_server_params_t *sparams,
|
|
|
|
ext_server_resp_t *err) {
|
2023-11-29 19:00:37 +00:00
|
|
|
s.llama_server_init(sparams, err);
|
|
|
|
}
|
|
|
|
|
2023-12-20 18:36:01 +00:00
|
|
|
inline void dynamic_shim_llama_server_start(struct dynamic_llama_server s) {
|
2023-11-29 19:00:37 +00:00
|
|
|
s.llama_server_start();
|
|
|
|
}
|
|
|
|
|
2023-12-20 18:36:01 +00:00
|
|
|
inline void dynamic_shim_llama_server_stop(struct dynamic_llama_server s) {
|
2023-11-29 19:00:37 +00:00
|
|
|
s.llama_server_stop();
|
|
|
|
}
|
|
|
|
|
2023-12-20 18:36:01 +00:00
|
|
|
inline void dynamic_shim_llama_server_completion(struct dynamic_llama_server s,
|
|
|
|
const char *json_req,
|
|
|
|
ext_server_resp_t *resp) {
|
2023-11-29 19:00:37 +00:00
|
|
|
s.llama_server_completion(json_req, resp);
|
|
|
|
}
|
|
|
|
|
2023-12-20 18:36:01 +00:00
|
|
|
inline void dynamic_shim_llama_server_completion_next_result(
|
|
|
|
struct dynamic_llama_server s, const int task_id,
|
2023-11-29 19:00:37 +00:00
|
|
|
ext_server_task_result_t *result) {
|
|
|
|
s.llama_server_completion_next_result(task_id, result);
|
|
|
|
}
|
|
|
|
|
2023-12-20 18:36:01 +00:00
|
|
|
inline void dynamic_shim_llama_server_completion_cancel(
|
|
|
|
struct dynamic_llama_server s, const int task_id, ext_server_resp_t *err) {
|
2023-11-29 19:00:37 +00:00
|
|
|
s.llama_server_completion_cancel(task_id, err);
|
|
|
|
}
|
2023-12-20 18:36:01 +00:00
|
|
|
inline void dynamic_shim_llama_server_release_task_result(
|
|
|
|
struct dynamic_llama_server s, ext_server_task_result_t *result) {
|
2023-11-29 19:00:37 +00:00
|
|
|
s.llama_server_release_task_result(result);
|
|
|
|
}
|
|
|
|
|
2023-12-20 18:36:01 +00:00
|
|
|
inline void dynamic_shim_llama_server_tokenize(struct dynamic_llama_server s,
|
|
|
|
const char *json_req,
|
|
|
|
char **json_resp,
|
|
|
|
ext_server_resp_t *err) {
|
2023-11-29 19:00:37 +00:00
|
|
|
s.llama_server_tokenize(json_req, json_resp, err);
|
|
|
|
}
|
|
|
|
|
2023-12-20 18:36:01 +00:00
|
|
|
inline void dynamic_shim_llama_server_detokenize(struct dynamic_llama_server s,
|
|
|
|
const char *json_req,
|
|
|
|
char **json_resp,
|
|
|
|
ext_server_resp_t *err) {
|
2023-11-29 19:00:37 +00:00
|
|
|
s.llama_server_detokenize(json_req, json_resp, err);
|
|
|
|
}
|
|
|
|
|
2023-12-20 18:36:01 +00:00
|
|
|
inline void dynamic_shim_llama_server_embedding(struct dynamic_llama_server s,
|
|
|
|
const char *json_req,
|
|
|
|
char **json_resp,
|
|
|
|
ext_server_resp_t *err) {
|
2023-11-29 19:00:37 +00:00
|
|
|
s.llama_server_embedding(json_req, json_resp, err);
|
|
|
|
}
|
|
|
|
|
2023-12-20 18:36:01 +00:00
|
|
|
inline void dynamic_shim_llama_server_release_json_resp(
|
|
|
|
struct dynamic_llama_server s, char **json_resp) {
|
2023-11-29 19:00:37 +00:00
|
|
|
s.llama_server_release_json_resp(json_resp);
|
|
|
|
}
|