From a05b4da80a67f6079e54cac4cd769cf877022639 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Mon, 18 Dec 2023 18:40:36 -0500 Subject: [PATCH] fix: float32 is not JSON serializable when streaming logits. --- llama_cpp/llama.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py index 1ce1f0b..c2c0455 100644 --- a/llama_cpp/llama.py +++ b/llama_cpp/llama.py @@ -1555,7 +1555,7 @@ class Llama: ) token_offset = len(prompt_tokens) + returned_tokens logits = self._scores[token_offset - 1, :] - current_logprobs = Llama.logits_to_logprobs(logits) + current_logprobs = Llama.logits_to_logprobs(logits).tolist() sorted_logprobs = list( sorted( zip(current_logprobs, range(len(current_logprobs))), @@ -1674,7 +1674,7 @@ class Llama: ) token_offset = len(prompt_tokens) + returned_tokens - 1 logits = self._scores[token_offset, :] - current_logprobs = Llama.logits_to_logprobs(logits) + current_logprobs = Llama.logits_to_logprobs(logits).tolist() sorted_logprobs = list( sorted( zip(current_logprobs, range(len(current_logprobs))),