ollama/llm/patches/01-cache.diff

22 lines
654 B
Diff
Raw Normal View History

diff --git a/examples/server/server.cpp b/examples/server/server.cpp
index 2b2f4a0f..afac49af 100644
--- a/examples/server/server.cpp
+++ b/examples/server/server.cpp
@@ -997,13 +997,15 @@ struct llama_server_context
slot.n_sent_text += result.text_to_send.size();
// add the token to slot queue and cache
}
- slot.add_token_string(result);
+
if (slot.params.stream)
{
send_partial_response(slot, result);
}
}
+ slot.add_token_string(result);
+
if (incomplete)
{
slot.has_next_token = true;