From 822a0e36eb15457fb04c1a42d34cf0db938a9789 Mon Sep 17 00:00:00 2001
From: Jeffrey Morgan <jmorganca@gmail.com>
Date: Fri, 28 Jul 2023 10:56:21 -0400
Subject: [PATCH] lower batch size to 512

---
 api/types.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/api/types.go b/api/types.go
index e8f91270..2e3b0578 100644
--- a/api/types.go
+++ b/api/types.go
@@ -190,7 +190,7 @@ func DefaultOptions() Options {
 		UseNUMA: false,
 
 		NumCtx:   2048,
-		NumBatch: 1024,
+		NumBatch: 512,
 		NumGPU:   1,
 		NumGQA:   1,
 		LowVRAM:  false,