baalajimaestro/ollama
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

configurable rope frequency parameters

Browse source
This commit is contained in:
Michael Yang 2023-08-03 22:10:49 -07:00
parent e3fb1fd3f1
commit b9f4d67554
2 changed files with 27 additions and 21 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

46
api/types.go
View file

@ -147,19 +147,21 @@ type Options struct {
UseNUMA bool `json:"numa,omitempty"` UseNUMA bool `json:"numa,omitempty"`
// Model options // Model options
NumCtx int `json:"num_ctx,omitempty"` NumCtx int `json:"num_ctx,omitempty"`
NumKeep int `json:"num_keep,omitempty"` NumKeep int `json:"num_keep,omitempty"`
NumBatch int `json:"num_batch,omitempty"` NumBatch int `json:"num_batch,omitempty"`
NumGQA int `json:"num_gqa,omitempty"` NumGQA int `json:"num_gqa,omitempty"`
NumGPU int `json:"num_gpu,omitempty"` NumGPU int `json:"num_gpu,omitempty"`
MainGPU int `json:"main_gpu,omitempty"` MainGPU int `json:"main_gpu,omitempty"`
LowVRAM bool `json:"low_vram,omitempty"` LowVRAM bool `json:"low_vram,omitempty"`
F16KV bool `json:"f16_kv,omitempty"` F16KV bool `json:"f16_kv,omitempty"`
LogitsAll bool `json:"logits_all,omitempty"` LogitsAll bool `json:"logits_all,omitempty"`
VocabOnly bool `json:"vocab_only,omitempty"` VocabOnly bool `json:"vocab_only,omitempty"`
UseMMap bool `json:"use_mmap,omitempty"` UseMMap bool `json:"use_mmap,omitempty"`
UseMLock bool `json:"use_mlock,omitempty"` UseMLock bool `json:"use_mlock,omitempty"`
EmbeddingOnly bool `json:"embedding_only,omitempty"` EmbeddingOnly bool `json:"embedding_only,omitempty"`
RopeFrequencyBase float32 `json:"rope_frequency_base,omitempty"`
RopeFrequencyScale float32 `json:"rope_frequency_scale,omitempty"`
// Predict options // Predict options
RepeatLastN int `json:"repeat_last_n,omitempty"` RepeatLastN int `json:"repeat_last_n,omitempty"`
@ -261,14 +263,16 @@ func DefaultOptions() Options {
UseNUMA: false, UseNUMA: false,
NumCtx: 2048, NumCtx: 2048,
NumBatch: 512, NumBatch: 512,
NumGPU: 1, NumGPU: 1,
NumGQA: 1, NumGQA: 1,
LowVRAM: false, LowVRAM: false,
F16KV: true, F16KV: true,
UseMMap: true, UseMMap: true,
UseMLock: false, UseMLock: false,
RopeFrequencyBase: 10000.0,
RopeFrequencyScale: 1.0,
RepeatLastN: 64, RepeatLastN: 64,
RepeatPenalty: 1.1, RepeatPenalty: 1.1,

2
llama/llama.go
View file

@ -142,6 +142,8 @@ func New(model string, opts api.Options) (*LLM, error) {
params.use_mmap = C.bool(llm.UseMMap) params.use_mmap = C.bool(llm.UseMMap)
params.use_mlock = C.bool(llm.UseMLock) params.use_mlock = C.bool(llm.UseMLock)
params.embedding = C.bool(llm.EmbeddingOnly) params.embedding = C.bool(llm.EmbeddingOnly)
params.rope_freq_base = C.float(llm.RopeFrequencyBase)
params.rope_freq_scale = C.float(llm.RopeFrequencyScale)
llm.params = &params llm.params = &params
cModel := C.CString(model) cModel := C.CString(model)