baalajimaestro/ollama
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

chore: update tokenizer.go (#4571)

Browse source 
PreTokenziers -> PreTokenizers
This commit is contained in:
Ikko Eltociear Ashimine 2024-05-22 16:25:23 +09:00 committed by GitHub
parent 9f18b88a06
commit 955c317cab
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
1 changed files with 2 additions and 2 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
convert/tokenizer.go
View file

@ -18,7 +18,7 @@ type Tokenizer struct {
Model TokenizerModel `json:"model"` Model TokenizerModel `json:"model"`
PreTokenizer struct { PreTokenizer struct {
PreTokenziers []struct { PreTokenizers []struct {
Type string `json:"type"` Type string `json:"type"`
Pattern struct { Pattern struct {
Regex string `json:"Regex"` Regex string `json:"Regex"`
@ -84,7 +84,7 @@ func parseTokens(dirpath string) (pre string, tokens []Token, merges []string, e
} }
sha256sum := sha256.New() sha256sum := sha256.New()
for _, pt := range t.PreTokenizer.PreTokenziers { for _, pt := range t.PreTokenizer.PreTokenizers {
switch pt.Type { switch pt.Type {
case "Split": case "Split":
if pt.Pattern.Regex != "" { if pt.Pattern.Regex != "" {