chore: update tokenizer.go (#4571)

PreTokenziers -> PreTokenizers
This commit is contained in:
Ikko Eltociear Ashimine 2024-05-22 16:25:23 +09:00 committed by GitHub
parent 9f18b88a06
commit 955c317cab
No known key found for this signature in database
GPG key ID: B5690EEEBB952194

View file

@ -18,7 +18,7 @@ type Tokenizer struct {
Model TokenizerModel `json:"model"`
PreTokenizer struct {
PreTokenziers []struct {
PreTokenizers []struct {
Type string `json:"type"`
Pattern struct {
Regex string `json:"Regex"`
@ -84,7 +84,7 @@ func parseTokens(dirpath string) (pre string, tokens []Token, merges []string, e
}
sha256sum := sha256.New()
for _, pt := range t.PreTokenizer.PreTokenziers {
for _, pt := range t.PreTokenizer.PreTokenizers {
switch pt.Type {
case "Split":
if pt.Pattern.Regex != "" {