From 955c317cabe1344c9f0ed7a71e33f6b4f0919e5e Mon Sep 17 00:00:00 2001
From: Ikko Eltociear Ashimine <eltociear@gmail.com>
Date: Wed, 22 May 2024 16:25:23 +0900
Subject: [PATCH] chore: update tokenizer.go (#4571)

PreTokenziers -> PreTokenizers
---
 convert/tokenizer.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/convert/tokenizer.go b/convert/tokenizer.go
index e0fe0bb7..efeb5491 100644
--- a/convert/tokenizer.go
+++ b/convert/tokenizer.go
@@ -18,7 +18,7 @@ type Tokenizer struct {
 	Model       TokenizerModel `json:"model"`
 
 	PreTokenizer struct {
-		PreTokenziers []struct {
+		PreTokenizers []struct {
 			Type    string `json:"type"`
 			Pattern struct {
 				Regex string `json:"Regex"`
@@ -84,7 +84,7 @@ func parseTokens(dirpath string) (pre string, tokens []Token, merges []string, e
 	}
 
 	sha256sum := sha256.New()
-	for _, pt := range t.PreTokenizer.PreTokenziers {
+	for _, pt := range t.PreTokenizer.PreTokenizers {
 		switch pt.Type {
 		case "Split":
 			if pt.Pattern.Regex != "" {