parent
9f18b88a06
commit
955c317cab
1 changed files with 2 additions and 2 deletions
|
@ -18,7 +18,7 @@ type Tokenizer struct {
|
|||
Model TokenizerModel `json:"model"`
|
||||
|
||||
PreTokenizer struct {
|
||||
PreTokenziers []struct {
|
||||
PreTokenizers []struct {
|
||||
Type string `json:"type"`
|
||||
Pattern struct {
|
||||
Regex string `json:"Regex"`
|
||||
|
@ -84,7 +84,7 @@ func parseTokens(dirpath string) (pre string, tokens []Token, merges []string, e
|
|||
}
|
||||
|
||||
sha256sum := sha256.New()
|
||||
for _, pt := range t.PreTokenizer.PreTokenziers {
|
||||
for _, pt := range t.PreTokenizer.PreTokenizers {
|
||||
switch pt.Type {
|
||||
case "Split":
|
||||
if pt.Pattern.Regex != "" {
|
||||
|
|
Loading…
Reference in a new issue