baalajimaestro/ollama
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

clean up convert tokenizer

Browse source
This commit is contained in:
Michael Yang 2024-08-27 10:45:39 -07:00
parent 3eb08377f8
commit eae3af6807
2 changed files with 2 additions and 3 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
convert/convert_test.go
View file

@ -89,7 +89,7 @@ func TestMain(m *testing.M) {
os.Exit(m.Run()) os.Exit(m.Run())
} }
func TestConvertFull(t *testing.T) { func TestConvertModel(t *testing.T) {
cases := []string{ cases := []string{
"Meta-Llama-3-8B-Instruct", "Meta-Llama-3-8B-Instruct",
"Meta-Llama-3.1-8B-Instruct", "Meta-Llama-3.1-8B-Instruct",

3
convert/tokenizer.go
View file

@ -154,7 +154,6 @@ func parseTokenizer(fsys fs.FS, specialTokenTypes []string) (*Tokenizer, error)
} }
type tokenizer struct { type tokenizer struct {
Version string `json:"version"`
AddedTokens []token `json:"added_tokens"` AddedTokens []token `json:"added_tokens"`
Model struct { Model struct {
Type string `json:"type"` Type string `json:"type"`
@ -252,7 +251,7 @@ func parseVocabulary(fsys fs.FS) (*Vocabulary, error) {
return pattern.Func(fsys) return pattern.Func(fsys)
} }
return nil, errors.New("unknown tensor format") return nil, errors.New("unknown tokenizer format")
} }
type SpecialVocabulary struct { type SpecialVocabulary struct {