baalajimaestro/ollama
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

parser: keep seeking until eof

Browse source
This commit is contained in:
Jeffrey Morgan 2023-07-20 00:37:52 -07:00
parent d9f54300c3
commit e6bae8d916
1 changed files with 5 additions and 11 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

16
parser/parser.go
View file

@ -71,22 +71,16 @@ func scanModelfile(data []byte, atEOF bool) (advance int, token []byte, err erro
if start := bytes.Index(data, []byte(`"""`)); start >= 0 && start < newline { if start := bytes.Index(data, []byte(`"""`)); start >= 0 && start < newline {
end := bytes.Index(data[start+3:], []byte(`"""`)) end := bytes.Index(data[start+3:], []byte(`"""`))
if end < 0 { if end < 0 {
return 0, nil, errors.New(`unterminated multiline string: """`) if atEOF {
return 0, nil, errors.New(`unterminated multiline string: """`)
} else {
return 0, nil, nil
}
} }
n := start + 3 + end + 3 n := start + 3 + end + 3
return n, bytes.Replace(data[:n], []byte(`"""`), []byte(""), 2), nil return n, bytes.Replace(data[:n], []byte(`"""`), []byte(""), 2), nil
} }
if start := bytes.Index(data, []byte(`'''`)); start >= 0 && start < newline {
end := bytes.Index(data[start+3:], []byte(`'''`))
if end < 0 {
return 0, nil, errors.New("unterminated multiline string: '''")
}
n := start + 3 + end + 3
return n, bytes.Replace(data[:n], []byte("'''"), []byte(""), 2), nil
}
return bufio.ScanLines(data, atEOF) return bufio.ScanLines(data, atEOF)
} }