From 5de568bffed863760c796c9df165ccdfd6d22ce2 Mon Sep 17 00:00:00 2001 From: Matt Williams Date: Fri, 10 Nov 2023 08:28:52 -0600 Subject: [PATCH 001/103] Add a simple log analysis example Signed-off-by: Matt Williams --- examples/python-loganalysis/Modelfile | 8 +++ examples/python-loganalysis/loganalysis.py | 51 ++++++++++++++++++++ examples/python-loganalysis/logtest.logfile | 32 ++++++++++++ examples/python-loganalysis/readme.md | 46 ++++++++++++++++++ examples/python-loganalysis/requirements.txt | 1 + 5 files changed, 138 insertions(+) create mode 100644 examples/python-loganalysis/Modelfile create mode 100644 examples/python-loganalysis/loganalysis.py create mode 100644 examples/python-loganalysis/logtest.logfile create mode 100644 examples/python-loganalysis/readme.md create mode 100644 examples/python-loganalysis/requirements.txt diff --git a/examples/python-loganalysis/Modelfile b/examples/python-loganalysis/Modelfile new file mode 100644 index 00000000..5237cb6e --- /dev/null +++ b/examples/python-loganalysis/Modelfile @@ -0,0 +1,8 @@ +FROM codebooga:latest + +SYSTEM """ +You are a log file analyzer. You will receive a set of lines from a log file for some software application, find the errors and other interesting aspects of the logs, and explain them so a new user can understand what they mean. If there are any steps they can do to resolve them, list the steps in your answer. +""" + +PARAMETER TEMPERATURE 0.3 + diff --git a/examples/python-loganalysis/loganalysis.py b/examples/python-loganalysis/loganalysis.py new file mode 100644 index 00000000..ed1f173a --- /dev/null +++ b/examples/python-loganalysis/loganalysis.py @@ -0,0 +1,51 @@ +import sys +import re +import requests +import json + +prelines = 10 +postlines = 10 + +def find_errors_in_log_file(): + if len(sys.argv) < 2: + print("Usage: python loganalysis.py ") + return + + log_file_path = sys.argv[1] + with open(log_file_path, 'r') as log_file: + log_lines = log_file.readlines() + + error_lines = [] + for i, line in enumerate(log_lines): + if re.search('error', line, re.IGNORECASE): + error_lines.append(i) + + error_logs = [] + for error_line in error_lines: + start_index = max(0, error_line - prelines) + end_index = min(len(log_lines), error_line + postlines) + error_logs.extend(log_lines[start_index:end_index]) + + return error_logs + +error_logs = find_errors_in_log_file() + +data = { + "prompt": "\n".join(error_logs), + "model": "mattw/loganalyzer" +} + + +response = requests.post("http://localhost:11434/api/generate", json=data, stream=True) +for line in response.iter_lines(): + if line: + json_data = json.loads(line) + if json_data['done'] == False: + print(json_data['response'], end='') + + + + + + + diff --git a/examples/python-loganalysis/logtest.logfile b/examples/python-loganalysis/logtest.logfile new file mode 100644 index 00000000..e4181bfe --- /dev/null +++ b/examples/python-loganalysis/logtest.logfile @@ -0,0 +1,32 @@ +2023-11-10 07:17:40 /docker-entrypoint.sh: /docker-entrypoint.d/ is not empty, will attempt to perform configuration +2023-11-10 07:17:40 /docker-entrypoint.sh: Looking for shell scripts in /docker-entrypoint.d/ +2023-11-10 07:17:40 /docker-entrypoint.sh: Launching /docker-entrypoint.d/10-listen-on-ipv6-by-default.sh +2023-11-10 07:17:40 10-listen-on-ipv6-by-default.sh: info: Getting the checksum of /etc/nginx/conf.d/default.conf +2023-11-10 07:17:40 10-listen-on-ipv6-by-default.sh: info: Enabled listen on IPv6 in /etc/nginx/conf.d/default.conf +2023-11-10 07:17:40 /docker-entrypoint.sh: Sourcing /docker-entrypoint.d/15-local-resolvers.envsh +2023-11-10 07:17:40 /docker-entrypoint.sh: Launching /docker-entrypoint.d/20-envsubst-on-templates.sh +2023-11-10 07:17:40 /docker-entrypoint.sh: Launching /docker-entrypoint.d/30-tune-worker-processes.sh +2023-11-10 07:17:40 /docker-entrypoint.sh: Configuration complete; ready for start up +2023-11-10 07:17:40 2023/11/10 13:17:40 [notice] 1#1: using the "epoll" event method +2023-11-10 07:17:40 2023/11/10 13:17:40 [notice] 1#1: nginx/1.25.3 +2023-11-10 07:17:40 2023/11/10 13:17:40 [notice] 1#1: built by gcc 12.2.0 (Debian 12.2.0-14) +2023-11-10 07:17:40 2023/11/10 13:17:40 [notice] 1#1: OS: Linux 6.4.16-linuxkit +2023-11-10 07:17:40 2023/11/10 13:17:40 [notice] 1#1: getrlimit(RLIMIT_NOFILE): 1048576:1048576 +2023-11-10 07:17:40 2023/11/10 13:17:40 [notice] 1#1: start worker processes +2023-11-10 07:17:40 2023/11/10 13:17:40 [notice] 1#1: start worker process 29 +2023-11-10 07:17:40 2023/11/10 13:17:40 [notice] 1#1: start worker process 30 +2023-11-10 07:17:40 2023/11/10 13:17:40 [notice] 1#1: start worker process 31 +2023-11-10 07:17:40 2023/11/10 13:17:40 [notice] 1#1: start worker process 32 +2023-11-10 07:17:40 2023/11/10 13:17:40 [notice] 1#1: start worker process 33 +2023-11-10 07:17:40 2023/11/10 13:17:40 [notice] 1#1: start worker process 34 +2023-11-10 07:17:40 2023/11/10 13:17:40 [notice] 1#1: start worker process 35 +2023-11-10 07:17:40 2023/11/10 13:17:40 [notice] 1#1: start worker process 36 +2023-11-10 07:17:40 2023/11/10 13:17:40 [notice] 1#1: start worker process 37 +2023-11-10 07:17:40 2023/11/10 13:17:40 [notice] 1#1: start worker process 38 +2023-11-10 07:17:44 192.168.65.1 - - [10/Nov/2023:13:17:43 +0000] "GET / HTTP/1.1" 200 615 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36" "-" +2023-11-10 07:17:44 2023/11/10 13:17:44 [error] 29#29: *1 open() "/usr/share/nginx/html/favicon.ico" failed (2: No such file or directory), client: 192.168.65.1, server: localhost, request: "GET /favicon.ico HTTP/1.1", host: "localhost:8080", referrer: "http://localhost:8080/" +2023-11-10 07:17:44 192.168.65.1 - - [10/Nov/2023:13:17:44 +0000] "GET /favicon.ico HTTP/1.1" 404 555 "http://localhost:8080/" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36" "-" +2023-11-10 07:17:50 2023/11/10 13:17:50 [error] 29#29: *1 open() "/usr/share/nginx/html/ahstat" failed (2: No such file or directory), client: 192.168.65.1, server: localhost, request: "GET /ahstat HTTP/1.1", host: "localhost:8080" +2023-11-10 07:17:50 192.168.65.1 - - [10/Nov/2023:13:17:50 +0000] "GET /ahstat HTTP/1.1" 404 555 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36" "-" +2023-11-10 07:18:53 2023/11/10 13:18:53 [error] 29#29: *1 open() "/usr/share/nginx/html/ahstat" failed (2: No such file or directory), client: 192.168.65.1, server: localhost, request: "GET /ahstat HTTP/1.1", host: "localhost:8080" +2023-11-10 07:18:53 192.168.65.1 - - [10/Nov/2023:13:18:53 +0000] "GET /ahstat HTTP/1.1" 404 555 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36" "-" diff --git a/examples/python-loganalysis/readme.md b/examples/python-loganalysis/readme.md new file mode 100644 index 00000000..cba22993 --- /dev/null +++ b/examples/python-loganalysis/readme.md @@ -0,0 +1,46 @@ +# Log Analysis example + +This example shows one possible way to create a log file analyzer. To use it, run: + +`python loganalysis.py ` + +You can try this with the `logtest.logfile` file included in this directory. + +## Review the code + +The first part of this example is a Modelfile that takes `codebooga` and applies a new System Prompt: + +```plaintext +SYSTEM """ +You are a log file analyzer. You will receive a set of lines from a log file for some software application, find the errors and other interesting aspects of the logs, and explain them so a new user can understand what they mean. If there are any steps they can do to resolve them, list the steps in your answer. +""" +``` + +This model is available at https://ollama.ai/mattw/loganalyzer. You can customize it and add to your own namespace using the command `ollama create -f ` then `ollama push `. + +Then loganalysis.py scans all the lines in the given log file and searches for the word 'error'. When the word is found, the 10 lines before and after are set as the prompt for a call to the Generate API. + +```python +data = { + "prompt": "\n".join(error_logs), + "model": "mattw/loganalyzer" +} +``` + +Finally, the streamed output is parsed and the response field in the output is printed to the line. + +```python +response = requests.post("http://localhost:11434/api/generate", json=data, stream=True) +for line in response.iter_lines(): + if line: + json_data = json.loads(line) + if json_data['done'] == False: + print(json_data['response'], end='') + +``` + +## Next Steps + +There is a lot more that can be done here. This is a simple way to detect errors, looking for the word error. Perhaps it would be interesting to find anomalous activity in the logs. It could be interesting to create embeddings for each line and compare them, looking for similar lines. Or look into applying Levenshtein Distance algorithms to find similar lines to help identify the anomalous lines. + +Also try different models and different prompts to analyze the data. You could consider adding RAG to this to help understand newer log formats. diff --git a/examples/python-loganalysis/requirements.txt b/examples/python-loganalysis/requirements.txt new file mode 100644 index 00000000..9688b8ec --- /dev/null +++ b/examples/python-loganalysis/requirements.txt @@ -0,0 +1 @@ +Requests==2.31.0 From e4f59ba073ee55dd4d720db8a8883220859488b1 Mon Sep 17 00:00:00 2001 From: Matt Williams Date: Fri, 10 Nov 2023 08:55:17 -0600 Subject: [PATCH 002/103] better streaming plus gif Signed-off-by: Matt Williams --- examples/python-loganalysis/loganalysis.py | 2 +- examples/python-loganalysis/readme.md | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/examples/python-loganalysis/loganalysis.py b/examples/python-loganalysis/loganalysis.py index ed1f173a..0b7ee5fe 100644 --- a/examples/python-loganalysis/loganalysis.py +++ b/examples/python-loganalysis/loganalysis.py @@ -41,7 +41,7 @@ for line in response.iter_lines(): if line: json_data = json.loads(line) if json_data['done'] == False: - print(json_data['response'], end='') + print(json_data['response'], end='', flush=True) diff --git a/examples/python-loganalysis/readme.md b/examples/python-loganalysis/readme.md index cba22993..58180444 100644 --- a/examples/python-loganalysis/readme.md +++ b/examples/python-loganalysis/readme.md @@ -1,5 +1,7 @@ # Log Analysis example +![loganalyzer 2023-11-10 08_53_29](https://github.com/jmorganca/ollama/assets/633681/ad30f1fc-321f-4953-8914-e30e24db9921) + This example shows one possible way to create a log file analyzer. To use it, run: `python loganalysis.py ` From 73f3448ede681df8eed25d2a3402ce892069ae29 Mon Sep 17 00:00:00 2001 From: Matt Williams Date: Fri, 10 Nov 2023 16:33:56 -0600 Subject: [PATCH 003/103] add example showing use of JSON format Signed-off-by: Matt Williams --- .../predefinedschema.py | 31 ++++++++++++++++ .../randomaddresses.py | 35 +++++++++++++++++++ examples/python-json-datagenerator/readme.md | 13 +++++++ .../requirements.txt | 1 + 4 files changed, 80 insertions(+) create mode 100644 examples/python-json-datagenerator/predefinedschema.py create mode 100644 examples/python-json-datagenerator/randomaddresses.py create mode 100644 examples/python-json-datagenerator/readme.md create mode 100644 examples/python-json-datagenerator/requirements.txt diff --git a/examples/python-json-datagenerator/predefinedschema.py b/examples/python-json-datagenerator/predefinedschema.py new file mode 100644 index 00000000..56d8afa2 --- /dev/null +++ b/examples/python-json-datagenerator/predefinedschema.py @@ -0,0 +1,31 @@ +import requests +import json +import random + +model = "llama2" +template = { + "firstName": "", + "lastName": "", + "address": { + "theStreet": "", + "theCity": "", + "theState": "", + "theZipCode": "" + }, + "phoneNumber": "" +} + +prompt = f"generate one realisticly believable sample data set of a persons first name, last name, address in the US, and phone number. \nUse the following template: {json.dumps(template)}." + +data = { + "prompt": prompt, + "model": model, + "format": "json", + "stream": False, + "options": {"temperature": 2.5, "top_p": 0.99, "top_k": 100}, +} + +print(f"Generating a sample user") +response = requests.post("http://localhost:11434/api/generate", json=data, stream=False) +json_data = json.loads(response.text) +print(json.dumps(json.loads(json_data["response"]), indent=2)) diff --git a/examples/python-json-datagenerator/randomaddresses.py b/examples/python-json-datagenerator/randomaddresses.py new file mode 100644 index 00000000..d5780c88 --- /dev/null +++ b/examples/python-json-datagenerator/randomaddresses.py @@ -0,0 +1,35 @@ +import requests +import json +import random + +countries = [ + "the US", + "the UK", + "the Netherlands", + "Germany", + "Mexico", + "Canada", + "France", +] +country = random.choice(countries) +model = "llama2" + +prompt = ( + "generate one realisticly believable sample data set of a persons first name, last name, address in the" + + country + + ", and phone number. Do not use common names. Respond using JSON. Key names should with no backslashes, values should use plain ascii with no special characters." +) + +data = { + "prompt": prompt, + "model": model, + "format": "json", + "stream": False, + "options": {"temperature": 2.5, "top_p": 0.99, "top_k": 100}, +} + +print(f"Generating a sample user in {country}") +response = requests.post("http://localhost:11434/api/generate", json=data, stream=False) +json_data = json.loads(response.text) + +print(json.dumps(json.loads(json_data["response"]), indent=2)) diff --git a/examples/python-json-datagenerator/readme.md b/examples/python-json-datagenerator/readme.md new file mode 100644 index 00000000..c311c42c --- /dev/null +++ b/examples/python-json-datagenerator/readme.md @@ -0,0 +1,13 @@ +# JSON Output Example + +New in version 0.1.9 is support for JSON output. There are two python scripts in this example. `randomaddresses.py` generates random addresses from different countries. `predefinedschema.py` sets a template for the model to fill in. + +## Review the Code + +Both programs are basically the same, with a different prompt for each, demonstrating two different ideas. The key part of getting JSON out of a model is to state in the prompt or system prompt that it should respond using JSON, and specifying the `format` as `json` in the data body. + +When running `randomaddresses.py` you will see that the schema changes and adapts to the chosen country. + +In `predefinedschema.py`, a template has been specified in the prompt as well. It's been defined as JSON and then dumped into the prompt string to make it easier to work with. + +Both examples turn streaming off so that we end up with the completed JSON all at once. We need to convert the `response.text` to JSON so that when we output it as a string we can set the indent spacing to make the output attractive. diff --git a/examples/python-json-datagenerator/requirements.txt b/examples/python-json-datagenerator/requirements.txt new file mode 100644 index 00000000..9688b8ec --- /dev/null +++ b/examples/python-json-datagenerator/requirements.txt @@ -0,0 +1 @@ +Requests==2.31.0 From b6817a83d89476f4eb3e45e039522f3e1b140b56 Mon Sep 17 00:00:00 2001 From: Matt Williams Date: Fri, 10 Nov 2023 16:41:48 -0600 Subject: [PATCH 004/103] Add gif and finish readme Signed-off-by: Matt Williams --- .../randomaddresses.py | 6 +---- examples/python-json-datagenerator/readme.md | 23 ++++++++++++++++++- 2 files changed, 23 insertions(+), 6 deletions(-) diff --git a/examples/python-json-datagenerator/randomaddresses.py b/examples/python-json-datagenerator/randomaddresses.py index d5780c88..afb97690 100644 --- a/examples/python-json-datagenerator/randomaddresses.py +++ b/examples/python-json-datagenerator/randomaddresses.py @@ -14,11 +14,7 @@ countries = [ country = random.choice(countries) model = "llama2" -prompt = ( - "generate one realisticly believable sample data set of a persons first name, last name, address in the" - + country - + ", and phone number. Do not use common names. Respond using JSON. Key names should with no backslashes, values should use plain ascii with no special characters." -) +prompt = f"generate one realisticly believable sample data set of a persons first name, last name, address in {country}, and phone number. Do not use common names. Respond using JSON. Key names should with no backslashes, values should use plain ascii with no special characters." data = { "prompt": prompt, diff --git a/examples/python-json-datagenerator/readme.md b/examples/python-json-datagenerator/readme.md index c311c42c..872833f8 100644 --- a/examples/python-json-datagenerator/readme.md +++ b/examples/python-json-datagenerator/readme.md @@ -1,13 +1,34 @@ # JSON Output Example -New in version 0.1.9 is support for JSON output. There are two python scripts in this example. `randomaddresses.py` generates random addresses from different countries. `predefinedschema.py` sets a template for the model to fill in. +![llmjson 2023-11-10 15_31_31](https://github.com/jmorganca/ollama/assets/633681/e599d986-9b4a-4118-81a4-4cfe7e22da25) + +There are two python scripts in this example. `randomaddresses.py` generates random addresses from different countries. `predefinedschema.py` sets a template for the model to fill in. ## Review the Code Both programs are basically the same, with a different prompt for each, demonstrating two different ideas. The key part of getting JSON out of a model is to state in the prompt or system prompt that it should respond using JSON, and specifying the `format` as `json` in the data body. +```python +prompt = f"generate one realisticly believable sample data set of a persons first name, last name, address in {country}, and phone number. Do not use common names. Respond using JSON. Key names should with no backslashes, values should use plain ascii with no special characters." + +data = { + "prompt": prompt, + "model": model, + "format": "json", + "stream": False, + "options": {"temperature": 2.5, "top_p": 0.99, "top_k": 100}, +} +``` + When running `randomaddresses.py` you will see that the schema changes and adapts to the chosen country. In `predefinedschema.py`, a template has been specified in the prompt as well. It's been defined as JSON and then dumped into the prompt string to make it easier to work with. Both examples turn streaming off so that we end up with the completed JSON all at once. We need to convert the `response.text` to JSON so that when we output it as a string we can set the indent spacing to make the output attractive. + +```python +response = requests.post("http://localhost:11434/api/generate", json=data, stream=False) +json_data = json.loads(response.text) + +print(json.dumps(json.loads(json_data["response"]), indent=2)) +``` From 96bf9cafa79b85671b518d47cf89f213f25de0cb Mon Sep 17 00:00:00 2001 From: Matt Williams Date: Tue, 14 Nov 2023 10:30:17 -0800 Subject: [PATCH 005/103] Update examples/python-loganalysis/loganalysis.py Co-authored-by: Bruce MacDonald --- examples/python-loganalysis/loganalysis.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/examples/python-loganalysis/loganalysis.py b/examples/python-loganalysis/loganalysis.py index 0b7ee5fe..336e2e8f 100644 --- a/examples/python-loganalysis/loganalysis.py +++ b/examples/python-loganalysis/loganalysis.py @@ -43,9 +43,3 @@ for line in response.iter_lines(): if json_data['done'] == False: print(json_data['response'], end='', flush=True) - - - - - - From eced0d52abeab6f1e1beba4686ac89eaf57d6438 Mon Sep 17 00:00:00 2001 From: Matt Williams Date: Tue, 14 Nov 2023 10:30:30 -0800 Subject: [PATCH 006/103] Update examples/python-loganalysis/loganalysis.py Co-authored-by: Bruce MacDonald --- examples/python-loganalysis/loganalysis.py | 1 + 1 file changed, 1 insertion(+) diff --git a/examples/python-loganalysis/loganalysis.py b/examples/python-loganalysis/loganalysis.py index 336e2e8f..e9385065 100644 --- a/examples/python-loganalysis/loganalysis.py +++ b/examples/python-loganalysis/loganalysis.py @@ -3,6 +3,7 @@ import re import requests import json +# prelines and postlines represent the number of lines of context to include in the output around the error prelines = 10 postlines = 10 From 64b7e0c218d8faac6533e729da720248f2196d15 Mon Sep 17 00:00:00 2001 From: Matt Williams Date: Tue, 14 Nov 2023 10:31:05 -0800 Subject: [PATCH 007/103] Update examples/python-loganalysis/loganalysis.py Co-authored-by: Bruce MacDonald --- examples/python-loganalysis/loganalysis.py | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/examples/python-loganalysis/loganalysis.py b/examples/python-loganalysis/loganalysis.py index e9385065..2b7ddd48 100644 --- a/examples/python-loganalysis/loganalysis.py +++ b/examples/python-loganalysis/loganalysis.py @@ -16,16 +16,12 @@ def find_errors_in_log_file(): with open(log_file_path, 'r') as log_file: log_lines = log_file.readlines() - error_lines = [] - for i, line in enumerate(log_lines): - if re.search('error', line, re.IGNORECASE): - error_lines.append(i) - - error_logs = [] - for error_line in error_lines: - start_index = max(0, error_line - prelines) - end_index = min(len(log_lines), error_line + postlines) - error_logs.extend(log_lines[start_index:end_index]) +error_logs = [] + for i, line in enumerate(log_lines): + if "error" in line.lower(): + start_index = max(0, i - prelines) + end_index = min(len(log_lines), i + postlines + 1) + error_logs.extend(log_lines[start_index:end_index]) return error_logs From f4edc302a8535f81cba7417abc2b5a0d5eea1e20 Mon Sep 17 00:00:00 2001 From: Matt Williams Date: Tue, 14 Nov 2023 10:31:22 -0800 Subject: [PATCH 008/103] Update examples/python-loganalysis/readme.md Co-authored-by: Bruce MacDonald --- examples/python-loganalysis/readme.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/python-loganalysis/readme.md b/examples/python-loganalysis/readme.md index 58180444..fbfb89a1 100644 --- a/examples/python-loganalysis/readme.md +++ b/examples/python-loganalysis/readme.md @@ -45,4 +45,4 @@ for line in response.iter_lines(): There is a lot more that can be done here. This is a simple way to detect errors, looking for the word error. Perhaps it would be interesting to find anomalous activity in the logs. It could be interesting to create embeddings for each line and compare them, looking for similar lines. Or look into applying Levenshtein Distance algorithms to find similar lines to help identify the anomalous lines. -Also try different models and different prompts to analyze the data. You could consider adding RAG to this to help understand newer log formats. +Also try different models and different prompts to analyze the data. You could consider adding retrieval augmented generation (RAG) to this to help understand newer log formats. From f748331aa346c65913fb304036202acd7dad5525 Mon Sep 17 00:00:00 2001 From: Matt Williams Date: Tue, 14 Nov 2023 10:32:45 -0800 Subject: [PATCH 009/103] Update examples/python-json-datagenerator/predefinedschema.py Co-authored-by: Bruce MacDonald --- examples/python-json-datagenerator/predefinedschema.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/python-json-datagenerator/predefinedschema.py b/examples/python-json-datagenerator/predefinedschema.py index 56d8afa2..7f7abd48 100644 --- a/examples/python-json-datagenerator/predefinedschema.py +++ b/examples/python-json-datagenerator/predefinedschema.py @@ -15,7 +15,7 @@ template = { "phoneNumber": "" } -prompt = f"generate one realisticly believable sample data set of a persons first name, last name, address in the US, and phone number. \nUse the following template: {json.dumps(template)}." +prompt = f"generate one realistically believable sample data set of a persons first name, last name, address in the US, and phone number. \nUse the following template: {json.dumps(template)}." data = { "prompt": prompt, From acde0819d97fa82af7d95da5ae64e0b4fdf6923a Mon Sep 17 00:00:00 2001 From: Matt Williams Date: Tue, 14 Nov 2023 10:33:02 -0800 Subject: [PATCH 010/103] Update examples/python-json-datagenerator/randomaddresses.py Co-authored-by: Bruce MacDonald --- examples/python-json-datagenerator/randomaddresses.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/python-json-datagenerator/randomaddresses.py b/examples/python-json-datagenerator/randomaddresses.py index afb97690..ec6c920d 100644 --- a/examples/python-json-datagenerator/randomaddresses.py +++ b/examples/python-json-datagenerator/randomaddresses.py @@ -14,7 +14,7 @@ countries = [ country = random.choice(countries) model = "llama2" -prompt = f"generate one realisticly believable sample data set of a persons first name, last name, address in {country}, and phone number. Do not use common names. Respond using JSON. Key names should with no backslashes, values should use plain ascii with no special characters." +prompt = f"generate one realistically believable sample data set of a persons first name, last name, address in {country}, and phone number. Do not use common names. Respond using JSON. Key names should have no backslashes, values should use plain ascii with no special characters." data = { "prompt": prompt, From 69795d2db083e723d3c3f0b2b8b5f6673462e13e Mon Sep 17 00:00:00 2001 From: Matt Williams Date: Tue, 14 Nov 2023 10:33:16 -0800 Subject: [PATCH 011/103] Update examples/python-json-datagenerator/readme.md Co-authored-by: Bruce MacDonald --- examples/python-json-datagenerator/readme.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/python-json-datagenerator/readme.md b/examples/python-json-datagenerator/readme.md index 872833f8..0d825543 100644 --- a/examples/python-json-datagenerator/readme.md +++ b/examples/python-json-datagenerator/readme.md @@ -9,7 +9,7 @@ There are two python scripts in this example. `randomaddresses.py` generates ran Both programs are basically the same, with a different prompt for each, demonstrating two different ideas. The key part of getting JSON out of a model is to state in the prompt or system prompt that it should respond using JSON, and specifying the `format` as `json` in the data body. ```python -prompt = f"generate one realisticly believable sample data set of a persons first name, last name, address in {country}, and phone number. Do not use common names. Respond using JSON. Key names should with no backslashes, values should use plain ascii with no special characters." +prompt = f"generate one realistically believable sample data set of a persons first name, last name, address in {country}, and phone number. Do not use common names. Respond using JSON. Key names should with no backslashes, values should use plain ascii with no special characters." data = { "prompt": prompt, From 47ffb81db7d000f8b68cec2f7845e9d6c0756966 Mon Sep 17 00:00:00 2001 From: Matt Williams Date: Tue, 14 Nov 2023 10:33:34 -0800 Subject: [PATCH 012/103] Update examples/python-json-datagenerator/readme.md Co-authored-by: Bruce MacDonald --- examples/python-json-datagenerator/readme.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/python-json-datagenerator/readme.md b/examples/python-json-datagenerator/readme.md index 0d825543..2dc958e7 100644 --- a/examples/python-json-datagenerator/readme.md +++ b/examples/python-json-datagenerator/readme.md @@ -24,7 +24,7 @@ When running `randomaddresses.py` you will see that the schema changes and adapt In `predefinedschema.py`, a template has been specified in the prompt as well. It's been defined as JSON and then dumped into the prompt string to make it easier to work with. -Both examples turn streaming off so that we end up with the completed JSON all at once. We need to convert the `response.text` to JSON so that when we output it as a string we can set the indent spacing to make the output attractive. +Both examples turn streaming off so that we end up with the completed JSON all at once. We need to convert the `response.text` to JSON so that when we output it as a string we can set the indent spacing to make the output easy to read. ```python response = requests.post("http://localhost:11434/api/generate", json=data, stream=False) From 01ea6002c49e8bf97901c94a3f655eba9e783cd4 Mon Sep 17 00:00:00 2001 From: Michael Yang Date: Tue, 14 Nov 2023 14:57:41 -0800 Subject: [PATCH 013/103] replace go-humanize with format.HumanBytes --- cmd/cmd.go | 3 +-- format/bytes.go | 6 +++--- go.mod | 1 - go.sum | 2 -- 4 files changed, 4 insertions(+), 8 deletions(-) diff --git a/cmd/cmd.go b/cmd/cmd.go index d7839558..8fc6e4c4 100644 --- a/cmd/cmd.go +++ b/cmd/cmd.go @@ -20,7 +20,6 @@ import ( "syscall" "time" - "github.com/dustin/go-humanize" "github.com/olekukonko/tablewriter" "github.com/spf13/cobra" "golang.org/x/crypto/ssh" @@ -173,7 +172,7 @@ func ListHandler(cmd *cobra.Command, args []string) error { for _, m := range models.Models { if len(args) == 0 || strings.HasPrefix(m.Name, args[0]) { - data = append(data, []string{m.Name, m.Digest[:12], humanize.Bytes(uint64(m.Size)), format.HumanTime(m.ModifiedAt, "Never")}) + data = append(data, []string{m.Name, m.Digest[:12], format.HumanBytes(m.Size), format.HumanTime(m.ModifiedAt, "Never")}) } } diff --git a/format/bytes.go b/format/bytes.go index ca5ac640..1cf5a762 100644 --- a/format/bytes.go +++ b/format/bytes.go @@ -12,11 +12,11 @@ const ( func HumanBytes(b int64) string { switch { case b > GigaByte: - return fmt.Sprintf("%d GB", b/GigaByte) + return fmt.Sprintf("%.1f GB", float64(b)/GigaByte) case b > MegaByte: - return fmt.Sprintf("%d MB", b/MegaByte) + return fmt.Sprintf("%.1f MB", float64(b)/MegaByte) case b > KiloByte: - return fmt.Sprintf("%d KB", b/KiloByte) + return fmt.Sprintf("%.1f KB", float64(b)/KiloByte) default: return fmt.Sprintf("%d B", b) } diff --git a/go.mod b/go.mod index b0f117bd..6d4c629a 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,6 @@ module github.com/jmorganca/ollama go 1.20 require ( - github.com/dustin/go-humanize v1.0.1 github.com/emirpasic/gods v1.18.1 github.com/gin-gonic/gin v1.9.1 github.com/mattn/go-runewidth v0.0.14 diff --git a/go.sum b/go.sum index 0b94adbc..dab05d3d 100644 --- a/go.sum +++ b/go.sum @@ -9,8 +9,6 @@ github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ3 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= -github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc= github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ= github.com/gabriel-vasile/mimetype v1.4.2 h1:w5qFW6JKBz9Y393Y4q372O9A7cUSequkh1Q7OhCmWKU= From 85951d25efd839e1456c1c35bfaec2c9bd931562 Mon Sep 17 00:00:00 2001 From: bnodnarb <97063458+bnodnarb@users.noreply.github.com> Date: Wed, 15 Nov 2023 07:32:37 -1000 Subject: [PATCH 014/103] Created tutorial for running Ollama on NVIDIA Jetson devices (#1098) --- docs/tutorials.md | 3 ++- docs/tutorials/nvidia-jetson.md | 38 +++++++++++++++++++++++++++++++++ 2 files changed, 40 insertions(+), 1 deletion(-) create mode 100644 docs/tutorials/nvidia-jetson.md diff --git a/docs/tutorials.md b/docs/tutorials.md index bf8adf8d..0f520c95 100644 --- a/docs/tutorials.md +++ b/docs/tutorials.md @@ -4,5 +4,6 @@ Here is a list of ways you can use Ollama with other tools to build interesting - [Using LangChain with Ollama in JavaScript](./tutorials/langchainjs.md) - [Using LangChain with Ollama in Python](./tutorials/langchainpy.md) +- [Running Ollama on NVIDIA Jetson Devices](./tutorials/nvidia-jetson.md) -Also be sure to check out the [examples](../examples) directory for more ways to use Ollama. \ No newline at end of file +Also be sure to check out the [examples](../examples) directory for more ways to use Ollama. diff --git a/docs/tutorials/nvidia-jetson.md b/docs/tutorials/nvidia-jetson.md new file mode 100644 index 00000000..85cf741c --- /dev/null +++ b/docs/tutorials/nvidia-jetson.md @@ -0,0 +1,38 @@ +# Running Ollama on NVIDIA Jetson Devices + +With some minor configuration, Ollama runs well on [NVIDIA Jetson Devices](https://www.nvidia.com/en-us/autonomous-machines/embedded-systems/). The following has been tested on [JetPack 5.1.2](https://developer.nvidia.com/embedded/jetpack). + +NVIDIA Jetson devices are Linux-based embedded AI computers that are purpose-built for AI applications. + +Jetsons have an integrated GPU that is wired directly to the memory controller of the machine. For this reason, the `nvidia-smi` command is unrecognized, and Ollama proceeds to operate in "CPU only" +mode. This can be verified by using a monitoring tool like jtop. + +In order to address this, we simply pass the path to the Jetson's pre-installed CUDA libraries into `ollama serve` (while in a tmux session). We then hardcode the num_gpu parameters into a cloned +version of our target model. + +Prerequisites: + +- curl +- tmux + +Here are the steps: + +- Install Ollama via standard Linux command (ignore the 404 error): `curl https://ollama.ai/install.sh | sh` +- Stop the Ollama service: `sudo systemctl stop ollama` +- Start Ollama serve in a tmux session called ollama_jetson and reference the CUDA libraries path: `tmux has-session -t ollama_jetson 2>/dev/null || tmux new-session -d -s ollama_jetson +'LD_LIBRARY_PATH=/usr/local/cuda/lib64 ollama serve'` +- Pull the model you want to use (e.g. mistral): `ollama pull mistral` +- Create a new Modelfile specifically for enabling GPU support on the Jetson: `touch ModelfileMistralJetson` +- In the ModelfileMistralJetson file, specify the FROM model and the num_gpu PARAMETER as shown below: + +``` +FROM mistral +PARAMETER num_gpu 999 +``` + +- Create a new model from your Modelfile: `ollama create mistral-jetson -f ./ModelfileMistralJetson` +- Run the new model: `ollama run mistral-jetson` + +If you run a monitoring tool like jtop you should now see that Ollama is using the Jetson's integrated GPU. + +And that's it! From f61f340279661582f96f315c2e8d0a798587128d Mon Sep 17 00:00:00 2001 From: Matt Williams Date: Wed, 15 Nov 2023 15:05:13 -0800 Subject: [PATCH 015/103] FAQ: answer a few faq questions (#1128) * faq: does ollama share my prompts Signed-off-by: Matt Williams * faq: ollama and openai Signed-off-by: Matt Williams * faq: vscode plugins Signed-off-by: Matt Williams * faq: send a doc to Ollama Signed-off-by: Matt Williams * extra spacing Signed-off-by: Matt Williams * Update faq.md * Update faq.md --------- Signed-off-by: Matt Williams Co-authored-by: Michael --- docs/faq.md | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/docs/faq.md b/docs/faq.md index 915d0cc0..28ed5cf6 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -1,5 +1,10 @@ # FAQ +- [How can I view the logs?](#how-can-i-view-the-logs) +- [How can I expose Ollama on my network?](#how-can-i-expose-ollama-on-my-network) +- [How can I allow additional web origins to access Ollama?](#how-can-i-allow-additional-web-origins-to-access-ollama) +- [Where are models stored?](#where-are-models-stored) + ## How can I view the logs? On macOS: @@ -74,8 +79,6 @@ systemctl restart ollama - macOS: Raw model data is stored under `~/.ollama/models`. - Linux: Raw model data is stored under `/usr/share/ollama/.ollama/models` - - Below the models directory you will find a structure similar to the following: ```shell @@ -96,3 +99,11 @@ The manifest lists all the layers used in this model. You will see a `media type ### How can I change where Ollama stores models? To modify where models are stored, you can use the `OLLAMA_MODELS` environment variable. Note that on Linux this means defining `OLLAMA_MODELS` in a drop-in `/etc/systemd/system/ollama.service.d` service file, reloading systemd, and restarting the ollama service. + +## Does Ollama send my prompts and answers back to Ollama.ai to use in any way? + +No. Anything you do with Ollama, such as generate a response from the model, stays with you. We don't collect any data about how you use the model. You are always in control of your own data. + +## How can I use Ollama in VSCode to help me code? + +There is already a large collection of plugins available for VSCode as well as other editors that leverage Ollama. You can see the list of plugins at the bottom of the main repository readme. From b0d14ed51c459d3c63649ebcaa8c581431a946ae Mon Sep 17 00:00:00 2001 From: Michael Yang Date: Tue, 14 Nov 2023 12:30:34 -0800 Subject: [PATCH 016/103] refactor create model --- server/images.go | 330 +++++++++++++++++++++-------------------------- server/routes.go | 16 ++- 2 files changed, 162 insertions(+), 184 deletions(-) diff --git a/server/images.go b/server/images.go index 8d784fef..42def270 100644 --- a/server/images.go +++ b/server/images.go @@ -248,200 +248,172 @@ func filenameWithPath(path, f string) (string, error) { return f, nil } -func CreateModel(ctx context.Context, name string, path string, fn func(resp api.ProgressResponse)) error { - mp := ParseModelPath(name) - - var manifest *ManifestV2 - var err error - var noprune string - - // build deleteMap to prune unused layers - deleteMap := make(map[string]bool) - - if noprune = os.Getenv("OLLAMA_NOPRUNE"); noprune == "" { - manifest, _, err = GetManifest(mp) - if err != nil && !errors.Is(err, os.ErrNotExist) { - return err - } - - if manifest != nil { - for _, l := range manifest.Layers { - deleteMap[l.Digest] = true - } - deleteMap[manifest.Config.Digest] = true - } - } - - mf, err := os.Open(path) +func realpath(p string) string { + abspath, err := filepath.Abs(p) if err != nil { - fn(api.ProgressResponse{Status: fmt.Sprintf("couldn't open modelfile '%s'", path)}) - return fmt.Errorf("failed to open file: %w", err) + return p } - defer mf.Close() - fn(api.ProgressResponse{Status: "parsing modelfile"}) - commands, err := parser.Parse(mf) + home, err := os.UserHomeDir() if err != nil { - return err + return abspath } + if p == "~" { + return home + } else if strings.HasPrefix(p, "~/") { + return filepath.Join(home, p[2:]) + } + + return abspath +} + +func CreateModel(ctx context.Context, name string, commands []parser.Command, fn func(resp api.ProgressResponse)) error { config := ConfigV2{ - Architecture: "amd64", OS: "linux", + Architecture: "amd64", } + deleteMap := make(map[string]struct{}) + var layers []*LayerReader + params := make(map[string][]string) - var sourceParams map[string]any + fromParams := make(map[string]any) + for _, c := range commands { - log.Printf("[%s] - %s\n", c.Name, c.Args) + log.Printf("[%s] - %s", c.Name, c.Args) + mediatype := fmt.Sprintf("application/vnd.ollama.image.%s", c.Name) + switch c.Name { case "model": - fn(api.ProgressResponse{Status: "looking for model"}) - - mp := ParseModelPath(c.Args) - mf, _, err := GetManifest(mp) + bin, err := os.Open(realpath(c.Args)) if err != nil { - modelFile, err := filenameWithPath(path, c.Args) - if err != nil { + // not a file on disk so must be a model reference + modelpath := ParseModelPath(c.Args) + manifest, _, err := GetManifest(modelpath) + switch { + case errors.Is(err, os.ErrNotExist): + fn(api.ProgressResponse{Status: "pulling model"}) + if err := PullModel(ctx, c.Args, &RegistryOptions{}, fn); err != nil { + return err + } + + manifest, _, err = GetManifest(modelpath) + if err != nil { + return err + } + case err != nil: return err } - if _, err := os.Stat(modelFile); err != nil { - // the model file does not exist, try pulling it - if errors.Is(err, os.ErrNotExist) { - fn(api.ProgressResponse{Status: "pulling model file"}) - if err := PullModel(ctx, c.Args, &RegistryOptions{}, fn); err != nil { - return err - } - mf, _, err = GetManifest(mp) - if err != nil { - return fmt.Errorf("failed to open file after pull: %v", err) - } - } else { - return err - } - } else { - // create a model from this specified file - fn(api.ProgressResponse{Status: "creating model layer"}) - file, err := os.Open(modelFile) - if err != nil { - return fmt.Errorf("failed to open file: %v", err) - } - defer file.Close() - ggml, err := llm.DecodeGGML(file) - if err != nil { - return err - } - - config.ModelFormat = ggml.Name() - config.ModelFamily = ggml.ModelFamily() - config.ModelType = ggml.ModelType() - config.FileType = ggml.FileType() - - // reset the file - file.Seek(0, io.SeekStart) - - l, err := CreateLayer(file) - if err != nil { - return fmt.Errorf("failed to create layer: %v", err) - } - l.MediaType = "application/vnd.ollama.image.model" - layers = append(layers, l) - } - } - - if mf != nil { fn(api.ProgressResponse{Status: "reading model metadata"}) - sourceBlobPath, err := GetBlobsPath(mf.Config.Digest) + fromConfigPath, err := GetBlobsPath(manifest.Config.Digest) if err != nil { return err } - sourceBlob, err := os.Open(sourceBlobPath) + fromConfigFile, err := os.Open(fromConfigPath) if err != nil { return err } - defer sourceBlob.Close() + defer fromConfigFile.Close() - var source ConfigV2 - if err := json.NewDecoder(sourceBlob).Decode(&source); err != nil { + var fromConfig ConfigV2 + if err := json.NewDecoder(fromConfigFile).Decode(&fromConfig); err != nil { return err } - // copy the model metadata - config.ModelFamily = source.ModelFamily - config.ModelType = source.ModelType - config.ModelFormat = source.ModelFormat - config.FileType = source.FileType + config.ModelFormat = fromConfig.ModelFormat + config.ModelFamily = fromConfig.ModelFamily + config.ModelType = fromConfig.ModelType + config.FileType = fromConfig.FileType - for _, l := range mf.Layers { - if l.MediaType == "application/vnd.ollama.image.params" { - sourceParamsBlobPath, err := GetBlobsPath(l.Digest) + for _, layer := range manifest.Layers { + deleteMap[layer.Digest] = struct{}{} + if layer.MediaType == "application/vnd.ollama.image.params" { + fromParamsPath, err := GetBlobsPath(layer.Digest) if err != nil { return err } - sourceParamsBlob, err := os.Open(sourceParamsBlobPath) + fromParamsFile, err := os.Open(fromParamsPath) if err != nil { return err } - defer sourceParamsBlob.Close() + defer fromParamsFile.Close() - if err := json.NewDecoder(sourceParamsBlob).Decode(&sourceParams); err != nil { + if err := json.NewDecoder(fromParamsFile).Decode(&fromParams); err != nil { return err } } - newLayer, err := GetLayerWithBufferFromLayer(l) + layer, err := GetLayerWithBufferFromLayer(layer) if err != nil { return err } - newLayer.From = mp.GetShortTagname() - layers = append(layers, newLayer) - } - } - case "adapter": - fn(api.ProgressResponse{Status: fmt.Sprintf("creating model %s layer", c.Name)}) - fp, err := filenameWithPath(path, c.Args) + layer.From = modelpath.GetShortTagname() + layers = append(layers, layer) + } + + deleteMap[manifest.Config.Digest] = struct{}{} + continue + } + defer bin.Close() + + fn(api.ProgressResponse{Status: "creating model layer"}) + ggml, err := llm.DecodeGGML(bin) if err != nil { return err } - // create a model from this specified file - fn(api.ProgressResponse{Status: "creating model layer"}) + config.ModelFormat = ggml.Name() + config.ModelFamily = ggml.ModelFamily() + config.ModelType = ggml.ModelType() + config.FileType = ggml.FileType() - file, err := os.Open(fp) + bin.Seek(0, io.SeekStart) + layer, err := CreateLayer(bin) if err != nil { - return fmt.Errorf("failed to open file: %v", err) + return err } - defer file.Close() - l, err := CreateLayer(file) + layer.MediaType = mediatype + layers = append(layers, layer) + case "adapter": + fn(api.ProgressResponse{Status: "creating adapter layer"}) + bin, err := os.Open(realpath(c.Args)) if err != nil { - return fmt.Errorf("failed to create layer: %v", err) + return err } - l.MediaType = "application/vnd.ollama.image.adapter" - layers = append(layers, l) - case "license": - fn(api.ProgressResponse{Status: fmt.Sprintf("creating model %s layer", c.Name)}) - mediaType := fmt.Sprintf("application/vnd.ollama.image.%s", c.Name) + defer bin.Close() - layer, err := CreateLayer(strings.NewReader(c.Args)) + layer, err := CreateLayer(bin) if err != nil { return err } if layer.Size > 0 { - layer.MediaType = mediaType + layer.MediaType = mediatype layers = append(layers, layer) } - case "template", "system", "prompt": - fn(api.ProgressResponse{Status: fmt.Sprintf("creating model %s layer", c.Name)}) - // remove the layer if one exists - mediaType := fmt.Sprintf("application/vnd.ollama.image.%s", c.Name) - layers = removeLayerFromLayers(layers, mediaType) + case "license": + fn(api.ProgressResponse{Status: "creating license layer"}) + layer, err := CreateLayer(strings.NewReader(c.Args)) + if err != nil { + return err + } + + if layer.Size > 0 { + layer.MediaType = mediatype + layers = append(layers, layer) + } + case "template", "system": + fn(api.ProgressResponse{Status: fmt.Sprintf("creating %s layer", c.Name)}) + + // remove duplicates layers + layers = removeLayerFromLayers(layers, mediatype) layer, err := CreateLayer(strings.NewReader(c.Args)) if err != nil { @@ -449,48 +421,47 @@ func CreateModel(ctx context.Context, name string, path string, fn func(resp api } if layer.Size > 0 { - layer.MediaType = mediaType + layer.MediaType = mediatype layers = append(layers, layer) } default: - // runtime parameters, build a list of args for each parameter to allow multiple values to be specified (ex: multiple stop sequences) params[c.Name] = append(params[c.Name], c.Args) } } - // Create a single layer for the parameters if len(params) > 0 { - fn(api.ProgressResponse{Status: "creating parameter layer"}) + fn(api.ProgressResponse{Status: "creating parameters layer"}) - layers = removeLayerFromLayers(layers, "application/vnd.ollama.image.params") formattedParams, err := formatParams(params) if err != nil { - return fmt.Errorf("couldn't create params json: %v", err) + return err } - for k, v := range sourceParams { + for k, v := range fromParams { if _, ok := formattedParams[k]; !ok { formattedParams[k] = v } } if config.ModelType == "65B" { - if numGQA, ok := formattedParams["num_gqa"].(int); ok && numGQA == 8 { + if gqa, ok := formattedParams["gqa"].(int); ok && gqa == 8 { config.ModelType = "70B" } } - bts, err := json.Marshal(formattedParams) + var b bytes.Buffer + if err := json.NewEncoder(&b).Encode(formattedParams); err != nil { + return err + } + + fn(api.ProgressResponse{Status: "creating config layer"}) + layer, err := CreateLayer(bytes.NewReader(b.Bytes())) if err != nil { return err } - l, err := CreateLayer(bytes.NewReader(bts)) - if err != nil { - return fmt.Errorf("failed to create layer: %v", err) - } - l.MediaType = "application/vnd.ollama.image.params" - layers = append(layers, l) + layer.MediaType = "application/vnd.ollama.image.params" + layers = append(layers, layer) } digests, err := getLayerDigests(layers) @@ -498,36 +469,31 @@ func CreateModel(ctx context.Context, name string, path string, fn func(resp api return err } - var manifestLayers []*Layer - for _, l := range layers { - manifestLayers = append(manifestLayers, &l.Layer) - delete(deleteMap, l.Layer.Digest) - } - - // Create a layer for the config object - fn(api.ProgressResponse{Status: "creating config layer"}) - cfg, err := createConfigLayer(config, digests) + configLayer, err := createConfigLayer(config, digests) if err != nil { return err } - layers = append(layers, cfg) - delete(deleteMap, cfg.Layer.Digest) + + layers = append(layers, configLayer) + delete(deleteMap, configLayer.Digest) if err := SaveLayers(layers, fn, false); err != nil { return err } - // Create the manifest + var contentLayers []*Layer + for _, layer := range layers { + contentLayers = append(contentLayers, &layer.Layer) + delete(deleteMap, layer.Digest) + } + fn(api.ProgressResponse{Status: "writing manifest"}) - err = CreateManifest(name, cfg, manifestLayers) - if err != nil { + if err := CreateManifest(name, configLayer, contentLayers); err != nil { return err } - if noprune == "" { - fn(api.ProgressResponse{Status: "removing any unused layers"}) - err = deleteUnusedLayers(nil, deleteMap, false) - if err != nil { + if noprune := os.Getenv("OLLAMA_NOPRUNE"); noprune == "" { + if err := deleteUnusedLayers(nil, deleteMap, false); err != nil { return err } } @@ -739,7 +705,7 @@ func CopyModel(src, dest string) error { return nil } -func deleteUnusedLayers(skipModelPath *ModelPath, deleteMap map[string]bool, dryRun bool) error { +func deleteUnusedLayers(skipModelPath *ModelPath, deleteMap map[string]struct{}, dryRun bool) error { fp, err := GetManifestPath() if err != nil { return err @@ -779,21 +745,19 @@ func deleteUnusedLayers(skipModelPath *ModelPath, deleteMap map[string]bool, dry } // only delete the files which are still in the deleteMap - for k, v := range deleteMap { - if v { - fp, err := GetBlobsPath(k) - if err != nil { - log.Printf("couldn't get file path for '%s': %v", k, err) + for k := range deleteMap { + fp, err := GetBlobsPath(k) + if err != nil { + log.Printf("couldn't get file path for '%s': %v", k, err) + continue + } + if !dryRun { + if err := os.Remove(fp); err != nil { + log.Printf("couldn't remove file '%s': %v", fp, err) continue } - if !dryRun { - if err := os.Remove(fp); err != nil { - log.Printf("couldn't remove file '%s': %v", fp, err) - continue - } - } else { - log.Printf("wanted to remove: %s", fp) - } + } else { + log.Printf("wanted to remove: %s", fp) } } @@ -801,7 +765,7 @@ func deleteUnusedLayers(skipModelPath *ModelPath, deleteMap map[string]bool, dry } func PruneLayers() error { - deleteMap := make(map[string]bool) + deleteMap := make(map[string]struct{}) p, err := GetBlobsPath("") if err != nil { return err @@ -818,7 +782,7 @@ func PruneLayers() error { if runtime.GOOS == "windows" { name = strings.ReplaceAll(name, "-", ":") } - deleteMap[name] = true + deleteMap[name] = struct{}{} } log.Printf("total blobs: %d", len(deleteMap)) @@ -873,11 +837,11 @@ func DeleteModel(name string) error { return err } - deleteMap := make(map[string]bool) + deleteMap := make(map[string]struct{}) for _, layer := range manifest.Layers { - deleteMap[layer.Digest] = true + deleteMap[layer.Digest] = struct{}{} } - deleteMap[manifest.Config.Digest] = true + deleteMap[manifest.Config.Digest] = struct{}{} err = deleteUnusedLayers(&mp, deleteMap, false) if err != nil { @@ -1013,7 +977,7 @@ func PullModel(ctx context.Context, name string, regOpts *RegistryOptions, fn fu var noprune string // build deleteMap to prune unused layers - deleteMap := make(map[string]bool) + deleteMap := make(map[string]struct{}) if noprune = os.Getenv("OLLAMA_NOPRUNE"); noprune == "" { manifest, _, err = GetManifest(mp) @@ -1023,9 +987,9 @@ func PullModel(ctx context.Context, name string, regOpts *RegistryOptions, fn fu if manifest != nil { for _, l := range manifest.Layers { - deleteMap[l.Digest] = true + deleteMap[l.Digest] = struct{}{} } - deleteMap[manifest.Config.Digest] = true + deleteMap[manifest.Config.Digest] = struct{}{} } } diff --git a/server/routes.go b/server/routes.go index a543b10e..e53bad43 100644 --- a/server/routes.go +++ b/server/routes.go @@ -26,6 +26,7 @@ import ( "github.com/jmorganca/ollama/api" "github.com/jmorganca/ollama/llm" + "github.com/jmorganca/ollama/parser" "github.com/jmorganca/ollama/version" ) @@ -414,6 +415,19 @@ func CreateModelHandler(c *gin.Context) { return } + modelfile, err := os.Open(req.Path) + if err != nil { + c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()}) + return + } + defer modelfile.Close() + + commands, err := parser.Parse(modelfile) + if err != nil { + c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()}) + return + } + ch := make(chan any) go func() { defer close(ch) @@ -424,7 +438,7 @@ func CreateModelHandler(c *gin.Context) { ctx, cancel := context.WithCancel(c.Request.Context()) defer cancel() - if err := CreateModel(ctx, req.Name, req.Path, fn); err != nil { + if err := CreateModel(ctx, req.Name, commands, fn); err != nil { ch <- gin.H{"error": err.Error()} } }() From 3ca56b5adafb6a46465024df1d9a4d52a6ae4f2f Mon Sep 17 00:00:00 2001 From: Michael Yang Date: Tue, 14 Nov 2023 13:45:07 -0800 Subject: [PATCH 017/103] add create modelfile field --- api/types.go | 7 ++++--- server/routes.go | 22 ++++++++++++++++------ 2 files changed, 20 insertions(+), 9 deletions(-) diff --git a/api/types.go b/api/types.go index ffa5b7ca..2a36a1f6 100644 --- a/api/types.go +++ b/api/types.go @@ -99,9 +99,10 @@ type EmbeddingResponse struct { } type CreateRequest struct { - Name string `json:"name"` - Path string `json:"path"` - Stream *bool `json:"stream,omitempty"` + Name string `json:"name"` + Path string `json:"path"` + Modelfile string `json:"modelfile"` + Stream *bool `json:"stream,omitempty"` } type DeleteRequest struct { diff --git a/server/routes.go b/server/routes.go index e53bad43..65a96911 100644 --- a/server/routes.go +++ b/server/routes.go @@ -410,17 +410,27 @@ func CreateModelHandler(c *gin.Context) { return } - if req.Name == "" || req.Path == "" { - c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "name and path are required"}) + if req.Name == "" { + c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "name is required"}) return } - modelfile, err := os.Open(req.Path) - if err != nil { - c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()}) + if req.Path == "" && req.Modelfile == "" { + c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "path or modelfile are required"}) return } - defer modelfile.Close() + + var modelfile io.Reader = strings.NewReader(req.Modelfile) + if req.Path != "" && req.Modelfile == "" { + bin, err := os.Open(req.Path) + if err != nil { + c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": fmt.Sprintf("error reading modelfile: %s", err)}) + return + } + defer bin.Close() + + modelfile = bin + } commands, err := parser.Parse(modelfile) if err != nil { From 1552cee59f6080fc8b74e81317e94381d2e1844a Mon Sep 17 00:00:00 2001 From: Michael Yang Date: Tue, 14 Nov 2023 14:07:40 -0800 Subject: [PATCH 018/103] client create modelfile --- api/client.go | 27 +++++++++++++++++- api/types.go | 4 +++ cmd/cmd.go | 73 +++++++++++++++++++++++++++++++++++++++--------- server/routes.go | 57 +++++++++++++++++++++++++++++++++++++ 4 files changed, 147 insertions(+), 14 deletions(-) diff --git a/api/client.go b/api/client.go index 974c08eb..262918b3 100644 --- a/api/client.go +++ b/api/client.go @@ -5,6 +5,7 @@ import ( "bytes" "context" "encoding/json" + "errors" "fmt" "io" "net" @@ -95,11 +96,19 @@ func (c *Client) do(ctx context.Context, method, path string, reqData, respData var reqBody io.Reader var data []byte var err error - if reqData != nil { + + switch reqData := reqData.(type) { + case io.Reader: + // reqData is already an io.Reader + reqBody = reqData + case nil: + // noop + default: data, err = json.Marshal(reqData) if err != nil { return err } + reqBody = bytes.NewReader(data) } @@ -287,3 +296,19 @@ func (c *Client) Heartbeat(ctx context.Context) error { } return nil } + +func (c *Client) CreateBlob(ctx context.Context, digest string, r io.Reader) (string, error) { + var response CreateBlobResponse + if err := c.do(ctx, http.MethodGet, fmt.Sprintf("/api/blobs/%s/path", digest), nil, &response); err != nil { + var statusError StatusError + if !errors.As(err, &statusError) || statusError.StatusCode != http.StatusNotFound { + return "", err + } + + if err := c.do(ctx, http.MethodPost, fmt.Sprintf("/api/blobs/%s", digest), r, &response); err != nil { + return "", err + } + } + + return response.Path, nil +} diff --git a/api/types.go b/api/types.go index 2a36a1f6..347c4f84 100644 --- a/api/types.go +++ b/api/types.go @@ -105,6 +105,10 @@ type CreateRequest struct { Stream *bool `json:"stream,omitempty"` } +type CreateBlobResponse struct { + Path string `json:"path"` +} + type DeleteRequest struct { Name string `json:"name"` } diff --git a/cmd/cmd.go b/cmd/cmd.go index 8fc6e4c4..30c6bcf6 100644 --- a/cmd/cmd.go +++ b/cmd/cmd.go @@ -1,9 +1,11 @@ package cmd import ( + "bytes" "context" "crypto/ed25519" "crypto/rand" + "crypto/sha256" "encoding/pem" "errors" "fmt" @@ -27,6 +29,7 @@ import ( "github.com/jmorganca/ollama/api" "github.com/jmorganca/ollama/format" + "github.com/jmorganca/ollama/parser" "github.com/jmorganca/ollama/progressbar" "github.com/jmorganca/ollama/readline" "github.com/jmorganca/ollama/server" @@ -45,17 +48,65 @@ func CreateHandler(cmd *cobra.Command, args []string) error { return err } - var spinner *Spinner + modelfile, err := os.ReadFile(filename) + if err != nil { + return err + } + + spinner := NewSpinner("transferring context") + go spinner.Spin(100 * time.Millisecond) + + commands, err := parser.Parse(bytes.NewReader(modelfile)) + if err != nil { + return err + } + + home, err := os.UserHomeDir() + if err != nil { + return err + } + + for _, c := range commands { + switch c.Name { + case "model", "adapter": + path := c.Args + if path == "~" { + path = home + } else if strings.HasPrefix(path, "~/") { + path = filepath.Join(home, path[2:]) + } + + bin, err := os.Open(path) + if errors.Is(err, os.ErrNotExist) && c.Name == "model" { + // value might be a model reference and not a real file + } else if err != nil { + return err + } + defer bin.Close() + + hash := sha256.New() + if _, err := io.Copy(hash, bin); err != nil { + return err + } + bin.Seek(0, io.SeekStart) + + digest := fmt.Sprintf("sha256:%x", hash.Sum(nil)) + path, err = client.CreateBlob(cmd.Context(), digest, bin) + if err != nil { + return err + } + + modelfile = bytes.ReplaceAll(modelfile, []byte(c.Args), []byte(path)) + } + } var currentDigest string var bar *progressbar.ProgressBar - request := api.CreateRequest{Name: args[0], Path: filename} + request := api.CreateRequest{Name: args[0], Path: filename, Modelfile: string(modelfile)} fn := func(resp api.ProgressResponse) error { if resp.Digest != currentDigest && resp.Digest != "" { - if spinner != nil { - spinner.Stop() - } + spinner.Stop() currentDigest = resp.Digest // pulling bar = progressbar.DefaultBytes( @@ -67,9 +118,7 @@ func CreateHandler(cmd *cobra.Command, args []string) error { bar.Set64(resp.Completed) } else { currentDigest = "" - if spinner != nil { - spinner.Stop() - } + spinner.Stop() spinner = NewSpinner(resp.Status) go spinner.Spin(100 * time.Millisecond) } @@ -81,11 +130,9 @@ func CreateHandler(cmd *cobra.Command, args []string) error { return err } - if spinner != nil { - spinner.Stop() - if spinner.description != "success" { - return errors.New("unexpected end to create model") - } + spinner.Stop() + if spinner.description != "success" { + return errors.New("unexpected end to create model") } return nil diff --git a/server/routes.go b/server/routes.go index 65a96911..c12a7cda 100644 --- a/server/routes.go +++ b/server/routes.go @@ -2,6 +2,7 @@ package server import ( "context" + "crypto/sha256" "encoding/json" "errors" "fmt" @@ -649,6 +650,60 @@ func CopyModelHandler(c *gin.Context) { } } +func GetBlobHandler(c *gin.Context) { + path, err := GetBlobsPath(c.Param("digest")) + if err != nil { + c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()}) + return + } + + if _, err := os.Stat(path); err != nil { + c.AbortWithStatusJSON(http.StatusNotFound, gin.H{"error": fmt.Sprintf("blob %q not found", c.Param("digest"))}) + return + } + + c.JSON(http.StatusOK, api.CreateBlobResponse{Path: path}) +} + +func CreateBlobHandler(c *gin.Context) { + hash := sha256.New() + temp, err := os.CreateTemp("", c.Param("digest")) + if err != nil { + c.AbortWithStatusJSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) + return + } + defer temp.Close() + defer os.Remove(temp.Name()) + + if _, err := io.Copy(temp, io.TeeReader(c.Request.Body, hash)); err != nil { + c.AbortWithStatusJSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) + return + } + + if fmt.Sprintf("sha256:%x", hash.Sum(nil)) != c.Param("digest") { + c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "digest does not match body"}) + return + } + + if err := temp.Close(); err != nil { + c.AbortWithStatusJSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) + return + } + + targetPath, err := GetBlobsPath(c.Param("digest")) + if err != nil { + c.AbortWithStatusJSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) + return + } + + if err := os.Rename(temp.Name(), targetPath); err != nil { + c.AbortWithStatusJSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) + return + } + + c.JSON(http.StatusOK, api.CreateBlobResponse{Path: targetPath}) +} + var defaultAllowOrigins = []string{ "localhost", "127.0.0.1", @@ -708,6 +763,7 @@ func Serve(ln net.Listener, allowOrigins []string) error { r.POST("/api/copy", CopyModelHandler) r.DELETE("/api/delete", DeleteModelHandler) r.POST("/api/show", ShowModelHandler) + r.POST("/api/blobs/:digest", CreateBlobHandler) for _, method := range []string{http.MethodGet, http.MethodHead} { r.Handle(method, "/", func(c *gin.Context) { @@ -715,6 +771,7 @@ func Serve(ln net.Listener, allowOrigins []string) error { }) r.Handle(method, "/api/tags", ListModelsHandler) + r.Handle(method, "/api/blobs/:digest/path", GetBlobHandler) } log.Printf("Listening on %s (version %s)", ln.Addr(), version.Version) From a07c935d345bef1bce4f3411da0e4b69fa9bf266 Mon Sep 17 00:00:00 2001 From: Michael Yang Date: Tue, 14 Nov 2023 14:27:51 -0800 Subject: [PATCH 019/103] ignore non blobs --- server/images.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/server/images.go b/server/images.go index 42def270..893fc4f2 100644 --- a/server/images.go +++ b/server/images.go @@ -782,7 +782,9 @@ func PruneLayers() error { if runtime.GOOS == "windows" { name = strings.ReplaceAll(name, "-", ":") } - deleteMap[name] = struct{}{} + if strings.HasPrefix(name, "sha256:") { + deleteMap[name] = struct{}{} + } } log.Printf("total blobs: %d", len(deleteMap)) From cac11c9137294961adbb08a6c279ab652af3bcdc Mon Sep 17 00:00:00 2001 From: Michael Yang Date: Tue, 14 Nov 2023 14:44:10 -0800 Subject: [PATCH 020/103] update api docs --- docs/api.md | 33 +++++++++++++++++++++++++++++++-- 1 file changed, 31 insertions(+), 2 deletions(-) diff --git a/docs/api.md b/docs/api.md index 08402266..aa3ef034 100644 --- a/docs/api.md +++ b/docs/api.md @@ -4,6 +4,7 @@ - [Generate a completion](#generate-a-completion) - [Create a Model](#create-a-model) +- [Create a Blob](#create-a-blob) - [List Local Models](#list-local-models) - [Show Model Information](#show-model-information) - [Copy a Model](#copy-a-model) @@ -292,12 +293,13 @@ curl -X POST http://localhost:11434/api/generate -d '{ POST /api/create ``` -Create a model from a [`Modelfile`](./modelfile.md) +Create a model from a [`Modelfile`](./modelfile.md). It is recommended to set `modelfile` to the content of the Modelfile rather than just set `path`. This is a requirement for remote create. Remote model creation should also create any file blobs, fields such as `FROM` and `ADAPTER`, explicitly with the server using [Create a Blob](#create-a-blob) and the value to the path indicated in the response. ### Parameters - `name`: name of the model to create - `path`: path to the Modelfile +- `modelfile`: contents of the Modelfile - `stream`: (optional) if `false` the response will be returned as a single response object, rather than a stream of objects ### Examples @@ -307,7 +309,8 @@ Create a model from a [`Modelfile`](./modelfile.md) ```shell curl -X POST http://localhost:11434/api/create -d '{ "name": "mario", - "path": "~/Modelfile" + "path": "~/Modelfile", + "modelfile": "FROM llama2" }' ``` @@ -321,6 +324,32 @@ A stream of JSON objects. When finished, `status` is `success`. } ``` +## Create a Blob + +```shell +POST /api/blobs/:digest +``` + +Create a blob from a file. Returns the server file path. + +### Query Parameters + +- `digest`: the expected SHA256 digest of the file + +### Examples + +```shell +curl -X POST http://localhost:11434/api/blobs/sha256:29fdb92e57cf0827ded04ae6461b5931d01fa595843f55d36f5b275a52087dd2 -d @llama2-13b-q4_0.gguf +``` + +### Response + +```json +{ + "path": "/home/user/.ollama/models/blobs/sha256:29fdb92e57cf0827ded04ae6461b5931d01fa595843f55d36f5b275a52087dd2" +} +``` + ## List Local Models ```shell From d660eebf22c11c5b13bc990aaa4f9bf538bc5480 Mon Sep 17 00:00:00 2001 From: Michael Yang Date: Wed, 15 Nov 2023 10:57:09 -0800 Subject: [PATCH 021/103] fix create from model tag --- cmd/cmd.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/cmd.go b/cmd/cmd.go index 30c6bcf6..d68c5868 100644 --- a/cmd/cmd.go +++ b/cmd/cmd.go @@ -78,7 +78,7 @@ func CreateHandler(cmd *cobra.Command, args []string) error { bin, err := os.Open(path) if errors.Is(err, os.ErrNotExist) && c.Name == "model" { - // value might be a model reference and not a real file + continue } else if err != nil { return err } From 1901044b075517fc48a298d044227db9666e3904 Mon Sep 17 00:00:00 2001 From: Michael Yang Date: Wed, 15 Nov 2023 10:59:38 -0800 Subject: [PATCH 022/103] use checksum reference --- api/client.go | 13 ++++++------- cmd/cmd.go | 5 ++--- server/images.go | 9 +++++++++ server/routes.go | 5 +++-- 4 files changed, 20 insertions(+), 12 deletions(-) diff --git a/api/client.go b/api/client.go index 262918b3..44af222c 100644 --- a/api/client.go +++ b/api/client.go @@ -297,18 +297,17 @@ func (c *Client) Heartbeat(ctx context.Context) error { return nil } -func (c *Client) CreateBlob(ctx context.Context, digest string, r io.Reader) (string, error) { - var response CreateBlobResponse - if err := c.do(ctx, http.MethodGet, fmt.Sprintf("/api/blobs/%s/path", digest), nil, &response); err != nil { +func (c *Client) CreateBlob(ctx context.Context, digest string, r io.Reader) error { + if err := c.do(ctx, http.MethodHead, fmt.Sprintf("/api/blobs/%s", digest), nil, nil); err != nil { var statusError StatusError if !errors.As(err, &statusError) || statusError.StatusCode != http.StatusNotFound { - return "", err + return err } - if err := c.do(ctx, http.MethodPost, fmt.Sprintf("/api/blobs/%s", digest), r, &response); err != nil { - return "", err + if err := c.do(ctx, http.MethodPost, fmt.Sprintf("/api/blobs/%s", digest), r, nil); err != nil { + return err } } - return response.Path, nil + return nil } diff --git a/cmd/cmd.go b/cmd/cmd.go index d68c5868..008c6b38 100644 --- a/cmd/cmd.go +++ b/cmd/cmd.go @@ -91,12 +91,11 @@ func CreateHandler(cmd *cobra.Command, args []string) error { bin.Seek(0, io.SeekStart) digest := fmt.Sprintf("sha256:%x", hash.Sum(nil)) - path, err = client.CreateBlob(cmd.Context(), digest, bin) - if err != nil { + if err = client.CreateBlob(cmd.Context(), digest, bin); err != nil { return err } - modelfile = bytes.ReplaceAll(modelfile, []byte(c.Args), []byte(path)) + modelfile = bytes.ReplaceAll(modelfile, []byte(c.Args), []byte("@"+digest)) } } diff --git a/server/images.go b/server/images.go index 893fc4f2..47ded90c 100644 --- a/server/images.go +++ b/server/images.go @@ -287,6 +287,15 @@ func CreateModel(ctx context.Context, name string, commands []parser.Command, fn switch c.Name { case "model": + if strings.HasPrefix(c.Args, "@") { + blobPath, err := GetBlobsPath(strings.TrimPrefix(c.Args, "@")) + if err != nil { + return err + } + + c.Args = blobPath + } + bin, err := os.Open(realpath(c.Args)) if err != nil { // not a file on disk so must be a model reference diff --git a/server/routes.go b/server/routes.go index c12a7cda..b3998e5c 100644 --- a/server/routes.go +++ b/server/routes.go @@ -650,7 +650,7 @@ func CopyModelHandler(c *gin.Context) { } } -func GetBlobHandler(c *gin.Context) { +func HeadBlobHandler(c *gin.Context) { path, err := GetBlobsPath(c.Param("digest")) if err != nil { c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()}) @@ -771,9 +771,10 @@ func Serve(ln net.Listener, allowOrigins []string) error { }) r.Handle(method, "/api/tags", ListModelsHandler) - r.Handle(method, "/api/blobs/:digest/path", GetBlobHandler) } + r.HEAD("/api/blobs/:digest", HeadBlobHandler) + log.Printf("Listening on %s (version %s)", ln.Addr(), version.Version) s := &http.Server{ Handler: r, From 71d71d09889e1c2cea16ecf6c20d9c30f35a12e0 Mon Sep 17 00:00:00 2001 From: Michael Yang Date: Wed, 15 Nov 2023 11:01:32 -0800 Subject: [PATCH 023/103] update docs --- docs/api.md | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/docs/api.md b/docs/api.md index aa3ef034..9975c4e3 100644 --- a/docs/api.md +++ b/docs/api.md @@ -4,7 +4,6 @@ - [Generate a completion](#generate-a-completion) - [Create a Model](#create-a-model) -- [Create a Blob](#create-a-blob) - [List Local Models](#list-local-models) - [Show Model Information](#show-model-information) - [Copy a Model](#copy-a-model) @@ -298,7 +297,7 @@ Create a model from a [`Modelfile`](./modelfile.md). It is recommended to set `m ### Parameters - `name`: name of the model to create -- `path`: path to the Modelfile +- `path`: path to the Modelfile (deprecated: please use modelfile instead) - `modelfile`: contents of the Modelfile - `stream`: (optional) if `false` the response will be returned as a single response object, rather than a stream of objects @@ -324,7 +323,7 @@ A stream of JSON objects. When finished, `status` is `success`. } ``` -## Create a Blob +### Create a Blob ```shell POST /api/blobs/:digest @@ -332,17 +331,17 @@ POST /api/blobs/:digest Create a blob from a file. Returns the server file path. -### Query Parameters +#### Query Parameters - `digest`: the expected SHA256 digest of the file -### Examples +#### Examples ```shell curl -X POST http://localhost:11434/api/blobs/sha256:29fdb92e57cf0827ded04ae6461b5931d01fa595843f55d36f5b275a52087dd2 -d @llama2-13b-q4_0.gguf ``` -### Response +#### Response ```json { From bc22d5a38b22d894468362307304e931be91c2b0 Mon Sep 17 00:00:00 2001 From: Michael Yang Date: Wed, 15 Nov 2023 13:55:37 -0800 Subject: [PATCH 024/103] no blob response --- api/types.go | 4 ---- server/routes.go | 4 ++-- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/api/types.go b/api/types.go index 347c4f84..2a36a1f6 100644 --- a/api/types.go +++ b/api/types.go @@ -105,10 +105,6 @@ type CreateRequest struct { Stream *bool `json:"stream,omitempty"` } -type CreateBlobResponse struct { - Path string `json:"path"` -} - type DeleteRequest struct { Name string `json:"name"` } diff --git a/server/routes.go b/server/routes.go index b3998e5c..9ebe273c 100644 --- a/server/routes.go +++ b/server/routes.go @@ -662,7 +662,7 @@ func HeadBlobHandler(c *gin.Context) { return } - c.JSON(http.StatusOK, api.CreateBlobResponse{Path: path}) + c.Status(http.StatusOK) } func CreateBlobHandler(c *gin.Context) { @@ -701,7 +701,7 @@ func CreateBlobHandler(c *gin.Context) { return } - c.JSON(http.StatusOK, api.CreateBlobResponse{Path: targetPath}) + c.Status(http.StatusCreated) } var defaultAllowOrigins = []string{ From 652d90e1c76e48a56c96c77528cffebdcc632c60 Mon Sep 17 00:00:00 2001 From: Michael Yang Date: Wed, 15 Nov 2023 15:15:36 -0800 Subject: [PATCH 025/103] Update server/images.go Co-authored-by: Bruce MacDonald --- server/images.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/images.go b/server/images.go index 47ded90c..d8ff0fd8 100644 --- a/server/images.go +++ b/server/images.go @@ -421,7 +421,7 @@ func CreateModel(ctx context.Context, name string, commands []parser.Command, fn case "template", "system": fn(api.ProgressResponse{Status: fmt.Sprintf("creating %s layer", c.Name)}) - // remove duplicates layers + // remove duplicate layers layers = removeLayerFromLayers(layers, mediatype) layer, err := CreateLayer(strings.NewReader(c.Args)) From 8ee4cbea0f01017a4289d75125bfa773328ac9e7 Mon Sep 17 00:00:00 2001 From: Jeffrey Morgan Date: Wed, 15 Nov 2023 18:16:27 -0500 Subject: [PATCH 026/103] Remove table of contents in `faq.md` --- docs/faq.md | 5 ----- 1 file changed, 5 deletions(-) diff --git a/docs/faq.md b/docs/faq.md index 28ed5cf6..b2eb6b34 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -1,10 +1,5 @@ # FAQ -- [How can I view the logs?](#how-can-i-view-the-logs) -- [How can I expose Ollama on my network?](#how-can-i-expose-ollama-on-my-network) -- [How can I allow additional web origins to access Ollama?](#how-can-i-allow-additional-web-origins-to-access-ollama) -- [Where are models stored?](#where-are-models-stored) - ## How can I view the logs? On macOS: From ecd71347ab36b1dea3bbcfefe1c8b3dfecf08188 Mon Sep 17 00:00:00 2001 From: Jeffrey Morgan Date: Wed, 15 Nov 2023 18:17:13 -0500 Subject: [PATCH 027/103] Update faq.md --- docs/faq.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/faq.md b/docs/faq.md index b2eb6b34..29eafb90 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -99,6 +99,6 @@ To modify where models are stored, you can use the `OLLAMA_MODELS` environment v No. Anything you do with Ollama, such as generate a response from the model, stays with you. We don't collect any data about how you use the model. You are always in control of your own data. -## How can I use Ollama in VSCode to help me code? +## How can I use Ollama in Visual Studio Code? -There is already a large collection of plugins available for VSCode as well as other editors that leverage Ollama. You can see the list of plugins at the bottom of the main repository readme. +There is already a large collection of plugins available for VSCode as well as other editors that leverage Ollama. You can see the list of [extensions & plugins](https://github.com/jmorganca/ollama#extensions--plugins) at the bottom of the main repository readme. From b28a30f7ba7505a379f45c45b88efa97c5a22e5e Mon Sep 17 00:00:00 2001 From: Michael Date: Wed, 15 Nov 2023 18:23:36 -0500 Subject: [PATCH 028/103] Update examples/python-json-datagenerator/predefinedschema.py Co-authored-by: Bruce MacDonald --- examples/python-json-datagenerator/predefinedschema.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/examples/python-json-datagenerator/predefinedschema.py b/examples/python-json-datagenerator/predefinedschema.py index 7f7abd48..abc399c4 100644 --- a/examples/python-json-datagenerator/predefinedschema.py +++ b/examples/python-json-datagenerator/predefinedschema.py @@ -7,10 +7,10 @@ template = { "firstName": "", "lastName": "", "address": { - "theStreet": "", - "theCity": "", - "theState": "", - "theZipCode": "" + "street": "", + "city": "", + "state": "", + "zipCode": "" }, "phoneNumber": "" } From 30ae6e731e2b77ae082e80500b9504d908e476d1 Mon Sep 17 00:00:00 2001 From: Michael Date: Wed, 15 Nov 2023 18:24:50 -0500 Subject: [PATCH 029/103] Update randomaddresses.py --- examples/python-json-datagenerator/randomaddresses.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/python-json-datagenerator/randomaddresses.py b/examples/python-json-datagenerator/randomaddresses.py index ec6c920d..5f27448f 100644 --- a/examples/python-json-datagenerator/randomaddresses.py +++ b/examples/python-json-datagenerator/randomaddresses.py @@ -3,8 +3,8 @@ import json import random countries = [ - "the US", - "the UK", + "United States", + "United Kingdom", "the Netherlands", "Germany", "Mexico", From 54f92f01cbeb16ca2baff5720f24b08d1c543e7f Mon Sep 17 00:00:00 2001 From: Michael Yang Date: Wed, 15 Nov 2023 15:22:12 -0800 Subject: [PATCH 030/103] update docs --- docs/api.md | 36 +++++++++++++++++++++++++++++------- server/routes.go | 3 +-- 2 files changed, 30 insertions(+), 9 deletions(-) diff --git a/docs/api.md b/docs/api.md index 9975c4e3..9bb4d378 100644 --- a/docs/api.md +++ b/docs/api.md @@ -323,6 +323,30 @@ A stream of JSON objects. When finished, `status` is `success`. } ``` +### Check if a Blob Exists + +```shell +HEAD /api/blobs/:digest +``` + +Check if a blob is known to the server. + +#### Query Parameters + +- `digest`: the SHA256 digest of the blob + +#### Examples + +##### Request + +```shell +curl -I http://localhost:11434/api/blobs/sha256:29fdb92e57cf0827ded04ae6461b5931d01fa595843f55d36f5b275a52087dd2 +``` + +##### Response + +Return 200 OK if the blob exists, 404 Not Found if it does not. + ### Create a Blob ```shell @@ -337,17 +361,15 @@ Create a blob from a file. Returns the server file path. #### Examples +##### Request + ```shell -curl -X POST http://localhost:11434/api/blobs/sha256:29fdb92e57cf0827ded04ae6461b5931d01fa595843f55d36f5b275a52087dd2 -d @llama2-13b-q4_0.gguf +curl -T model.bin -X POST http://localhost:11434/api/blobs/sha256:29fdb92e57cf0827ded04ae6461b5931d01fa595843f55d36f5b275a52087dd2 ``` -#### Response +##### Response -```json -{ - "path": "/home/user/.ollama/models/blobs/sha256:29fdb92e57cf0827ded04ae6461b5931d01fa595843f55d36f5b275a52087dd2" -} -``` +Return 201 Created if the blob was successfully created. ## List Local Models diff --git a/server/routes.go b/server/routes.go index 9ebe273c..58145576 100644 --- a/server/routes.go +++ b/server/routes.go @@ -764,6 +764,7 @@ func Serve(ln net.Listener, allowOrigins []string) error { r.DELETE("/api/delete", DeleteModelHandler) r.POST("/api/show", ShowModelHandler) r.POST("/api/blobs/:digest", CreateBlobHandler) + r.HEAD("/api/blobs/:digest", HeadBlobHandler) for _, method := range []string{http.MethodGet, http.MethodHead} { r.Handle(method, "/", func(c *gin.Context) { @@ -773,8 +774,6 @@ func Serve(ln net.Listener, allowOrigins []string) error { r.Handle(method, "/api/tags", ListModelsHandler) } - r.HEAD("/api/blobs/:digest", HeadBlobHandler) - log.Printf("Listening on %s (version %s)", ln.Addr(), version.Version) s := &http.Server{ Handler: r, From 5f301ece1d1ac948210b4a0c8a332a654afbf96a Mon Sep 17 00:00:00 2001 From: Dane Madsen Date: Fri, 17 Nov 2023 02:27:53 +1000 Subject: [PATCH 031/103] Add Maid to Community Integrations (#1120) --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 0dabc59c..ab19a727 100644 --- a/README.md +++ b/README.md @@ -216,6 +216,10 @@ See the [API documentation](./docs/api.md) for all endpoints. ## Community Integrations +### Mobile + +- [Maid](https://github.com/danemadsen/Maid) (Mobile Artificial Intelligence Distribution) + ### Web & Desktop - [HTML UI](https://github.com/rtcfirefly/ollama-ui) From 30141b42e91019e3219dd2d5759455f10c81cc3e Mon Sep 17 00:00:00 2001 From: Piero Savastano Date: Thu, 16 Nov 2023 17:30:54 +0100 Subject: [PATCH 032/103] Add Cheshire Cat to community integrations (#1124) --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index ab19a727..47f745fd 100644 --- a/README.md +++ b/README.md @@ -229,6 +229,7 @@ See the [API documentation](./docs/api.md) for all endpoints. - [Web UI](https://github.com/ollama-webui/ollama-webui) - [Ollamac](https://github.com/kevinhermawan/Ollamac) - [big-AGI](https://github.com/enricoros/big-agi/blob/main/docs/config-ollama.md) +- [Cheshire Cat assistant framework](https://github.com/cheshire-cat-ai/core) ### Terminal From b5f158f046e13c58240ecbca7c9dbca14db981ad Mon Sep 17 00:00:00 2001 From: Michael Yang Date: Thu, 16 Nov 2023 08:43:37 -0800 Subject: [PATCH 033/103] add faq for proxies (#1147) --- docs/faq.md | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/docs/faq.md b/docs/faq.md index 29eafb90..d143e6fc 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -102,3 +102,34 @@ No. Anything you do with Ollama, such as generate a response from the model, sta ## How can I use Ollama in Visual Studio Code? There is already a large collection of plugins available for VSCode as well as other editors that leverage Ollama. You can see the list of [extensions & plugins](https://github.com/jmorganca/ollama#extensions--plugins) at the bottom of the main repository readme. + +## How do I use Ollama behind a proxy? + +Ollama is compatible with proxy servers if `HTTP_PROXY` or `HTTPS_PROXY` are configured. When using either variables, ensure it is set where `ollama serve` can access the values. + +When using `HTTPS_PROXY`, ensure the proxy certificate is installed as a system certificate. + +On macOS: + +```bash +HTTPS_PROXY=http://proxy.example.com ollama serve +``` + +On Linux: + +```bash +echo "Environment=HTTPS_PROXY=https://proxy.example.com" >>/etc/systemd/system/ollama.service.d/environment.conf +``` + +Reload `systemd` and restart Ollama: + +```bash +systemctl daemon-reload +systemctl restart ollama +``` + +### How do I use Ollama behind a proxy in Docker? + +Ollama Docker container can be configured to use a proxy by passing `-e HTTPS_PROXY=https://proxy.example.com` when starting the container. Ensure the certificate is installed as a system certificate when using HTTPS. + +Alternatively, Docker daemon can be configured to use a proxy. Instructions are available for Docker Desktop on [macOS](https://docs.docker.com/desktop/settings/mac/#proxies), [Windows](https://docs.docker.com/desktop/settings/windows/#proxies), and [Linux](https://docs.docker.com/desktop/settings/linux/#proxies), and Docker [daemon with systemd](https://docs.docker.com/config/daemon/systemd/#httphttps-proxy). From 75295b952899daaa1ad188088c5f37e62a414b55 Mon Sep 17 00:00:00 2001 From: yanndegat Date: Thu, 16 Nov 2023 21:53:06 +0100 Subject: [PATCH 034/103] install: fix enable contrib on debian 12 (#1151) On debian 12, sources definitions have moved from /etc/apt/sources.list to /etc/apt/sources.list.d/debian.sources --- scripts/install.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scripts/install.sh b/scripts/install.sh index 0cb0f915..219f60fd 100644 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -181,6 +181,9 @@ install_cuda_driver_apt() { debian) status 'Enabling contrib sources...' $SUDO sed 's/main/contrib/' < /etc/apt/sources.list | $SUDO tee /etc/apt/sources.list.d/contrib.list > /dev/null + if [ -f "/etc/apt/sources.list.d/debian.sources" ]; then + $SUDO sed 's/main/contrib/' < /etc/apt/sources.list.d/debian.sources | $SUDO tee /etc/apt/sources.list.d/contrib.sources > /dev/null + fi ;; esac From e33ef391cd937f1e46e7f257e7722e3014c79981 Mon Sep 17 00:00:00 2001 From: Michael Yang Date: Thu, 16 Nov 2023 11:44:09 -0800 Subject: [PATCH 035/103] fix push scope error for inherited model --- server/upload.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/upload.go b/server/upload.go index 04575560..8b483618 100644 --- a/server/upload.go +++ b/server/upload.go @@ -55,7 +55,7 @@ func (b *blobUpload) Prepare(ctx context.Context, requestURL *url.URL, opts *Reg if b.From != "" { values := requestURL.Query() values.Add("mount", b.Digest) - values.Add("from", b.From) + values.Add("from", ParseModelPath(b.From).GetNamespaceRepository()) requestURL.RawQuery = values.Encode() } @@ -260,7 +260,7 @@ func (b *blobUpload) uploadChunk(ctx context.Context, method string, requestURL return err } - return fmt.Errorf("http status %d %s: %s", resp.StatusCode, resp.Status, body) + return fmt.Errorf("http status %s: %s", resp.Status, body) } if method == http.MethodPatch { From a5ccf742c1ab0af3f350a42a5a4e511780c238b3 Mon Sep 17 00:00:00 2001 From: Michael Yang Date: Thu, 16 Nov 2023 12:18:03 -0800 Subject: [PATCH 036/103] fix cross repo mounts --- server/upload.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/server/upload.go b/server/upload.go index 8b483618..306b6e63 100644 --- a/server/upload.go +++ b/server/upload.go @@ -77,6 +77,14 @@ func (b *blobUpload) Prepare(ctx context.Context, requestURL *url.URL, opts *Reg b.Total = fi.Size() + // http.StatusCreated indicates a blob has been mounted + // ref: https://distribution.github.io/distribution/spec/api/#cross-repository-blob-mount + if resp.StatusCode == http.StatusCreated { + b.Completed.Store(b.Total) + b.done = true + return nil + } + var size = b.Total / numUploadParts switch { case size < minUploadPartSize: From 4b3f4bc7d9019daef4f1e20c0e7b3ad02eb6d496 Mon Sep 17 00:00:00 2001 From: Bruce MacDonald Date: Thu, 16 Nov 2023 16:44:18 -0500 Subject: [PATCH 037/103] return failure details when unauthorized to push (#1131) Co-authored-by: Jeffrey Morgan --- server/images.go | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/server/images.go b/server/images.go index d8ff0fd8..7febb1ef 100644 --- a/server/images.go +++ b/server/images.go @@ -954,6 +954,9 @@ func PushModel(ctx context.Context, name string, regOpts *RegistryOptions, fn fu for _, layer := range layers { if err := uploadBlob(ctx, mp, layer, regOpts, fn); err != nil { log.Printf("error uploading blob: %v", err) + if errors.Is(err, errUnauthorized) { + return fmt.Errorf("unable to push %s, make sure this namespace exists and you are authorized to push to it", ParseModelPath(name).GetNamespaceRepository()) + } return err } } @@ -1140,7 +1143,10 @@ func GetSHA256Digest(r io.Reader) (string, int64) { return fmt.Sprintf("sha256:%x", h.Sum(nil)), n } +var errUnauthorized = fmt.Errorf("unauthorized") + func makeRequestWithRetry(ctx context.Context, method string, requestURL *url.URL, headers http.Header, body io.ReadSeeker, regOpts *RegistryOptions) (*http.Response, error) { + lastErr := errMaxRetriesExceeded for try := 0; try < maxRetries; try++ { resp, err := makeRequest(ctx, method, requestURL, headers, body, regOpts) if err != nil { @@ -1161,8 +1167,7 @@ func makeRequestWithRetry(ctx context.Context, method string, requestURL *url.UR if body != nil { body.Seek(0, io.SeekStart) } - - continue + lastErr = errUnauthorized case resp.StatusCode == http.StatusNotFound: return nil, os.ErrNotExist case resp.StatusCode >= http.StatusBadRequest: @@ -1177,7 +1182,7 @@ func makeRequestWithRetry(ctx context.Context, method string, requestURL *url.UR } } - return nil, errMaxRetriesExceeded + return nil, lastErr } func makeRequest(ctx context.Context, method string, requestURL *url.URL, headers http.Header, body io.Reader, regOpts *RegistryOptions) (*http.Response, error) { From dbe6e77472126174ad151b2ce5467bbca33a4d51 Mon Sep 17 00:00:00 2001 From: Jeffrey Morgan Date: Thu, 16 Nov 2023 16:46:38 -0500 Subject: [PATCH 038/103] Update README.md --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 47f745fd..acc5d697 100644 --- a/README.md +++ b/README.md @@ -216,10 +216,6 @@ See the [API documentation](./docs/api.md) for all endpoints. ## Community Integrations -### Mobile - -- [Maid](https://github.com/danemadsen/Maid) (Mobile Artificial Intelligence Distribution) - ### Web & Desktop - [HTML UI](https://github.com/rtcfirefly/ollama-ui) @@ -252,6 +248,10 @@ See the [API documentation](./docs/api.md) for all endpoints. - [OllamaKit for Swift](https://github.com/kevinhermawan/OllamaKit) - [Ollama for Dart](https://github.com/breitburg/dart-ollama) +### Mobile + +- [Maid](https://github.com/danemadsen/Maid) (Mobile Artificial Intelligence Distribution) + ### Extensions & Plugins - [Raycast extension](https://github.com/MassimilianoPasquini97/raycast_ollama) From ee307937fd59fa654d4b19ebdcc90d04c74af2e0 Mon Sep 17 00:00:00 2001 From: Michael Yang Date: Thu, 16 Nov 2023 16:46:43 -0800 Subject: [PATCH 039/103] update faq --- docs/faq.md | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/docs/faq.md b/docs/faq.md index d143e6fc..a2f3768a 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -130,6 +130,21 @@ systemctl restart ollama ### How do I use Ollama behind a proxy in Docker? -Ollama Docker container can be configured to use a proxy by passing `-e HTTPS_PROXY=https://proxy.example.com` when starting the container. Ensure the certificate is installed as a system certificate when using HTTPS. +Ollama Docker container can be configured to use a proxy by passing `-e HTTPS_PROXY=https://proxy.example.com` when starting the container. Alternatively, Docker daemon can be configured to use a proxy. Instructions are available for Docker Desktop on [macOS](https://docs.docker.com/desktop/settings/mac/#proxies), [Windows](https://docs.docker.com/desktop/settings/windows/#proxies), and [Linux](https://docs.docker.com/desktop/settings/linux/#proxies), and Docker [daemon with systemd](https://docs.docker.com/config/daemon/systemd/#httphttps-proxy). + +Ensure the certificate is installed as a system certificate when using HTTPS. This may require a new Docker image when using a self-signed certificate. + +```dockerfile +FROM ollama/ollama +COPY my-ca.pem /usr/local/share/ca-certificates/my-ca.crt +RUN update-ca-certificate +``` + +Build and run this image: + +```shell +docker build -t ollama-with-ca . +docker run -d -e HTTPS_PROXY=https://my.proxy.example.com -p 11434:11434 ollama-with-ca +``` From c13bde962d2188ae1726bd04f600bf1a6f6545ca Mon Sep 17 00:00:00 2001 From: Michael Yang Date: Thu, 16 Nov 2023 16:48:38 -0800 Subject: [PATCH 040/103] Update docs/faq.md Co-authored-by: Jeffrey Morgan --- docs/faq.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/faq.md b/docs/faq.md index a2f3768a..8596f573 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -130,7 +130,7 @@ systemctl restart ollama ### How do I use Ollama behind a proxy in Docker? -Ollama Docker container can be configured to use a proxy by passing `-e HTTPS_PROXY=https://proxy.example.com` when starting the container. +The Ollama Docker container image can be configured to use a proxy by passing `-e HTTPS_PROXY=https://proxy.example.com` when starting the container. Alternatively, Docker daemon can be configured to use a proxy. Instructions are available for Docker Desktop on [macOS](https://docs.docker.com/desktop/settings/mac/#proxies), [Windows](https://docs.docker.com/desktop/settings/windows/#proxies), and [Linux](https://docs.docker.com/desktop/settings/linux/#proxies), and Docker [daemon with systemd](https://docs.docker.com/config/daemon/systemd/#httphttps-proxy). From 32add8577d3cd82ad29ec06c8a31cc71dbe40afd Mon Sep 17 00:00:00 2001 From: Michael Yang Date: Thu, 16 Nov 2023 16:52:40 -0800 Subject: [PATCH 041/103] placeholder environment variables --- scripts/install.sh | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/scripts/install.sh b/scripts/install.sh index 219f60fd..8ff5eaee 100644 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -97,6 +97,16 @@ Environment="PATH=$PATH" [Install] WantedBy=default.target EOF + + mkdir -p /etc/systemd/system/ollama.service.d + cat </dev/null +[Service] +#Environment="OLLAMA_HOST=" +#Environment="OLLAMA_ORIGINS=" +#Environment="OLLAMA_MODELS=" +#Environment="HTTPS_PROXY=" +EOF + SYSTEMCTL_RUNNING="$(systemctl is-system-running || true)" case $SYSTEMCTL_RUNNING in running|degraded) From d8842b4d4b307b1d030ce0c4dba292fbab6ca8da Mon Sep 17 00:00:00 2001 From: Michael Yang Date: Thu, 16 Nov 2023 17:06:21 -0800 Subject: [PATCH 042/103] update faq --- docs/faq.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/faq.md b/docs/faq.md index 8596f573..f95137fe 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -32,11 +32,11 @@ Create a `systemd` drop-in directory and set `Environment=OLLAMA_HOST` ```bash mkdir -p /etc/systemd/system/ollama.service.d -echo "[Service]" >>/etc/systemd/system/ollama.service.d/environment.conf +echo '[Service]' >>/etc/systemd/system/ollama.service.d/environment.conf ``` ```bash -echo "Environment=OLLAMA_HOST=0.0.0.0:11434" >>/etc/systemd/system/ollama.service.d/environment.conf +echo 'Environment="OLLAMA_HOST=0.0.0.0:11434"' >>/etc/systemd/system/ollama.service.d/environment.conf ``` Reload `systemd` and restart Ollama: @@ -59,7 +59,7 @@ OLLAMA_ORIGINS=http://192.168.1.1:*,https://example.com ollama serve On Linux: ```bash -echo "Environment=OLLAMA_ORIGINS=http://129.168.1.1:*,https://example.com" >>/etc/systemd/system/ollama.service.d/environment.conf +echo 'Environment="OLLAMA_ORIGINS=http://129.168.1.1:*,https://example.com"' >>/etc/systemd/system/ollama.service.d/environment.conf ``` Reload `systemd` and restart Ollama: @@ -118,7 +118,7 @@ HTTPS_PROXY=http://proxy.example.com ollama serve On Linux: ```bash -echo "Environment=HTTPS_PROXY=https://proxy.example.com" >>/etc/systemd/system/ollama.service.d/environment.conf +echo 'Environment="HTTPS_PROXY=https://proxy.example.com"' >>/etc/systemd/system/ollama.service.d/environment.conf ``` Reload `systemd` and restart Ollama: From 41434a7cdcf33918ae2d37eb23d819ef7361e843 Mon Sep 17 00:00:00 2001 From: Jeffrey Morgan Date: Thu, 16 Nov 2023 22:13:25 -0500 Subject: [PATCH 043/103] build intel mac with correct binary and compile flags --- llm/llama.cpp/generate_darwin_amd64.go | 2 +- llm/llama.go | 7 ++++--- scripts/build_darwin.sh | 1 + 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/llm/llama.cpp/generate_darwin_amd64.go b/llm/llama.cpp/generate_darwin_amd64.go index 7cbc36f5..804b29e1 100644 --- a/llm/llama.cpp/generate_darwin_amd64.go +++ b/llm/llama.cpp/generate_darwin_amd64.go @@ -14,6 +14,6 @@ package llm //go:generate git submodule update --force gguf //go:generate git -C gguf apply ../patches/0001-update-default-log-target.patch //go:generate git -C gguf apply ../patches/0001-metal-handle-ggml_scale-for-n-4-0-close-3754.patch -//go:generate cmake -S gguf -B gguf/build/cpu -DLLAMA_ACCELERATE=on -DLLAMA_K_QUANTS=on -DCMAKE_SYSTEM_PROCESSOR=x86_64 -DCMAKE_OSX_ARCHITECTURES=x86_64 -DCMAKE_OSX_DEPLOYMENT_TARGET=11.0 +//go:generate cmake -S gguf -B gguf/build/cpu -DLLAMA_METAL=off -DLLAMA_ACCELERATE=on -DLLAMA_K_QUANTS=on -DCMAKE_SYSTEM_PROCESSOR=x86_64 -DCMAKE_OSX_ARCHITECTURES=x86_64 -DCMAKE_OSX_DEPLOYMENT_TARGET=11.0 -DLLAMA_NATIVE=off -DLLAMA_AVX=on -DLLAMA_AVX2=off -DLLAMA_AVX512=off -DLLAMA_FMA=off -DLLAMA_F16C=off //go:generate cmake --build gguf/build/cpu --target server --config Release //go:generate mv gguf/build/cpu/bin/server gguf/build/cpu/bin/ollama-runner diff --git a/llm/llama.go b/llm/llama.go index d8859393..a7561f8d 100644 --- a/llm/llama.go +++ b/llm/llama.go @@ -71,9 +71,10 @@ func chooseRunners(workDir, runnerType string) []ModelRunner { // IMPORTANT: the order of the runners in the array is the priority order switch runtime.GOOS { case "darwin": - runners = []ModelRunner{ - {Path: path.Join(buildPath, "metal", "bin", "ollama-runner")}, - {Path: path.Join(buildPath, "cpu", "bin", "ollama-runner")}, + if runtime.GOARCH == "arm64" { + runners = []ModelRunner{{Path: path.Join(buildPath, "metal", "bin", "ollama-runner")}} + } else { + runners = []ModelRunner{{Path: path.Join(buildPath, "cpu", "bin", "ollama-runner")}} } case "linux": runners = []ModelRunner{ diff --git a/scripts/build_darwin.sh b/scripts/build_darwin.sh index 54aef9a4..c35a3d8d 100755 --- a/scripts/build_darwin.sh +++ b/scripts/build_darwin.sh @@ -10,6 +10,7 @@ mkdir -p dist for TARGETARCH in arm64 amd64; do GOOS=darwin GOARCH=$TARGETARCH go generate ./... GOOS=darwin GOARCH=$TARGETARCH go build -o dist/ollama-darwin-$TARGETARCH + rm -rf llm/llama.cpp/*/build done lipo -create -output dist/ollama dist/ollama-darwin-* From 92656a74b798b2ca0712f566056a34a6b085cc8b Mon Sep 17 00:00:00 2001 From: Jeffrey Morgan Date: Fri, 17 Nov 2023 07:17:51 -0500 Subject: [PATCH 044/103] Use `llama2` as the model in `api.md` --- docs/api.md | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/docs/api.md b/docs/api.md index 9bb4d378..d98b1323 100644 --- a/docs/api.md +++ b/docs/api.md @@ -114,7 +114,7 @@ To calculate how fast the response is generated in tokens per second (token/s), ```shell curl -X POST http://localhost:11434/api/generate -d '{ - "model": "llama2:7b", + "model": "llama2", "prompt": "Why is the sky blue?", "stream": false }' @@ -126,7 +126,7 @@ If `stream` is set to `false`, the response will be a single JSON object: ```json { - "model": "llama2:7b", + "model": "llama2", "created_at": "2023-08-04T19:22:45.499127Z", "response": "The sky is blue because it is the color of the sky.", "context": [1, 2, 3], @@ -225,7 +225,7 @@ If you want to set custom options for the model at runtime rather than in the Mo ```shell curl -X POST http://localhost:11434/api/generate -d '{ - "model": "llama2:7b", + "model": "llama2", "prompt": "Why is the sky blue?", "stream": false, "options": { @@ -270,7 +270,7 @@ curl -X POST http://localhost:11434/api/generate -d '{ ```json { - "model": "llama2:7b", + "model": "llama2", "created_at": "2023-08-04T19:22:45.499127Z", "response": "The sky is blue because it is the color of the sky.", "context": [1, 2, 3], @@ -395,7 +395,7 @@ A single JSON object will be returned. { "models": [ { - "name": "llama2:7b", + "name": "llama2", "modified_at": "2023-08-02T17:02:23.713454393-07:00", "size": 3791730596 }, @@ -426,7 +426,7 @@ Show details about a model including modelfile, template, parameters, license, a ```shell curl http://localhost:11434/api/show -d '{ - "name": "llama2:7b" + "name": "llama2" }' ``` @@ -455,7 +455,7 @@ Copy a model. Creates a model with another name from an existing model. ```shell curl http://localhost:11434/api/copy -d '{ - "source": "llama2:7b", + "source": "llama2", "destination": "llama2-backup" }' ``` @@ -510,7 +510,7 @@ Download a model from the ollama library. Cancelled pulls are resumed from where ```shell curl -X POST http://localhost:11434/api/pull -d '{ - "name": "llama2:7b" + "name": "llama2" }' ``` @@ -650,7 +650,7 @@ Advanced parameters: ```shell curl -X POST http://localhost:11434/api/embeddings -d '{ - "model": "llama2:7b", + "model": "llama2", "prompt": "Here is an article about llamas..." }' ``` From 81092147c4b2aef13be6a11009be7a9474e9985c Mon Sep 17 00:00:00 2001 From: Jeffrey Morgan Date: Fri, 17 Nov 2023 09:50:38 -0500 Subject: [PATCH 045/103] remove unnecessary `-X POST` from example `curl` commands --- README.md | 2 +- docs/api.md | 23 +++++++++++------------ 2 files changed, 12 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index acc5d697..2e94a418 100644 --- a/README.md +++ b/README.md @@ -206,7 +206,7 @@ Ollama has a REST API for running and managing models. For example, to generate text from a model: ``` -curl -X POST http://localhost:11434/api/generate -d '{ +curl http://localhost:11434/api/generate -d '{ "model": "llama2", "prompt":"Why is the sky blue?" }' diff --git a/docs/api.md b/docs/api.md index d98b1323..6f34ed7f 100644 --- a/docs/api.md +++ b/docs/api.md @@ -58,7 +58,7 @@ Enable JSON mode by setting the `format` parameter to `json` and specifying the #### Request ```shell -curl -X POST http://localhost:11434/api/generate -d '{ +curl http://localhost:11434/api/generate -d '{ "model": "llama2", "prompt": "Why is the sky blue?" }' @@ -113,7 +113,7 @@ To calculate how fast the response is generated in tokens per second (token/s), #### Request (No streaming) ```shell -curl -X POST http://localhost:11434/api/generate -d '{ +curl http://localhost:11434/api/generate -d '{ "model": "llama2", "prompt": "Why is the sky blue?", "stream": false @@ -147,7 +147,7 @@ If `stream` is set to `false`, the response will be a single JSON object: In some cases you may wish to bypass the templating system and provide a full prompt. In this case, you can use the `raw` parameter to disable formatting and context. ```shell -curl -X POST http://localhost:11434/api/generate -d '{ +curl http://localhost:11434/api/generate -d '{ "model": "mistral", "prompt": "[INST] why is the sky blue? [/INST]", "raw": true, @@ -175,7 +175,7 @@ curl -X POST http://localhost:11434/api/generate -d '{ #### Request (JSON mode) ```shell -curl -X POST http://localhost:11434/api/generate -d '{ +curl http://localhost:11434/api/generate -d '{ "model": "llama2", "prompt": "What color is the sky at different times of the day? Respond using JSON", "format": "json", @@ -224,7 +224,7 @@ The value of `response` will be a string containing JSON similar to: If you want to set custom options for the model at runtime rather than in the Modelfile, you can do so with the `options` parameter. This example sets every available option, but you can set any of them individually and omit the ones you do not want to override. ```shell -curl -X POST http://localhost:11434/api/generate -d '{ +curl http://localhost:11434/api/generate -d '{ "model": "llama2", "prompt": "Why is the sky blue?", "stream": false, @@ -297,19 +297,18 @@ Create a model from a [`Modelfile`](./modelfile.md). It is recommended to set `m ### Parameters - `name`: name of the model to create -- `path`: path to the Modelfile (deprecated: please use modelfile instead) - `modelfile`: contents of the Modelfile - `stream`: (optional) if `false` the response will be returned as a single response object, rather than a stream of objects +- `path` (deprecated): path to the Modelfile ### Examples #### Request ```shell -curl -X POST http://localhost:11434/api/create -d '{ +curl http://localhost:11434/api/create -d '{ "name": "mario", - "path": "~/Modelfile", - "modelfile": "FROM llama2" + "modelfile": "FROM llama2\nSYSTEM You are mario from Super Mario Bros." }' ``` @@ -509,7 +508,7 @@ Download a model from the ollama library. Cancelled pulls are resumed from where #### Request ```shell -curl -X POST http://localhost:11434/api/pull -d '{ +curl http://localhost:11434/api/pull -d '{ "name": "llama2" }' ``` @@ -581,7 +580,7 @@ Upload a model to a model library. Requires registering for ollama.ai and adding #### Request ```shell -curl -X POST http://localhost:11434/api/push -d '{ +curl http://localhost:11434/api/push -d '{ "name": "mattw/pygmalion:latest" }' ``` @@ -649,7 +648,7 @@ Advanced parameters: #### Request ```shell -curl -X POST http://localhost:11434/api/embeddings -d '{ +curl http://localhost:11434/api/embeddings -d '{ "model": "llama2", "prompt": "Here is an article about llamas..." }' From 90860b6a7e1cd8ce6a200ff39f9779041d3726e3 Mon Sep 17 00:00:00 2001 From: Michael Yang Date: Fri, 17 Nov 2023 08:42:58 -0800 Subject: [PATCH 046/103] update faq (#1176) --- docs/faq.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/faq.md b/docs/faq.md index f95137fe..1beb95a1 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -148,3 +148,9 @@ Build and run this image: docker build -t ollama-with-ca . docker run -d -e HTTPS_PROXY=https://my.proxy.example.com -p 11434:11434 ollama-with-ca ``` + +### How do I use Ollama with GPU acceleration in Docker? + +The Ollama Docker container can be configured with GPU acceleration in Linux or Windows (with WSL2). This requires the [nvidia-container-toolkit](https://github.com/NVIDIA/nvidia-container-toolkit). + +GPU acceleration is not available for Docker Desktop in macOS due to the lack of GPU passthrough and emulation. From c82ead4d01411ac5b6b863e782faae066c00f2ad Mon Sep 17 00:00:00 2001 From: Michael Yang Date: Fri, 17 Nov 2023 09:00:14 -0800 Subject: [PATCH 047/103] faq: fix heading and add more details --- docs/faq.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/faq.md b/docs/faq.md index 1beb95a1..ce32beb2 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -149,8 +149,8 @@ docker build -t ollama-with-ca . docker run -d -e HTTPS_PROXY=https://my.proxy.example.com -p 11434:11434 ollama-with-ca ``` -### How do I use Ollama with GPU acceleration in Docker? +## How do I use Ollama with GPU acceleration in Docker? -The Ollama Docker container can be configured with GPU acceleration in Linux or Windows (with WSL2). This requires the [nvidia-container-toolkit](https://github.com/NVIDIA/nvidia-container-toolkit). +The Ollama Docker container can be configured with GPU acceleration in Linux or Windows (with WSL2). This requires the [nvidia-container-toolkit](https://github.com/NVIDIA/nvidia-container-toolkit). See [ollama/ollama](https://hub.docker.com/r/ollama/ollama) for more details. GPU acceleration is not available for Docker Desktop in macOS due to the lack of GPU passthrough and emulation. From a3053b66d215beaf5f06b7b6790a2979122a37fe Mon Sep 17 00:00:00 2001 From: Michael Yang Date: Sun, 12 Nov 2023 13:13:52 -0800 Subject: [PATCH 048/103] add jupyter notebook example --- examples/jupyter-notebook/ollama.ipynb | 113 +++++++++++++++++++++++++ 1 file changed, 113 insertions(+) create mode 100644 examples/jupyter-notebook/ollama.ipynb diff --git a/examples/jupyter-notebook/ollama.ipynb b/examples/jupyter-notebook/ollama.ipynb new file mode 100644 index 00000000..fec8a7f1 --- /dev/null +++ b/examples/jupyter-notebook/ollama.ipynb @@ -0,0 +1,113 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "38d57674-b3d5-40f3-ab83-9109df3a7821", + "metadata": {}, + "source": [ + "# Ollama Jupyter Notebook\n", + "\n", + "Ollama is the easiest way to run large language models (LLMs) locally. You can deploy it to macOS by installing the the macOS application, Linux by running the install script (below), and Docker or Kubernetes by pulling the official Ollama Docker image.\n", + "\n", + "For best results, this notebook should be run on a Linux node with GPU or an environment like Google Colab." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "93f59dcb-c588-41b8-a792-55d88ade739c", + "metadata": {}, + "outputs": [], + "source": [ + "# Download and run the Ollama Linux install script\n", + "!curl https://ollama.ai/install.sh | sh" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "658c147e-c7f8-490e-910e-62b80f577dda", + "metadata": {}, + "outputs": [], + "source": [ + "!pip install aiohttp pyngrok\n", + "\n", + "import os\n", + "import asyncio\n", + "from aiohttp import ClientSession\n", + "\n", + "# Set LD_LIBRARY_PATH so the system NVIDIA library becomes preferred\n", + "# over the built-in library. This is particularly important for \n", + "# Google Colab which installs older drivers\n", + "os.environ.update({'LD_LIBRARY_PATH': '/usr/lib64-nvidia'})\n", + "\n", + "async def run(cmd):\n", + " '''\n", + " run is a helper function to run subcommands asynchronously.\n", + " '''\n", + " print('>>> starting', *cmd)\n", + " p = await asyncio.subprocess.create_subprocess_exec(\n", + " *cmd,\n", + " stdout=asyncio.subprocess.PIPE,\n", + " stderr=asyncio.subprocess.PIPE,\n", + " )\n", + "\n", + " async def pipe(lines):\n", + " async for line in lines:\n", + " print(line.strip().decode('utf-8'))\n", + "\n", + " await asyncio.gather(\n", + " pipe(p.stdout),\n", + " pipe(p.stderr),\n", + " )\n", + "\n", + "\n", + "await asyncio.gather(\n", + " run(['ollama', 'serve']),\n", + " run(['ngrok', 'http', '--log', 'stderr', '11434']),\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "e7735a55-9aad-4caf-8683-52e2163ba53b", + "metadata": {}, + "source": [ + "The previous cell starts two processes, `ollama` and `ngrok`. The log output will show a line like the following which describes the external address.\n", + "\n", + "```\n", + "t=2023-11-12T22:55:56+0000 lvl=info msg=\"started tunnel\" obj=tunnels name=command_line addr=http://localhost:11434 url=https://8249-34-125-179-11.ngrok.io\n", + "```\n", + "\n", + "The external address in this case is `https://8249-34-125-179-11.ngrok.io` which can be passed into `OLLAMA_HOST` to access this instance.\n", + "\n", + "```bash\n", + "export OLLAMA_HOST=https://8249-34-125-179-11.ngrok.io\n", + "ollama list\n", + "ollama run mistral\n", + "```" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.6" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} From f7f6d6c6936e73ded989c877ed2b06ff124326bd Mon Sep 17 00:00:00 2001 From: Michael Yang Date: Tue, 14 Nov 2023 10:58:28 -0800 Subject: [PATCH 049/103] Update examples/jupyter-notebook/ollama.ipynb Co-authored-by: Bruce MacDonald --- examples/jupyter-notebook/ollama.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/jupyter-notebook/ollama.ipynb b/examples/jupyter-notebook/ollama.ipynb index fec8a7f1..b00f8862 100644 --- a/examples/jupyter-notebook/ollama.ipynb +++ b/examples/jupyter-notebook/ollama.ipynb @@ -9,7 +9,7 @@ "\n", "Ollama is the easiest way to run large language models (LLMs) locally. You can deploy it to macOS by installing the the macOS application, Linux by running the install script (below), and Docker or Kubernetes by pulling the official Ollama Docker image.\n", "\n", - "For best results, this notebook should be run on a Linux node with GPU or an environment like Google Colab." + "For best results, this notebook should be run on a Linux node with a GPU or an environment like Google Colab." ] }, { From 4936b5bb37289419dee96239cc65c75aad34f7b2 Mon Sep 17 00:00:00 2001 From: Michael Yang Date: Fri, 17 Nov 2023 09:30:02 -0800 Subject: [PATCH 050/103] add jupyter readme --- examples/jupyter-notebook/README.md | 5 +++++ examples/jupyter-notebook/ollama.ipynb | 15 ++------------- 2 files changed, 7 insertions(+), 13 deletions(-) create mode 100644 examples/jupyter-notebook/README.md diff --git a/examples/jupyter-notebook/README.md b/examples/jupyter-notebook/README.md new file mode 100644 index 00000000..fba6802f --- /dev/null +++ b/examples/jupyter-notebook/README.md @@ -0,0 +1,5 @@ +# Ollama Jupyter Notebook + +This example downloads and installs Ollama in a Jupyter instance such as Google Colab. It will start the Ollama service and expose an endpoint using `ngrok` which can be used to communicate with the Ollama instance remotely. + +For best results, use an instance with GPU accelerator. diff --git a/examples/jupyter-notebook/ollama.ipynb b/examples/jupyter-notebook/ollama.ipynb index b00f8862..d57e2057 100644 --- a/examples/jupyter-notebook/ollama.ipynb +++ b/examples/jupyter-notebook/ollama.ipynb @@ -1,17 +1,5 @@ { "cells": [ - { - "cell_type": "markdown", - "id": "38d57674-b3d5-40f3-ab83-9109df3a7821", - "metadata": {}, - "source": [ - "# Ollama Jupyter Notebook\n", - "\n", - "Ollama is the easiest way to run large language models (LLMs) locally. You can deploy it to macOS by installing the the macOS application, Linux by running the install script (below), and Docker or Kubernetes by pulling the official Ollama Docker image.\n", - "\n", - "For best results, this notebook should be run on a Linux node with a GPU or an environment like Google Colab." - ] - }, { "cell_type": "code", "execution_count": null, @@ -20,7 +8,8 @@ "outputs": [], "source": [ "# Download and run the Ollama Linux install script\n", - "!curl https://ollama.ai/install.sh | sh" + "!curl https://ollama.ai/install.sh | sh\n", + "!command -v systemctl >/dev/null && sudo systemctl stop ollama" ] }, { From f91bb2f7f00e5c82b30f08f56fd99b6f7c735e00 Mon Sep 17 00:00:00 2001 From: Michael Yang Date: Mon, 13 Nov 2023 15:12:15 -0800 Subject: [PATCH 051/103] remove progressbar --- cmd/cmd.go | 96 +--- cmd/spinner.go | 44 -- progressbar/LICENSE | 21 - progressbar/README.md | 121 ---- progressbar/progressbar.go | 1098 ------------------------------------ progressbar/spinners.go | 80 --- 6 files changed, 5 insertions(+), 1455 deletions(-) delete mode 100644 cmd/spinner.go delete mode 100644 progressbar/LICENSE delete mode 100644 progressbar/README.md delete mode 100644 progressbar/progressbar.go delete mode 100644 progressbar/spinners.go diff --git a/cmd/cmd.go b/cmd/cmd.go index 008c6b38..5d31eae2 100644 --- a/cmd/cmd.go +++ b/cmd/cmd.go @@ -30,7 +30,6 @@ import ( "github.com/jmorganca/ollama/api" "github.com/jmorganca/ollama/format" "github.com/jmorganca/ollama/parser" - "github.com/jmorganca/ollama/progressbar" "github.com/jmorganca/ollama/readline" "github.com/jmorganca/ollama/server" "github.com/jmorganca/ollama/version" @@ -53,9 +52,6 @@ func CreateHandler(cmd *cobra.Command, args []string) error { return err } - spinner := NewSpinner("transferring context") - go spinner.Spin(100 * time.Millisecond) - commands, err := parser.Parse(bytes.NewReader(modelfile)) if err != nil { return err @@ -99,29 +95,9 @@ func CreateHandler(cmd *cobra.Command, args []string) error { } } - var currentDigest string - var bar *progressbar.ProgressBar - request := api.CreateRequest{Name: args[0], Path: filename, Modelfile: string(modelfile)} fn := func(resp api.ProgressResponse) error { - if resp.Digest != currentDigest && resp.Digest != "" { - spinner.Stop() - currentDigest = resp.Digest - // pulling - bar = progressbar.DefaultBytes( - resp.Total, - resp.Status, - ) - bar.Set64(resp.Completed) - } else if resp.Digest == currentDigest && resp.Digest != "" { - bar.Set64(resp.Completed) - } else { - currentDigest = "" - spinner.Stop() - spinner = NewSpinner(resp.Status) - go spinner.Spin(100 * time.Millisecond) - } - + log.Printf("progress(%s): %s", resp.Digest, resp.Status) return nil } @@ -129,11 +105,6 @@ func CreateHandler(cmd *cobra.Command, args []string) error { return err } - spinner.Stop() - if spinner.description != "success" { - return errors.New("unexpected end to create model") - } - return nil } @@ -170,37 +141,13 @@ func PushHandler(cmd *cobra.Command, args []string) error { return err } - var currentDigest string - var bar *progressbar.ProgressBar - request := api.PushRequest{Name: args[0], Insecure: insecure} fn := func(resp api.ProgressResponse) error { - if resp.Digest != currentDigest && resp.Digest != "" { - currentDigest = resp.Digest - bar = progressbar.DefaultBytes( - resp.Total, - fmt.Sprintf("pushing %s...", resp.Digest[7:19]), - ) - - bar.Set64(resp.Completed) - } else if resp.Digest == currentDigest && resp.Digest != "" { - bar.Set64(resp.Completed) - } else { - currentDigest = "" - fmt.Println(resp.Status) - } + log.Printf("progress(%s): %s", resp.Digest, resp.Status) return nil } - if err := client.Push(context.Background(), &request, fn); err != nil { - return err - } - - if bar != nil && !bar.IsFinished() { - return errors.New("unexpected end to push model") - } - - return nil + return client.Push(context.Background(), &request, fn) } func ListHandler(cmd *cobra.Command, args []string) error { @@ -359,38 +306,13 @@ func pull(model string, insecure bool) error { return err } - var currentDigest string - var bar *progressbar.ProgressBar - request := api.PullRequest{Name: model, Insecure: insecure} fn := func(resp api.ProgressResponse) error { - if resp.Digest != currentDigest && resp.Digest != "" { - currentDigest = resp.Digest - bar = progressbar.DefaultBytes( - resp.Total, - fmt.Sprintf("pulling %s...", resp.Digest[7:19]), - ) - - bar.Set64(resp.Completed) - } else if resp.Digest == currentDigest && resp.Digest != "" { - bar.Set64(resp.Completed) - } else { - currentDigest = "" - fmt.Println(resp.Status) - } - + log.Printf("progress(%s): %s", resp.Digest, resp.Status) return nil } - if err := client.Pull(context.Background(), &request, fn); err != nil { - return err - } - - if bar != nil && !bar.IsFinished() { - return errors.New("unexpected end to pull model") - } - - return nil + return client.Pull(context.Background(), &request, fn) } func RunGenerate(cmd *cobra.Command, args []string) error { @@ -442,9 +364,6 @@ func generate(cmd *cobra.Command, model, prompt string, wordWrap bool, format st return err } - spinner := NewSpinner("") - go spinner.Spin(60 * time.Millisecond) - var latest api.GenerateResponse generateContext, ok := cmd.Context().Value(generateContextKey("context")).([]int) @@ -475,10 +394,6 @@ func generate(cmd *cobra.Command, model, prompt string, wordWrap bool, format st request := api.GenerateRequest{Model: model, Prompt: prompt, Context: generateContext, Format: format} fn := func(response api.GenerateResponse) error { - if !spinner.IsFinished() { - spinner.Finish() - } - latest = response if wordWrap { @@ -511,7 +426,6 @@ func generate(cmd *cobra.Command, model, prompt string, wordWrap bool, format st if err := client.Generate(cancelCtx, &request, fn); err != nil { if strings.Contains(err.Error(), "context canceled") && abort { - spinner.Finish() return nil } return err diff --git a/cmd/spinner.go b/cmd/spinner.go deleted file mode 100644 index 53751c74..00000000 --- a/cmd/spinner.go +++ /dev/null @@ -1,44 +0,0 @@ -package cmd - -import ( - "fmt" - "os" - "time" - - "github.com/jmorganca/ollama/progressbar" -) - -type Spinner struct { - description string - *progressbar.ProgressBar -} - -func NewSpinner(description string) *Spinner { - return &Spinner{ - description: description, - ProgressBar: progressbar.NewOptions(-1, - progressbar.OptionSetWriter(os.Stderr), - progressbar.OptionThrottle(60*time.Millisecond), - progressbar.OptionSpinnerType(14), - progressbar.OptionSetRenderBlankState(true), - progressbar.OptionSetElapsedTime(false), - progressbar.OptionClearOnFinish(), - progressbar.OptionSetDescription(description), - ), - } -} - -func (s *Spinner) Spin(tick time.Duration) { - for range time.Tick(tick) { - if s.IsFinished() { - break - } - - s.Add(1) - } -} - -func (s *Spinner) Stop() { - s.Finish() - fmt.Println(s.description) -} diff --git a/progressbar/LICENSE b/progressbar/LICENSE deleted file mode 100644 index 0ca97652..00000000 --- a/progressbar/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -MIT License - -Copyright (c) 2017 Zack - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/progressbar/README.md b/progressbar/README.md deleted file mode 100644 index b9096dbd..00000000 --- a/progressbar/README.md +++ /dev/null @@ -1,121 +0,0 @@ -# progressbar - -[![CI](https://github.com/schollz/progressbar/actions/workflows/ci.yml/badge.svg?branch=main&event=push)](https://github.com/schollz/progressbar/actions/workflows/ci.yml) -[![go report card](https://goreportcard.com/badge/github.com/schollz/progressbar)](https://goreportcard.com/report/github.com/schollz/progressbar) -[![coverage](https://img.shields.io/badge/coverage-84%25-brightgreen.svg)](https://gocover.io/github.com/schollz/progressbar) -[![godocs](https://godoc.org/github.com/schollz/progressbar?status.svg)](https://godoc.org/github.com/schollz/progressbar/v3) - -A very simple thread-safe progress bar which should work on every OS without problems. I needed a progressbar for [croc](https://github.com/schollz/croc) and everything I tried had problems, so I made another one. In order to be OS agnostic I do not plan to support [multi-line outputs](https://github.com/schollz/progressbar/issues/6). - - -## Install - -``` -go get -u github.com/schollz/progressbar/v3 -``` - -## Usage - -### Basic usage - -```golang -bar := progressbar.Default(100) -for i := 0; i < 100; i++ { - bar.Add(1) - time.Sleep(40 * time.Millisecond) -} -``` - -which looks like: - -![Example of basic bar](examples/basic/basic.gif) - - -### I/O operations - -The `progressbar` implements an `io.Writer` so it can automatically detect the number of bytes written to a stream, so you can use it as a progressbar for an `io.Reader`. - -```golang -req, _ := http.NewRequest("GET", "https://dl.google.com/go/go1.14.2.src.tar.gz", nil) -resp, _ := http.DefaultClient.Do(req) -defer resp.Body.Close() - -f, _ := os.OpenFile("go1.14.2.src.tar.gz", os.O_CREATE|os.O_WRONLY, 0644) -defer f.Close() - -bar := progressbar.DefaultBytes( - resp.ContentLength, - "downloading", -) -io.Copy(io.MultiWriter(f, bar), resp.Body) -``` - -which looks like: - -![Example of download bar](examples/download/download.gif) - - -### Progress bar with unknown length - -A progressbar with unknown length is a spinner. Any bar with -1 length will automatically convert it to a spinner with a customizable spinner type. For example, the above code can be run and set the `resp.ContentLength` to `-1`. - -which looks like: - -![Example of download bar with unknown length](examples/download-unknown/download-unknown.gif) - - -### Customization - -There is a lot of customization that you can do - change the writer, the color, the width, description, theme, etc. See [all the options](https://pkg.go.dev/github.com/schollz/progressbar/v3?tab=doc#Option). - -```golang -bar := progressbar.NewOptions(1000, - progressbar.OptionSetWriter(ansi.NewAnsiStdout()), - progressbar.OptionEnableColorCodes(true), - progressbar.OptionShowBytes(true), - progressbar.OptionSetWidth(15), - progressbar.OptionSetDescription("[cyan][1/3][reset] Writing moshable file..."), - progressbar.OptionSetTheme(progressbar.Theme{ - Saucer: "[green]=[reset]", - SaucerHead: "[green]>[reset]", - SaucerPadding: " ", - BarStart: "[", - BarEnd: "]", - })) -for i := 0; i < 1000; i++ { - bar.Add(1) - time.Sleep(5 * time.Millisecond) -} -``` - -which looks like: - -![Example of customized bar](examples/customization/customization.gif) - - -## Contributing - -Pull requests are welcome. Feel free to... - -- Revise documentation -- Add new features -- Fix bugs -- Suggest improvements - -## Thanks - -Thanks [@Dynom](https://github.com/dynom) for massive improvements in version 2.0! - -Thanks [@CrushedPixel](https://github.com/CrushedPixel) for adding descriptions and color code support! - -Thanks [@MrMe42](https://github.com/MrMe42) for adding some minor features! - -Thanks [@tehstun](https://github.com/tehstun) for some great PRs! - -Thanks [@Benzammour](https://github.com/Benzammour) and [@haseth](https://github.com/haseth) for helping create v3! - -Thanks [@briandowns](https://github.com/briandowns) for compiling the list of spinners. - -## License - -MIT diff --git a/progressbar/progressbar.go b/progressbar/progressbar.go deleted file mode 100644 index c1a36c43..00000000 --- a/progressbar/progressbar.go +++ /dev/null @@ -1,1098 +0,0 @@ -package progressbar - -import ( - "errors" - "fmt" - "io" - "math" - "os" - "regexp" - "strings" - "sync" - "time" - - "github.com/mattn/go-runewidth" - "github.com/mitchellh/colorstring" - "golang.org/x/term" -) - -// ProgressBar is a thread-safe, simple -// progress bar -type ProgressBar struct { - state state - config config - lock sync.Mutex -} - -// State is the basic properties of the bar -type State struct { - CurrentPercent float64 - CurrentBytes float64 - SecondsSince float64 - SecondsLeft float64 - KBsPerSecond float64 -} - -type state struct { - currentNum int64 - currentPercent int - lastPercent int - currentSaucerSize int - isAltSaucerHead bool - - lastShown time.Time - startTime time.Time - - counterTime time.Time - counterNumSinceLast int64 - counterLastTenRates []float64 - - maxLineWidth int - currentBytes float64 - finished bool - exit bool // Progress bar exit halfway - - rendered string -} - -type config struct { - max int64 // max number of the counter - maxHumanized string - maxHumanizedSuffix string - width int - writer io.Writer - theme Theme - renderWithBlankState bool - description string - iterationString string - ignoreLength bool // ignoreLength if max bytes not known - - // whether the output is expected to contain color codes - colorCodes bool - - // show rate of change in kB/sec or MB/sec - showBytes bool - // show the iterations per second - showIterationsPerSecond bool - showIterationsCount bool - - // whether the progress bar should show elapsed time. - // always enabled if predictTime is true. - elapsedTime bool - - showElapsedTimeOnFinish bool - - // whether the progress bar should attempt to predict the finishing - // time of the progress based on the start time and the average - // number of seconds between increments. - predictTime bool - - // minimum time to wait in between updates - throttleDuration time.Duration - - // clear bar once finished - clearOnFinish bool - - // spinnerType should be a number between 0-75 - spinnerType int - - // spinnerTypeOptionUsed remembers if the spinnerType was changed manually - spinnerTypeOptionUsed bool - - // spinner represents the spinner as a slice of string - spinner []string - - // fullWidth specifies whether to measure and set the bar to a specific width - fullWidth bool - - // invisible doesn't render the bar at all, useful for debugging - invisible bool - - onCompletion func() - - // whether the render function should make use of ANSI codes to reduce console I/O - useANSICodes bool - - // showDescriptionAtLineEnd specifies whether description should be written at line end instead of line start - showDescriptionAtLineEnd bool -} - -// Theme defines the elements of the bar -type Theme struct { - Saucer string - AltSaucerHead string - SaucerHead string - SaucerPadding string - BarStart string - BarEnd string -} - -// Option is the type all options need to adhere to -type Option func(p *ProgressBar) - -// OptionSetWidth sets the width of the bar -func OptionSetWidth(s int) Option { - return func(p *ProgressBar) { - p.config.width = s - } -} - -// OptionSpinnerType sets the type of spinner used for indeterminate bars -func OptionSpinnerType(spinnerType int) Option { - return func(p *ProgressBar) { - p.config.spinnerTypeOptionUsed = true - p.config.spinnerType = spinnerType - } -} - -// OptionSpinnerCustom sets the spinner used for indeterminate bars to the passed -// slice of string -func OptionSpinnerCustom(spinner []string) Option { - return func(p *ProgressBar) { - p.config.spinner = spinner - } -} - -// OptionSetTheme sets the elements the bar is constructed of -func OptionSetTheme(t Theme) Option { - return func(p *ProgressBar) { - p.config.theme = t - } -} - -// OptionSetVisibility sets the visibility -func OptionSetVisibility(visibility bool) Option { - return func(p *ProgressBar) { - p.config.invisible = !visibility - } -} - -// OptionFullWidth sets the bar to be full width -func OptionFullWidth() Option { - return func(p *ProgressBar) { - p.config.fullWidth = true - } -} - -// OptionSetWriter sets the output writer (defaults to os.StdOut) -func OptionSetWriter(w io.Writer) Option { - return func(p *ProgressBar) { - p.config.writer = w - } -} - -// OptionSetRenderBlankState sets whether or not to render a 0% bar on construction -func OptionSetRenderBlankState(r bool) Option { - return func(p *ProgressBar) { - p.config.renderWithBlankState = r - } -} - -// OptionSetDescription sets the description of the bar to render in front of it -func OptionSetDescription(description string) Option { - return func(p *ProgressBar) { - p.config.description = description - } -} - -// OptionEnableColorCodes enables or disables support for color codes -// using mitchellh/colorstring -func OptionEnableColorCodes(colorCodes bool) Option { - return func(p *ProgressBar) { - p.config.colorCodes = colorCodes - } -} - -// OptionSetElapsedTime will enable elapsed time. Always enabled if OptionSetPredictTime is true. -func OptionSetElapsedTime(elapsedTime bool) Option { - return func(p *ProgressBar) { - p.config.elapsedTime = elapsedTime - } -} - -// OptionSetPredictTime will also attempt to predict the time remaining. -func OptionSetPredictTime(predictTime bool) Option { - return func(p *ProgressBar) { - p.config.predictTime = predictTime - } -} - -// OptionShowCount will also print current count out of total -func OptionShowCount() Option { - return func(p *ProgressBar) { - p.config.showIterationsCount = true - } -} - -// OptionShowIts will also print the iterations/second -func OptionShowIts() Option { - return func(p *ProgressBar) { - p.config.showIterationsPerSecond = true - } -} - -// OptionShowElapsedOnFinish will keep the display of elapsed time on finish -func OptionShowElapsedTimeOnFinish() Option { - return func(p *ProgressBar) { - p.config.showElapsedTimeOnFinish = true - } -} - -// OptionSetItsString sets what's displayed for iterations a second. The default is "it" which would display: "it/s" -func OptionSetItsString(iterationString string) Option { - return func(p *ProgressBar) { - p.config.iterationString = iterationString - } -} - -// OptionThrottle will wait the specified duration before updating again. The default -// duration is 0 seconds. -func OptionThrottle(duration time.Duration) Option { - return func(p *ProgressBar) { - p.config.throttleDuration = duration - } -} - -// OptionClearOnFinish will clear the bar once its finished -func OptionClearOnFinish() Option { - return func(p *ProgressBar) { - p.config.clearOnFinish = true - } -} - -// OptionOnCompletion will invoke cmpl function once its finished -func OptionOnCompletion(cmpl func()) Option { - return func(p *ProgressBar) { - p.config.onCompletion = cmpl - } -} - -// OptionShowBytes will update the progress bar -// configuration settings to display/hide kBytes/Sec -func OptionShowBytes(val bool) Option { - return func(p *ProgressBar) { - p.config.showBytes = val - } -} - -// OptionUseANSICodes will use more optimized terminal i/o. -// -// Only useful in environments with support for ANSI escape sequences. -func OptionUseANSICodes(val bool) Option { - return func(p *ProgressBar) { - p.config.useANSICodes = val - } -} - -// OptionShowDescriptionAtLineEnd defines whether description should be written at line end instead of line start -func OptionShowDescriptionAtLineEnd() Option { - return func(p *ProgressBar) { - p.config.showDescriptionAtLineEnd = true - } -} - -var defaultTheme = Theme{Saucer: "█", SaucerPadding: " ", BarStart: "▕", BarEnd: "▏"} - -// NewOptions constructs a new instance of ProgressBar, with any options you specify -func NewOptions(max int, options ...Option) *ProgressBar { - return NewOptions64(int64(max), options...) -} - -// NewOptions64 constructs a new instance of ProgressBar, with any options you specify -func NewOptions64(max int64, options ...Option) *ProgressBar { - b := ProgressBar{ - state: getBasicState(), - config: config{ - writer: os.Stdout, - theme: defaultTheme, - iterationString: "it", - width: 40, - max: max, - throttleDuration: 0 * time.Nanosecond, - elapsedTime: true, - predictTime: true, - spinnerType: 9, - invisible: false, - }, - } - - for _, o := range options { - o(&b) - } - - if b.config.spinnerType < 0 || b.config.spinnerType > 75 { - panic("invalid spinner type, must be between 0 and 75") - } - - // ignoreLength if max bytes not known - if b.config.max == -1 { - b.config.ignoreLength = true - b.config.max = int64(b.config.width) - b.config.predictTime = false - } - - b.config.maxHumanized, b.config.maxHumanizedSuffix = humanizeBytes(float64(b.config.max)) - - if b.config.renderWithBlankState { - b.RenderBlank() - } - - return &b -} - -func getBasicState() state { - now := time.Now() - return state{ - startTime: now, - lastShown: now, - counterTime: now, - } -} - -// New returns a new ProgressBar -// with the specified maximum -func New(max int) *ProgressBar { - return NewOptions(max) -} - -// DefaultBytes provides a progressbar to measure byte -// throughput with recommended defaults. -// Set maxBytes to -1 to use as a spinner. -func DefaultBytes(maxBytes int64, description ...string) *ProgressBar { - desc := "" - if len(description) > 0 { - desc = description[0] - } - return NewOptions64( - maxBytes, - OptionSetDescription(desc), - OptionSetWriter(os.Stderr), - OptionShowBytes(true), - OptionSetWidth(10), - OptionThrottle(65*time.Millisecond), - OptionShowCount(), - OptionOnCompletion(func() { - fmt.Fprint(os.Stderr, "\n") - }), - OptionSpinnerType(14), - OptionFullWidth(), - OptionSetRenderBlankState(true), - ) -} - -// DefaultBytesSilent is the same as DefaultBytes, but does not output anywhere. -// String() can be used to get the output instead. -func DefaultBytesSilent(maxBytes int64, description ...string) *ProgressBar { - // Mostly the same bar as DefaultBytes - - desc := "" - if len(description) > 0 { - desc = description[0] - } - return NewOptions64( - maxBytes, - OptionSetDescription(desc), - OptionSetWriter(io.Discard), - OptionShowBytes(true), - OptionSetWidth(10), - OptionThrottle(65*time.Millisecond), - OptionShowCount(), - OptionSpinnerType(14), - OptionFullWidth(), - ) -} - -// Default provides a progressbar with recommended defaults. -// Set max to -1 to use as a spinner. -func Default(max int64, description ...string) *ProgressBar { - desc := "" - if len(description) > 0 { - desc = description[0] - } - return NewOptions64( - max, - OptionSetDescription(desc), - OptionSetWriter(os.Stderr), - OptionSetWidth(10), - OptionThrottle(65*time.Millisecond), - OptionShowCount(), - OptionShowIts(), - OptionOnCompletion(func() { - fmt.Fprint(os.Stderr, "\n") - }), - OptionSpinnerType(14), - OptionFullWidth(), - OptionSetRenderBlankState(true), - ) -} - -// DefaultSilent is the same as Default, but does not output anywhere. -// String() can be used to get the output instead. -func DefaultSilent(max int64, description ...string) *ProgressBar { - // Mostly the same bar as Default - - desc := "" - if len(description) > 0 { - desc = description[0] - } - return NewOptions64( - max, - OptionSetDescription(desc), - OptionSetWriter(io.Discard), - OptionSetWidth(10), - OptionThrottle(65*time.Millisecond), - OptionShowCount(), - OptionShowIts(), - OptionSpinnerType(14), - OptionFullWidth(), - ) -} - -// String returns the current rendered version of the progress bar. -// It will never return an empty string while the progress bar is running. -func (p *ProgressBar) String() string { - return p.state.rendered -} - -// RenderBlank renders the current bar state, you can use this to render a 0% state -func (p *ProgressBar) RenderBlank() error { - if p.config.invisible { - return nil - } - if p.state.currentNum == 0 { - p.state.lastShown = time.Time{} - } - return p.render() -} - -// Reset will reset the clock that is used -// to calculate current time and the time left. -func (p *ProgressBar) Reset() { - p.lock.Lock() - defer p.lock.Unlock() - - p.state = getBasicState() -} - -// Finish will fill the bar to full -func (p *ProgressBar) Finish() error { - p.lock.Lock() - p.state.currentNum = p.config.max - p.lock.Unlock() - return p.Add(0) -} - -// Exit will exit the bar to keep current state -func (p *ProgressBar) Exit() error { - p.lock.Lock() - defer p.lock.Unlock() - - p.state.exit = true - if p.config.onCompletion != nil { - p.config.onCompletion() - } - return nil -} - -// Add will add the specified amount to the progressbar -func (p *ProgressBar) Add(num int) error { - return p.Add64(int64(num)) -} - -// Set will set the bar to a current number -func (p *ProgressBar) Set(num int) error { - return p.Set64(int64(num)) -} - -// Set64 will set the bar to a current number -func (p *ProgressBar) Set64(num int64) error { - p.lock.Lock() - toAdd := num - int64(p.state.currentBytes) - p.lock.Unlock() - return p.Add64(toAdd) -} - -// Add64 will add the specified amount to the progressbar -func (p *ProgressBar) Add64(num int64) error { - if p.config.invisible { - return nil - } - p.lock.Lock() - defer p.lock.Unlock() - - if p.state.exit { - return nil - } - - // error out since OptionSpinnerCustom will always override a manually set spinnerType - if p.config.spinnerTypeOptionUsed && len(p.config.spinner) > 0 { - return errors.New("OptionSpinnerType and OptionSpinnerCustom cannot be used together") - } - - if p.config.max == 0 { - return errors.New("max must be greater than 0") - } - - if p.state.currentNum < p.config.max { - if p.config.ignoreLength { - p.state.currentNum = (p.state.currentNum + num) % p.config.max - } else { - p.state.currentNum += num - } - } - - p.state.currentBytes += float64(num) - - // reset the countdown timer every second to take rolling average - p.state.counterNumSinceLast += num - if time.Since(p.state.counterTime).Seconds() > 0.5 { - p.state.counterLastTenRates = append(p.state.counterLastTenRates, float64(p.state.counterNumSinceLast)/time.Since(p.state.counterTime).Seconds()) - if len(p.state.counterLastTenRates) > 10 { - p.state.counterLastTenRates = p.state.counterLastTenRates[1:] - } - p.state.counterTime = time.Now() - p.state.counterNumSinceLast = 0 - } - - percent := float64(p.state.currentNum) / float64(p.config.max) - p.state.currentSaucerSize = int(percent * float64(p.config.width)) - p.state.currentPercent = int(percent * 100) - updateBar := p.state.currentPercent != p.state.lastPercent && p.state.currentPercent > 0 - - p.state.lastPercent = p.state.currentPercent - if p.state.currentNum > p.config.max { - return errors.New("current number exceeds max") - } - - // always update if show bytes/second or its/second - if updateBar || p.config.showIterationsPerSecond || p.config.showIterationsCount { - return p.render() - } - - return nil -} - -// Clear erases the progress bar from the current line -func (p *ProgressBar) Clear() error { - return clearProgressBar(p.config, p.state) -} - -// Describe will change the description shown before the progress, which -// can be changed on the fly (as for a slow running process). -func (p *ProgressBar) Describe(description string) { - p.lock.Lock() - defer p.lock.Unlock() - p.config.description = description - if p.config.invisible { - return - } - p.render() -} - -// New64 returns a new ProgressBar -// with the specified maximum -func New64(max int64) *ProgressBar { - return NewOptions64(max) -} - -// GetMax returns the max of a bar -func (p *ProgressBar) GetMax() int { - return int(p.config.max) -} - -// GetMax64 returns the current max -func (p *ProgressBar) GetMax64() int64 { - return p.config.max -} - -// ChangeMax takes in a int -// and changes the max value -// of the progress bar -func (p *ProgressBar) ChangeMax(newMax int) { - p.ChangeMax64(int64(newMax)) -} - -// ChangeMax64 is basically -// the same as ChangeMax, -// but takes in a int64 -// to avoid casting -func (p *ProgressBar) ChangeMax64(newMax int64) { - p.config.max = newMax - - if p.config.showBytes { - p.config.maxHumanized, p.config.maxHumanizedSuffix = humanizeBytes(float64(p.config.max)) - } - - p.Add(0) // re-render -} - -// IsFinished returns true if progress bar is completed -func (p *ProgressBar) IsFinished() bool { - return p.state.finished -} - -// render renders the progress bar, updating the maximum -// rendered line width. this function is not thread-safe, -// so it must be called with an acquired lock. -func (p *ProgressBar) render() error { - // make sure that the rendering is not happening too quickly - // but always show if the currentNum reaches the max - if time.Since(p.state.lastShown).Nanoseconds() < p.config.throttleDuration.Nanoseconds() && - p.state.currentNum < p.config.max { - return nil - } - - if !p.config.useANSICodes { - // first, clear the existing progress bar - err := clearProgressBar(p.config, p.state) - if err != nil { - return err - } - } - - // check if the progress bar is finished - if !p.state.finished && p.state.currentNum >= p.config.max { - p.state.finished = true - if !p.config.clearOnFinish { - renderProgressBar(p.config, &p.state) - } - if p.config.onCompletion != nil { - p.config.onCompletion() - } - } - if p.state.finished { - // when using ANSI codes we don't pre-clean the current line - if p.config.useANSICodes && p.config.clearOnFinish { - err := clearProgressBar(p.config, p.state) - if err != nil { - return err - } - } - return nil - } - - // then, re-render the current progress bar - w, err := renderProgressBar(p.config, &p.state) - if err != nil { - return err - } - - if w > p.state.maxLineWidth { - p.state.maxLineWidth = w - } - - p.state.lastShown = time.Now() - - return nil -} - -// State returns the current state -func (p *ProgressBar) State() State { - p.lock.Lock() - defer p.lock.Unlock() - s := State{} - s.CurrentPercent = float64(p.state.currentNum) / float64(p.config.max) - s.CurrentBytes = p.state.currentBytes - s.SecondsSince = time.Since(p.state.startTime).Seconds() - if p.state.currentNum > 0 { - s.SecondsLeft = s.SecondsSince / float64(p.state.currentNum) * (float64(p.config.max) - float64(p.state.currentNum)) - } - s.KBsPerSecond = float64(p.state.currentBytes) / 1000.0 / s.SecondsSince - return s -} - -// regex matching ansi escape codes -var ansiRegex = regexp.MustCompile(`\x1b\[[0-9;]*[a-zA-Z]`) - -func getStringWidth(c config, str string, colorize bool) int { - if c.colorCodes { - // convert any color codes in the progress bar into the respective ANSI codes - str = colorstring.Color(str) - } - - // the width of the string, if printed to the console - // does not include the carriage return character - cleanString := strings.Replace(str, "\r", "", -1) - - if c.colorCodes { - // the ANSI codes for the colors do not take up space in the console output, - // so they do not count towards the output string width - cleanString = ansiRegex.ReplaceAllString(cleanString, "") - } - - // get the amount of runes in the string instead of the - // character count of the string, as some runes span multiple characters. - // see https://stackoverflow.com/a/12668840/2733724 - stringWidth := runewidth.StringWidth(cleanString) - return stringWidth -} - -func renderProgressBar(c config, s *state) (int, error) { - var sb strings.Builder - - averageRate := average(s.counterLastTenRates) - if len(s.counterLastTenRates) == 0 || s.finished { - // if no average samples, or if finished, - // then average rate should be the total rate - if t := time.Since(s.startTime).Seconds(); t > 0 { - averageRate = s.currentBytes / t - } else { - averageRate = 0 - } - } - - // show iteration count in "current/total" iterations format - if c.showIterationsCount { - if sb.Len() == 0 { - sb.WriteString("(") - } else { - sb.WriteString(", ") - } - if !c.ignoreLength { - if c.showBytes { - currentHumanize, currentSuffix := humanizeBytes(s.currentBytes) - if currentSuffix == c.maxHumanizedSuffix { - sb.WriteString(fmt.Sprintf("%s/%s%s", - currentHumanize, c.maxHumanized, c.maxHumanizedSuffix)) - } else { - sb.WriteString(fmt.Sprintf("%s%s/%s%s", - currentHumanize, currentSuffix, c.maxHumanized, c.maxHumanizedSuffix)) - } - } else { - sb.WriteString(fmt.Sprintf("%.0f/%d", s.currentBytes, c.max)) - } - } else { - if c.showBytes { - currentHumanize, currentSuffix := humanizeBytes(s.currentBytes) - sb.WriteString(fmt.Sprintf("%s%s", currentHumanize, currentSuffix)) - } else { - sb.WriteString(fmt.Sprintf("%.0f/%s", s.currentBytes, "-")) - } - } - } - - // show rolling average rate - if c.showBytes && averageRate > 0 && !math.IsInf(averageRate, 1) { - if sb.Len() == 0 { - sb.WriteString("(") - } else { - sb.WriteString(", ") - } - currentHumanize, currentSuffix := humanizeBytes(averageRate) - sb.WriteString(fmt.Sprintf("%s%s/s", currentHumanize, currentSuffix)) - } - - // show iterations rate - if c.showIterationsPerSecond { - if sb.Len() == 0 { - sb.WriteString("(") - } else { - sb.WriteString(", ") - } - if averageRate > 1 { - sb.WriteString(fmt.Sprintf("%0.0f %s/s", averageRate, c.iterationString)) - } else if averageRate*60 > 1 { - sb.WriteString(fmt.Sprintf("%0.0f %s/min", 60*averageRate, c.iterationString)) - } else { - sb.WriteString(fmt.Sprintf("%0.0f %s/hr", 3600*averageRate, c.iterationString)) - } - } - if sb.Len() > 0 { - sb.WriteString(")") - } - - leftBrac, rightBrac, saucer, saucerHead := "", "", "", "" - - // show time prediction in "current/total" seconds format - switch { - case c.predictTime: - rightBracNum := (time.Duration((1/averageRate)*(float64(c.max)-float64(s.currentNum))) * time.Second) - if rightBracNum.Seconds() < 0 { - rightBracNum = 0 * time.Second - } - rightBrac = rightBracNum.String() - fallthrough - case c.elapsedTime: - leftBrac = (time.Duration(time.Since(s.startTime).Seconds()) * time.Second).String() - } - - if c.fullWidth && !c.ignoreLength { - width, err := termWidth() - if err != nil { - width = 80 - } - - amend := 1 // an extra space at eol - switch { - case leftBrac != "" && rightBrac != "": - amend = 4 // space, square brackets and colon - case leftBrac != "" && rightBrac == "": - amend = 4 // space and square brackets and another space - case leftBrac == "" && rightBrac != "": - amend = 3 // space and square brackets - } - if c.showDescriptionAtLineEnd { - amend += 1 // another space - } - - c.width = width - getStringWidth(c, c.description, true) - 10 - amend - sb.Len() - len(leftBrac) - len(rightBrac) - s.currentSaucerSize = int(float64(s.currentPercent) / 100.0 * float64(c.width)) - } - if s.currentSaucerSize > 0 { - if c.ignoreLength { - saucer = strings.Repeat(c.theme.SaucerPadding, s.currentSaucerSize-1) - } else { - saucer = strings.Repeat(c.theme.Saucer, s.currentSaucerSize-1) - } - - // Check if an alternate saucer head is set for animation - if c.theme.AltSaucerHead != "" && s.isAltSaucerHead { - saucerHead = c.theme.AltSaucerHead - s.isAltSaucerHead = false - } else if c.theme.SaucerHead == "" || s.currentSaucerSize == c.width { - // use the saucer for the saucer head if it hasn't been set - // to preserve backwards compatibility - saucerHead = c.theme.Saucer - } else { - saucerHead = c.theme.SaucerHead - s.isAltSaucerHead = true - } - } - - /* - Progress Bar format - Description % |------ | (kb/s) (iteration count) (iteration rate) (predict time) - - or if showDescriptionAtLineEnd is enabled - % |------ | (kb/s) (iteration count) (iteration rate) (predict time) Description - */ - - repeatAmount := c.width - s.currentSaucerSize - if repeatAmount < 0 { - repeatAmount = 0 - } - - str := "" - - if c.ignoreLength { - selectedSpinner := spinners[c.spinnerType] - if len(c.spinner) > 0 { - selectedSpinner = c.spinner - } - spinner := selectedSpinner[int(math.Round(math.Mod(float64(time.Since(s.startTime).Milliseconds()/100), float64(len(selectedSpinner)))))] - if c.elapsedTime { - if c.showDescriptionAtLineEnd { - str = fmt.Sprintf("\r%s %s [%s] %s ", - spinner, - sb.String(), - leftBrac, - c.description) - } else { - str = fmt.Sprintf("\r%s %s %s [%s] ", - spinner, - c.description, - sb.String(), - leftBrac) - } - } else { - if c.showDescriptionAtLineEnd { - str = fmt.Sprintf("\r%s %s %s ", - spinner, - sb.String(), - c.description) - } else { - str = fmt.Sprintf("\r%s %s %s ", - spinner, - c.description, - sb.String()) - } - } - } else if rightBrac == "" { - str = fmt.Sprintf("%4d%% %s%s%s%s%s %s", - s.currentPercent, - c.theme.BarStart, - saucer, - saucerHead, - strings.Repeat(c.theme.SaucerPadding, repeatAmount), - c.theme.BarEnd, - sb.String()) - - if s.currentPercent == 100 && c.showElapsedTimeOnFinish { - str = fmt.Sprintf("%s [%s]", str, leftBrac) - } - - if c.showDescriptionAtLineEnd { - str = fmt.Sprintf("\r%s %s ", str, c.description) - } else { - str = fmt.Sprintf("\r%s%s ", c.description, str) - } - } else { - if s.currentPercent == 100 { - str = fmt.Sprintf("%4d%% %s%s%s%s%s %s", - s.currentPercent, - c.theme.BarStart, - saucer, - saucerHead, - strings.Repeat(c.theme.SaucerPadding, repeatAmount), - c.theme.BarEnd, - sb.String()) - - if c.showElapsedTimeOnFinish { - str = fmt.Sprintf("%s [%s]", str, leftBrac) - } - - if c.showDescriptionAtLineEnd { - str = fmt.Sprintf("\r%s %s", str, c.description) - } else { - str = fmt.Sprintf("\r%s%s", c.description, str) - } - } else { - str = fmt.Sprintf("%4d%% %s%s%s%s%s %s [%s:%s]", - s.currentPercent, - c.theme.BarStart, - saucer, - saucerHead, - strings.Repeat(c.theme.SaucerPadding, repeatAmount), - c.theme.BarEnd, - sb.String(), - leftBrac, - rightBrac) - - if c.showDescriptionAtLineEnd { - str = fmt.Sprintf("\r%s %s", str, c.description) - } else { - str = fmt.Sprintf("\r%s%s", c.description, str) - } - } - } - - if c.colorCodes { - // convert any color codes in the progress bar into the respective ANSI codes - str = colorstring.Color(str) - } - - s.rendered = str - - return getStringWidth(c, str, false), writeString(c, str) -} - -func clearProgressBar(c config, s state) error { - if s.maxLineWidth == 0 { - return nil - } - if c.useANSICodes { - // write the "clear current line" ANSI escape sequence - return writeString(c, "\033[2K\r") - } - // fill the empty content - // to overwrite the progress bar and jump - // back to the beginning of the line - str := fmt.Sprintf("\r%s\r", strings.Repeat(" ", s.maxLineWidth)) - return writeString(c, str) - // the following does not show correctly if the previous line is longer than subsequent line - // return writeString(c, "\r") -} - -func writeString(c config, str string) error { - if _, err := io.WriteString(c.writer, str); err != nil { - return err - } - - if f, ok := c.writer.(*os.File); ok { - // ignore any errors in Sync(), as stdout - // can't be synced on some operating systems - // like Debian 9 (Stretch) - f.Sync() - } - - return nil -} - -// Reader is the progressbar io.Reader struct -type Reader struct { - io.Reader - bar *ProgressBar -} - -// NewReader return a new Reader with a given progress bar. -func NewReader(r io.Reader, bar *ProgressBar) Reader { - return Reader{ - Reader: r, - bar: bar, - } -} - -// Read will read the data and add the number of bytes to the progressbar -func (r *Reader) Read(p []byte) (n int, err error) { - n, err = r.Reader.Read(p) - r.bar.Add(n) - return -} - -// Close the reader when it implements io.Closer -func (r *Reader) Close() (err error) { - if closer, ok := r.Reader.(io.Closer); ok { - return closer.Close() - } - r.bar.Finish() - return -} - -// Write implement io.Writer -func (p *ProgressBar) Write(b []byte) (n int, err error) { - n = len(b) - p.Add(n) - return -} - -// Read implement io.Reader -func (p *ProgressBar) Read(b []byte) (n int, err error) { - n = len(b) - p.Add(n) - return -} - -func (p *ProgressBar) Close() (err error) { - p.Finish() - return -} - -func average(xs []float64) float64 { - total := 0.0 - for _, v := range xs { - total += v - } - return total / float64(len(xs)) -} - -func humanizeBytes(s float64) (string, string) { - sizes := []string{" B", " kB", " MB", " GB", " TB", " PB", " EB"} - base := 1000.0 - if s < 10 { - return fmt.Sprintf("%2.0f", s), sizes[0] - } - e := math.Floor(logn(float64(s), base)) - suffix := sizes[int(e)] - val := math.Floor(float64(s)/math.Pow(base, e)*10+0.5) / 10 - f := "%.0f" - if val < 10 { - f = "%.1f" - } - - return fmt.Sprintf(f, val), suffix -} - -func logn(n, b float64) float64 { - return math.Log(n) / math.Log(b) -} - -// termWidth function returns the visible width of the current terminal -// and can be redefined for testing -var termWidth = func() (width int, err error) { - width, _, err = term.GetSize(int(os.Stdout.Fd())) - if err == nil { - return width, nil - } - - return 0, err -} diff --git a/progressbar/spinners.go b/progressbar/spinners.go deleted file mode 100644 index c3ccd01f..00000000 --- a/progressbar/spinners.go +++ /dev/null @@ -1,80 +0,0 @@ -package progressbar - -var spinners = map[int][]string{ - 0: {"←", "↖", "↑", "↗", "→", "↘", "↓", "↙"}, - 1: {"▁", "▃", "▄", "▅", "▆", "▇", "█", "▇", "▆", "▅", "▄", "▃", "▁"}, - 2: {"▖", "▘", "▝", "▗"}, - 3: {"┤", "┘", "┴", "└", "├", "┌", "┬", "┐"}, - 4: {"◢", "◣", "◤", "◥"}, - 5: {"◰", "◳", "◲", "◱"}, - 6: {"◴", "◷", "◶", "◵"}, - 7: {"◐", "◓", "◑", "◒"}, - 8: {".", "o", "O", "@", "*"}, - 9: {"|", "/", "-", "\\"}, - 10: {"◡◡", "⊙⊙", "◠◠"}, - 11: {"⣾", "⣽", "⣻", "⢿", "⡿", "⣟", "⣯", "⣷"}, - 12: {">))'>", " >))'>", " >))'>", " >))'>", " >))'>", " <'((<", " <'((<", " <'((<"}, - 13: {"⠁", "⠂", "⠄", "⡀", "⢀", "⠠", "⠐", "⠈"}, - 14: {"⠋", "⠙", "⠹", "⠸", "⠼", "⠴", "⠦", "⠧", "⠇", "⠏"}, - 15: {"a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", "n", "o", "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z"}, - 16: {"▉", "▊", "▋", "▌", "▍", "▎", "▏", "▎", "▍", "▌", "▋", "▊", "▉"}, - 17: {"■", "□", "▪", "▫"}, - 18: {"←", "↑", "→", "↓"}, - 19: {"╫", "╪"}, - 20: {"⇐", "⇖", "⇑", "⇗", "⇒", "⇘", "⇓", "⇙"}, - 21: {"⠁", "⠁", "⠉", "⠙", "⠚", "⠒", "⠂", "⠂", "⠒", "⠲", "⠴", "⠤", "⠄", "⠄", "⠤", "⠠", "⠠", "⠤", "⠦", "⠖", "⠒", "⠐", "⠐", "⠒", "⠓", "⠋", "⠉", "⠈", "⠈"}, - 22: {"⠈", "⠉", "⠋", "⠓", "⠒", "⠐", "⠐", "⠒", "⠖", "⠦", "⠤", "⠠", "⠠", "⠤", "⠦", "⠖", "⠒", "⠐", "⠐", "⠒", "⠓", "⠋", "⠉", "⠈"}, - 23: {"⠁", "⠉", "⠙", "⠚", "⠒", "⠂", "⠂", "⠒", "⠲", "⠴", "⠤", "⠄", "⠄", "⠤", "⠴", "⠲", "⠒", "⠂", "⠂", "⠒", "⠚", "⠙", "⠉", "⠁"}, - 24: {"⠋", "⠙", "⠚", "⠒", "⠂", "⠂", "⠒", "⠲", "⠴", "⠦", "⠖", "⠒", "⠐", "⠐", "⠒", "⠓", "⠋"}, - 25: {"ｦ", "ｧ", "ｨ", "ｩ", "ｪ", "ｫ", "ｬ", "ｭ", "ｮ", "ｯ", "ｱ", "ｲ", "ｳ", "ｴ", "ｵ", "ｶ", "ｷ", "ｸ", "ｹ", "ｺ", "ｻ", "ｼ", "ｽ", "ｾ", "ｿ", "ﾀ", "ﾁ", "ﾂ", "ﾃ", "ﾄ", "ﾅ", "ﾆ", "ﾇ", "ﾈ", "ﾉ", "ﾊ", "ﾋ", "ﾌ", "ﾍ", "ﾎ", "ﾏ", "ﾐ", "ﾑ", "ﾒ", "ﾓ", "ﾔ", "ﾕ", "ﾖ", "ﾗ", "ﾘ", "ﾙ", "ﾚ", "ﾛ", "ﾜ", "ﾝ"}, - 26: {".", "..", "..."}, - 27: {"▁", "▂", "▃", "▄", "▅", "▆", "▇", "█", "▉", "▊", "▋", "▌", "▍", "▎", "▏", "▏", "▎", "▍", "▌", "▋", "▊", "▉", "█", "▇", "▆", "▅", "▄", "▃", "▂", "▁"}, - 28: {".", "o", "O", "°", "O", "o", "."}, - 29: {"+", "x"}, - 30: {"v", "<", "^", ">"}, - 31: {">>--->", " >>--->", " >>--->", " >>--->", " >>--->", " <---<<", " <---<<", " <---<<", " <---<<", "<---<<"}, - 32: {"|", "||", "|||", "||||", "|||||", "|||||||", "||||||||", "|||||||", "||||||", "|||||", "||||", "|||", "||", "|"}, - 33: {"[ ]", "[= ]", "[== ]", "[=== ]", "[==== ]", "[===== ]", "[====== ]", "[======= ]", "[======== ]", "[========= ]", "[==========]"}, - 34: {"(*---------)", "(-*--------)", "(--*-------)", "(---*------)", "(----*-----)", "(-----*----)", "(------*---)", "(-------*--)", "(--------*-)", "(---------*)"}, - 35: {"█▒▒▒▒▒▒▒▒▒", "███▒▒▒▒▒▒▒", "█████▒▒▒▒▒", "███████▒▒▒", "██████████"}, - 36: {"[ ]", "[=> ]", "[===> ]", "[=====> ]", "[======> ]", "[========> ]", "[==========> ]", "[============> ]", "[==============> ]", "[================> ]", "[==================> ]", "[===================>]"}, - 37: {"ဝ", "၀"}, - 38: {"▌", "▀", "▐▄"}, - 39: {"🌍", "🌎", "🌏"}, - 40: {"◜", "◝", "◞", "◟"}, - 41: {"⬒", "⬔", "⬓", "⬕"}, - 42: {"⬖", "⬘", "⬗", "⬙"}, - 43: {"[>>> >]", "[]>>>> []", "[] >>>> []", "[] >>>> []", "[] >>>> []", "[] >>>>[]", "[>> >>]"}, - 44: {"♠", "♣", "♥", "♦"}, - 45: {"➞", "➟", "➠", "➡", "➠", "➟"}, - 46: {" | ", ` \ `, "_ ", ` \ `, " | ", " / ", " _", " / "}, - 47: {" . . . .", ". . . .", ". . . .", ". . . .", ". . . . ", ". . . . ."}, - 48: {" | ", " / ", " _ ", ` \ `, " | ", ` \ `, " _ ", " / "}, - 49: {"⎺", "⎻", "⎼", "⎽", "⎼", "⎻"}, - 50: {"▹▹▹▹▹", "▸▹▹▹▹", "▹▸▹▹▹", "▹▹▸▹▹", "▹▹▹▸▹", "▹▹▹▹▸"}, - 51: {"[ ]", "[ =]", "[ ==]", "[ ===]", "[====]", "[=== ]", "[== ]", "[= ]"}, - 52: {"( ● )", "( ● )", "( ● )", "( ● )", "( ●)", "( ● )", "( ● )", "( ● )", "( ● )"}, - 53: {"✶", "✸", "✹", "✺", "✹", "✷"}, - 54: {"▐|\\____________▌", "▐_|\\___________▌", "▐__|\\__________▌", "▐___|\\_________▌", "▐____|\\________▌", "▐_____|\\_______▌", "▐______|\\______▌", "▐_______|\\_____▌", "▐________|\\____▌", "▐_________|\\___▌", "▐__________|\\__▌", "▐___________|\\_▌", "▐____________|\\▌", "▐____________/|▌", "▐___________/|_▌", "▐__________/|__▌", "▐_________/|___▌", "▐________/|____▌", "▐_______/|_____▌", "▐______/|______▌", "▐_____/|_______▌", "▐____/|________▌", "▐___/|_________▌", "▐__/|__________▌", "▐_/|___________▌", "▐/|____________▌"}, - 55: {"▐⠂ ▌", "▐⠈ ▌", "▐ ⠂ ▌", "▐ ⠠ ▌", "▐ ⡀ ▌", "▐ ⠠ ▌", "▐ ⠂ ▌", "▐ ⠈ ▌", "▐ ⠂ ▌", "▐ ⠠ ▌", "▐ ⡀ ▌", "▐ ⠠ ▌", "▐ ⠂ ▌", "▐ ⠈ ▌", "▐ ⠂▌", "▐ ⠠▌", "▐ ⡀▌", "▐ ⠠ ▌", "▐ ⠂ ▌", "▐ ⠈ ▌", "▐ ⠂ ▌", "▐ ⠠ ▌", "▐ ⡀ ▌", "▐ ⠠ ▌", "▐ ⠂ ▌", "▐ ⠈ ▌", "▐ ⠂ ▌", "▐ ⠠ ▌", "▐ ⡀ ▌", "▐⠠ ▌"}, - 56: {"¿", "?"}, - 57: {"⢹", "⢺", "⢼", "⣸", "⣇", "⡧", "⡗", "⡏"}, - 58: {"⢄", "⢂", "⢁", "⡁", "⡈", "⡐", "⡠"}, - 59: {". ", ".. ", "...", " ..", " .", " "}, - 60: {".", "o", "O", "°", "O", "o", "."}, - 61: {"▓", "▒", "░"}, - 62: {"▌", "▀", "▐", "▄"}, - 63: {"⊶", "⊷"}, - 64: {"▪", "▫"}, - 65: {"□", "■"}, - 66: {"▮", "▯"}, - 67: {"-", "=", "≡"}, - 68: {"d", "q", "p", "b"}, - 69: {"∙∙∙", "●∙∙", "∙●∙", "∙∙●", "∙∙∙"}, - 70: {"🌑 ", "🌒 ", "🌓 ", "🌔 ", "🌕 ", "🌖 ", "🌗 ", "🌘 "}, - 71: {"☗", "☖"}, - 72: {"⧇", "⧆"}, - 73: {"◉", "◎"}, - 74: {"㊂", "㊀", "㊁"}, - 75: {"⦾", "⦿"}, -} From 9f04e5a8eaf24b0391766fc4904f7f1f08d2d1e8 Mon Sep 17 00:00:00 2001 From: Michael Yang Date: Tue, 14 Nov 2023 16:33:09 -0800 Subject: [PATCH 052/103] format bytes --- format/bytes.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/format/bytes.go b/format/bytes.go index 1cf5a762..471bdf49 100644 --- a/format/bytes.go +++ b/format/bytes.go @@ -7,10 +7,13 @@ const ( KiloByte = Byte * 1000 MegaByte = KiloByte * 1000 GigaByte = MegaByte * 1000 + TeraByte = GigaByte * 1000 ) func HumanBytes(b int64) string { switch { + case b > TeraByte: + return fmt.Sprintf("%.1f TB", float64(b)/TeraByte) case b > GigaByte: return fmt.Sprintf("%.1f GB", float64(b)/GigaByte) case b > MegaByte: From c4a3ccd7ace38a9e9c54d83edb035742d0eff962 Mon Sep 17 00:00:00 2001 From: Michael Yang Date: Tue, 14 Nov 2023 16:33:16 -0800 Subject: [PATCH 053/103] progress --- progress/bar.go | 118 +++++++++++++++++++++++++++++++++++++++++++ progress/progress.go | 65 ++++++++++++++++++++++++ progress/spinner.go | 102 +++++++++++++++++++++++++++++++++++++ 3 files changed, 285 insertions(+) create mode 100644 progress/bar.go create mode 100644 progress/progress.go create mode 100644 progress/spinner.go diff --git a/progress/bar.go b/progress/bar.go new file mode 100644 index 00000000..9a81fd0f --- /dev/null +++ b/progress/bar.go @@ -0,0 +1,118 @@ +package progress + +import ( + "fmt" + "os" + "strings" + "time" + + "github.com/jmorganca/ollama/format" + "golang.org/x/term" +) + +type Bar struct { + message string + messageWidth int + + maxValue int64 + initialValue int64 + currentValue int64 + + started time.Time + stopped time.Time +} + +func NewBar(message string, maxValue, initialValue int64) *Bar { + return &Bar{ + message: message, + messageWidth: -1, + maxValue: maxValue, + initialValue: initialValue, + currentValue: initialValue, + started: time.Now(), + } +} + +func (b *Bar) String() string { + termWidth, _, err := term.GetSize(int(os.Stderr.Fd())) + if err != nil { + panic(err) + } + + var pre, mid, suf strings.Builder + + if b.message != "" { + message := strings.TrimSpace(b.message) + if b.messageWidth > 0 && len(message) > b.messageWidth { + message = message[:b.messageWidth] + } + + fmt.Fprintf(&pre, "%s", message) + if b.messageWidth-pre.Len() >= 0 { + pre.WriteString(strings.Repeat(" ", b.messageWidth-pre.Len())) + } + + pre.WriteString(" ") + } + + fmt.Fprintf(&pre, "%.1f%% ", b.percent()) + + fmt.Fprintf(&suf, "(%s/%s, %s/s, %s)", + format.HumanBytes(b.currentValue), + format.HumanBytes(b.maxValue), + format.HumanBytes(int64(b.rate())), + b.elapsed()) + + mid.WriteString("[") + + // pad 3 for last = or > and "] " + f := termWidth - pre.Len() - mid.Len() - suf.Len() - 3 + n := int(float64(f) * b.percent() / 100) + if n > 0 { + mid.WriteString(strings.Repeat("=", n)) + } + + if b.currentValue >= b.maxValue { + mid.WriteString("=") + } else { + mid.WriteString(">") + } + + if f-n > 0 { + mid.WriteString(strings.Repeat(" ", f-n)) + } + + mid.WriteString("] ") + + return pre.String() + mid.String() + suf.String() +} + +func (b *Bar) Set(value int64) { + if value >= b.maxValue { + value = b.maxValue + b.stopped = time.Now() + } + + b.currentValue = value +} + +func (b *Bar) percent() float64 { + if b.maxValue > 0 { + return float64(b.currentValue) / float64(b.maxValue) * 100 + } + + return 0 +} + +func (b *Bar) rate() float64 { + return (float64(b.currentValue) - float64(b.initialValue)) / b.elapsed().Seconds() +} + +func (b *Bar) elapsed() time.Duration { + stopped := b.stopped + if stopped.IsZero() { + stopped = time.Now() + } + + return stopped.Sub(b.started).Round(time.Second) +} diff --git a/progress/progress.go b/progress/progress.go new file mode 100644 index 00000000..5002b8d2 --- /dev/null +++ b/progress/progress.go @@ -0,0 +1,65 @@ +package progress + +import ( + "fmt" + "io" + "sync" + "time" +) + +type State interface { + String() string +} + +type Progress struct { + mu sync.Mutex + pos int + w io.Writer + + ticker *time.Ticker + states []State +} + +func NewProgress(w io.Writer) *Progress { + p := &Progress{pos: -1, w: w} + go p.start() + return p +} + +func (p *Progress) Stop() { + if p.ticker != nil { + p.ticker.Stop() + p.ticker = nil + p.render() + } +} + +func (p *Progress) Add(key string, state State) { + p.mu.Lock() + defer p.mu.Unlock() + + p.states = append(p.states, state) +} + +func (p *Progress) render() error { + p.mu.Lock() + defer p.mu.Unlock() + + fmt.Fprintf(p.w, "\033[%dA", p.pos) + for _, state := range p.states { + fmt.Fprintln(p.w, state.String()) + } + + if len(p.states) > 0 { + p.pos = len(p.states) + } + + return nil +} + +func (p *Progress) start() { + p.ticker = time.NewTicker(100 * time.Millisecond) + for range p.ticker.C { + p.render() + } +} diff --git a/progress/spinner.go b/progress/spinner.go new file mode 100644 index 00000000..bc46bc02 --- /dev/null +++ b/progress/spinner.go @@ -0,0 +1,102 @@ +package progress + +import ( + "fmt" + "os" + "strings" + "time" + + "golang.org/x/term" +) + +type Spinner struct { + message string + messageWidth int + + parts []string + + value int + + ticker *time.Ticker + started time.Time + stopped time.Time +} + +func NewSpinner(message string) *Spinner { + s := &Spinner{ + message: message, + parts: []string{ + "⠋", "⠙", "⠹", "⠸", "⠼", "⠴", "⠦", "⠧", "⠇", "⠏", + }, + started: time.Now(), + } + go s.start() + return s +} + +func (s *Spinner) String() string { + termWidth, _, err := term.GetSize(int(os.Stderr.Fd())) + if err != nil { + panic(err) + } + + var pre strings.Builder + if len(s.message) > 0 { + message := strings.TrimSpace(s.message) + if s.messageWidth > 0 && len(message) > s.messageWidth { + message = message[:s.messageWidth] + } + + fmt.Fprintf(&pre, "%s", message) + if s.messageWidth-pre.Len() >= 0 { + pre.WriteString(strings.Repeat(" ", s.messageWidth-pre.Len())) + } + + pre.WriteString(" ") + } + + var pad int + if s.stopped.IsZero() { + // spinner has a string length of 3 but a rune length of 1 + // in order to align correctly, we need to pad with (3 - 1) = 2 spaces + spinner := s.parts[s.value] + pre.WriteString(spinner) + pad = len(spinner) - len([]rune(spinner)) + } + + var suf strings.Builder + fmt.Fprintf(&suf, "(%s)", s.elapsed()) + + var mid strings.Builder + f := termWidth - pre.Len() - mid.Len() - suf.Len() + pad + if f > 0 { + mid.WriteString(strings.Repeat(" ", f)) + } + + return pre.String() + mid.String() + suf.String() +} + +func (s *Spinner) start() { + s.ticker = time.NewTicker(100 * time.Millisecond) + for range s.ticker.C { + s.value = (s.value + 1) % len(s.parts) + if !s.stopped.IsZero() { + return + } + } +} + +func (s *Spinner) Stop() { + if s.stopped.IsZero() { + s.stopped = time.Now() + } +} + +func (s *Spinner) elapsed() time.Duration { + stopped := s.stopped + if stopped.IsZero() { + stopped = time.Now() + } + + return stopped.Sub(s.started).Round(time.Second) +} From 1c0e092eadc9f56abd745d31ff5c57e91fddd45e Mon Sep 17 00:00:00 2001 From: Michael Yang Date: Tue, 14 Nov 2023 16:33:24 -0800 Subject: [PATCH 054/103] progress cmd --- cmd/cmd.go | 132 ++++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 120 insertions(+), 12 deletions(-) diff --git a/cmd/cmd.go b/cmd/cmd.go index 5d31eae2..4df3b004 100644 --- a/cmd/cmd.go +++ b/cmd/cmd.go @@ -30,6 +30,7 @@ import ( "github.com/jmorganca/ollama/api" "github.com/jmorganca/ollama/format" "github.com/jmorganca/ollama/parser" + "github.com/jmorganca/ollama/progress" "github.com/jmorganca/ollama/readline" "github.com/jmorganca/ollama/server" "github.com/jmorganca/ollama/version" @@ -47,6 +48,15 @@ func CreateHandler(cmd *cobra.Command, args []string) error { return err } + p := progress.NewProgress(os.Stderr) + defer p.Stop() + + bars := make(map[string]*progress.Bar) + + status := "transferring context" + spinner := progress.NewSpinner(status) + p.Add(status, spinner) + modelfile, err := os.ReadFile(filename) if err != nil { return err @@ -95,16 +105,38 @@ func CreateHandler(cmd *cobra.Command, args []string) error { } } - request := api.CreateRequest{Name: args[0], Path: filename, Modelfile: string(modelfile)} fn := func(resp api.ProgressResponse) error { - log.Printf("progress(%s): %s", resp.Digest, resp.Status) + if resp.Digest != "" { + spinner.Stop() + + bar, ok := bars[resp.Digest] + if !ok { + bar = progress.NewBar(resp.Status, resp.Total, resp.Completed) + bars[resp.Digest] = bar + p.Add(resp.Digest, bar) + } + + bar.Set(resp.Completed) + } else if status != resp.Status { + spinner.Stop() + + status = resp.Status + spinner = progress.NewSpinner(status) + p.Add(status, spinner) + } + return nil } + request := api.CreateRequest{Name: args[0], Path: filename, Modelfile: string(modelfile)} if err := client.Create(context.Background(), &request, fn); err != nil { return err } + if spinner != nil { + spinner.Stop() + } + return nil } @@ -141,13 +173,53 @@ func PushHandler(cmd *cobra.Command, args []string) error { return err } - request := api.PushRequest{Name: args[0], Insecure: insecure} + p := progress.NewProgress(os.Stderr) + defer p.Stop() + + bars := make(map[string]*progress.Bar) + + var status string + var spinner *progress.Spinner + fn := func(resp api.ProgressResponse) error { - log.Printf("progress(%s): %s", resp.Digest, resp.Status) + if resp.Digest != "" { + if spinner != nil { + spinner.Stop() + spinner = nil + } + + bar, ok := bars[resp.Digest] + if !ok { + bar = progress.NewBar(resp.Status, resp.Total, resp.Completed) + bars[resp.Digest] = bar + p.Add(resp.Digest, bar) + } + + bar.Set(resp.Completed) + } else if status != resp.Status { + if spinner != nil { + spinner.Stop() + spinner = nil + } + + status = resp.Status + spinner = progress.NewSpinner(status) + p.Add(status, spinner) + } + return nil } - return client.Push(context.Background(), &request, fn) + request := api.PushRequest{Name: args[0], Insecure: insecure} + if err := client.Push(context.Background(), &request, fn); err != nil { + return err + } + + if spinner != nil { + spinner.Stop() + } + + return nil } func ListHandler(cmd *cobra.Command, args []string) error { @@ -297,22 +369,58 @@ func PullHandler(cmd *cobra.Command, args []string) error { return err } - return pull(args[0], insecure) -} - -func pull(model string, insecure bool) error { client, err := api.ClientFromEnvironment() if err != nil { return err } - request := api.PullRequest{Name: model, Insecure: insecure} + p := progress.NewProgress(os.Stderr) + defer p.Stop() + + bars := make(map[string]*progress.Bar) + + var status string + var spinner *progress.Spinner + fn := func(resp api.ProgressResponse) error { - log.Printf("progress(%s): %s", resp.Digest, resp.Status) + if resp.Digest != "" { + if spinner != nil { + spinner.Stop() + spinner = nil + } + + bar, ok := bars[resp.Digest] + if !ok { + bar = progress.NewBar(resp.Status, resp.Total, resp.Completed) + bars[resp.Digest] = bar + p.Add(resp.Digest, bar) + } + + bar.Set(resp.Completed) + } else if status != resp.Status { + if spinner != nil { + spinner.Stop() + spinner = nil + } + + status = resp.Status + spinner = progress.NewSpinner(status) + p.Add(status, spinner) + } + return nil } - return client.Pull(context.Background(), &request, fn) + request := api.PullRequest{Name: args[0], Insecure: insecure} + if err := client.Pull(context.Background(), &request, fn); err != nil { + return err + } + + if spinner != nil { + spinner.Stop() + } + + return nil } func RunGenerate(cmd *cobra.Command, args []string) error { From 4dcf7a59b13c3a28868cd82d4addaeef5d0159b0 Mon Sep 17 00:00:00 2001 From: Michael Yang Date: Tue, 14 Nov 2023 16:58:51 -0800 Subject: [PATCH 055/103] generate progress --- cmd/cmd.go | 10 ++++++++++ progress/progress.go | 28 +++++++++++++++++++++++++++- 2 files changed, 37 insertions(+), 1 deletion(-) diff --git a/cmd/cmd.go b/cmd/cmd.go index 4df3b004..bc128d74 100644 --- a/cmd/cmd.go +++ b/cmd/cmd.go @@ -472,6 +472,13 @@ func generate(cmd *cobra.Command, model, prompt string, wordWrap bool, format st return err } + p := progress.NewProgress(os.Stderr) + defer p.Stop() + + spinner := progress.NewSpinner("") + defer spinner.Stop() + p.Add("", spinner) + var latest api.GenerateResponse generateContext, ok := cmd.Context().Value(generateContextKey("context")).([]int) @@ -502,6 +509,9 @@ func generate(cmd *cobra.Command, model, prompt string, wordWrap bool, format st request := api.GenerateRequest{Model: model, Prompt: prompt, Context: generateContext, Format: format} fn := func(response api.GenerateResponse) error { + spinner.Stop() + p.StopAndClear() + latest = response if wordWrap { diff --git a/progress/progress.go b/progress/progress.go index 5002b8d2..0aa1e911 100644 --- a/progress/progress.go +++ b/progress/progress.go @@ -3,8 +3,12 @@ package progress import ( "fmt" "io" + "os" + "strings" "sync" "time" + + "golang.org/x/term" ) type State interface { @@ -26,12 +30,34 @@ func NewProgress(w io.Writer) *Progress { return p } -func (p *Progress) Stop() { +func (p *Progress) Stop() bool { if p.ticker != nil { p.ticker.Stop() p.ticker = nil p.render() + return true } + + return false +} + +func (p *Progress) StopAndClear() bool { + stopped := p.Stop() + if stopped { + termWidth, _, err := term.GetSize(int(os.Stderr.Fd())) + if err != nil { + panic(err) + } + + // clear the progress bar by: + // 1. reset to beginning of line + // 2. move up to the first line of the progress bar + // 3. fill the terminal width with spaces + // 4. reset to beginning of line + fmt.Fprintf(p.w, "\r\033[%dA%s\r", p.pos, strings.Repeat(" ", termWidth)) + } + + return stopped } func (p *Progress) Add(key string, state State) { From d6ecaa2cbfccec94d172d02dde1a6fceca15a88d Mon Sep 17 00:00:00 2001 From: Michael Yang Date: Tue, 14 Nov 2023 17:03:18 -0800 Subject: [PATCH 056/103] update progress responses --- server/download.go | 4 ++-- server/upload.go | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/server/download.go b/server/download.go index 13e8ee18..64ee5623 100644 --- a/server/download.go +++ b/server/download.go @@ -285,7 +285,7 @@ func (b *blobDownload) Wait(ctx context.Context, fn func(api.ProgressResponse)) } fn(api.ProgressResponse{ - Status: fmt.Sprintf("downloading %s", b.Digest), + Status: fmt.Sprintf("downloading %s", b.Digest[7:19]), Digest: b.Digest, Total: b.Total, Completed: b.Completed.Load(), @@ -322,7 +322,7 @@ func downloadBlob(ctx context.Context, opts downloadOpts) error { return err default: opts.fn(api.ProgressResponse{ - Status: fmt.Sprintf("downloading %s", opts.digest), + Status: fmt.Sprintf("downloading %s", opts.digest[7:19]), Digest: opts.digest, Total: fi.Size(), Completed: fi.Size(), diff --git a/server/upload.go b/server/upload.go index 306b6e63..19b0af91 100644 --- a/server/upload.go +++ b/server/upload.go @@ -301,7 +301,7 @@ func (b *blobUpload) Wait(ctx context.Context, fn func(api.ProgressResponse)) er } fn(api.ProgressResponse{ - Status: fmt.Sprintf("uploading %s", b.Digest), + Status: fmt.Sprintf("uploading %s", b.Digest[7:19]), Digest: b.Digest, Total: b.Total, Completed: b.Completed.Load(), @@ -352,7 +352,7 @@ func uploadBlob(ctx context.Context, mp ModelPath, layer *Layer, opts *RegistryO default: defer resp.Body.Close() fn(api.ProgressResponse{ - Status: fmt.Sprintf("uploading %s", layer.Digest), + Status: fmt.Sprintf("uploading %s", layer.Digest[7:19]), Digest: layer.Digest, Total: layer.Size, Completed: layer.Size, From 7ea905871a5bf54953f19af21b5ccd6022bdc635 Mon Sep 17 00:00:00 2001 From: Michael Yang Date: Wed, 15 Nov 2023 14:04:57 -0800 Subject: [PATCH 057/103] only move cursor up if pos > 0 --- progress/progress.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/progress/progress.go b/progress/progress.go index 0aa1e911..e3d207cc 100644 --- a/progress/progress.go +++ b/progress/progress.go @@ -25,7 +25,7 @@ type Progress struct { } func NewProgress(w io.Writer) *Progress { - p := &Progress{pos: -1, w: w} + p := &Progress{w: w} go p.start() return p } @@ -71,7 +71,10 @@ func (p *Progress) render() error { p.mu.Lock() defer p.mu.Unlock() - fmt.Fprintf(p.w, "\033[%dA", p.pos) + if p.pos > 0 { + fmt.Fprintf(p.w, "\033[%dA", p.pos) + } + for _, state := range p.states { fmt.Fprintln(p.w, state.String()) } From 4d677ee3894c28190c424c9359aae3c6aedae13d Mon Sep 17 00:00:00 2001 From: Michael Yang Date: Wed, 15 Nov 2023 14:52:21 -0800 Subject: [PATCH 058/103] no divide by zero --- progress/bar.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/progress/bar.go b/progress/bar.go index 9a81fd0f..f69fb87b 100644 --- a/progress/bar.go +++ b/progress/bar.go @@ -105,7 +105,12 @@ func (b *Bar) percent() float64 { } func (b *Bar) rate() float64 { - return (float64(b.currentValue) - float64(b.initialValue)) / b.elapsed().Seconds() + elapsed := b.elapsed() + if elapsed.Seconds() > 0 { + return (float64(b.currentValue) - float64(b.initialValue)) / elapsed.Seconds() + } + + return 0 } func (b *Bar) elapsed() time.Duration { From 976068369b3ec293bb2769975fcd64b4c50e1585 Mon Sep 17 00:00:00 2001 From: Michael Yang Date: Wed, 15 Nov 2023 16:59:49 -0800 Subject: [PATCH 059/103] stop all spinners on progress stop --- cmd/cmd.go | 52 +++++++++++++++----------------------------- progress/progress.go | 6 +++++ 2 files changed, 24 insertions(+), 34 deletions(-) diff --git a/cmd/cmd.go b/cmd/cmd.go index bc128d74..e3dd3837 100644 --- a/cmd/cmd.go +++ b/cmd/cmd.go @@ -53,7 +53,7 @@ func CreateHandler(cmd *cobra.Command, args []string) error { bars := make(map[string]*progress.Bar) - status := "transferring context" + status := fmt.Sprintf("creating %s", args[0]) spinner := progress.NewSpinner(status) p.Add(status, spinner) @@ -72,6 +72,12 @@ func CreateHandler(cmd *cobra.Command, args []string) error { return err } + spinner.Stop() + + status = "transferring context" + spinner = progress.NewSpinner(status) + p.Add(status, spinner) + for _, c := range commands { switch c.Name { case "model", "adapter": @@ -133,10 +139,6 @@ func CreateHandler(cmd *cobra.Command, args []string) error { return err } - if spinner != nil { - spinner.Stop() - } - return nil } @@ -178,15 +180,13 @@ func PushHandler(cmd *cobra.Command, args []string) error { bars := make(map[string]*progress.Bar) - var status string - var spinner *progress.Spinner + status := fmt.Sprintf("pushing %s", args[0]) + spinner := progress.NewSpinner(status) + p.Add(status, spinner) fn := func(resp api.ProgressResponse) error { if resp.Digest != "" { - if spinner != nil { - spinner.Stop() - spinner = nil - } + spinner.Stop() bar, ok := bars[resp.Digest] if !ok { @@ -197,10 +197,7 @@ func PushHandler(cmd *cobra.Command, args []string) error { bar.Set(resp.Completed) } else if status != resp.Status { - if spinner != nil { - spinner.Stop() - spinner = nil - } + spinner.Stop() status = resp.Status spinner = progress.NewSpinner(status) @@ -215,10 +212,7 @@ func PushHandler(cmd *cobra.Command, args []string) error { return err } - if spinner != nil { - spinner.Stop() - } - + spinner.Stop() return nil } @@ -379,15 +373,13 @@ func PullHandler(cmd *cobra.Command, args []string) error { bars := make(map[string]*progress.Bar) - var status string - var spinner *progress.Spinner + status := fmt.Sprintf("pulling %s", args[0]) + spinner := progress.NewSpinner(status) + p.Add(status, spinner) fn := func(resp api.ProgressResponse) error { if resp.Digest != "" { - if spinner != nil { - spinner.Stop() - spinner = nil - } + spinner.Stop() bar, ok := bars[resp.Digest] if !ok { @@ -398,10 +390,7 @@ func PullHandler(cmd *cobra.Command, args []string) error { bar.Set(resp.Completed) } else if status != resp.Status { - if spinner != nil { - spinner.Stop() - spinner = nil - } + spinner.Stop() status = resp.Status spinner = progress.NewSpinner(status) @@ -416,10 +405,6 @@ func PullHandler(cmd *cobra.Command, args []string) error { return err } - if spinner != nil { - spinner.Stop() - } - return nil } @@ -476,7 +461,6 @@ func generate(cmd *cobra.Command, model, prompt string, wordWrap bool, format st defer p.Stop() spinner := progress.NewSpinner("") - defer spinner.Stop() p.Add("", spinner) var latest api.GenerateResponse diff --git a/progress/progress.go b/progress/progress.go index e3d207cc..97acf82c 100644 --- a/progress/progress.go +++ b/progress/progress.go @@ -31,6 +31,12 @@ func NewProgress(w io.Writer) *Progress { } func (p *Progress) Stop() bool { + for _, state := range p.states { + if spinner, ok := state.(*Spinner); ok { + spinner.Stop() + } + } + if p.ticker != nil { p.ticker.Stop() p.ticker = nil From 3cb07d2773c6a3229c774355a4cdf56e84de5680 Mon Sep 17 00:00:00 2001 From: Michael Yang Date: Fri, 17 Nov 2023 10:13:13 -0800 Subject: [PATCH 060/103] simplify StopAndClear --- progress/progress.go | 26 ++++++++++++-------------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/progress/progress.go b/progress/progress.go index 97acf82c..8c195be9 100644 --- a/progress/progress.go +++ b/progress/progress.go @@ -3,12 +3,8 @@ package progress import ( "fmt" "io" - "os" - "strings" "sync" "time" - - "golang.org/x/term" ) type State interface { @@ -48,19 +44,18 @@ func (p *Progress) Stop() bool { } func (p *Progress) StopAndClear() bool { + fmt.Fprint(p.w, "\033[?25l") + defer fmt.Fprint(p.w, "\033[?25h") + stopped := p.Stop() if stopped { - termWidth, _, err := term.GetSize(int(os.Stderr.Fd())) - if err != nil { - panic(err) - } - // clear the progress bar by: - // 1. reset to beginning of line - // 2. move up to the first line of the progress bar - // 3. fill the terminal width with spaces - // 4. reset to beginning of line - fmt.Fprintf(p.w, "\r\033[%dA%s\r", p.pos, strings.Repeat(" ", termWidth)) + // 1. for each line in the progress: + // a. move the cursor up one line + // b. clear the line + for i := 0; i < p.pos; i++ { + fmt.Fprint(p.w, "\033[A\033[2K") + } } return stopped @@ -77,6 +72,9 @@ func (p *Progress) render() error { p.mu.Lock() defer p.mu.Unlock() + fmt.Fprint(p.w, "\033[?25l") + defer fmt.Fprint(p.w, "\033[?25h") + if p.pos > 0 { fmt.Fprintf(p.w, "\033[%dA", p.pos) } From 0b19e24d8186b1766ba341269ef8e12ee5181004 Mon Sep 17 00:00:00 2001 From: Bruce MacDonald Date: Fri, 17 Nov 2023 14:22:35 -0500 Subject: [PATCH 061/103] only retry once on auth failure (#1175) --- server/images.go | 55 ++++++++++++++++++++++++------------------------ 1 file changed, 27 insertions(+), 28 deletions(-) diff --git a/server/images.go b/server/images.go index 7febb1ef..e5adb64a 100644 --- a/server/images.go +++ b/server/images.go @@ -1146,43 +1146,42 @@ func GetSHA256Digest(r io.Reader) (string, int64) { var errUnauthorized = fmt.Errorf("unauthorized") func makeRequestWithRetry(ctx context.Context, method string, requestURL *url.URL, headers http.Header, body io.ReadSeeker, regOpts *RegistryOptions) (*http.Response, error) { - lastErr := errMaxRetriesExceeded - for try := 0; try < maxRetries; try++ { - resp, err := makeRequest(ctx, method, requestURL, headers, body, regOpts) + resp, err := makeRequest(ctx, method, requestURL, headers, body, regOpts) + if err != nil { + if !errors.Is(err, context.Canceled) { + log.Printf("request failed: %v", err) + } + return nil, err + } + + switch { + case resp.StatusCode == http.StatusUnauthorized: + // Handle authentication error with one retry + auth := resp.Header.Get("www-authenticate") + authRedir := ParseAuthRedirectString(auth) + token, err := getAuthToken(ctx, authRedir) if err != nil { - log.Printf("couldn't start upload: %v", err) return nil, err } - - switch { - case resp.StatusCode == http.StatusUnauthorized: - auth := resp.Header.Get("www-authenticate") - authRedir := ParseAuthRedirectString(auth) - token, err := getAuthToken(ctx, authRedir) + regOpts.Token = token + if body != nil { + _, err = body.Seek(0, io.SeekStart) if err != nil { return nil, err } - - regOpts.Token = token - if body != nil { - body.Seek(0, io.SeekStart) - } - lastErr = errUnauthorized - case resp.StatusCode == http.StatusNotFound: - return nil, os.ErrNotExist - case resp.StatusCode >= http.StatusBadRequest: - body, err := io.ReadAll(resp.Body) - if err != nil { - return nil, fmt.Errorf("%d: %s", resp.StatusCode, err) - } - - return nil, fmt.Errorf("%d: %s", resp.StatusCode, body) - default: - return resp, nil } + return makeRequest(ctx, method, requestURL, headers, body, regOpts) + case resp.StatusCode == http.StatusNotFound: + return nil, os.ErrNotExist + case resp.StatusCode >= http.StatusBadRequest: + responseBody, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("%d: %s", resp.StatusCode, err) + } + return nil, fmt.Errorf("%d: %s", resp.StatusCode, responseBody) } - return nil, lastErr + return resp, nil } func makeRequest(ctx context.Context, method string, requestURL *url.URL, headers http.Header, body io.Reader, regOpts *RegistryOptions) (*http.Response, error) { From a185b29719a5ed93d1a7b18ad9d0b489e2a312f0 Mon Sep 17 00:00:00 2001 From: Jeffrey Morgan Date: Fri, 17 Nov 2023 18:00:41 -0500 Subject: [PATCH 062/103] fix install script error on linux --- scripts/install.sh | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/scripts/install.sh b/scripts/install.sh index 8ff5eaee..219f60fd 100644 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -97,16 +97,6 @@ Environment="PATH=$PATH" [Install] WantedBy=default.target EOF - - mkdir -p /etc/systemd/system/ollama.service.d - cat </dev/null -[Service] -#Environment="OLLAMA_HOST=" -#Environment="OLLAMA_ORIGINS=" -#Environment="OLLAMA_MODELS=" -#Environment="HTTPS_PROXY=" -EOF - SYSTEMCTL_RUNNING="$(systemctl is-system-running || true)" case $SYSTEMCTL_RUNNING in running|degraded) From c6e6c8ee7e86b774436a18e72bb3b5f1c495ae43 Mon Sep 17 00:00:00 2001 From: Michael Yang Date: Fri, 17 Nov 2023 15:21:57 -0800 Subject: [PATCH 063/103] fix cross device rename --- server/routes.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/server/routes.go b/server/routes.go index 58145576..6c1386b0 100644 --- a/server/routes.go +++ b/server/routes.go @@ -666,8 +666,14 @@ func HeadBlobHandler(c *gin.Context) { } func CreateBlobHandler(c *gin.Context) { + targetPath, err := GetBlobsPath(c.Param("digest")) + if err != nil { + c.AbortWithStatusJSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) + return + } + hash := sha256.New() - temp, err := os.CreateTemp("", c.Param("digest")) + temp, err := os.CreateTemp(filepath.Dir(targetPath), c.Param("digest")) if err != nil { c.AbortWithStatusJSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) return @@ -690,12 +696,6 @@ func CreateBlobHandler(c *gin.Context) { return } - targetPath, err := GetBlobsPath(c.Param("digest")) - if err != nil { - c.AbortWithStatusJSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) - return - } - if err := os.Rename(temp.Name(), targetPath); err != nil { c.AbortWithStatusJSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) return From 85e4441c6a47e0a74b2bdb752882460d07e2322a Mon Sep 17 00:00:00 2001 From: Jeffrey Morgan Date: Sat, 18 Nov 2023 08:51:24 -0500 Subject: [PATCH 064/103] cache docker builds --- .dockerignore | 1 + .gitignore | 1 + scripts/build_docker.sh | 2 ++ scripts/push_docker.sh | 1 + 4 files changed, 5 insertions(+) diff --git a/.dockerignore b/.dockerignore index a9253852..150c8f6e 100644 --- a/.dockerignore +++ b/.dockerignore @@ -6,3 +6,4 @@ scripts llm/llama.cpp/ggml llm/llama.cpp/gguf .env +.cache diff --git a/.gitignore b/.gitignore index 1e9ab3f4..feb68d6c 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,4 @@ dist ollama ggml-metal.metal +.cache diff --git a/scripts/build_docker.sh b/scripts/build_docker.sh index 1f612def..9cfb4113 100755 --- a/scripts/build_docker.sh +++ b/scripts/build_docker.sh @@ -10,6 +10,8 @@ docker buildx build \ --platform=linux/arm64,linux/amd64 \ --build-arg=VERSION \ --build-arg=GOFLAGS \ + --cache-from type=local,src=.cache \ + --cache-to type=local,dest=.cache \ -f Dockerfile \ -t ollama \ . diff --git a/scripts/push_docker.sh b/scripts/push_docker.sh index 3f3fb213..31865c2b 100755 --- a/scripts/push_docker.sh +++ b/scripts/push_docker.sh @@ -10,6 +10,7 @@ docker buildx build \ --platform=linux/arm64,linux/amd64 \ --build-arg=VERSION \ --build-arg=GOFLAGS \ + --cache-from type=local,src=.cache \ -f Dockerfile \ -t ollama/ollama -t ollama/ollama:$VERSION \ . From bab9494176d6fd68a558da0e95f6cd8461b18348 Mon Sep 17 00:00:00 2001 From: Jeffrey Morgan Date: Sat, 18 Nov 2023 09:39:52 -0500 Subject: [PATCH 065/103] add `-` separator to temp file created on `ollama create` --- server/routes.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/routes.go b/server/routes.go index 6c1386b0..cc53dc60 100644 --- a/server/routes.go +++ b/server/routes.go @@ -673,7 +673,7 @@ func CreateBlobHandler(c *gin.Context) { } hash := sha256.New() - temp, err := os.CreateTemp(filepath.Dir(targetPath), c.Param("digest")) + temp, err := os.CreateTemp(filepath.Dir(targetPath), c.Param("digest")+"-") if err != nil { c.AbortWithStatusJSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) return From 984714f13182a6b32a8d456427536bbe51d6032e Mon Sep 17 00:00:00 2001 From: Jeffrey Morgan Date: Sat, 18 Nov 2023 09:40:10 -0500 Subject: [PATCH 066/103] update status text when transfering blob on `ollama create` --- cmd/cmd.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/cmd.go b/cmd/cmd.go index e3dd3837..d4c5f92c 100644 --- a/cmd/cmd.go +++ b/cmd/cmd.go @@ -74,7 +74,7 @@ func CreateHandler(cmd *cobra.Command, args []string) error { spinner.Stop() - status = "transferring context" + status = "transferring model data" spinner = progress.NewSpinner(status) p.Add(status, spinner) From 43a726149df828e1557608bdffa97efc806ee448 Mon Sep 17 00:00:00 2001 From: Bruce MacDonald Date: Fri, 17 Nov 2023 18:05:28 -0500 Subject: [PATCH 067/103] fix potentially inaccurate error message --- llm/llama.go | 2 +- server/routes.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/llm/llama.go b/llm/llama.go index a7561f8d..35a2c556 100644 --- a/llm/llama.go +++ b/llm/llama.go @@ -226,7 +226,7 @@ type llama struct { } var ( - errNvidiaSMI = errors.New("nvidia-smi command failed") + errNvidiaSMI = errors.New("warning: gpu support may not be enabled, check you have installed GPU drivers: nvidia-smi command failed") errAvailableVRAM = errors.New("not enough VRAM available, falling back to CPU only") ) diff --git a/server/routes.go b/server/routes.go index cc53dc60..8a5a5a24 100644 --- a/server/routes.go +++ b/server/routes.go @@ -794,7 +794,7 @@ func Serve(ln net.Listener, allowOrigins []string) error { if runtime.GOOS == "linux" { // check compatibility to log warnings if _, err := llm.CheckVRAM(); err != nil { - log.Printf("Warning: GPU support may not be enabled, check you have installed GPU drivers: %v", err) + log.Printf(err.Error()) } } From 36a3bbf65f32dbdabf1b79f3975d8a6e8d95e0d7 Mon Sep 17 00:00:00 2001 From: Jeffrey Morgan Date: Sat, 18 Nov 2023 21:24:59 -0500 Subject: [PATCH 068/103] Update llm/llama.go --- llm/llama.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llm/llama.go b/llm/llama.go index 35a2c556..4481e97d 100644 --- a/llm/llama.go +++ b/llm/llama.go @@ -226,7 +226,7 @@ type llama struct { } var ( - errNvidiaSMI = errors.New("warning: gpu support may not be enabled, check you have installed GPU drivers: nvidia-smi command failed") + errNvidiaSMI = errors.New("warning: gpu support may not be enabled, check that you have installed GPU drivers: nvidia-smi command failed") errAvailableVRAM = errors.New("not enough VRAM available, falling back to CPU only") ) From 12e046f12a38df2abb17b2a6ff3c8ca08cbb1943 Mon Sep 17 00:00:00 2001 From: Jeffrey Morgan Date: Sat, 18 Nov 2023 21:29:32 -0500 Subject: [PATCH 069/103] remove unused function --- server/images.go | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/server/images.go b/server/images.go index e5adb64a..976ba8ff 100644 --- a/server/images.go +++ b/server/images.go @@ -228,26 +228,6 @@ func GetModel(name string) (*Model, error) { return model, nil } -func filenameWithPath(path, f string) (string, error) { - // if filePath starts with ~/, replace it with the user's home directory. - if strings.HasPrefix(f, fmt.Sprintf("~%s", string(os.PathSeparator))) { - parts := strings.Split(f, string(os.PathSeparator)) - home, err := os.UserHomeDir() - if err != nil { - return "", fmt.Errorf("failed to open file: %v", err) - } - - f = filepath.Join(home, filepath.Join(parts[1:]...)) - } - - // if filePath is not an absolute path, make it relative to the modelfile path - if !filepath.IsAbs(f) { - f = filepath.Join(filepath.Dir(path), f) - } - - return f, nil -} - func realpath(p string) string { abspath, err := filepath.Abs(p) if err != nil { From 1657c6abc7f2061debb285797ecf8d1d52c0ddce Mon Sep 17 00:00:00 2001 From: Jeffrey Morgan Date: Sat, 18 Nov 2023 22:59:26 -0500 Subject: [PATCH 070/103] add note to specify JSON in the prompt when using JSON mode --- docs/api.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/api.md b/docs/api.md index 6f34ed7f..99378ac3 100644 --- a/docs/api.md +++ b/docs/api.md @@ -51,7 +51,9 @@ Advanced parameters (optional): ### JSON mode -Enable JSON mode by setting the `format` parameter to `json` and specifying the model should use JSON in the `prompt`. This will structure the response as valid JSON. See the JSON mode [example](#request-json-mode) below. +Enable JSON mode by setting the `format` parameter to `json`. This will structure the response as valid JSON. See the JSON mode [example](#request-json-mode) below. + +> Note: it's important to instruct the model to use JSON in the `prompt`. Otherwise, the model may generate large amounts whitespace. ### Examples From 02524a56ff167f6307454abdd3c63d090fdef392 Mon Sep 17 00:00:00 2001 From: Jeffrey Morgan Date: Sun, 19 Nov 2023 00:19:53 -0500 Subject: [PATCH 071/103] check retry for authorization error --- server/images.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/server/images.go b/server/images.go index 976ba8ff..8d2af15b 100644 --- a/server/images.go +++ b/server/images.go @@ -1131,6 +1131,7 @@ func makeRequestWithRetry(ctx context.Context, method string, requestURL *url.UR if !errors.Is(err, context.Canceled) { log.Printf("request failed: %v", err) } + return nil, err } @@ -1150,7 +1151,13 @@ func makeRequestWithRetry(ctx context.Context, method string, requestURL *url.UR return nil, err } } - return makeRequest(ctx, method, requestURL, headers, body, regOpts) + + resp, err := makeRequest(ctx, method, requestURL, headers, body, regOpts) + if resp.StatusCode == http.StatusUnauthorized { + return nil, errUnauthorized + } + + return resp, err case resp.StatusCode == http.StatusNotFound: return nil, os.ErrNotExist case resp.StatusCode >= http.StatusBadRequest: From e1d705649675c7011b7166240f68659e8e1f0047 Mon Sep 17 00:00:00 2001 From: Jeffrey Morgan Date: Sun, 19 Nov 2023 09:20:22 -0500 Subject: [PATCH 072/103] update progress statuses --- server/download.go | 4 ++-- server/upload.go | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/server/download.go b/server/download.go index 64ee5623..095bd074 100644 --- a/server/download.go +++ b/server/download.go @@ -285,7 +285,7 @@ func (b *blobDownload) Wait(ctx context.Context, fn func(api.ProgressResponse)) } fn(api.ProgressResponse{ - Status: fmt.Sprintf("downloading %s", b.Digest[7:19]), + Status: fmt.Sprintf("pulling %s", b.Digest[7:19]), Digest: b.Digest, Total: b.Total, Completed: b.Completed.Load(), @@ -322,7 +322,7 @@ func downloadBlob(ctx context.Context, opts downloadOpts) error { return err default: opts.fn(api.ProgressResponse{ - Status: fmt.Sprintf("downloading %s", opts.digest[7:19]), + Status: fmt.Sprintf("pulling %s", opts.digest[7:19]), Digest: opts.digest, Total: fi.Size(), Completed: fi.Size(), diff --git a/server/upload.go b/server/upload.go index 19b0af91..b6bc1d8f 100644 --- a/server/upload.go +++ b/server/upload.go @@ -301,7 +301,7 @@ func (b *blobUpload) Wait(ctx context.Context, fn func(api.ProgressResponse)) er } fn(api.ProgressResponse{ - Status: fmt.Sprintf("uploading %s", b.Digest[7:19]), + Status: fmt.Sprintf("pushing %s", b.Digest[7:19]), Digest: b.Digest, Total: b.Total, Completed: b.Completed.Load(), @@ -352,7 +352,7 @@ func uploadBlob(ctx context.Context, mp ModelPath, layer *Layer, opts *RegistryO default: defer resp.Body.Close() fn(api.ProgressResponse{ - Status: fmt.Sprintf("uploading %s", layer.Digest[7:19]), + Status: fmt.Sprintf("pushing %s", layer.Digest[7:19]), Digest: layer.Digest, Total: layer.Size, Completed: layer.Size, From 04cbf5ccc0e837ff4caf6e801e6456a994d2b19c Mon Sep 17 00:00:00 2001 From: Jeffrey Morgan Date: Sun, 19 Nov 2023 09:54:33 -0500 Subject: [PATCH 073/103] progress bar styling improvements --- progress/bar.go | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/progress/bar.go b/progress/bar.go index f69fb87b..5ebc203d 100644 --- a/progress/bar.go +++ b/progress/bar.go @@ -55,7 +55,7 @@ func (b *Bar) String() string { pre.WriteString(" ") } - fmt.Fprintf(&pre, "%.1f%% ", b.percent()) + fmt.Fprintf(&pre, "%.0f%% ", b.percent()) fmt.Fprintf(&suf, "(%s/%s, %s/s, %s)", format.HumanBytes(b.currentValue), @@ -63,26 +63,20 @@ func (b *Bar) String() string { format.HumanBytes(int64(b.rate())), b.elapsed()) - mid.WriteString("[") + mid.WriteString("▕") - // pad 3 for last = or > and "] " - f := termWidth - pre.Len() - mid.Len() - suf.Len() - 3 + f := termWidth - pre.Len() - suf.Len() - 2 n := int(float64(f) * b.percent() / 100) - if n > 0 { - mid.WriteString(strings.Repeat("=", n)) - } - if b.currentValue >= b.maxValue { - mid.WriteString("=") - } else { - mid.WriteString(">") + if n > 0 { + mid.WriteString(strings.Repeat("█", n)) } if f-n > 0 { mid.WriteString(strings.Repeat(" ", f-n)) } - mid.WriteString("] ") + mid.WriteString("▏") return pre.String() + mid.String() + suf.String() } From 95b9acd3244577c4a021a3b91579241fabc0c969 Mon Sep 17 00:00:00 2001 From: Jeffrey Morgan Date: Sun, 19 Nov 2023 11:00:43 -0500 Subject: [PATCH 074/103] improve pull percentage rendering --- progress/bar.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/progress/bar.go b/progress/bar.go index 5ebc203d..6be4ab2b 100644 --- a/progress/bar.go +++ b/progress/bar.go @@ -2,6 +2,7 @@ package progress import ( "fmt" + "math" "os" "strings" "time" @@ -55,7 +56,7 @@ func (b *Bar) String() string { pre.WriteString(" ") } - fmt.Fprintf(&pre, "%.0f%% ", b.percent()) + fmt.Fprintf(&pre, "%3.0f%% ", math.Floor(b.percent())) fmt.Fprintf(&suf, "(%s/%s, %s/s, %s)", format.HumanBytes(b.currentValue), From c06b9b7304a53ed5526dc51fc4fcafc45d92be54 Mon Sep 17 00:00:00 2001 From: Jeffrey Morgan Date: Sun, 19 Nov 2023 13:43:21 -0500 Subject: [PATCH 075/103] update progress rendering to be closer to `v0.1.10` --- cmd/cmd.go | 37 ++++++++++++++++++------------------ progress/bar.go | 3 ++- progress/progress.go | 31 +++++++++++++++++------------- progress/spinner.go | 45 ++++++++------------------------------------ 4 files changed, 46 insertions(+), 70 deletions(-) diff --git a/cmd/cmd.go b/cmd/cmd.go index d4c5f92c..68636dcf 100644 --- a/cmd/cmd.go +++ b/cmd/cmd.go @@ -53,10 +53,6 @@ func CreateHandler(cmd *cobra.Command, args []string) error { bars := make(map[string]*progress.Bar) - status := fmt.Sprintf("creating %s", args[0]) - spinner := progress.NewSpinner(status) - p.Add(status, spinner) - modelfile, err := os.ReadFile(filename) if err != nil { return err @@ -72,10 +68,8 @@ func CreateHandler(cmd *cobra.Command, args []string) error { return err } - spinner.Stop() - - status = "transferring model data" - spinner = progress.NewSpinner(status) + status := "transferring model data" + spinner := progress.NewSpinner(status) p.Add(status, spinner) for _, c := range commands { @@ -179,14 +173,14 @@ func PushHandler(cmd *cobra.Command, args []string) error { defer p.Stop() bars := make(map[string]*progress.Bar) - - status := fmt.Sprintf("pushing %s", args[0]) - spinner := progress.NewSpinner(status) - p.Add(status, spinner) + var status string + var spinner *progress.Spinner fn := func(resp api.ProgressResponse) error { if resp.Digest != "" { - spinner.Stop() + if spinner != nil { + spinner.Stop() + } bar, ok := bars[resp.Digest] if !ok { @@ -197,7 +191,9 @@ func PushHandler(cmd *cobra.Command, args []string) error { bar.Set(resp.Completed) } else if status != resp.Status { - spinner.Stop() + if spinner != nil { + spinner.Stop() + } status = resp.Status spinner = progress.NewSpinner(status) @@ -373,13 +369,14 @@ func PullHandler(cmd *cobra.Command, args []string) error { bars := make(map[string]*progress.Bar) - status := fmt.Sprintf("pulling %s", args[0]) - spinner := progress.NewSpinner(status) - p.Add(status, spinner) + var status string + var spinner *progress.Spinner fn := func(resp api.ProgressResponse) error { if resp.Digest != "" { - spinner.Stop() + if spinner != nil { + spinner.Stop() + } bar, ok := bars[resp.Digest] if !ok { @@ -390,7 +387,9 @@ func PullHandler(cmd *cobra.Command, args []string) error { bar.Set(resp.Completed) } else if status != resp.Status { - spinner.Stop() + if spinner != nil { + spinner.Stop() + } status = resp.Status spinner = progress.NewSpinner(status) diff --git a/progress/bar.go b/progress/bar.go index 6be4ab2b..a8383f5c 100644 --- a/progress/bar.go +++ b/progress/bar.go @@ -66,7 +66,8 @@ func (b *Bar) String() string { mid.WriteString("▕") - f := termWidth - pre.Len() - suf.Len() - 2 + // add 3 extra spaces: 2 boundary characters and 1 space at the end + f := termWidth - pre.Len() - suf.Len() - 3 n := int(float64(f) * b.percent() / 100) if n > 0 { diff --git a/progress/progress.go b/progress/progress.go index 8c195be9..3c488307 100644 --- a/progress/progress.go +++ b/progress/progress.go @@ -12,9 +12,10 @@ type State interface { } type Progress struct { - mu sync.Mutex + mu sync.Mutex + w io.Writer + pos int - w io.Writer ticker *time.Ticker states []State @@ -37,6 +38,7 @@ func (p *Progress) Stop() bool { p.ticker.Stop() p.ticker = nil p.render() + fmt.Fprint(p.w, "\n") return true } @@ -50,11 +52,8 @@ func (p *Progress) StopAndClear() bool { stopped := p.Stop() if stopped { // clear the progress bar by: - // 1. for each line in the progress: - // a. move the cursor up one line - // b. clear the line for i := 0; i < p.pos; i++ { - fmt.Fprint(p.w, "\033[A\033[2K") + fmt.Fprint(p.w, "\033[A\033[2K\033[1G") } } @@ -75,17 +74,23 @@ func (p *Progress) render() error { fmt.Fprint(p.w, "\033[?25l") defer fmt.Fprint(p.w, "\033[?25h") - if p.pos > 0 { - fmt.Fprintf(p.w, "\033[%dA", p.pos) + // clear already rendered progress lines + for i := 0; i < p.pos; i++ { + if i > 0 { + fmt.Fprint(p.w, "\033[A") + } + fmt.Fprint(p.w, "\033[2K\033[1G") } - for _, state := range p.states { - fmt.Fprintln(p.w, state.String()) + // render progress lines + for i, state := range p.states { + fmt.Fprint(p.w, state.String()) + if i < len(p.states)-1 { + fmt.Fprint(p.w, "\n") + } } - if len(p.states) > 0 { - p.pos = len(p.states) - } + p.pos = len(p.states) return nil } diff --git a/progress/spinner.go b/progress/spinner.go index bc46bc02..62ed4f09 100644 --- a/progress/spinner.go +++ b/progress/spinner.go @@ -2,11 +2,8 @@ package progress import ( "fmt" - "os" "strings" "time" - - "golang.org/x/term" ) type Spinner struct { @@ -35,45 +32,28 @@ func NewSpinner(message string) *Spinner { } func (s *Spinner) String() string { - termWidth, _, err := term.GetSize(int(os.Stderr.Fd())) - if err != nil { - panic(err) - } - - var pre strings.Builder + var sb strings.Builder if len(s.message) > 0 { message := strings.TrimSpace(s.message) if s.messageWidth > 0 && len(message) > s.messageWidth { message = message[:s.messageWidth] } - fmt.Fprintf(&pre, "%s", message) - if s.messageWidth-pre.Len() >= 0 { - pre.WriteString(strings.Repeat(" ", s.messageWidth-pre.Len())) + fmt.Fprintf(&sb, "%s", message) + if s.messageWidth-sb.Len() >= 0 { + sb.WriteString(strings.Repeat(" ", s.messageWidth-sb.Len())) } - pre.WriteString(" ") + sb.WriteString(" ") } - var pad int if s.stopped.IsZero() { - // spinner has a string length of 3 but a rune length of 1 - // in order to align correctly, we need to pad with (3 - 1) = 2 spaces spinner := s.parts[s.value] - pre.WriteString(spinner) - pad = len(spinner) - len([]rune(spinner)) + sb.WriteString(spinner) + sb.WriteString(" ") } - var suf strings.Builder - fmt.Fprintf(&suf, "(%s)", s.elapsed()) - - var mid strings.Builder - f := termWidth - pre.Len() - mid.Len() - suf.Len() + pad - if f > 0 { - mid.WriteString(strings.Repeat(" ", f)) - } - - return pre.String() + mid.String() + suf.String() + return sb.String() } func (s *Spinner) start() { @@ -91,12 +71,3 @@ func (s *Spinner) Stop() { s.stopped = time.Now() } } - -func (s *Spinner) elapsed() time.Duration { - stopped := s.stopped - if stopped.IsZero() { - stopped = time.Now() - } - - return stopped.Sub(s.started).Round(time.Second) -} From 258addc79911e3035cc77e17709f5264f359b6ba Mon Sep 17 00:00:00 2001 From: Jeffrey Morgan Date: Sun, 19 Nov 2023 13:46:19 -0500 Subject: [PATCH 076/103] fix comment in `progress.go` --- progress/progress.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/progress/progress.go b/progress/progress.go index 3c488307..56ae1913 100644 --- a/progress/progress.go +++ b/progress/progress.go @@ -51,7 +51,7 @@ func (p *Progress) StopAndClear() bool { stopped := p.Stop() if stopped { - // clear the progress bar by: + // clear all progress lines for i := 0; i < p.pos; i++ { fmt.Fprint(p.w, "\033[A\033[2K\033[1G") } From cb42589792d66cff7b7674f8411e87e7090a3f69 Mon Sep 17 00:00:00 2001 From: Michael Yang Date: Fri, 17 Nov 2023 13:17:55 -0800 Subject: [PATCH 077/103] adjust download/upload parts --- server/download.go | 10 ++++++---- server/upload.go | 8 +++++--- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/server/download.go b/server/download.go index 095bd074..3a5071f2 100644 --- a/server/download.go +++ b/server/download.go @@ -53,8 +53,8 @@ type blobDownloadPart struct { const ( numDownloadParts = 64 - minDownloadPartSize int64 = 32 * 1000 * 1000 - maxDownloadPartSize int64 = 256 * 1000 * 1000 + minDownloadPartSize int64 = 100 * format.MegaByte + maxDownloadPartSize int64 = 1000 * format.MegaByte ) func (p *blobDownloadPart) Name() string { @@ -158,7 +158,9 @@ func (b *blobDownload) run(ctx context.Context, requestURL *url.URL, opts *Regis // return immediately if the context is canceled or the device is out of space return err case err != nil: - log.Printf("%s part %d attempt %d failed: %v, retrying", b.Digest[7:19], i, try, err) + sleep := 200*time.Millisecond + time.Duration(try)*time.Second/4 + log.Printf("%s part %d attempt %d failed: %v, retrying in %s", b.Digest[7:19], i, try, err, sleep) + time.Sleep(sleep) continue default: if try > 0 { @@ -304,7 +306,7 @@ type downloadOpts struct { fn func(api.ProgressResponse) } -const maxRetries = 3 +const maxRetries = 10 var errMaxRetriesExceeded = errors.New("max retries exceeded") diff --git a/server/upload.go b/server/upload.go index b6bc1d8f..5ef22d14 100644 --- a/server/upload.go +++ b/server/upload.go @@ -42,8 +42,8 @@ type blobUpload struct { const ( numUploadParts = 64 - minUploadPartSize int64 = 95 * 1000 * 1000 - maxUploadPartSize int64 = 1000 * 1000 * 1000 + minUploadPartSize int64 = 100 * format.MegaByte + maxUploadPartSize int64 = 1000 * format.MegaByte ) func (b *blobUpload) Prepare(ctx context.Context, requestURL *url.URL, opts *RegistryOptions) error { @@ -153,7 +153,9 @@ func (b *blobUpload) Run(ctx context.Context, opts *RegistryOptions) { case errors.Is(err, errMaxRetriesExceeded): return err case err != nil: - log.Printf("%s part %d attempt %d failed: %v, retrying", b.Digest[7:19], part.N, try, err) + sleep := 200*time.Millisecond + time.Duration(try)*time.Second/4 + log.Printf("%s part %d attempt %d failed: %v, retrying in %s", b.Digest[7:19], part.N, try, err, sleep) + time.Sleep(sleep) continue } From 42c2e3a624cd836979da07fea5ac179eadfcfae7 Mon Sep 17 00:00:00 2001 From: Michael Yang Date: Fri, 17 Nov 2023 23:52:11 -0800 Subject: [PATCH 078/103] upload: retry complete upload --- server/upload.go | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/server/upload.go b/server/upload.go index 5ef22d14..666bb4f4 100644 --- a/server/upload.go +++ b/server/upload.go @@ -190,14 +190,21 @@ func (b *blobUpload) Run(ctx context.Context, opts *RegistryOptions) { headers.Set("Content-Type", "application/octet-stream") headers.Set("Content-Length", "0") - resp, err := makeRequestWithRetry(ctx, http.MethodPut, requestURL, headers, nil, opts) - if err != nil { - b.err = err + for try := 0; try < maxRetries; try++ { + resp, err := makeRequestWithRetry(ctx, http.MethodPut, requestURL, headers, nil, opts) + if err != nil { + b.err = err + sleep := 200*time.Millisecond + time.Duration(try)*time.Second/4 + log.Printf("%s complete upload attempt %d failed: %v, retrying in %s", b.Digest[7:19], try, err, sleep) + time.Sleep(sleep) + continue + } + defer resp.Body.Close() + + b.err = nil + b.done = true return } - defer resp.Body.Close() - - b.done = true } func (b *blobUpload) uploadChunk(ctx context.Context, method string, requestURL *url.URL, part *blobUploadPart, opts *RegistryOptions) error { From ac5076ce1ed64c62d2d40701d9dd53d45fa4a1aa Mon Sep 17 00:00:00 2001 From: Jeffrey Morgan Date: Sat, 18 Nov 2023 19:25:22 -0500 Subject: [PATCH 079/103] exponential backoff up to 30s --- server/download.go | 11 ++++------- server/upload.go | 8 ++++++-- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/server/download.go b/server/download.go index 3a5071f2..3c823f32 100644 --- a/server/download.go +++ b/server/download.go @@ -7,6 +7,7 @@ import ( "fmt" "io" "log" + "math" "net/http" "net/url" "os" @@ -147,7 +148,6 @@ func (b *blobDownload) run(ctx context.Context, requestURL *url.URL, opts *Regis continue } - i := i g.Go(func() error { var err error for try := 0; try < maxRetries; try++ { @@ -158,14 +158,11 @@ func (b *blobDownload) run(ctx context.Context, requestURL *url.URL, opts *Regis // return immediately if the context is canceled or the device is out of space return err case err != nil: - sleep := 200*time.Millisecond + time.Duration(try)*time.Second/4 - log.Printf("%s part %d attempt %d failed: %v, retrying in %s", b.Digest[7:19], i, try, err, sleep) + sleep := time.Second * time.Duration(math.Pow(2, float64(try))) + log.Printf("%s part %d attempt %d failed: %v, retrying in %s", b.Digest[7:19], part.N, try, err, sleep) time.Sleep(sleep) continue default: - if try > 0 { - log.Printf("%s part %d completed after %d retries", b.Digest[7:19], i, try) - } return nil } } @@ -306,7 +303,7 @@ type downloadOpts struct { fn func(api.ProgressResponse) } -const maxRetries = 10 +const maxRetries = 6 var errMaxRetriesExceeded = errors.New("max retries exceeded") diff --git a/server/upload.go b/server/upload.go index 666bb4f4..3666d7d5 100644 --- a/server/upload.go +++ b/server/upload.go @@ -8,6 +8,7 @@ import ( "hash" "io" "log" + "math" "net/http" "net/url" "os" @@ -153,7 +154,7 @@ func (b *blobUpload) Run(ctx context.Context, opts *RegistryOptions) { case errors.Is(err, errMaxRetriesExceeded): return err case err != nil: - sleep := 200*time.Millisecond + time.Duration(try)*time.Second/4 + sleep := time.Second * time.Duration(math.Pow(2, float64(try))) log.Printf("%s part %d attempt %d failed: %v, retrying in %s", b.Digest[7:19], part.N, try, err, sleep) time.Sleep(sleep) continue @@ -244,6 +245,7 @@ func (b *blobUpload) uploadChunk(ctx context.Context, method string, requestURL return err } + // retry uploading to the redirect URL for try := 0; try < maxRetries; try++ { err = b.uploadChunk(ctx, http.MethodPut, redirectURL, part, nil) switch { @@ -252,7 +254,9 @@ func (b *blobUpload) uploadChunk(ctx context.Context, method string, requestURL case errors.Is(err, errMaxRetriesExceeded): return err case err != nil: - log.Printf("%s part %d attempt %d failed: %v, retrying", b.Digest[7:19], part.N, try, err) + sleep := time.Second * time.Duration(math.Pow(2, float64(try))) + log.Printf("%s part %d attempt %d failed: %v, retrying in %s", b.Digest[7:19], part.N, try, err, sleep) + time.Sleep(sleep) continue } From f6b317e8c9556b1865dd87f509acdc80097e5198 Mon Sep 17 00:00:00 2001 From: Jeffrey Morgan Date: Sat, 18 Nov 2023 20:36:34 -0500 Subject: [PATCH 080/103] fix sending too little data in chunk upload body --- server/upload.go | 42 +++++++++++++++++++++--------------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/server/upload.go b/server/upload.go index 3666d7d5..f4dda888 100644 --- a/server/upload.go +++ b/server/upload.go @@ -123,19 +123,6 @@ func (b *blobUpload) Run(ctx context.Context, opts *RegistryOptions) { defer blobUploadManager.Delete(b.Digest) ctx, b.CancelFunc = context.WithCancel(ctx) - p, err := GetBlobsPath(b.Digest) - if err != nil { - b.err = err - return - } - - f, err := os.Open(p) - if err != nil { - b.err = err - return - } - defer f.Close() - g, inner := errgroup.WithContext(ctx) g.SetLimit(numUploadParts) for i := range b.Parts { @@ -146,7 +133,6 @@ func (b *blobUpload) Run(ctx context.Context, opts *RegistryOptions) { g.Go(func() error { var err error for try := 0; try < maxRetries; try++ { - part.ReadSeeker = io.NewSectionReader(f, part.Offset, part.Size) err = b.uploadChunk(inner, http.MethodPatch, requestURL, part, opts) switch { case errors.Is(err, context.Canceled): @@ -209,18 +195,27 @@ func (b *blobUpload) Run(ctx context.Context, opts *RegistryOptions) { } func (b *blobUpload) uploadChunk(ctx context.Context, method string, requestURL *url.URL, part *blobUploadPart, opts *RegistryOptions) error { - part.Reset() - headers := make(http.Header) headers.Set("Content-Type", "application/octet-stream") headers.Set("Content-Length", fmt.Sprintf("%d", part.Size)) - headers.Set("X-Redirect-Uploads", "1") if method == http.MethodPatch { + headers.Set("X-Redirect-Uploads", "1") headers.Set("Content-Range", fmt.Sprintf("%d-%d", part.Offset, part.Offset+part.Size-1)) } - resp, err := makeRequest(ctx, method, requestURL, headers, io.TeeReader(part.ReadSeeker, io.MultiWriter(part, part.Hash)), opts) + p, err := GetBlobsPath(b.Digest) + if err != nil { + return err + } + + f, err := os.Open(p) + if err != nil { + return err + } + defer f.Close() + + resp, err := makeRequest(ctx, method, requestURL, headers, io.TeeReader(io.NewSectionReader(f, part.Offset, part.Size), part), opts) if err != nil { return err } @@ -335,7 +330,6 @@ type blobUploadPart struct { written int64 - io.ReadSeeker *blobUpload } @@ -343,14 +337,20 @@ func (p *blobUploadPart) Write(b []byte) (n int, err error) { n = len(b) p.written += int64(n) p.Completed.Add(int64(n)) + + if p.Hash == nil { + p.Hash = md5.New() + } + + p.Hash.Write(b) + return n, nil } func (p *blobUploadPart) Reset() { - p.Seek(0, io.SeekStart) p.Completed.Add(-int64(p.written)) p.written = 0 - p.Hash = md5.New() + p.Hash.Reset() } func uploadBlob(ctx context.Context, mp ModelPath, layer *Layer, opts *RegistryOptions, fn func(api.ProgressResponse)) error { From 9a8c21ac3db4dd5bc4f37e9c21218f2c2d6fc604 Mon Sep 17 00:00:00 2001 From: Jeffrey Morgan Date: Sat, 18 Nov 2023 20:59:55 -0500 Subject: [PATCH 081/103] use exponential everywhere --- server/upload.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/upload.go b/server/upload.go index f4dda888..1ac87af3 100644 --- a/server/upload.go +++ b/server/upload.go @@ -181,7 +181,7 @@ func (b *blobUpload) Run(ctx context.Context, opts *RegistryOptions) { resp, err := makeRequestWithRetry(ctx, http.MethodPut, requestURL, headers, nil, opts) if err != nil { b.err = err - sleep := 200*time.Millisecond + time.Duration(try)*time.Second/4 + sleep := time.Second * time.Duration(math.Pow(2, float64(try))) log.Printf("%s complete upload attempt %d failed: %v, retrying in %s", b.Digest[7:19], try, err, sleep) time.Sleep(sleep) continue From 1bd594b2fab7a6945f90df2e7caa4d5701d4a48e Mon Sep 17 00:00:00 2001 From: Jeffrey Morgan Date: Sat, 18 Nov 2023 22:08:21 -0500 Subject: [PATCH 082/103] revert to using one open file for blob uploads --- server/upload.go | 37 ++++++++++++++++++------------------- 1 file changed, 18 insertions(+), 19 deletions(-) diff --git a/server/upload.go b/server/upload.go index 1ac87af3..7e104bc2 100644 --- a/server/upload.go +++ b/server/upload.go @@ -36,6 +36,8 @@ type blobUpload struct { context.CancelFunc + file *os.File + done bool err error references atomic.Int32 @@ -101,7 +103,7 @@ func (b *blobUpload) Prepare(ctx context.Context, requestURL *url.URL, opts *Reg } // set part.N to the current number of parts - b.Parts = append(b.Parts, blobUploadPart{blobUpload: b, N: len(b.Parts), Offset: offset, Size: size}) + b.Parts = append(b.Parts, blobUploadPart{blobUpload: b, N: len(b.Parts), Hash: md5.New(), Offset: offset, Size: size}) offset += size } @@ -123,6 +125,19 @@ func (b *blobUpload) Run(ctx context.Context, opts *RegistryOptions) { defer blobUploadManager.Delete(b.Digest) ctx, b.CancelFunc = context.WithCancel(ctx) + p, err := GetBlobsPath(b.Digest) + if err != nil { + b.err = err + return + } + + b.file, err = os.Open(p) + if err != nil { + b.err = err + return + } + defer b.file.Close() + g, inner := errgroup.WithContext(ctx) g.SetLimit(numUploadParts) for i := range b.Parts { @@ -204,18 +219,8 @@ func (b *blobUpload) uploadChunk(ctx context.Context, method string, requestURL headers.Set("Content-Range", fmt.Sprintf("%d-%d", part.Offset, part.Offset+part.Size-1)) } - p, err := GetBlobsPath(b.Digest) - if err != nil { - return err - } - - f, err := os.Open(p) - if err != nil { - return err - } - defer f.Close() - - resp, err := makeRequest(ctx, method, requestURL, headers, io.TeeReader(io.NewSectionReader(f, part.Offset, part.Size), part), opts) + sr := io.NewSectionReader(b.file, part.Offset, part.Size) + resp, err := makeRequest(ctx, method, requestURL, headers, io.TeeReader(sr, part), opts) if err != nil { return err } @@ -337,13 +342,7 @@ func (p *blobUploadPart) Write(b []byte) (n int, err error) { n = len(b) p.written += int64(n) p.Completed.Add(int64(n)) - - if p.Hash == nil { - p.Hash = md5.New() - } - p.Hash.Write(b) - return n, nil } From 72cd33641012680a954fe2df0e4c2dcfb40bc880 Mon Sep 17 00:00:00 2001 From: Jeffrey Morgan Date: Sun, 19 Nov 2023 00:08:18 -0500 Subject: [PATCH 083/103] dont retry on upload complete context cancel --- server/upload.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/server/upload.go b/server/upload.go index 7e104bc2..abd4e0e7 100644 --- a/server/upload.go +++ b/server/upload.go @@ -196,6 +196,10 @@ func (b *blobUpload) Run(ctx context.Context, opts *RegistryOptions) { resp, err := makeRequestWithRetry(ctx, http.MethodPut, requestURL, headers, nil, opts) if err != nil { b.err = err + if errors.Is(err, context.Canceled) { + return + } + sleep := time.Second * time.Duration(math.Pow(2, float64(try))) log.Printf("%s complete upload attempt %d failed: %v, retrying in %s", b.Digest[7:19], try, err, sleep) time.Sleep(sleep) From 9d73d3a6b543c51c274fc90d077b12055ee30d9a Mon Sep 17 00:00:00 2001 From: Jeffrey Morgan Date: Sun, 19 Nov 2023 14:16:13 -0500 Subject: [PATCH 084/103] add back `part.Reset()` --- server/upload.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/server/upload.go b/server/upload.go index abd4e0e7..7858d984 100644 --- a/server/upload.go +++ b/server/upload.go @@ -155,6 +155,7 @@ func (b *blobUpload) Run(ctx context.Context, opts *RegistryOptions) { case errors.Is(err, errMaxRetriesExceeded): return err case err != nil: + part.Reset() sleep := time.Second * time.Duration(math.Pow(2, float64(try))) log.Printf("%s part %d attempt %d failed: %v, retrying in %s", b.Digest[7:19], part.N, try, err, sleep) time.Sleep(sleep) @@ -258,6 +259,7 @@ func (b *blobUpload) uploadChunk(ctx context.Context, method string, requestURL case errors.Is(err, errMaxRetriesExceeded): return err case err != nil: + part.Reset() sleep := time.Second * time.Duration(math.Pow(2, float64(try))) log.Printf("%s part %d attempt %d failed: %v, retrying in %s", b.Digest[7:19], part.N, try, err, sleep) time.Sleep(sleep) From 13ba6df5abba724fdd06b5671c1707d4098592b2 Mon Sep 17 00:00:00 2001 From: Jeffrey Morgan Date: Sun, 19 Nov 2023 23:20:21 -0500 Subject: [PATCH 085/103] enable cpu instructions on intel macs --- llm/llama.cpp/generate_darwin_amd64.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/llm/llama.cpp/generate_darwin_amd64.go b/llm/llama.cpp/generate_darwin_amd64.go index 804b29e1..779d1239 100644 --- a/llm/llama.cpp/generate_darwin_amd64.go +++ b/llm/llama.cpp/generate_darwin_amd64.go @@ -7,13 +7,13 @@ package llm //go:generate git -C ggml apply ../patches/0002-34B-model-support.patch //go:generate git -C ggml apply ../patches/0003-metal-fix-synchronization-in-new-matrix-multiplicati.patch //go:generate git -C ggml apply ../patches/0004-metal-add-missing-barriers-for-mul-mat-2699.patch -//go:generate cmake -S ggml -B ggml/build/cpu -DLLAMA_ACCELERATE=on -DLLAMA_K_QUANTS=on -DCMAKE_SYSTEM_PROCESSOR=x86_64 -DCMAKE_OSX_ARCHITECTURES=x86_64 -DCMAKE_OSX_DEPLOYMENT_TARGET=11.0 +//go:generate cmake -S ggml -B ggml/build/cpu -DLLAMA_ACCELERATE=on -DLLAMA_K_QUANTS=on -DCMAKE_SYSTEM_NAME=Darwin -DCMAKE_SYSTEM_PROCESSOR=x86_64 -DCMAKE_OSX_ARCHITECTURES=x86_64 -DCMAKE_OSX_DEPLOYMENT_TARGET=11.0 //go:generate cmake --build ggml/build/cpu --target server --config Release //go:generate mv ggml/build/cpu/bin/server ggml/build/cpu/bin/ollama-runner //go:generate git submodule update --force gguf //go:generate git -C gguf apply ../patches/0001-update-default-log-target.patch //go:generate git -C gguf apply ../patches/0001-metal-handle-ggml_scale-for-n-4-0-close-3754.patch -//go:generate cmake -S gguf -B gguf/build/cpu -DLLAMA_METAL=off -DLLAMA_ACCELERATE=on -DLLAMA_K_QUANTS=on -DCMAKE_SYSTEM_PROCESSOR=x86_64 -DCMAKE_OSX_ARCHITECTURES=x86_64 -DCMAKE_OSX_DEPLOYMENT_TARGET=11.0 -DLLAMA_NATIVE=off -DLLAMA_AVX=on -DLLAMA_AVX2=off -DLLAMA_AVX512=off -DLLAMA_FMA=off -DLLAMA_F16C=off +//go:generate cmake -S gguf -B gguf/build/cpu -DLLAMA_METAL=off -DLLAMA_ACCELERATE=on -DLLAMA_K_QUANTS=on -DCMAKE_SYSTEM_NAME=Darwin -DCMAKE_SYSTEM_PROCESSOR=x86_64 -DCMAKE_OSX_ARCHITECTURES=x86_64 -DCMAKE_OSX_DEPLOYMENT_TARGET=11.0 -DLLAMA_NATIVE=off -DLLAMA_AVX=on -DLLAMA_AVX2=on -DLLAMA_AVX512=off -DLLAMA_FMA=on -DLLAMA_F16C=on //go:generate cmake --build gguf/build/cpu --target server --config Release //go:generate mv gguf/build/cpu/bin/server gguf/build/cpu/bin/ollama-runner From e6ad4813d3ae1ac229aed71c5e609f829f61a25d Mon Sep 17 00:00:00 2001 From: Jeffrey Morgan Date: Sun, 19 Nov 2023 23:50:45 -0500 Subject: [PATCH 086/103] dont crash when redirecting stderr --- progress/bar.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/progress/bar.go b/progress/bar.go index a8383f5c..f5a2622a 100644 --- a/progress/bar.go +++ b/progress/bar.go @@ -37,7 +37,7 @@ func NewBar(message string, maxValue, initialValue int64) *Bar { func (b *Bar) String() string { termWidth, _, err := term.GetSize(int(os.Stderr.Fd())) if err != nil { - panic(err) + termWidth = 80 } var pre, mid, suf strings.Builder From 6bbd6e26fb81637de90779760bf744f4dc3ad8b2 Mon Sep 17 00:00:00 2001 From: Jeffrey Morgan Date: Mon, 20 Nov 2023 00:49:08 -0500 Subject: [PATCH 087/103] fix temporary newline created and removed with spinner in `ollama run` --- cmd/cmd.go | 3 +-- progress/progress.go | 18 ++++++++++++++---- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/cmd/cmd.go b/cmd/cmd.go index 68636dcf..abf9e940 100644 --- a/cmd/cmd.go +++ b/cmd/cmd.go @@ -457,7 +457,7 @@ func generate(cmd *cobra.Command, model, prompt string, wordWrap bool, format st } p := progress.NewProgress(os.Stderr) - defer p.Stop() + defer p.StopAndClear() spinner := progress.NewSpinner("") p.Add("", spinner) @@ -492,7 +492,6 @@ func generate(cmd *cobra.Command, model, prompt string, wordWrap bool, format st request := api.GenerateRequest{Model: model, Prompt: prompt, Context: generateContext, Format: format} fn := func(response api.GenerateResponse) error { - spinner.Stop() p.StopAndClear() latest = response diff --git a/progress/progress.go b/progress/progress.go index 56ae1913..78917e9c 100644 --- a/progress/progress.go +++ b/progress/progress.go @@ -27,7 +27,7 @@ func NewProgress(w io.Writer) *Progress { return p } -func (p *Progress) Stop() bool { +func (p *Progress) stop() bool { for _, state := range p.states { if spinner, ok := state.(*Spinner); ok { spinner.Stop() @@ -38,22 +38,32 @@ func (p *Progress) Stop() bool { p.ticker.Stop() p.ticker = nil p.render() - fmt.Fprint(p.w, "\n") return true } return false } +func (p *Progress) Stop() bool { + stopped := p.stop() + if stopped { + fmt.Fprint(p.w, "\n") + } + return stopped +} + func (p *Progress) StopAndClear() bool { fmt.Fprint(p.w, "\033[?25l") defer fmt.Fprint(p.w, "\033[?25h") - stopped := p.Stop() + stopped := p.stop() if stopped { // clear all progress lines for i := 0; i < p.pos; i++ { - fmt.Fprint(p.w, "\033[A\033[2K\033[1G") + if i > 0 { + fmt.Fprint(p.w, "\033[A") + } + fmt.Fprint(p.w, "\033[2K\033[1G") } } From be48741308f3e101336a8c36c8cda153f144f2bb Mon Sep 17 00:00:00 2001 From: Eli Bendersky Date: Mon, 20 Nov 2023 07:35:07 -0800 Subject: [PATCH 088/103] README: link to LangChainGo for talking to ollama, with an example (#1206) --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 2e94a418..aa51eb07 100644 --- a/README.md +++ b/README.md @@ -239,6 +239,7 @@ See the [API documentation](./docs/api.md) for all endpoints. ### Libraries - [LangChain](https://python.langchain.com/docs/integrations/llms/ollama) and [LangChain.js](https://js.langchain.com/docs/modules/model_io/models/llms/integrations/ollama) with [example](https://js.langchain.com/docs/use_cases/question_answering/local_retrieval_qa) +- [LangChainGo](https://github.com/tmc/langchaingo/) with [example](https://github.com/tmc/langchaingo/tree/main/examples/ollama-completion-example) - [LlamaIndex](https://gpt-index.readthedocs.io/en/stable/examples/llm/ollama.html) - [LiteLLM](https://github.com/BerriAI/litellm) - [OllamaSharp for .NET](https://github.com/awaescher/OllamaSharp) From 0179d8eb6b49c3c15f1903f08906fd96cabc2228 Mon Sep 17 00:00:00 2001 From: Andy Brenneke Date: Mon, 20 Nov 2023 07:36:47 -0800 Subject: [PATCH 089/103] Add Rivet to Community Integrations (#1183) --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index aa51eb07..b14f4f76 100644 --- a/README.md +++ b/README.md @@ -263,3 +263,4 @@ See the [API documentation](./docs/api.md) for all endpoints. - [Dagger Chatbot](https://github.com/samalba/dagger-chatbot) - [Discord AI Bot](https://github.com/mekb-turtle/discord-ai-bot) - [Hass Ollama Conversation](https://github.com/ej52/hass-ollama-conversation) +- [Rivet plugin](https://github.com/abrenneke/rivet-plugin-ollama) From 331068b964770f304fe3447a8e86c73b600e0978 Mon Sep 17 00:00:00 2001 From: Huy Le Date: Mon, 20 Nov 2023 08:39:14 -0700 Subject: [PATCH 090/103] Adding `ogpt.nvim` into the list of plugins! (#1190) * adding ollama.nvim for visibility * adding an ogpt.nvim neovim plugin --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index b14f4f76..44f8afaa 100644 --- a/README.md +++ b/README.md @@ -234,6 +234,7 @@ See the [API documentation](./docs/api.md) for all endpoints. - [Emacs client](https://github.com/zweifisch/ollama) - [gen.nvim](https://github.com/David-Kunz/gen.nvim) - [ollama.nvim](https://github.com/nomnivore/ollama.nvim) +- [ogpt.nvim](https://github.com/huynle/ogpt.nvim) - [gptel Emacs client](https://github.com/karthink/gptel) ### Libraries From 2fdf1b5ff891383752593813e5133f815819f271 Mon Sep 17 00:00:00 2001 From: Toni Soriano Date: Mon, 20 Nov 2023 16:48:35 +0100 Subject: [PATCH 091/103] add laravel package to README.md (#1208) Co-authored-by: Toni --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 44f8afaa..1e4f57c2 100644 --- a/README.md +++ b/README.md @@ -249,6 +249,7 @@ See the [API documentation](./docs/api.md) for all endpoints. - [ModelFusion Typescript Library](https://modelfusion.dev/integration/model-provider/ollama) - [OllamaKit for Swift](https://github.com/kevinhermawan/OllamaKit) - [Ollama for Dart](https://github.com/breitburg/dart-ollama) +- [Ollama for Laravel](https://github.com/cloudstudio/ollama-laravel) ### Mobile From be61a81758ecd7990e0bd4cdd77093ffc4ecb1ca Mon Sep 17 00:00:00 2001 From: Purinda Gunasekara Date: Tue, 21 Nov 2023 02:52:52 +1100 Subject: [PATCH 092/103] main-gpu argument is not getting passed to llamacpp, fixed. (#1192) --- llm/llama.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/llm/llama.go b/llm/llama.go index 4481e97d..4eab751d 100644 --- a/llm/llama.go +++ b/llm/llama.go @@ -339,6 +339,7 @@ func newLlama(model string, adapters []string, runners []ModelRunner, numLayers "--model", model, "--ctx-size", fmt.Sprintf("%d", opts.NumCtx), "--batch-size", fmt.Sprintf("%d", opts.NumBatch), + "--main-gpu", fmt.Sprintf("%d", opts.MainGPU), "--n-gpu-layers", fmt.Sprintf("%d", numGPU), "--embedding", } @@ -544,6 +545,7 @@ func (llm *llama) Predict(ctx context.Context, prevContext []int, prompt string, "stream": true, "n_predict": llm.NumPredict, "n_keep": llm.NumKeep, + "main_gpu": llm.MainGPU, "temperature": llm.Temperature, "top_k": llm.TopK, "top_p": llm.TopP, From 93a108214c427829d34ff279976810aeba834b0e Mon Sep 17 00:00:00 2001 From: Jeffrey Morgan Date: Mon, 20 Nov 2023 09:13:53 -0500 Subject: [PATCH 093/103] only show decimal points for smaller file size numbers --- format/bytes.go | 37 ++++++++++++++++++++++++++++--------- 1 file changed, 28 insertions(+), 9 deletions(-) diff --git a/format/bytes.go b/format/bytes.go index 471bdf49..4e5d43d1 100644 --- a/format/bytes.go +++ b/format/bytes.go @@ -1,6 +1,9 @@ package format -import "fmt" +import ( + "fmt" + "math" +) const ( Byte = 1 @@ -11,16 +14,32 @@ const ( ) func HumanBytes(b int64) string { + var value float64 + var unit string + switch { - case b > TeraByte: - return fmt.Sprintf("%.1f TB", float64(b)/TeraByte) - case b > GigaByte: - return fmt.Sprintf("%.1f GB", float64(b)/GigaByte) - case b > MegaByte: - return fmt.Sprintf("%.1f MB", float64(b)/MegaByte) - case b > KiloByte: - return fmt.Sprintf("%.1f KB", float64(b)/KiloByte) + case b >= TeraByte: + value = float64(b) / TeraByte + unit = "TB" + case b >= GigaByte: + value = float64(b) / GigaByte + unit = "GB" + case b >= MegaByte: + value = float64(b) / MegaByte + unit = "MB" + case b >= KiloByte: + value = float64(b) / KiloByte + unit = "KB" default: return fmt.Sprintf("%d B", b) } + + switch { + case value >= 100: + return fmt.Sprintf("%d %s", int(value), unit) + case value != math.Trunc(value): + return fmt.Sprintf("%.1f %s", value, unit) + default: + return fmt.Sprintf("%d %s", int(value), unit) + } } From f10ac5de19a763cc52341985311df013f19c5cd5 Mon Sep 17 00:00:00 2001 From: Jeffrey Morgan Date: Mon, 20 Nov 2023 10:58:13 -0500 Subject: [PATCH 094/103] restore stats updated every second to progress bar --- progress/bar.go | 65 +++++++++++++++++++++++++++++++++++-------------- 1 file changed, 47 insertions(+), 18 deletions(-) diff --git a/progress/bar.go b/progress/bar.go index f5a2622a..baa4c5d4 100644 --- a/progress/bar.go +++ b/progress/bar.go @@ -11,6 +11,12 @@ import ( "golang.org/x/term" ) +type Stats struct { + rate int64 + value int64 + remaining time.Duration +} + type Bar struct { message string messageWidth int @@ -20,7 +26,9 @@ type Bar struct { currentValue int64 started time.Time - stopped time.Time + + stats Stats + statted time.Time } func NewBar(message string, maxValue, initialValue int64) *Bar { @@ -57,12 +65,22 @@ func (b *Bar) String() string { } fmt.Fprintf(&pre, "%3.0f%% ", math.Floor(b.percent())) + fmt.Fprintf(&suf, "(%s/%s", format.HumanBytes(b.currentValue), format.HumanBytes(b.maxValue)) - fmt.Fprintf(&suf, "(%s/%s, %s/s, %s)", - format.HumanBytes(b.currentValue), - format.HumanBytes(b.maxValue), - format.HumanBytes(int64(b.rate())), - b.elapsed()) + stats := b.Stats() + rate := int64(stats.rate) + if rate > 0 { + fmt.Fprintf(&suf, ", %s/s", format.HumanBytes(rate)) + } + + fmt.Fprintf(&suf, ")") + + elapsed := time.Since(b.started) + if b.percent() < 100 && rate > 0 { + fmt.Fprintf(&suf, " [%s:%s]", elapsed.Round(time.Second), stats.remaining) + } else { + fmt.Fprintf(&suf, " ") + } mid.WriteString("▕") @@ -86,7 +104,6 @@ func (b *Bar) String() string { func (b *Bar) Set(value int64) { if value >= b.maxValue { value = b.maxValue - b.stopped = time.Now() } b.currentValue = value @@ -100,20 +117,32 @@ func (b *Bar) percent() float64 { return 0 } -func (b *Bar) rate() float64 { - elapsed := b.elapsed() - if elapsed.Seconds() > 0 { - return (float64(b.currentValue) - float64(b.initialValue)) / elapsed.Seconds() +func (b *Bar) Stats() Stats { + if time.Since(b.statted) < time.Second || b.currentValue >= b.maxValue { + return b.stats } - return 0 -} + if b.statted.IsZero() { + b.stats = Stats{ + value: b.initialValue, + rate: 0, + remaining: 0, + } + } else { + rate := b.currentValue - b.stats.value + var remaining time.Duration + if rate > 0 { + remaining = time.Second * time.Duration((float64(b.maxValue-b.currentValue))/(float64(rate))) + } -func (b *Bar) elapsed() time.Duration { - stopped := b.stopped - if stopped.IsZero() { - stopped = time.Now() + b.stats = Stats{ + value: b.currentValue, + rate: rate, + remaining: remaining, + } } - return stopped.Sub(b.started).Round(time.Second) + b.statted = time.Now() + + return b.stats } From 6066c70eddd9133160da66942073e122bdd81cd8 Mon Sep 17 00:00:00 2001 From: Jeffrey Morgan Date: Mon, 20 Nov 2023 11:37:17 -0500 Subject: [PATCH 095/103] restore progress messages for older endpoints --- cmd/cmd.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cmd/cmd.go b/cmd/cmd.go index abf9e940..2c48ca80 100644 --- a/cmd/cmd.go +++ b/cmd/cmd.go @@ -111,7 +111,7 @@ func CreateHandler(cmd *cobra.Command, args []string) error { bar, ok := bars[resp.Digest] if !ok { - bar = progress.NewBar(resp.Status, resp.Total, resp.Completed) + bar = progress.NewBar(fmt.Sprintf("pulling %s...", resp.Digest[7:19]), resp.Total, resp.Completed) bars[resp.Digest] = bar p.Add(resp.Digest, bar) } @@ -184,7 +184,7 @@ func PushHandler(cmd *cobra.Command, args []string) error { bar, ok := bars[resp.Digest] if !ok { - bar = progress.NewBar(resp.Status, resp.Total, resp.Completed) + bar = progress.NewBar(fmt.Sprintf("pushing %s...", resp.Digest[7:19]), resp.Total, resp.Completed) bars[resp.Digest] = bar p.Add(resp.Digest, bar) } @@ -380,7 +380,7 @@ func PullHandler(cmd *cobra.Command, args []string) error { bar, ok := bars[resp.Digest] if !ok { - bar = progress.NewBar(resp.Status, resp.Total, resp.Completed) + bar = progress.NewBar(fmt.Sprintf("pulling %s...", resp.Digest[7:19]), resp.Total, resp.Completed) bars[resp.Digest] = bar p.Add(resp.Digest, bar) } From 433702f4218c81c1ef2565b5f765172cc2f90e96 Mon Sep 17 00:00:00 2001 From: Jeffrey Morgan Date: Mon, 20 Nov 2023 14:22:39 -0500 Subject: [PATCH 096/103] hide progress stats on completion --- progress/bar.go | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/progress/bar.go b/progress/bar.go index baa4c5d4..97bc15cf 100644 --- a/progress/bar.go +++ b/progress/bar.go @@ -118,17 +118,25 @@ func (b *Bar) percent() float64 { } func (b *Bar) Stats() Stats { - if time.Since(b.statted) < time.Second || b.currentValue >= b.maxValue { + if time.Since(b.statted) < time.Second { return b.stats } - if b.statted.IsZero() { + switch { + case b.statted.IsZero(): + case b.currentValue >= b.maxValue: + b.stats = Stats{ + value: b.maxValue, + rate: 0, + remaining: 0, + } + case b.statted.IsZero(): b.stats = Stats{ value: b.initialValue, rate: 0, remaining: 0, } - } else { + default: rate := b.currentValue - b.stats.value var remaining time.Duration if rate > 0 { From 8c4022b06b4ff593439ed8ef29e8362f3844172c Mon Sep 17 00:00:00 2001 From: Jeffrey Morgan Date: Mon, 20 Nov 2023 14:33:46 -0500 Subject: [PATCH 097/103] fix initial progress stats --- progress/bar.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/progress/bar.go b/progress/bar.go index 97bc15cf..31fb8aea 100644 --- a/progress/bar.go +++ b/progress/bar.go @@ -124,15 +124,14 @@ func (b *Bar) Stats() Stats { switch { case b.statted.IsZero(): - case b.currentValue >= b.maxValue: b.stats = Stats{ - value: b.maxValue, + value: b.initialValue, rate: 0, remaining: 0, } - case b.statted.IsZero(): + case b.currentValue >= b.maxValue: b.stats = Stats{ - value: b.initialValue, + value: b.maxValue, rate: 0, remaining: 0, } From f24741ff39d0c090b5ee515263da9eacdba24c9d Mon Sep 17 00:00:00 2001 From: James Braza Date: Mon, 20 Nov 2023 12:24:29 -0800 Subject: [PATCH 098/103] Documenting how to view `Modelfile`s (#723) * Documented viewing Modelfiles in ollama.ai/library * Moved Modelfile in ollama.ai down per request --- docs/modelfile.md | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/docs/modelfile.md b/docs/modelfile.md index 47386b67..6f1d8e88 100644 --- a/docs/modelfile.md +++ b/docs/modelfile.md @@ -41,6 +41,8 @@ INSTRUCTION arguments ## Examples +### Basic `Modelfile` + An example of a `Modelfile` creating a mario blueprint: ```modelfile @@ -63,6 +65,35 @@ To use this: More examples are available in the [examples directory](../examples). +### `Modelfile`s in [ollama.ai/library][1] + +There are two ways to view `Modelfile`s underlying the models in [ollama.ai/library][1]: + +- Option 1: view a details page from a model's tags page: + 1. Go to a particular model's tags (e.g. https://ollama.ai/library/llama2/tags) + 2. Click on a tag (e.g. https://ollama.ai/library/llama2:13b) + 3. Scroll down to "Layers" + - Note: if the [`FROM` instruction](#from-required) is not present, + it means the model was created from a local file +- Option 2: use `ollama show` to print the `Modelfile` like so: + + ```bash + > ollama show --modelfile llama2:13b + # Modelfile generated by "ollama show" + # To build a new Modelfile based on this one, replace the FROM line with: + # FROM llama2:13b + + FROM /root/.ollama/models/blobs/sha256:123abc + TEMPLATE """[INST] {{ if and .First .System }}<>{{ .System }}<> + + {{ end }}{{ .Prompt }} [/INST] """ + SYSTEM """""" + PARAMETER stop [INST] + PARAMETER stop [/INST] + PARAMETER stop <> + PARAMETER stop <> + ``` + ## Instructions ### FROM (Required) @@ -177,3 +208,5 @@ LICENSE """ - the **`Modelfile` is not case sensitive**. In the examples, we use uppercase for instructions to make it easier to distinguish it from arguments. - Instructions can be in any order. In the examples, we start with FROM instruction to keep it easily readable. + +[1]: https://ollama.ai/library From 35c4b5ec16b5fac3564f1ab98543ac46be769f9a Mon Sep 17 00:00:00 2001 From: Jeffrey Morgan Date: Mon, 20 Nov 2023 15:44:36 -0500 Subject: [PATCH 099/103] calculate hash separately from http request --- server/upload.go | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/server/upload.go b/server/upload.go index 7858d984..04cd5ac0 100644 --- a/server/upload.go +++ b/server/upload.go @@ -5,7 +5,6 @@ import ( "crypto/md5" "errors" "fmt" - "hash" "io" "log" "math" @@ -103,7 +102,7 @@ func (b *blobUpload) Prepare(ctx context.Context, requestURL *url.URL, opts *Reg } // set part.N to the current number of parts - b.Parts = append(b.Parts, blobUploadPart{blobUpload: b, N: len(b.Parts), Hash: md5.New(), Offset: offset, Size: size}) + b.Parts = append(b.Parts, blobUploadPart{blobUpload: b, N: len(b.Parts), Offset: offset, Size: size}) offset += size } @@ -178,8 +177,16 @@ func (b *blobUpload) Run(ctx context.Context, opts *RegistryOptions) { requestURL := <-b.nextURL var sb strings.Builder + + // calculate md5 checksum and add it to the commit request for _, part := range b.Parts { - sb.Write(part.Sum(nil)) + hash := md5.New() + if _, err := io.Copy(hash, io.NewSectionReader(b.file, part.Offset, part.Size)); err != nil { + b.err = err + return + } + + sb.Write(hash.Sum(nil)) } md5sum := md5.Sum([]byte(sb.String())) @@ -334,13 +341,10 @@ func (b *blobUpload) Wait(ctx context.Context, fn func(api.ProgressResponse)) er type blobUploadPart struct { // N is the part number - N int - Offset int64 - Size int64 - hash.Hash - + N int + Offset int64 + Size int64 written int64 - *blobUpload } @@ -348,14 +352,12 @@ func (p *blobUploadPart) Write(b []byte) (n int, err error) { n = len(b) p.written += int64(n) p.Completed.Add(int64(n)) - p.Hash.Write(b) return n, nil } func (p *blobUploadPart) Reset() { p.Completed.Add(-int64(p.written)) p.written = 0 - p.Hash.Reset() } func uploadBlob(ctx context.Context, mp ModelPath, layer *Layer, opts *RegistryOptions, fn func(api.ProgressResponse)) error { From 31ab453d37f08e7f13f0bda37d9963755de23632 Mon Sep 17 00:00:00 2001 From: Bruce MacDonald Date: Mon, 20 Nov 2023 16:43:48 -0500 Subject: [PATCH 100/103] resolve FROM path before sending modelfile (#1211) --- cmd/cmd.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cmd/cmd.go b/cmd/cmd.go index 2c48ca80..8990a652 100644 --- a/cmd/cmd.go +++ b/cmd/cmd.go @@ -82,6 +82,10 @@ func CreateHandler(cmd *cobra.Command, args []string) error { path = filepath.Join(home, path[2:]) } + if !filepath.IsAbs(path) { + path = filepath.Join(filepath.Dir(filename), path) + } + bin, err := os.Open(path) if errors.Is(err, os.ErrNotExist) && c.Name == "model" { continue From 19b7a4d7150cfa69d23e7066abb6cea8d4f79cfa Mon Sep 17 00:00:00 2001 From: Michael Yang Date: Mon, 20 Nov 2023 13:44:12 -0800 Subject: [PATCH 101/103] recent llama.cpp update added kernels for fp32, q5_0, and q5_1 --- llm/llm.go | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/llm/llm.go b/llm/llm.go index 22706da5..4901d9fe 100644 --- a/llm/llm.go +++ b/llm/llm.go @@ -41,20 +41,13 @@ func New(workDir, model string, adapters []string, opts api.Options) (LLM, error if runtime.GOOS == "darwin" { switch ggml.FileType() { - case "Q8_0": + case "F32", "Q5_0", "Q5_1", "Q8_0": if ggml.Name() != "gguf" && opts.NumGPU != 0 { // GGML Q8_0 do not support Metal API and will // cause the runner to segmentation fault so disable GPU log.Printf("WARNING: GPU disabled for F32, Q5_0, Q5_1, and Q8_0") opts.NumGPU = 0 } - case "F32", "Q5_0", "Q5_1": - if opts.NumGPU != 0 { - // F32, Q5_0, Q5_1, and Q8_0 do not support Metal API and will - // cause the runner to segmentation fault so disable GPU - log.Printf("WARNING: GPU disabled for F32, Q5_0, Q5_1, and Q8_0") - opts.NumGPU = 0 - } } var requiredMemory int64 From df07e4a0970b77886f3f1d3fde97eeaef24f1f91 Mon Sep 17 00:00:00 2001 From: Jeffrey Morgan Date: Mon, 20 Nov 2023 17:05:36 -0500 Subject: [PATCH 102/103] remove redundant filename parameter (#1213) --- cmd/cmd.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/cmd.go b/cmd/cmd.go index 8990a652..56d3471c 100644 --- a/cmd/cmd.go +++ b/cmd/cmd.go @@ -132,7 +132,7 @@ func CreateHandler(cmd *cobra.Command, args []string) error { return nil } - request := api.CreateRequest{Name: args[0], Path: filename, Modelfile: string(modelfile)} + request := api.CreateRequest{Name: args[0], Modelfile: string(modelfile)} if err := client.Create(context.Background(), &request, fn); err != nil { return err } From a3fcecf943d9105fb0636e26ecb1aa38ffd778a4 Mon Sep 17 00:00:00 2001 From: Jeffrey Morgan Date: Mon, 20 Nov 2023 19:54:04 -0500 Subject: [PATCH 103/103] only set `main_gpu` if value > 0 is provided --- llm/llama.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/llm/llama.go b/llm/llama.go index 4eab751d..7172f91e 100644 --- a/llm/llama.go +++ b/llm/llama.go @@ -339,11 +339,14 @@ func newLlama(model string, adapters []string, runners []ModelRunner, numLayers "--model", model, "--ctx-size", fmt.Sprintf("%d", opts.NumCtx), "--batch-size", fmt.Sprintf("%d", opts.NumBatch), - "--main-gpu", fmt.Sprintf("%d", opts.MainGPU), "--n-gpu-layers", fmt.Sprintf("%d", numGPU), "--embedding", } + if opts.MainGPU > 0 { + params = append(params, "--main-gpu", fmt.Sprintf("%d", opts.MainGPU)) + } + if opts.RopeFrequencyBase > 0 { params = append(params, "--rope-freq-base", fmt.Sprintf("%f", opts.RopeFrequencyBase)) }