102 lines
2.9 KiB
Text
102 lines
2.9 KiB
Text
{
|
|
"cells": [
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "93f59dcb-c588-41b8-a792-55d88ade739c",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"# Download and run the Ollama Linux install script\n",
|
|
"!curl -fsSL https://ollama.com/install.sh | sh\n",
|
|
"!command -v systemctl >/dev/null && sudo systemctl stop ollama"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "658c147e-c7f8-490e-910e-62b80f577dda",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"!pip install aiohttp pyngrok\n",
|
|
"\n",
|
|
"import os\n",
|
|
"import asyncio\n",
|
|
"from aiohttp import ClientSession\n",
|
|
"\n",
|
|
"# Set LD_LIBRARY_PATH so the system NVIDIA library becomes preferred\n",
|
|
"# over the built-in library. This is particularly important for \n",
|
|
"# Google Colab which installs older drivers\n",
|
|
"os.environ.update({'LD_LIBRARY_PATH': '/usr/lib64-nvidia'})\n",
|
|
"\n",
|
|
"async def run(cmd):\n",
|
|
" '''\n",
|
|
" run is a helper function to run subcommands asynchronously.\n",
|
|
" '''\n",
|
|
" print('>>> starting', *cmd)\n",
|
|
" p = await asyncio.subprocess.create_subprocess_exec(\n",
|
|
" *cmd,\n",
|
|
" stdout=asyncio.subprocess.PIPE,\n",
|
|
" stderr=asyncio.subprocess.PIPE,\n",
|
|
" )\n",
|
|
"\n",
|
|
" async def pipe(lines):\n",
|
|
" async for line in lines:\n",
|
|
" print(line.strip().decode('utf-8'))\n",
|
|
"\n",
|
|
" await asyncio.gather(\n",
|
|
" pipe(p.stdout),\n",
|
|
" pipe(p.stderr),\n",
|
|
" )\n",
|
|
"\n",
|
|
"\n",
|
|
"await asyncio.gather(\n",
|
|
" run(['ollama', 'serve']),\n",
|
|
" run(['ngrok', 'http', '--log', 'stderr', '11434']),\n",
|
|
")"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "e7735a55-9aad-4caf-8683-52e2163ba53b",
|
|
"metadata": {},
|
|
"source": [
|
|
"The previous cell starts two processes, `ollama` and `ngrok`. The log output will show a line like the following which describes the external address.\n",
|
|
"\n",
|
|
"```\n",
|
|
"t=2023-11-12T22:55:56+0000 lvl=info msg=\"started tunnel\" obj=tunnels name=command_line addr=http://localhost:11434 url=https://8249-34-125-179-11.ngrok.io\n",
|
|
"```\n",
|
|
"\n",
|
|
"The external address in this case is `https://8249-34-125-179-11.ngrok.io` which can be passed into `OLLAMA_HOST` to access this instance.\n",
|
|
"\n",
|
|
"```bash\n",
|
|
"export OLLAMA_HOST=https://8249-34-125-179-11.ngrok.io\n",
|
|
"ollama list\n",
|
|
"ollama run mistral\n",
|
|
"```"
|
|
]
|
|
}
|
|
],
|
|
"metadata": {
|
|
"kernelspec": {
|
|
"display_name": "Python 3 (ipykernel)",
|
|
"language": "python",
|
|
"name": "python3"
|
|
},
|
|
"language_info": {
|
|
"codemirror_mode": {
|
|
"name": "ipython",
|
|
"version": 3
|
|
},
|
|
"file_extension": ".py",
|
|
"mimetype": "text/x-python",
|
|
"name": "python",
|
|
"nbconvert_exporter": "python",
|
|
"pygments_lexer": "ipython3",
|
|
"version": "3.11.6"
|
|
}
|
|
},
|
|
"nbformat": 4,
|
|
"nbformat_minor": 5
|
|
}
|