vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

try fix clouflare tunnel (+2 squashed commit)

Browse source 
Squashed commit:

[87d96bf2] update remote option

[c30bc909] updated fixed colab (+1 squashed commits)

Squashed commits:

[97b77563] updated fixed colab (+2 squashed commit)

Squashed commit:

[d851b04c] replaced cloudflare manual dl with remotetunnel in colab

[90ff1790] updated lite
This commit is contained in:
Concedo 2023-11-02 21:16:12 +08:00
parent 6dbb8d82b0
commit b0c7b88eac
3 changed files with 36 additions and 26 deletions
Download patch file Download diff file Expand all files Collapse all files

13
colab.ipynb
View file

@ -42,7 +42,7 @@
"source": [ "source": [
"#@title <b>v-- Enter your model below and then click this to start Koboldcpp</b>\r\n", "#@title <b>v-- Enter your model below and then click this to start Koboldcpp</b>\r\n",
"\r\n", "\r\n",
"Model = \"https://huggingface.co/KoboldAI/LLaMA2-13B-Tiefighter-GGUF/resolve/main/LLaMA2-13B-Tiefighter.Q4_K_M.gguf\" #@param [\"https://huggingface.co/KoboldAI/LLaMA2-13B-Tiefighter-GGUF/resolve/main/LLaMA2-13B-Tiefighter.Q4_K_M.gguf\",\"https://huggingface.co/TheBloke/MythoMax-L2-13B-GGUF/resolve/main/mythomax-l2-13b.Q4_K_M.gguf\",\"https://huggingface.co/TheBloke/ReMM-SLERP-L2-13B-GGUF/resolve/main/remm-slerp-l2-13b.Q4_K_M.gguf\",\"https://huggingface.co/TheBloke/Xwin-LM-13B-v0.2-GGUF/resolve/main/xwin-lm-13b-v0.2.Q4_K_M.gguf\",\"https://huggingface.co/TheBloke/Stheno-L2-13B-GGUF/resolve/main/stheno-l2-13b.Q4_K_M.gguf\",\"https://huggingface.co/TheBloke/Mistral-7B-Instruct-v0.1-GGUF\"]{allow-input: true}\r\n", "Model = \"https://huggingface.co/KoboldAI/LLaMA2-13B-Tiefighter-GGUF/resolve/main/LLaMA2-13B-Tiefighter.Q4_K_M.gguf\" #@param [\"https://huggingface.co/KoboldAI/LLaMA2-13B-Tiefighter-GGUF/resolve/main/LLaMA2-13B-Tiefighter.Q4_K_M.gguf\",\"https://huggingface.co/TheBloke/MythoMax-L2-13B-GGUF/resolve/main/mythomax-l2-13b.Q4_K_M.gguf\",\"https://huggingface.co/TheBloke/ReMM-SLERP-L2-13B-GGUF/resolve/main/remm-slerp-l2-13b.Q4_K_M.gguf\",\"https://huggingface.co/TheBloke/Xwin-LM-13B-v0.2-GGUF/resolve/main/xwin-lm-13b-v0.2.Q4_K_M.gguf\",\"https://huggingface.co/TheBloke/Stheno-L2-13B-GGUF/resolve/main/stheno-l2-13b.Q4_K_M.gguf\",\"https://huggingface.co/TheBloke/Mistral-7B-Instruct-v0.1-GGUF/resolve/main/mistral-7b-instruct-v0.1.Q4_K_S.gguf\"]{allow-input: true}\r\n",
"Layers = 43 #@param [43]{allow-input: true}\r\n", "Layers = 43 #@param [43]{allow-input: true}\r\n",
"ContextSize = 4096 #@param [4096] {allow-input: true}\r\n", "ContextSize = 4096 #@param [4096] {allow-input: true}\r\n",
"\r\n", "\r\n",
@ -52,17 +52,12 @@
"kvers = !(cat koboldcpp.py | grep 'KcppVersion = ' | cut -d '\"' -f2)\r\n", "kvers = !(cat koboldcpp.py | grep 'KcppVersion = ' | cut -d '\"' -f2)\r\n",
"kvers = kvers[0]\r\n", "kvers = kvers[0]\r\n",
"!echo Finding prebuilt binary for {kvers}\r\n", "!echo Finding prebuilt binary for {kvers}\r\n",
"!wget -O koboldcpp_cublas.so -c https://kcppcolab.concedo.workers.dev/?{kvers}\r\n", "!wget -O dlfile.tmp -c https://kcppcolab.concedo.workers.dev/?{kvers} && mv dlfile.tmp koboldcpp_cublas.so\r\n",
"!test -f koboldcpp_cublas.so && echo Prebuilt Binary Exists || echo Prebuilt Binary Does Not Exist\r\n", "!test -f koboldcpp_cublas.so && echo Prebuilt Binary Exists || echo Prebuilt Binary Does Not Exist\r\n",
"!test -f koboldcpp_cublas.so && echo Build Skipped || make koboldcpp_cublas LLAMA_CUBLAS=1\r\n", "!test -f koboldcpp_cublas.so && echo Build Skipped || make koboldcpp_cublas LLAMA_CUBLAS=1\r\n",
"!cp koboldcpp_cublas.so koboldcpp_cublas.dat\r\n", "!cp koboldcpp_cublas.so koboldcpp_cublas.dat\r\n",
"!wget $Model -O model.ggml\r\n", "!wget $Model -O model.ggml\r\n",
"!wget -c https://github.com/cloudflare/cloudflared/releases/latest/download/cloudflared-linux-amd64\r\n", "!python koboldcpp.py model.ggml --usecublas 0 mmq --multiuser --gpulayers $Layers --contextsize $ContextSize --hordeconfig concedo 1 1 --remotetunnel\r\n"
"!chmod +x cloudflared-linux-amd64\r\n",
"!nohup ./cloudflared-linux-amd64 tunnel --url http://localhost:5001 &\r\n",
"!sleep 5\r\n",
"!cat nohup.out\r\n",
"!python koboldcpp.py model.ggml --usecublas 0 mmq --multiuser --gpulayers $Layers --contextsize $ContextSize --hordeconfig concedo 1 1 --onready \"echo Connect to the link below && cat nohup.out | grep trycloudflare.com && rm nohup.out\"\r\n"
] ]
} }
], ],

7
klite.embd
View file

@ -6,7 +6,7 @@ It requires no dependencies, installation or setup.
Just copy this single static HTML file anywhere and open it in a browser, or from a webserver. Just copy this single static HTML file anywhere and open it in a browser, or from a webserver.
Please go to https://github.com/LostRuins/lite.koboldai.net for updates on Kobold Lite. Please go to https://github.com/LostRuins/lite.koboldai.net for updates on Kobold Lite.
Kobold Lite is under the AGPL v3.0 License unless otherwise exempted. Please do not remove this line. Kobold Lite is under the AGPL v3.0 License unless otherwise exempted. Please do not remove this line.
Current version: 89 Current version: 90
-Concedo -Concedo
--> -->
@ -2943,7 +2943,10 @@ Current version: 89
console.error('Error:', error); console.error('Error:', error);
clear_poll_flags(); clear_poll_flags();
render_gametext(); render_gametext();
msgbox("Error while submitting prompt: " + error); if(error.name!="AbortError") //aborts are silent
{
msgbox("Error while submitting prompt: " + error);
}
}); });
} }
} }

42
koboldcpp.py
View file

@ -1808,36 +1808,48 @@ def setuptunnel():
def run_tunnel(): def run_tunnel():
tunnelproc = None tunnelproc = None
tunneloutput = ""
time.sleep(0.2)
if os.name == 'nt': if os.name == 'nt':
print("Starting Cloudflare Tunnel for Windows...") print("Starting Cloudflare Tunnel for Windows...")
tunnelproc = subprocess.Popen(f"cloudflared.exe tunnel --url localhost:{args.port}", text=True, encoding='utf-8', shell=False, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE) tunnelproc = subprocess.Popen(f"cloudflared.exe tunnel --url localhost:{args.port}", text=True, encoding='utf-8', shell=True, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE)
else: else:
print("Starting Cloudflare Tunnel for Linux...") print("Starting Cloudflare Tunnel for Linux...")
tunnelproc = subprocess.Popen(f"cloudflared-linux-amd64 tunnel --url http://localhost:{args.port}", text=True, encoding='utf-8', shell=False, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE) tunnelproc = subprocess.Popen(f"cloudflared-linux-amd64 tunnel --url http://localhost:{args.port}", text=True, encoding='utf-8', shell=True, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE)
time.sleep(5) time.sleep(5.4)
pattern = r'https://[\w\.-]+\.trycloudflare\.com' def tunnel_reader():
while True: nonlocal tunnelproc,tunneloutput
line = tunnelproc.stderr.readline() #cloudflare writes to stderr for some reason pattern = r'https://[\w\.-]+\.trycloudflare\.com'
if not line: while True:
break # No more data to read line = tunnelproc.stderr.readline() #cloudflare writes to stderr for some reason
found = re.findall(pattern, line) if not line:
for x in found: return
print(f"Your remote tunnel is ready, please connect to {x}") found = re.findall(pattern, line)
tunnelproc.wait() for x in found:
break tunneloutput = x
return
tunnel_reader_thread = threading.Thread(target=tunnel_reader)
tunnel_reader_thread.start()
time.sleep(0.4)
if tunneloutput!="":
print(f"Your remote tunnel is ready, please connect to {tunneloutput}")
else:
print(f"Error: Could not create cloudflare tunnel.")
tunnelproc.wait() tunnelproc.wait()
if os.name == 'nt': if os.name == 'nt':
print("Downloading Cloudflare Tunnel for Windows...") print("Downloading Cloudflare Tunnel for Windows...")
subprocess.run("curl -L https://github.com/cloudflare/cloudflared/releases/latest/download/cloudflared-windows-amd64.exe -o cloudflared.exe", capture_output=True, text=True, check=True, encoding='utf-8') subprocess.run("curl -L https://github.com/cloudflare/cloudflared/releases/latest/download/cloudflared-windows-amd64.exe -o cloudflared.exe", shell=True, capture_output=True, text=True, check=True, encoding='utf-8')
else: else:
print("Downloading Cloudflare Tunnel for Linux...") print("Downloading Cloudflare Tunnel for Linux...")
subprocess.run("curl -L https://github.com/cloudflare/cloudflared/releases/latest/download/cloudflared-linux-amd64 -o 'cloudflared-linux-amd64'", capture_output=True, text=True, check=True, encoding='utf-8') subprocess.run("curl -L https://github.com/cloudflare/cloudflared/releases/latest/download/cloudflared-linux-amd64 -o 'cloudflared-linux-amd64'", shell=True, capture_output=True, text=True, check=True, encoding='utf-8')
subprocess.run("chmod +x 'cloudflared-linux-amd64'", shell=True) subprocess.run("chmod +x 'cloudflared-linux-amd64'", shell=True)
tunnel_thread = threading.Thread(target=run_tunnel) tunnel_thread = threading.Thread(target=run_tunnel)
tunnel_thread.start() tunnel_thread.start()
except Exception as ex: except Exception as ex:
print("Remote Tunnel Failed!") print("Remote Tunnel Failed!")
print(str(ex))
return None return None
def unload_libs(): def unload_libs():