From b0c7b88eac06f2c1194eb75d4d64d52e5a2208f6 Mon Sep 17 00:00:00 2001 From: Concedo <39025047+LostRuins@users.noreply.github.com> Date: Thu, 2 Nov 2023 21:16:12 +0800 Subject: [PATCH] try fix clouflare tunnel (+2 squashed commit) Squashed commit: [87d96bf2] update remote option [c30bc909] updated fixed colab (+1 squashed commits) Squashed commits: [97b77563] updated fixed colab (+2 squashed commit) Squashed commit: [d851b04c] replaced cloudflare manual dl with remotetunnel in colab [90ff1790] updated lite --- colab.ipynb | 13 ++++--------- klite.embd | 7 +++++-- koboldcpp.py | 42 +++++++++++++++++++++++++++--------------- 3 files changed, 36 insertions(+), 26 deletions(-) diff --git a/colab.ipynb b/colab.ipynb index a31a27abb..096c087b4 100644 --- a/colab.ipynb +++ b/colab.ipynb @@ -42,7 +42,7 @@ "source": [ "#@title v-- Enter your model below and then click this to start Koboldcpp\r\n", "\r\n", - "Model = \"https://huggingface.co/KoboldAI/LLaMA2-13B-Tiefighter-GGUF/resolve/main/LLaMA2-13B-Tiefighter.Q4_K_M.gguf\" #@param [\"https://huggingface.co/KoboldAI/LLaMA2-13B-Tiefighter-GGUF/resolve/main/LLaMA2-13B-Tiefighter.Q4_K_M.gguf\",\"https://huggingface.co/TheBloke/MythoMax-L2-13B-GGUF/resolve/main/mythomax-l2-13b.Q4_K_M.gguf\",\"https://huggingface.co/TheBloke/ReMM-SLERP-L2-13B-GGUF/resolve/main/remm-slerp-l2-13b.Q4_K_M.gguf\",\"https://huggingface.co/TheBloke/Xwin-LM-13B-v0.2-GGUF/resolve/main/xwin-lm-13b-v0.2.Q4_K_M.gguf\",\"https://huggingface.co/TheBloke/Stheno-L2-13B-GGUF/resolve/main/stheno-l2-13b.Q4_K_M.gguf\",\"https://huggingface.co/TheBloke/Mistral-7B-Instruct-v0.1-GGUF\"]{allow-input: true}\r\n", + "Model = \"https://huggingface.co/KoboldAI/LLaMA2-13B-Tiefighter-GGUF/resolve/main/LLaMA2-13B-Tiefighter.Q4_K_M.gguf\" #@param [\"https://huggingface.co/KoboldAI/LLaMA2-13B-Tiefighter-GGUF/resolve/main/LLaMA2-13B-Tiefighter.Q4_K_M.gguf\",\"https://huggingface.co/TheBloke/MythoMax-L2-13B-GGUF/resolve/main/mythomax-l2-13b.Q4_K_M.gguf\",\"https://huggingface.co/TheBloke/ReMM-SLERP-L2-13B-GGUF/resolve/main/remm-slerp-l2-13b.Q4_K_M.gguf\",\"https://huggingface.co/TheBloke/Xwin-LM-13B-v0.2-GGUF/resolve/main/xwin-lm-13b-v0.2.Q4_K_M.gguf\",\"https://huggingface.co/TheBloke/Stheno-L2-13B-GGUF/resolve/main/stheno-l2-13b.Q4_K_M.gguf\",\"https://huggingface.co/TheBloke/Mistral-7B-Instruct-v0.1-GGUF/resolve/main/mistral-7b-instruct-v0.1.Q4_K_S.gguf\"]{allow-input: true}\r\n", "Layers = 43 #@param [43]{allow-input: true}\r\n", "ContextSize = 4096 #@param [4096] {allow-input: true}\r\n", "\r\n", @@ -52,17 +52,12 @@ "kvers = !(cat koboldcpp.py | grep 'KcppVersion = ' | cut -d '\"' -f2)\r\n", "kvers = kvers[0]\r\n", "!echo Finding prebuilt binary for {kvers}\r\n", - "!wget -O koboldcpp_cublas.so -c https://kcppcolab.concedo.workers.dev/?{kvers}\r\n", + "!wget -O dlfile.tmp -c https://kcppcolab.concedo.workers.dev/?{kvers} && mv dlfile.tmp koboldcpp_cublas.so\r\n", "!test -f koboldcpp_cublas.so && echo Prebuilt Binary Exists || echo Prebuilt Binary Does Not Exist\r\n", "!test -f koboldcpp_cublas.so && echo Build Skipped || make koboldcpp_cublas LLAMA_CUBLAS=1\r\n", "!cp koboldcpp_cublas.so koboldcpp_cublas.dat\r\n", - "!wget $Model -O model.ggml\r\n", - "!wget -c https://github.com/cloudflare/cloudflared/releases/latest/download/cloudflared-linux-amd64\r\n", - "!chmod +x cloudflared-linux-amd64\r\n", - "!nohup ./cloudflared-linux-amd64 tunnel --url http://localhost:5001 &\r\n", - "!sleep 5\r\n", - "!cat nohup.out\r\n", - "!python koboldcpp.py model.ggml --usecublas 0 mmq --multiuser --gpulayers $Layers --contextsize $ContextSize --hordeconfig concedo 1 1 --onready \"echo Connect to the link below && cat nohup.out | grep trycloudflare.com && rm nohup.out\"\r\n" + "!wget $Model -O model.ggml\r\n", + "!python koboldcpp.py model.ggml --usecublas 0 mmq --multiuser --gpulayers $Layers --contextsize $ContextSize --hordeconfig concedo 1 1 --remotetunnel\r\n" ] } ], diff --git a/klite.embd b/klite.embd index 96a7396e9..2dab7b914 100644 --- a/klite.embd +++ b/klite.embd @@ -6,7 +6,7 @@ It requires no dependencies, installation or setup. Just copy this single static HTML file anywhere and open it in a browser, or from a webserver. Please go to https://github.com/LostRuins/lite.koboldai.net for updates on Kobold Lite. Kobold Lite is under the AGPL v3.0 License unless otherwise exempted. Please do not remove this line. -Current version: 89 +Current version: 90 -Concedo --> @@ -2943,7 +2943,10 @@ Current version: 89 console.error('Error:', error); clear_poll_flags(); render_gametext(); - msgbox("Error while submitting prompt: " + error); + if(error.name!="AbortError") //aborts are silent + { + msgbox("Error while submitting prompt: " + error); + } }); } } diff --git a/koboldcpp.py b/koboldcpp.py index 9fcd1065f..3f9003dc8 100755 --- a/koboldcpp.py +++ b/koboldcpp.py @@ -1808,36 +1808,48 @@ def setuptunnel(): def run_tunnel(): tunnelproc = None + tunneloutput = "" + time.sleep(0.2) if os.name == 'nt': print("Starting Cloudflare Tunnel for Windows...") - tunnelproc = subprocess.Popen(f"cloudflared.exe tunnel --url localhost:{args.port}", text=True, encoding='utf-8', shell=False, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE) + tunnelproc = subprocess.Popen(f"cloudflared.exe tunnel --url localhost:{args.port}", text=True, encoding='utf-8', shell=True, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE) else: print("Starting Cloudflare Tunnel for Linux...") - tunnelproc = subprocess.Popen(f"cloudflared-linux-amd64 tunnel --url http://localhost:{args.port}", text=True, encoding='utf-8', shell=False, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE) - time.sleep(5) - pattern = r'https://[\w\.-]+\.trycloudflare\.com' - while True: - line = tunnelproc.stderr.readline() #cloudflare writes to stderr for some reason - if not line: - break # No more data to read - found = re.findall(pattern, line) - for x in found: - print(f"Your remote tunnel is ready, please connect to {x}") - tunnelproc.wait() - break + tunnelproc = subprocess.Popen(f"cloudflared-linux-amd64 tunnel --url http://localhost:{args.port}", text=True, encoding='utf-8', shell=True, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE) + time.sleep(5.4) + def tunnel_reader(): + nonlocal tunnelproc,tunneloutput + pattern = r'https://[\w\.-]+\.trycloudflare\.com' + while True: + line = tunnelproc.stderr.readline() #cloudflare writes to stderr for some reason + if not line: + return + found = re.findall(pattern, line) + for x in found: + tunneloutput = x + return + + tunnel_reader_thread = threading.Thread(target=tunnel_reader) + tunnel_reader_thread.start() + time.sleep(0.4) + if tunneloutput!="": + print(f"Your remote tunnel is ready, please connect to {tunneloutput}") + else: + print(f"Error: Could not create cloudflare tunnel.") tunnelproc.wait() if os.name == 'nt': print("Downloading Cloudflare Tunnel for Windows...") - subprocess.run("curl -L https://github.com/cloudflare/cloudflared/releases/latest/download/cloudflared-windows-amd64.exe -o cloudflared.exe", capture_output=True, text=True, check=True, encoding='utf-8') + subprocess.run("curl -L https://github.com/cloudflare/cloudflared/releases/latest/download/cloudflared-windows-amd64.exe -o cloudflared.exe", shell=True, capture_output=True, text=True, check=True, encoding='utf-8') else: print("Downloading Cloudflare Tunnel for Linux...") - subprocess.run("curl -L https://github.com/cloudflare/cloudflared/releases/latest/download/cloudflared-linux-amd64 -o 'cloudflared-linux-amd64'", capture_output=True, text=True, check=True, encoding='utf-8') + subprocess.run("curl -L https://github.com/cloudflare/cloudflared/releases/latest/download/cloudflared-linux-amd64 -o 'cloudflared-linux-amd64'", shell=True, capture_output=True, text=True, check=True, encoding='utf-8') subprocess.run("chmod +x 'cloudflared-linux-amd64'", shell=True) tunnel_thread = threading.Thread(target=run_tunnel) tunnel_thread.start() except Exception as ex: print("Remote Tunnel Failed!") + print(str(ex)) return None def unload_libs():