vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

add documentation for koboldcpp

Browse source
This commit is contained in:
Concedo 2023-10-05 21:17:36 +08:00
parent 5beb773320
commit b4b5c35074
8 changed files with 814 additions and 15 deletions
Download patch file Download diff file Expand all files Collapse all files

781
kcpp_docs.embd Normal file
View file

File diff suppressed because one or more lines are too long

33
koboldcpp.py
View file

@ -376,10 +376,11 @@ class ServerRequestHandler(http.server.SimpleHTTPRequestHandler):
sys_version = ""
server_version = "ConcedoLlamaForKoboldServer"
def __init__(self, addr, port, embedded_kailite):
def __init__(self, addr, port, embedded_kailite, embedded_kcpp_docs):
self.addr = addr
self.port = port
self.embedded_kailite = embedded_kailite
self.embedded_kcpp_docs = embedded_kcpp_docs
def __call__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
@ -547,7 +548,7 @@ class ServerRequestHandler(http.server.SimpleHTTPRequestHandler):
response_body = (json.dumps({"values": []}).encode())
elif self.path.endswith(('/api/v1/info/version', '/api/latest/info/version')):
response_body = (json.dumps({"result":"1.2.4"}).encode())
response_body = (json.dumps({"result":"1.2.5"}).encode())
elif self.path.endswith(('/api/extra/true_max_context_length')): #do not advertise this to horde
response_body = (json.dumps({"value": maxctx}).encode())
@ -573,9 +574,17 @@ class ServerRequestHandler(http.server.SimpleHTTPRequestHandler):
response_body = (json.dumps({"object":"list","data":[{"id":"koboldcpp","object":"model","created":1,"owned_by":"koboldcpp","permission":[],"root":"koboldcpp"}]}).encode())
force_json = True
elif self.path=="/api":
if self.embedded_kcpp_docs is None:
response_body = (f"KoboldCpp partial API reference can be found at the wiki: https://github.com/LostRuins/koboldcpp/wiki").encode()
else:
response_body = self.embedded_kcpp_docs
elif self.path.endswith(('/api')) or self.path.endswith(('/api/v1')):
response_body = (json.dumps({"result":"KoboldCpp partial API reference can be found at https://link.concedo.workers.dev/koboldapi"}).encode())
self.path = "/api"
self.send_response(302)
self.send_header("Location", self.path)
self.end_headers()
return None
if response_body is None:
self.send_response(404)
@ -721,7 +730,7 @@ class ServerRequestHandler(http.server.SimpleHTTPRequestHandler):
self.send_header('Access-Control-Allow-Origin', '*')
self.send_header('Access-Control-Allow-Methods', '*')
self.send_header('Access-Control-Allow-Headers', '*')
if "/api" in self.path or force_json:
if ("/api" in self.path and self.path!="/api") or force_json:
if self.path.endswith("/stream"):
self.send_header('Content-type', 'text/event-stream')
self.send_header('Content-type', 'application/json')
@ -730,7 +739,7 @@ class ServerRequestHandler(http.server.SimpleHTTPRequestHandler):
return super(ServerRequestHandler, self).end_headers()
def RunServerMultiThreaded(addr, port, embedded_kailite = None):
def RunServerMultiThreaded(addr, port, embedded_kailite = None, embedded_kcpp_docs = None):
global exitcounter
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
@ -746,7 +755,7 @@ def RunServerMultiThreaded(addr, port, embedded_kailite = None):
def run(self):
global exitcounter
handler = ServerRequestHandler(addr, port, embedded_kailite)
handler = ServerRequestHandler(addr, port, embedded_kailite, embedded_kcpp_docs)
with http.server.HTTPServer((addr, port), handler, False) as self.httpd:
try:
self.httpd.socket = sock
@ -1594,6 +1603,7 @@ def main(launch_args,start_server=True):
global args
args = launch_args
embedded_kailite = None
embedded_kcpp_docs = None
if args.config and len(args.config)==1:
if isinstance(args.config[0], str) and os.path.exists(args.config[0]):
loadconfigfile(args.config[0])
@ -1703,6 +1713,13 @@ def main(launch_args,start_server=True):
except:
print("Could not find Kobold Lite. Embedded Kobold Lite will not be available.")
try:
basepath = os.path.abspath(os.path.dirname(__file__))
with open(os.path.join(basepath, "kcpp_docs.embd"), mode='rb') as f:
embedded_kcpp_docs = f.read()
except:
print("Could not find Embedded KoboldCpp API docs.")
if args.port_param!=defaultport:
args.port = args.port_param
print(f"Starting Kobold HTTP Server on port {args.port}")
@ -1735,7 +1752,7 @@ def main(launch_args,start_server=True):
if start_server:
print(f"Please connect to custom endpoint at {epurl}")
asyncio.run(RunServerMultiThreaded(args.host, args.port, embedded_kailite))
asyncio.run(RunServerMultiThreaded(args.host, args.port, embedded_kailite, embedded_kcpp_docs))
else:
print(f"Server was not started, main function complete. Idling.")

6
llama.cpp
View file

@ -8142,7 +8142,7 @@ int llama_token_to_piece(const struct llama_model * model, llama_token token, ch
buf[0] = llama_token_to_byte(model->vocab, token);
return 1;
} else {
GGML_ASSERT(false);
LLAMA_LOG_WARN("%s: Unknown Tokenization Error 1\n", __func__);
}
break;
}
@ -8158,12 +8158,12 @@ int llama_token_to_piece(const struct llama_model * model, llama_token token, ch
} else if (llama_is_control_token(model->vocab, token)) {
;
} else {
GGML_ASSERT(false);
LLAMA_LOG_WARN("%s: Unknown Tokenization Error 2\n", __func__);
}
break;
}
default:
GGML_ASSERT(false);
LLAMA_LOG_WARN("%s: Unknown Tokenization Error 3\n", __func__);
}
}
return 0;

2
make_old_pyinstaller.bat
View file

@ -1,4 +1,4 @@
echo This file is only for my own usage, please do not use it. I am lazy.
set PATH=d:\\MainApplications\\KoboldAIGPT\\KoboldAI-Horde-Bridge\\python;d:\\MainApplications\\KoboldAIGPT\\KoboldAI-Horde-Bridge\\python\\Scripts;%PATH%
PyInstaller --noconfirm --onefile --clean --console --collect-all customtkinter --icon "./niko.ico" --add-data "./klite.embd;." --add-data "./koboldcpp_default.dll;." --add-data "./koboldcpp_openblas.dll;." --add-data "./koboldcpp_failsafe.dll;." --add-data "./koboldcpp_noavx2.dll;." --add-data "./libopenblas.dll;." --add-data "./koboldcpp_clblast.dll;." --add-data "./clblast.dll;." --add-data "./rwkv_vocab.embd;." --add-data "./rwkv_world_vocab.embd;." "./koboldcpp.py" -n "koboldcpp_nocuda.exe"
PyInstaller --noconfirm --onefile --clean --console --collect-all customtkinter --icon "./niko.ico" --add-data "./klite.embd;." --add-data "./kcpp_docs.embd;." --add-data "./koboldcpp_default.dll;." --add-data "./koboldcpp_openblas.dll;." --add-data "./koboldcpp_failsafe.dll;." --add-data "./koboldcpp_noavx2.dll;." --add-data "./libopenblas.dll;." --add-data "./koboldcpp_clblast.dll;." --add-data "./clblast.dll;." --add-data "./rwkv_vocab.embd;." --add-data "./rwkv_world_vocab.embd;." "./koboldcpp.py" -n "koboldcpp_nocuda.exe"

2
make_old_pyinstaller_cuda.bat
View file

@ -1,4 +1,4 @@
echo This file is only for my own usage, please do not use it. I am lazy.
set PATH=d:\\MainApplications\\KoboldAIGPT\\KoboldAI-Horde-Bridge\\python;d:\\MainApplications\\KoboldAIGPT\\KoboldAI-Horde-Bridge\\python\\Scripts;%PATH%
PyInstaller --noconfirm --onefile --clean --console --collect-all customtkinter --icon "./nikogreen.ico" --add-data "./klite.embd;." --add-data "./koboldcpp_default.dll;." --add-data "./koboldcpp_openblas.dll;." --add-data "./koboldcpp_failsafe.dll;." --add-data "./koboldcpp_noavx2.dll;." --add-data "./libopenblas.dll;." --add-data "./koboldcpp_clblast.dll;." --add-data "./clblast.dll;." --add-data "./koboldcpp_cublas.dll;." --add-data "./cublas64_11.dll;." --add-data "./cublasLt64_11.dll;." --add-data "./cudart64_110.dll;." --add-data "./msvcp140.dll;." --add-data "./vcruntime140.dll;." --add-data "./vcruntime140_1.dll;." --add-data "./rwkv_vocab.embd;." --add-data "./rwkv_world_vocab.embd;." "./koboldcpp.py" -n "koboldcpp.exe"
PyInstaller --noconfirm --onefile --clean --console --collect-all customtkinter --icon "./nikogreen.ico" --add-data "./klite.embd;." --add-data "./kcpp_docs.embd;." --add-data "./koboldcpp_default.dll;." --add-data "./koboldcpp_openblas.dll;." --add-data "./koboldcpp_failsafe.dll;." --add-data "./koboldcpp_noavx2.dll;." --add-data "./libopenblas.dll;." --add-data "./koboldcpp_clblast.dll;." --add-data "./clblast.dll;." --add-data "./koboldcpp_cublas.dll;." --add-data "./cublas64_11.dll;." --add-data "./cublasLt64_11.dll;." --add-data "./cudart64_110.dll;." --add-data "./msvcp140.dll;." --add-data "./vcruntime140.dll;." --add-data "./vcruntime140_1.dll;." --add-data "./rwkv_vocab.embd;." --add-data "./rwkv_world_vocab.embd;." "./koboldcpp.py" -n "koboldcpp.exe"

2
make_pyinstaller.bat
View file

@ -1 +1 @@
PyInstaller --noconfirm --onefile --clean --console --collect-all customtkinter --icon "./niko.ico" --add-data "./klite.embd;." --add-data "./koboldcpp_default.dll;." --add-data "./koboldcpp_openblas.dll;." --add-data "./koboldcpp_failsafe.dll;." --add-data "./koboldcpp_noavx2.dll;." --add-data "./libopenblas.dll;." --add-data "./koboldcpp_clblast.dll;." --add-data "./clblast.dll;." --add-data "./rwkv_vocab.embd;." --add-data "./rwkv_world_vocab.embd;." "./koboldcpp.py" -n "koboldcpp.exe"
PyInstaller --noconfirm --onefile --clean --console --collect-all customtkinter --icon "./niko.ico" --add-data "./klite.embd;." --add-data "./kcpp_docs.embd;." --add-data "./koboldcpp_default.dll;." --add-data "./koboldcpp_openblas.dll;." --add-data "./koboldcpp_failsafe.dll;." --add-data "./koboldcpp_noavx2.dll;." --add-data "./libopenblas.dll;." --add-data "./koboldcpp_clblast.dll;." --add-data "./clblast.dll;." --add-data "./rwkv_vocab.embd;." --add-data "./rwkv_world_vocab.embd;." "./koboldcpp.py" -n "koboldcpp.exe"

1
make_pyinstaller.sh
View file

@ -2,6 +2,7 @@
pyinstaller --noconfirm --onefile --clean --console --collect-all customtkinter --icon "./niko.ico" \
--add-data "./klite.embd:." \
--add-data "./kcpp_docs.embd:." \
--add-data "./koboldcpp_default.so:." \
--add-data "./koboldcpp_openblas.so:." \
--add-data "./koboldcpp_failsafe.so:." \

2
make_pyinstaller_hybrid_henk.bat
View file

@ -2,4 +2,4 @@ cd /d "%~dp0"
copy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.4\bin\cudart64_110.dll" .\ /Y
copy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.4\bin\cublasLt64_11.dll" .\ /Y
copy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.4\bin\cublas64_11.dll" .\ /Y
PyInstaller --noconfirm --onefile --collect-all customtkinter --clean --console --icon ".\niko.ico" --add-data "./klite.embd;." --add-data "./koboldcpp_default.dll;." --add-data "./koboldcpp_openblas.dll;." --add-data "./koboldcpp_failsafe.dll;." --add-data "./koboldcpp_noavx2.dll;." --add-data "./libopenblas.dll;." --add-data "./koboldcpp_clblast.dll;." --add-data "./clblast.dll;." --add-data "./koboldcpp_cublas.dll;." --add-data "./cudart64_110.dll;." --add-data "./cublasLt64_11.dll;." --add-data "./cublas64_11.dll;." --add-data "./rwkv_vocab.embd;." --add-data "C:/Windows/System32/msvcp140.dll;." --add-data "C:/Windows/System32/vcruntime140_1.dll;." "./koboldcpp.py" -n "koboldcpp.exe"
PyInstaller --noconfirm --onefile --collect-all customtkinter --clean --console --icon ".\niko.ico" --add-data "./klite.embd;." --add-data "./kcpp_docs.embd;." --add-data "./koboldcpp_default.dll;." --add-data "./koboldcpp_openblas.dll;." --add-data "./koboldcpp_failsafe.dll;." --add-data "./koboldcpp_noavx2.dll;." --add-data "./libopenblas.dll;." --add-data "./koboldcpp_clblast.dll;." --add-data "./clblast.dll;." --add-data "./koboldcpp_cublas.dll;." --add-data "./cudart64_110.dll;." --add-data "./cublasLt64_11.dll;." --add-data "./cublas64_11.dll;." --add-data "./rwkv_vocab.embd;." --add-data "C:/Windows/System32/msvcp140.dll;." --add-data "C:/Windows/System32/vcruntime140_1.dll;." "./koboldcpp.py" -n "koboldcpp.exe"