improved remotelink cmd, fixed lib unload, updated class.py

Remote-Link.cmd

@@ -1,2 +1,18 @@
+: # This script will help setup a cloudflared tunnel for accessing KoboldCpp over the internet
+: # It should work out of the box on both linux and windows
+: # ======
+: # WINDOWS PORTION
+:<<BATCH
+    @echo off
+    echo Starting Cloudflare Tunnel for Windows
     curl -L https://github.com/cloudflare/cloudflared/releases/latest/download/cloudflared-windows-amd64.exe -o cloudflared.exe
     cloudflared.exe tunnel --url localhost:5001
+    GOTO ENDING
+BATCH
+: # LINUX PORTION
+echo 'Starting Cloudflare Tunnel for Linux'
+curl -L https://github.com/cloudflare/cloudflared/releases/latest/download/cloudflared-linux-amd64 -o 'cloudflared-linux-amd64' # 
+chmod +x 'cloudflared-linux-amd64' # 
+./cloudflared-linux-amd64 tunnel --url http://localhost:5001 # 
+exit # 
+:ENDING

class.py

@@ -188,6 +188,18 @@ class model_backend(InferenceModel):
                                     "extra_classes": "",
                                     'children': [{'text': 'False', 'value': False}, {'text': 'True', 'value': True}],
                                     })
+        requested_parameters.append({
+                                    "uitype": "text",
+                                    "unit": "text",
+                                    "label": "GPU ID",
+                                    "id": "kcpp_tensor_split_str",
+                                    "default": "1",
+                                    "check": {"value": "", 'check': "!="},
+                                    "tooltip": "Which GPU's do we use? For example:1 2",
+                                    "menu_path": "",
+                                    "refresh_model_inputs": False,
+                                    "extra_classes": ""
+                                    })
         requested_parameters.append({
                                     "uitype": "dropdown",
                                     "unit": "int",
@@ -202,18 +214,6 @@ class model_backend(InferenceModel):
                                     "extra_classes": "",
                                     'children': [{'text': 'False', 'value': 0}, {'text': 'True', 'value': 1}],
                                     })
-        requested_parameters.append({
-                                    "uitype": "text",
-                                    "unit": "text",
-                                    "label": "Tensor Split",
-                                    "id": "kcpp_tensor_split_str",
-                                    "default": self.kcpp_tensor_split_str,
-                                    "check": {"value": "", 'check': "!="},
-                                    "tooltip": "Tensor Split, values are space separated",
-                                    "menu_path": "",
-                                    "refresh_model_inputs": False,
-                                    "extra_classes": ""
-                                    })
         return requested_parameters
 
     def set_input_parameters(self, parameters):
@@ -232,6 +232,7 @@ class model_backend(InferenceModel):
             self.kcpp_tensor_split = []
             for s in splits:
                 self.kcpp_tensor_split.append(int(s))
+                print(self.kcpp_tensor_split)
 
         accel = parameters["kcpp_accelerator"]
         if accel==0:

koboldcpp.py

@@ -1680,8 +1680,9 @@ def unload_libs():
     dll_close = None
     if OS == "Windows":  # pragma: Windows
         from ctypes import wintypes
-        ctypes.windll.kernel32.FreeLibrary.argtypes = [wintypes.HMODULE]
         dll_close = ctypes.windll.kernel32.FreeLibrary
+        dll_close.argtypes = [wintypes.HMODULE]
+        dll_close.restype = ctypes.c_int
     elif OS == "Darwin":
         try:
             try:  # macOS 11 (Big Sur). Possibly also later macOS 10s.
@@ -1693,19 +1694,27 @@ def unload_libs():
             # not even in PATH.
             stdlib = ctypes.CDLL("/usr/lib/system/libsystem_c.dylib")
         dll_close = stdlib.dlclose
+        dll_close.argtypes = [ctypes.c_void_p]
+        dll_close.restype = ctypes.c_int
     elif OS == "Linux":
         try:
             stdlib = ctypes.CDLL("")
         except OSError:
             stdlib = ctypes.CDLL("libc.so") # Alpine Linux.
         dll_close = stdlib.dlclose
+        dll_close.argtypes = [ctypes.c_void_p]
+        dll_close.restype = ctypes.c_int
     elif sys.platform == "msys":
         # msys can also use `ctypes.CDLL("kernel32.dll").FreeLibrary()`.
         stdlib = ctypes.CDLL("msys-2.0.dll")
         dll_close = stdlib.dlclose
+        dll_close.argtypes = [ctypes.c_void_p]
+        dll_close.restype = ctypes.c_int
     elif sys.platform == "cygwin":
         stdlib = ctypes.CDLL("cygwin1.dll")
         dll_close = stdlib.dlclose
+        dll_close.argtypes = [ctypes.c_void_p]
+        dll_close.restype = ctypes.c_int
     elif OS == "FreeBSD":
         # FreeBSD uses `/usr/lib/libc.so.7` where `7` is another version number.
         # It is not in PATH but using its name instead of its path is somehow the
@@ -1716,6 +1725,18 @@ def unload_libs():
     if handle and dll_close:
         print("Unloading Libraries...")
         dll_close(handle._handle)
+        del handle.load_model
+        del handle.generate
+        del handle.new_token
+        del handle.get_stream_count
+        del handle.has_finished
+        del handle.get_last_eval_time
+        del handle.get_last_process_time
+        del handle.get_last_token_count
+        del handle.get_last_stop_reason
+        del handle.abort_generate
+        del handle.token_count
+        del handle.get_pending_output
         del handle
         handle = None