library unloading is working

class.py

@@ -233,6 +233,13 @@ class model_backend(InferenceModel):
             self.kcpp_nommap = True
         pass
 
+    def unload(self):
+        print("Attemping to unload library")
+        koboldcpp.unload_libs()
+        global kcpp_backend_loaded
+        kcpp_backend_loaded = False
+        pass
+
     def _load(self, save_model: bool, initial_load: bool) -> None:
         global kcpp_backend_loaded
         self.tokenizer = self._get_tokenizer("gpt2")

koboldcpp.py

@@ -1656,6 +1656,52 @@ def run_horde_worker(args, api_key, worker_name):
         time.sleep(2)
     sys.exit(2)
 
+def unload_libs():
+    global handle
+    import platform
+    OS = platform.system()
+    dll_close = None
+    if OS == "Windows":  # pragma: Windows
+        from ctypes import wintypes
+        ctypes.windll.kernel32.FreeLibrary.argtypes = [wintypes.HMODULE]
+        dll_close = ctypes.windll.kernel32.FreeLibrary
+    elif OS == "Darwin":
+        try:
+            try:  # macOS 11 (Big Sur). Possibly also later macOS 10s.
+                stdlib = ctypes.CDLL("libc.dylib")
+            except OSError:
+                stdlib = ctypes.CDLL("libSystem")
+        except OSError:
+            # Older macOSs. Not only is the name inconsistent but it's
+            # not even in PATH.
+            stdlib = ctypes.CDLL("/usr/lib/system/libsystem_c.dylib")
+        dll_close = stdlib.dlclose
+    elif OS == "Linux":
+        try:
+            stdlib = ctypes.CDLL("")
+        except OSError:
+            stdlib = ctypes.CDLL("libc.so") # Alpine Linux.
+        dll_close = stdlib.dlclose
+    elif sys.platform == "msys":
+        # msys can also use `ctypes.CDLL("kernel32.dll").FreeLibrary()`.
+        stdlib = ctypes.CDLL("msys-2.0.dll")
+        dll_close = stdlib.dlclose
+    elif sys.platform == "cygwin":
+        stdlib = ctypes.CDLL("cygwin1.dll")
+        dll_close = stdlib.dlclose
+    elif OS == "FreeBSD":
+        # FreeBSD uses `/usr/lib/libc.so.7` where `7` is another version number.
+        # It is not in PATH but using its name instead of its path is somehow the
+        # only way to open it. The name must include the .so.7 suffix.
+        stdlib = ctypes.CDLL("libc.so.7")
+        dll_close = stdlib.close
+
+    if handle and dll_close:
+        print("Unloading Libraries...")
+        dll_close(handle._handle)
+        del handle
+        handle = None
+
 def main(launch_args,start_server=True):
     global args
     args = launch_args