From 951614bfc6cdc4396a3346c410a9069794c0b188 Mon Sep 17 00:00:00 2001
From: Concedo <39025047+LostRuins@users.noreply.github.com>
Date: Mon, 18 Sep 2023 15:03:52 +0800
Subject: [PATCH] library unloading is working

---
 class.py     |  7 +++++++
 koboldcpp.py | 46 ++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 53 insertions(+)

diff --git a/class.py b/class.py
index 8fd1fdbe3..96fa97cbf 100644
--- a/class.py
+++ b/class.py
@@ -233,6 +233,13 @@ class model_backend(InferenceModel):
             self.kcpp_nommap = True
         pass
 
+    def unload(self):
+        print("Attemping to unload library")
+        koboldcpp.unload_libs()
+        global kcpp_backend_loaded
+        kcpp_backend_loaded = False
+        pass
+
     def _load(self, save_model: bool, initial_load: bool) -> None:
         global kcpp_backend_loaded
         self.tokenizer = self._get_tokenizer("gpt2")
diff --git a/koboldcpp.py b/koboldcpp.py
index 13c12ce9d..099985240 100755
--- a/koboldcpp.py
+++ b/koboldcpp.py
@@ -1656,6 +1656,52 @@ def run_horde_worker(args, api_key, worker_name):
         time.sleep(2)
     sys.exit(2)
 
+def unload_libs():
+    global handle
+    import platform
+    OS = platform.system()
+    dll_close = None
+    if OS == "Windows":  # pragma: Windows
+        from ctypes import wintypes
+        ctypes.windll.kernel32.FreeLibrary.argtypes = [wintypes.HMODULE]
+        dll_close = ctypes.windll.kernel32.FreeLibrary
+    elif OS == "Darwin":
+        try:
+            try:  # macOS 11 (Big Sur). Possibly also later macOS 10s.
+                stdlib = ctypes.CDLL("libc.dylib")
+            except OSError:
+                stdlib = ctypes.CDLL("libSystem")
+        except OSError:
+            # Older macOSs. Not only is the name inconsistent but it's
+            # not even in PATH.
+            stdlib = ctypes.CDLL("/usr/lib/system/libsystem_c.dylib")
+        dll_close = stdlib.dlclose
+    elif OS == "Linux":
+        try:
+            stdlib = ctypes.CDLL("")
+        except OSError:
+            stdlib = ctypes.CDLL("libc.so") # Alpine Linux.
+        dll_close = stdlib.dlclose
+    elif sys.platform == "msys":
+        # msys can also use `ctypes.CDLL("kernel32.dll").FreeLibrary()`.
+        stdlib = ctypes.CDLL("msys-2.0.dll")
+        dll_close = stdlib.dlclose
+    elif sys.platform == "cygwin":
+        stdlib = ctypes.CDLL("cygwin1.dll")
+        dll_close = stdlib.dlclose
+    elif OS == "FreeBSD":
+        # FreeBSD uses `/usr/lib/libc.so.7` where `7` is another version number.
+        # It is not in PATH but using its name instead of its path is somehow the
+        # only way to open it. The name must include the .so.7 suffix.
+        stdlib = ctypes.CDLL("libc.so.7")
+        dll_close = stdlib.close
+
+    if handle and dll_close:
+        print("Unloading Libraries...")
+        dll_close(handle._handle)
+        del handle
+        handle = None
+
 def main(launch_args,start_server=True):
     global args
     args = launch_args