Fix vocab space conversion logic

2023-08-20 10:36:57 -06:00 · 2023-08-20 10:36:57 -06:00 · 08959c88c2
commit 08959c88c2
parent f7e61fd1a9
1 changed files with 1 additions and 1 deletions
--- a/convert-llama-ggmlv3-to-gguf.py
+++ b/convert-llama-ggmlv3-to-gguf.py
@ -182,7 +182,7 @@ class GGMLToGGUF:
        toktypes = []
        for (tokid, (vbytes, vscore)) in enumerate(self.model.vocab.items):
            tt = 1
-            if len(vbytes) > 1 and vbytes[0] == 32:
+            if len(vbytes) > 0 and vbytes[0] == 32:
                vbytes = vbytes.replace(b' ', b'\xe2\x96\x81')
            elif len(vbytes) == 0:
                tt = 3