From 08959c88c21d02808b73198a038a96b7b34f2f5e Mon Sep 17 00:00:00 2001 From: KerfuffleV2 Date: Sun, 20 Aug 2023 10:36:57 -0600 Subject: [PATCH] Fix vocab space conversion logic --- convert-llama-ggmlv3-to-gguf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/convert-llama-ggmlv3-to-gguf.py b/convert-llama-ggmlv3-to-gguf.py index c522585af..58fd3a09a 100644 --- a/convert-llama-ggmlv3-to-gguf.py +++ b/convert-llama-ggmlv3-to-gguf.py @@ -182,7 +182,7 @@ class GGMLToGGUF: toktypes = [] for (tokid, (vbytes, vscore)) in enumerate(self.model.vocab.items): tt = 1 - if len(vbytes) > 1 and vbytes[0] == 32: + if len(vbytes) > 0 and vbytes[0] == 32: vbytes = vbytes.replace(b' ', b'\xe2\x96\x81') elif len(vbytes) == 0: tt = 3