From 58d663d32793fb476762212519341dfb80e17689 Mon Sep 17 00:00:00 2001
From: Aman Karmani <aman@tmm1.net>
Date: Thu, 6 Jul 2023 14:08:32 -0700
Subject: [PATCH] hack in empty tokens for unknown vocab

---
 convert.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/convert.py b/convert.py
index c593abeb4..5d2f8aa55 100644
--- a/convert.py
+++ b/convert.py
@@ -219,6 +219,8 @@ class XgenVocab:
         for index in range(0, self.vocab_size_base):
             token = self.xt._convert_id_to_token(index)
             yield (token, float(index))
+        for index in range(self.vocab_size_base, self.vocab_size):
+            yield (b'', float(index))
 
     def __repr__(self) -> str:
         return f"<XgenVocab with {self.vocab_size_base} base tokens and {len(self.added_tokens_list)} added tokens>"
@@ -977,6 +979,9 @@ def check_vocab_size(params: Params, vocab: Vocab) -> None:
             vocab.added_tokens_list = []
             vocab.vocab_size = vocab.vocab_size_base
             return
+        if isinstance(vocab, XgenVocab):
+            vocab.vocab_size = params.n_vocab
+            return
         msg = f"Vocab size mismatch (model has {params.n_vocab}, but {vocab.fname_tokenizer}"
         if vocab.fname_added_tokens is not None:
             msg += f" combined with {vocab.fname_added_tokens}"