Ported Starcoder and added some assertions

convert-starcoder-hf-to-gguf.py

@@ -20,28 +20,6 @@ if 'NO_LOCAL_GGUF' not in os.environ:
 import gguf
 
 
-def bytes_to_unicode():
-    # ref: https://github.com/openai/gpt-2/blob/master/src/encoder.py
-    """
-    Returns list of utf-8 byte and a corresponding list of unicode strings.
-    The reversible bpe codes work on unicode strings.
-    This means you need a large # of unicode characters in your vocab if you want to avoid UNKs.
-    When you're at something like a 10B token dataset you end up needing around 5K for decent coverage.
-    This is a significant percentage of your normal, say, 32K bpe vocab.
-    To avoid that, we want lookup tables between utf-8 bytes and unicode strings.
-    And avoids mapping to whitespace/control characters the bpe code barfs on.
-    """
-    bs = list(range(ord("!"), ord("~")+1))+list(range(ord("¡"), ord("¬")+1))+list(range(ord("®"), ord("ÿ")+1))
-    cs = bs[:]
-    n = 0
-    for b in range(2**8):
-        if b not in bs:
-            bs.append(b)
-            cs.append(2**8+n)
-            n += 1
-    return dict(zip(bs, (chr(n) for n in cs)))
-
-
 def count_model_parts(dir_model: Path) -> int:
     num_parts = 0
     for filename in os.listdir(dir_model):
@@ -117,6 +95,8 @@ gguf_writer.add_file_type(ftype)
 print("gguf: get tokenizer metadata")
 
 tokens: list[bytearray] = []
+scores: list[float] = []
+toktypes: list[int] = []
 
 tokenizer_json_file = dir_model / 'tokenizer.json'
 if not tokenizer_json_file.is_file():
@@ -139,28 +119,15 @@ vocab_size = hparams["vocab_size"] if "vocab_size" in hparams else len(tokenizer
 tokenizer = AutoTokenizer.from_pretrained(dir_model)
 
 reverse_vocab = {id: encoded_tok for encoded_tok, id in tokenizer.vocab.items()}
-byte_encoder = bytes_to_unicode()
-byte_decoder = {v: k for k, v in byte_encoder.items()}
 
 for i in range(vocab_size):
-    if i in reverse_vocab:
+    tokens.append(reverse_vocab[i])
-        try:
+    scores.append(0.0) # dummy
-            text = bytearray([byte_decoder[c] for c in reverse_vocab[i]])
+    toktypes.append(gguf.TokenType.NORMAL)
-        except KeyError:
-            text = bytearray()
-            for c in reverse_vocab[i]:
-                if ord(c) < 256:  # single byte character
-                    text.append(byte_decoder[ord(c)])
-                else:  # multibyte special token character
-                    text.extend(c.encode('utf-8'))
-    else:
-        print(f"Key {i} not in tokenizer vocabulary. Padding with an arbitrary token.")
-        pad_token = f"[PAD{i}]".encode("utf8")
-        text = bytearray(pad_token)
-
-    tokens.append(text)
 
 gguf_writer.add_token_list(tokens)
+gguf_writer.add_token_scores(scores)
+gguf_writer.add_token_types(toktypes)
 
 special_vocab = gguf.SpecialVocab(dir_model, load_merges = True)
 special_vocab.add_to_gguf(gguf_writer)

unicode.h

@@ -405,12 +405,15 @@ static int codepoint_type(std::string utf8) {
 static std::unordered_map<uint8_t, std::string> bytes_to_unicode_map_bpe() {
     std::unordered_map<uint8_t, std::string> map;
     for (int ch = u'!'; ch <= u'~'; ++ch) {
+        assert(0 <= ch && ch < 256);
         map[ch] = codepoint_to_utf8(ch);
     }
     for (int ch = u'¡'; ch <= u'¬'; ++ch) {
+        assert(0 <= ch && ch < 256);
         map[ch] = codepoint_to_utf8(ch);
     }
     for (int ch = u'®'; ch <= u'ÿ'; ++ch) {
+        assert(0 <= ch && ch < 256);
         map[ch] = codepoint_to_utf8(ch);
     }
     auto n = 0;
@@ -431,12 +434,15 @@ static std::string bytes_to_unicode_bpe(uint8_t byte) {
 static std::unordered_map<std::string, uint8_t> unicode_to_bytes_map_bpe() {
     std::unordered_map<std::string, uint8_t> map;
     for (int ch = u'!'; ch <= u'~'; ++ch) {
+        assert(0 <= ch && ch < 256);
         map[codepoint_to_utf8(ch)] = ch;
     }
     for (int ch = u'¡'; ch <= u'¬'; ++ch) {
+        assert(0 <= ch && ch < 256);
         map[codepoint_to_utf8(ch)] = ch;
     }
     for (int ch = u'®'; ch <= u'ÿ'; ++ch) {
+        assert(0 <= ch && ch < 256);
         map[codepoint_to_utf8(ch)] = ch;
     }
     auto n = 0;
@@ -449,7 +455,7 @@ static std::unordered_map<std::string, uint8_t> unicode_to_bytes_map_bpe() {
     return map;
 }
 
-static uint8_t unicode_to_bytes_bpe(const std::string& utf8) {
+static uint8_t unicode_to_bytes_bpe(const std::string & utf8) {
     static std::unordered_map<std::string, uint8_t> map = unicode_to_bytes_map_bpe();
     return map.at(utf8);
 }