From ebcbc45711786322b760d8c22c1aa3fe5a5a8c77 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sigbj=C3=B8rn=20Skj=C3=A6ret?= Date: Wed, 25 Sep 2024 16:03:50 +0200 Subject: [PATCH 1/4] Add inverse chat template metadata --- gguf-py/gguf/constants.py | 1 + gguf-py/gguf/gguf_writer.py | 3 +++ gguf-py/gguf/vocab.py | 11 +++++++++++ 3 files changed, 15 insertions(+) diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py index 560eee916..569dbd860 100644 --- a/gguf-py/gguf/constants.py +++ b/gguf-py/gguf/constants.py @@ -166,6 +166,7 @@ class Keys: CHAT_TEMPLATE = "tokenizer.chat_template" CHAT_TEMPLATE_N = "tokenizer.chat_template.{name}" CHAT_TEMPLATES = "tokenizer.chat_templates" + INVERSE_TEMPLATE = "tokenizer.inverse_template" # FIM/Infill special tokens constants PREFIX_ID = "tokenizer.ggml.prefix_token_id" SUFFIX_ID = "tokenizer.ggml.suffix_token_id" diff --git a/gguf-py/gguf/gguf_writer.py b/gguf-py/gguf/gguf_writer.py index bd059b45c..bd481fc0f 100644 --- a/gguf-py/gguf/gguf_writer.py +++ b/gguf-py/gguf/gguf_writer.py @@ -840,6 +840,9 @@ class GGUFWriter: self.add_string(Keys.Tokenizer.CHAT_TEMPLATE, value) + def add_inverse_template(self, value: str) -> None: + self.add_string(Keys.Tokenizer.INVERSE_TEMPLATE, value) + def add_prefix_token_id(self, id: int) -> None: self.add_uint32(Keys.Tokenizer.PREFIX_ID, id) diff --git a/gguf-py/gguf/vocab.py b/gguf-py/gguf/vocab.py index dc5749913..1e2fd366a 100644 --- a/gguf-py/gguf/vocab.py +++ b/gguf-py/gguf/vocab.py @@ -21,6 +21,7 @@ class SpecialVocab: add_special_token: dict[str, bool] special_token_ids: dict[str, int] chat_template: str | Sequence[Mapping[str, str]] | None + inverse_template: str | None def __init__( self, path: str | os.PathLike[str], load_merges: bool = False, @@ -33,6 +34,7 @@ class SpecialVocab: self.load_merges = load_merges self.merges = [] self.chat_template = None + self.inverse_template = None if special_token_types is not None: self.special_token_types = special_token_types else: @@ -71,6 +73,10 @@ class SpecialVocab: if not quiet: logger.info(f'Setting chat_template to {self.chat_template}') gw.add_chat_template(self.chat_template) + if self.inverse_template is not None: + if not quiet: + logger.info(f'Setting inverse_template to {self.inverse_template}') + gw.add_inverse_template(self.inverse_template) def _load(self, path: Path) -> None: self._try_load_from_tokenizer_json(path) @@ -137,6 +143,11 @@ class SpecialVocab: self.chat_template = chat_template else: logger.warning(f'Bad type for chat_template field in {tokenizer_config_file!r} - ignoring') + inverse_template = tokenizer_config.get('inverse_template') + if inverse_template is None or isinstance(inverse_template, str): + self.inverse_template = inverse_template + else: + logger.warning(f'Bad type for inverse_template field in {tokenizer_config_file!r} - ignoring') for typ in self.special_token_types: add_entry = tokenizer_config.get(f'add_{typ}_token') if isinstance(add_entry, bool): From 28393079faa905baf80a52f7e20bdfed653f7d08 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sigbj=C3=B8rn=20Skj=C3=A6ret?= Date: Wed, 25 Sep 2024 16:04:49 +0200 Subject: [PATCH 2/4] support inverse chat template --- gguf-py/scripts/gguf_new_metadata.py | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/gguf-py/scripts/gguf_new_metadata.py b/gguf-py/scripts/gguf_new_metadata.py index fce52a8c1..8f37a1a0c 100755 --- a/gguf-py/scripts/gguf_new_metadata.py +++ b/gguf-py/scripts/gguf_new_metadata.py @@ -85,7 +85,7 @@ def copy_with_new_metadata(reader: gguf.GGUFReader, writer: gguf.GGUFWriter, new continue # Skip old chat templates if we have new ones - if field.name.startswith(gguf.Keys.Tokenizer.CHAT_TEMPLATE) and gguf.Keys.Tokenizer.CHAT_TEMPLATE in new_metadata: + if (field.name.startswith(gguf.Keys.Tokenizer.CHAT_TEMPLATE) and gguf.Keys.Tokenizer.CHAT_TEMPLATE in new_metadata) or (field.name.startswith(gguf.Keys.Tokenizer.INVERSE_TEMPLATE) and gguf.Keys.Tokenizer.INVERSE_TEMPLATE in new_metadata): logger.debug(f'Skipping {field.name}') continue @@ -110,6 +110,11 @@ def copy_with_new_metadata(reader: gguf.GGUFReader, writer: gguf.GGUFWriter, new writer.add_chat_template(new_metadata[gguf.Keys.Tokenizer.CHAT_TEMPLATE].value) del new_metadata[gguf.Keys.Tokenizer.CHAT_TEMPLATE] + if gguf.Keys.Tokenizer.INVERSE_TEMPLATE in new_metadata: + logger.debug('Adding inverse template(s)') + writer.add_inverse_template(new_metadata[gguf.Keys.Tokenizer.INVERSE_TEMPLATE].value) + del new_metadata[gguf.Keys.Tokenizer.INVERSE_TEMPLATE] + for key, val in new_metadata.items(): logger.debug(f'Adding {key}: "{val.value}" {val.description}') writer.add_key_value(key, val.value, val.type) @@ -143,7 +148,8 @@ def main() -> None: parser.add_argument("--general-name", type=str, help="The models general.name", metavar='"name"') parser.add_argument("--general-description", type=str, help="The models general.description", metavar='"Description ..."') parser.add_argument("--chat-template", type=str, help="Chat template string (or JSON string containing templates)", metavar='"{% ... %} ..."') - parser.add_argument("--chat-template-config", type=Path, help="Config file containing chat template(s)", metavar='tokenizer_config.json') + parser.add_argument("--inverse-template", type=str, help="Inverse template string", metavar='"{% ... %} ..."') + parser.add_argument("--chat-template-config", type=Path, help="Config file containing chat and/or inverse template(s)", metavar='tokenizer_config.json') parser.add_argument("--pre-tokenizer", type=str, help="The models tokenizer.ggml.pre", metavar='"pre tokenizer"') parser.add_argument("--remove-metadata", action="append", type=str, help="Remove metadata (by key name) from output model", metavar='general.url') parser.add_argument("--special-token", action="append", type=str, help="Special token by value", nargs=2, metavar=(' | '.join(token_names.keys()), '""')) @@ -166,12 +172,18 @@ def main() -> None: if args.chat_template: new_metadata[gguf.Keys.Tokenizer.CHAT_TEMPLATE] = MetadataDetails(gguf.GGUFValueType.STRING, json.loads(args.chat_template) if args.chat_template.startswith('[') else args.chat_template) + if args.inverse_template: + new_metadata[gguf.Keys.Tokenizer.INVERSE_TEMPLATE] = MetadataDetails(gguf.GGUFValueType.STRING, args.inverse_template) + if args.chat_template_config: with open(args.chat_template_config, 'r') as fp: config = json.load(fp) - template = config.get('chat_template') - if template: - new_metadata[gguf.Keys.Tokenizer.CHAT_TEMPLATE] = MetadataDetails(gguf.GGUFValueType.STRING, template) + chat_template = config.get('chat_template') + inverse_template = config.get('inverse_template') + if chat_template: + new_metadata[gguf.Keys.Tokenizer.CHAT_TEMPLATE] = MetadataDetails(gguf.GGUFValueType.STRING, chat_template) + if inverse_template: + new_metadata[gguf.Keys.Tokenizer.INVERSE_TEMPLATE] = MetadataDetails(gguf.GGUFValueType.STRING, inverse_template) if args.pre_tokenizer: new_metadata[gguf.Keys.Tokenizer.PRE] = MetadataDetails(gguf.GGUFValueType.STRING, args.pre_tokenizer) From 87793d466674836e246a0e66402ed7e8874198ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sigbj=C3=B8rn=20Skj=C3=A6ret?= Date: Wed, 25 Sep 2024 16:06:07 +0200 Subject: [PATCH 3/4] bump version --- gguf-py/pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gguf-py/pyproject.toml b/gguf-py/pyproject.toml index 33cfe26b7..10e94876c 100644 --- a/gguf-py/pyproject.toml +++ b/gguf-py/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "gguf" -version = "0.10.0" +version = "0.11.0" description = "Read and write ML models in GGUF for GGML" authors = ["GGML "] packages = [ From 41efa86198ee9b822d9638b1330ea91bfcb1bb47 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sigbj=C3=B8rn=20Skj=C3=A6ret?= Date: Wed, 25 Sep 2024 16:17:22 +0200 Subject: [PATCH 4/4] singular --- gguf-py/scripts/gguf_new_metadata.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gguf-py/scripts/gguf_new_metadata.py b/gguf-py/scripts/gguf_new_metadata.py index 8f37a1a0c..a004f0dd1 100755 --- a/gguf-py/scripts/gguf_new_metadata.py +++ b/gguf-py/scripts/gguf_new_metadata.py @@ -111,7 +111,7 @@ def copy_with_new_metadata(reader: gguf.GGUFReader, writer: gguf.GGUFWriter, new del new_metadata[gguf.Keys.Tokenizer.CHAT_TEMPLATE] if gguf.Keys.Tokenizer.INVERSE_TEMPLATE in new_metadata: - logger.debug('Adding inverse template(s)') + logger.debug('Adding inverse template') writer.add_inverse_template(new_metadata[gguf.Keys.Tokenizer.INVERSE_TEMPLATE].value) del new_metadata[gguf.Keys.Tokenizer.INVERSE_TEMPLATE]