convert-hf : begin refactoring write_tensor
This commit is contained in:
parent
b8a7a5a90f
commit
47e02eb7bc
10 changed files with 386 additions and 852 deletions
|
@ -4,7 +4,7 @@ import json
|
|||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any, Callable
|
||||
from typing import Any, Callable, Sequence, Mapping, Iterable
|
||||
|
||||
from .gguf_writer import GGUFWriter
|
||||
|
||||
|
@ -13,11 +13,11 @@ class SpecialVocab:
|
|||
merges: list[str]
|
||||
add_special_token: dict[str, bool]
|
||||
special_token_ids: dict[str, int]
|
||||
chat_template: str | None
|
||||
chat_template: str | Sequence[Mapping[str, str]] | None
|
||||
|
||||
def __init__(
|
||||
self, path: str | os.PathLike[str], load_merges: bool = False,
|
||||
special_token_types: tuple[str, ...] | None = None,
|
||||
special_token_types: Iterable[str] | None = None,
|
||||
n_vocab: int | None = None,
|
||||
):
|
||||
self.special_token_ids = {}
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue