vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

convert : add necessary type: ignore comments

Browse source
This commit is contained in:
Cebtenzzre 2023-08-30 17:04:55 -04:00
parent e66e0be432
commit 8a8c1cb0f2
5 changed files with 6 additions and 6 deletions
Download patch file Download diff file Expand all files Collapse all files

2
convert-falcon-hf-to-gguf.py
View file

@ -14,7 +14,7 @@ from typing import Any
import gguf import gguf
import numpy as np import numpy as np
import torch import torch
from transformers import AutoTokenizer from transformers import AutoTokenizer # type: ignore[import]
def bytes_to_unicode(): def bytes_to_unicode():

2
convert-gptneox-hf-to-gguf.py
View file

@ -14,7 +14,7 @@ from typing import Any
import gguf import gguf
import numpy as np import numpy as np
import torch import torch
from transformers import AutoTokenizer from transformers import AutoTokenizer # type: ignore[import]
# ref: https://github.com/openai/gpt-2/blob/master/src/encoder.py # ref: https://github.com/openai/gpt-2/blob/master/src/encoder.py

2
convert-llama-7b-pth-to-gguf.py
View file

@ -16,7 +16,7 @@ from typing import TYPE_CHECKING, Any
import gguf import gguf
import numpy as np import numpy as np
import torch import torch
from sentencepiece import SentencePieceProcessor from sentencepiece import SentencePieceProcessor # type: ignore[import]
if TYPE_CHECKING: if TYPE_CHECKING:
from typing import TypeAlias from typing import TypeAlias

2
convert-llama-hf-to-gguf.py
View file

@ -14,7 +14,7 @@ from typing import TYPE_CHECKING, Any
import gguf import gguf
import numpy as np import numpy as np
import torch import torch
from sentencepiece import SentencePieceProcessor from sentencepiece import SentencePieceProcessor # type: ignore[import]
if TYPE_CHECKING: if TYPE_CHECKING:
from typing import TypeAlias from typing import TypeAlias

4
convert.py
View file

@ -27,7 +27,7 @@ from typing import IO, TYPE_CHECKING, Any, Callable, Generator, Iterable, Litera
import gguf import gguf
import numpy as np import numpy as np
from sentencepiece import SentencePieceProcessor # type: ignore from sentencepiece import SentencePieceProcessor # type: ignore[import]
if TYPE_CHECKING: if TYPE_CHECKING:
from typing import TypeAlias from typing import TypeAlias
@ -338,7 +338,7 @@ class BpeVocab:
def bpe_tokens(self) -> Iterable[tuple[bytes, float, gguf.TokenType]]: def bpe_tokens(self) -> Iterable[tuple[bytes, float, gguf.TokenType]]:
tokenizer = self.bpe_tokenizer tokenizer = self.bpe_tokenizer
from transformers.models.gpt2 import tokenization_gpt2 from transformers.models.gpt2 import tokenization_gpt2 # type: ignore[import]
byte_encoder = tokenization_gpt2.bytes_to_unicode() byte_encoder = tokenization_gpt2.bytes_to_unicode()
byte_decoder = {v: k for k, v in byte_encoder.items()} byte_decoder = {v: k for k, v in byte_encoder.items()}
for i, item in enumerate(tokenizer): for i, item in enumerate(tokenizer):