scripts: Use local gguf package when running from repo (#2927)
* scripts: Use local gguf when running from repo
This commit is contained in:
parent
e8422de39e
commit
aeefac4ff7
5 changed files with 22 additions and 5 deletions
|
@ -11,11 +11,14 @@ import sys
|
|||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import gguf
|
||||
import numpy as np
|
||||
import torch
|
||||
from transformers import AutoTokenizer # type: ignore[import]
|
||||
|
||||
if 'NO_LOCAL_GGUF' not in os.environ:
|
||||
sys.path.insert(1, str(Path(__file__).parent / 'gguf-py' / 'gguf'))
|
||||
import gguf
|
||||
|
||||
|
||||
def bytes_to_unicode():
|
||||
# ref: https://github.com/openai/gpt-2/blob/master/src/encoder.py
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue