convert : sort imports
This commit is contained in:
parent
b18159b803
commit
a09b674a98
8 changed files with 53 additions and 49 deletions
|
@ -3,19 +3,20 @@
|
|||
|
||||
from __future__ import annotations
|
||||
|
||||
import gguf
|
||||
import os
|
||||
import sys
|
||||
import struct
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import struct
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any, List
|
||||
|
||||
import gguf
|
||||
import numpy as np
|
||||
import torch
|
||||
import argparse
|
||||
|
||||
from typing import Any, List
|
||||
from pathlib import Path
|
||||
from transformers import AutoTokenizer
|
||||
|
||||
|
||||
def bytes_to_unicode():
|
||||
# ref: https://github.com/openai/gpt-2/blob/master/src/encoder.py
|
||||
"""
|
||||
|
|
|
@ -3,17 +3,17 @@
|
|||
|
||||
from __future__ import annotations
|
||||
|
||||
import gguf
|
||||
import os
|
||||
import sys
|
||||
import struct
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import struct
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any, List
|
||||
|
||||
import gguf
|
||||
import numpy as np
|
||||
import torch
|
||||
import argparse
|
||||
|
||||
from typing import Any, List
|
||||
from pathlib import Path
|
||||
from transformers import AutoTokenizer
|
||||
|
||||
# ref: https://github.com/openai/gpt-2/blob/master/src/encoder.py
|
||||
|
|
|
@ -5,17 +5,17 @@
|
|||
|
||||
from __future__ import annotations
|
||||
|
||||
import gguf
|
||||
import os
|
||||
import sys
|
||||
import struct
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import struct
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING, Any, List
|
||||
|
||||
import gguf
|
||||
import numpy as np
|
||||
import torch
|
||||
import argparse
|
||||
|
||||
from typing import TYPE_CHECKING, Any, List
|
||||
from pathlib import Path
|
||||
from sentencepiece import SentencePieceProcessor
|
||||
|
||||
if TYPE_CHECKING:
|
||||
|
|
|
@ -1,10 +1,12 @@
|
|||
#!/usr/bin/env python3
|
||||
import sys, struct, math, argparse
|
||||
import argparse
|
||||
import math
|
||||
import struct
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
|
||||
import gguf
|
||||
import numpy as np
|
||||
|
||||
# Note: Does not support GGML_QKK_64
|
||||
QK_K = 256
|
||||
|
|
|
@ -3,17 +3,17 @@
|
|||
|
||||
from __future__ import annotations
|
||||
|
||||
import gguf
|
||||
import os
|
||||
import sys
|
||||
import struct
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import struct
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING, Any, List, Optional
|
||||
|
||||
import gguf
|
||||
import numpy as np
|
||||
import torch
|
||||
import argparse
|
||||
|
||||
from typing import TYPE_CHECKING, Any, List, Optional
|
||||
from pathlib import Path
|
||||
from sentencepiece import SentencePieceProcessor
|
||||
|
||||
if TYPE_CHECKING:
|
||||
|
|
|
@ -6,7 +6,7 @@ import os
|
|||
import re
|
||||
import struct
|
||||
import sys
|
||||
from typing import Any, Dict, Sequence, BinaryIO
|
||||
from typing import Any, BinaryIO, Dict, Sequence
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
|
|
11
convert.py
11
convert.py
|
@ -1,10 +1,8 @@
|
|||
#!/usr/bin/env python3
|
||||
from __future__ import annotations
|
||||
|
||||
import gguf
|
||||
import argparse
|
||||
import concurrent.futures
|
||||
from concurrent.futures import ThreadPoolExecutor, ProcessPoolExecutor
|
||||
import copy
|
||||
import enum
|
||||
import faulthandler
|
||||
|
@ -21,12 +19,15 @@ import struct
|
|||
import sys
|
||||
import time
|
||||
import zipfile
|
||||
import numpy as np
|
||||
|
||||
from abc import ABCMeta, abstractmethod
|
||||
from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import (IO, TYPE_CHECKING, Any, Callable, Dict, Generator, Iterable, List, Literal, Optional, Sequence, Set, Tuple, Type, TypeVar, Union)
|
||||
from typing import (IO, TYPE_CHECKING, Any, Callable, Dict, Generator, Iterable, List, Literal, Optional, Sequence, Set,
|
||||
Tuple, Type, TypeVar, Union)
|
||||
|
||||
import gguf
|
||||
import numpy as np
|
||||
from sentencepiece import SentencePieceProcessor # type: ignore
|
||||
|
||||
if TYPE_CHECKING:
|
||||
|
|
|
@ -1,18 +1,18 @@
|
|||
#!/usr/bin/env python3
|
||||
from __future__ import annotations
|
||||
|
||||
import shutil
|
||||
import sys
|
||||
import struct
|
||||
import tempfile
|
||||
import numpy as np
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
import shutil
|
||||
import struct
|
||||
import sys
|
||||
import tempfile
|
||||
from enum import IntEnum, auto
|
||||
from io import BufferedWriter
|
||||
from typing import Any, BinaryIO, Callable, IO, Dict, List, Optional, Sequence, Tuple, Union
|
||||
from pathlib import Path
|
||||
from typing import IO, Any, BinaryIO, Callable, Dict, List, Optional, Sequence, Tuple, Union
|
||||
|
||||
import numpy as np
|
||||
|
||||
#
|
||||
# constants
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue