vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge branch 'master' into concedo

Browse source 
# Conflicts:
#	CMakeLists.txt
#	Makefile
This commit is contained in:
Concedo 2023-04-16 14:08:23 +08:00
commit 5a4d1b5d15
4 changed files with 529 additions and 373 deletions
Download patch file Download diff file Expand all files Collapse all files

6
convert.py
View file

@ -735,7 +735,7 @@ def lazy_load_safetensors_file(fp: IO[bytes], path: Path) -> ModelPlus:
header: Dict[str, Dict[str, Any]] = json.loads(fp.read(header_size))
# Use mmap for the actual data to avoid race conditions with the file offset.
mapped = memoryview(mmap.mmap(fp.fileno(), 0, access=mmap.ACCESS_READ))
byte_buf = mapped[fp.tell():]
byte_buf = mapped[8 + header_size:]
def convert(info: Dict[str, Any]) -> LazyTensor:
data_type = SAFETENSORS_DATA_TYPES[info['dtype']]
@ -761,7 +761,7 @@ def must_read(fp: IO[bytes], length: int) -> bytes:
return ret
def lazy_load_ggml_file(fp: IO[bytes], path: Path) -> ModelPlus:
def lazy_load_ggml_file(fp: io.BufferedReader, path: Path) -> ModelPlus:
magic = must_read(fp, 4)[::-1]
if magic in (b'ggmf', b'ggjt'):
version, = struct.unpack("i", must_read(fp, 4))
@ -795,7 +795,9 @@ def lazy_load_ggml_file(fp: IO[bytes], path: Path) -> ModelPlus:
model: LazyModel = {}
# Use mmap for the actual data to avoid race conditions with the file offset.
off = fp.raw.tell()
mapped = memoryview(mmap.mmap(fp.fileno(), 0, access=mmap.ACCESS_READ))
fp.raw.seek(off) # needed on Windows
def read_tensor() -> None: # this is a function so that variables captured in `load` don't change
shape_len, name_len, ftype = struct.unpack("iii", must_read(fp, 12))

2
examples/benchmark/benchmark-q4_0-matmult.c
View file

@ -24,7 +24,7 @@
float tensor_sum_elements(struct ggml_tensor * tensor) {
float sum = 0;
if (tensor->type==6) {
if (tensor->type==GGML_TYPE_F32) {
for (int j = 0; j < tensor->ne[1]; j++) {
for (int k = 0; k < tensor->ne[0]; k++) {
sum += ((float *) tensor->data)[j*tensor->ne[0]+k];

892
ggml.c
View file

File diff suppressed because it is too large Load diff

2
ggml.h
View file

@ -204,6 +204,7 @@ enum ggml_type {
GGML_TYPE_F16 = 1,
GGML_TYPE_Q4_0 = 2,
GGML_TYPE_Q4_1 = 3,
GGML_TYPE_Q8_0 = 4,
GGML_TYPE_I8,
GGML_TYPE_I16,
GGML_TYPE_I32,
@ -836,6 +837,7 @@ typedef struct {
dequantize_row_q_t dequantize_row_q;
quantize_row_q_t quantize_row_q;
quantize_row_q_t quantize_row_q_reference;
quantize_row_q_t quantize_row_q_dot;
vec_dot_q_t vec_dot_q;
} quantize_fns_t;