vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

reduce batch sizes and skip all intrinsic flags except AVX when building in compatibility mode.

Browse source
This commit is contained in:
Concedo 2023-04-13 11:32:05 +08:00
parent f4257a8eef
commit 5c22f7e4c4
3 changed files with 9 additions and 4 deletions
Download patch file Download diff file Expand all files Collapse all files

9
Makefile
View file

@ -72,8 +72,13 @@ endif
# feel free to update the Makefile for your architecture and send a pull request or issue # feel free to update the Makefile for your architecture and send a pull request or issue
ifeq ($(UNAME_M),$(filter $(UNAME_M),x86_64 i686)) ifeq ($(UNAME_M),$(filter $(UNAME_M),x86_64 i686))
# Use all CPU extensions that are available: # Use all CPU extensions that are available:
CFLAGS += -mf16c -mavx -msse3 CFLAGS += -mavx
BONUSCFLAGS += -mfma -mavx2 ifeq ($(OS),Windows_NT)
BONUSCFLAGS += -mfma -mavx2 -mf16c -msse3
else
# if not on windows, they are clearly building it themselves, so lets just use whatever is supported
CFLAGS += -march=native -mtune=native
endif
endif endif
ifneq ($(filter ppc64%,$(UNAME_M)),) ifneq ($(filter ppc64%,$(UNAME_M)),)
POWER9_M := $(shell grep "POWER9" /proc/cpuinfo) POWER9_M := $(shell grep "POWER9" /proc/cpuinfo)

2
gpttype_adapter.cpp
View file

@ -224,7 +224,7 @@ generation_outputs gpttype_generate(const generation_inputs inputs, generation_o
int original_threads = params.n_threads; int original_threads = params.n_threads;
if (blasmode) if (blasmode)
{ {
params.n_batch = 1024; params.n_batch = 512; //received reports of 1024 and above crashing on some models
params.n_threads = 1; params.n_threads = 1;
} }

2
llama_adapter.cpp
View file

@ -160,7 +160,7 @@ generation_outputs llama_generate(const generation_inputs inputs, generation_out
int original_threads = params.n_threads; int original_threads = params.n_threads;
if (blasmode) if (blasmode)
{ {
params.n_batch = 1024; params.n_batch = 512; //received reports of 1024 and above crashing on some models
params.n_threads = 1; params.n_threads = 1;
} }