vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Adds --mlock argument

Browse source
This commit is contained in:
Sergey Kucher 2023-05-02 16:19:37 +03:00
parent 5a10ea50da
commit 069b3d4c37
2 changed files with 52 additions and 48 deletions
Download patch file Download diff file Expand all files Collapse all files

2
.gitignore vendored
View file

@ -47,3 +47,5 @@ zig-cache/
ppl-*.txt ppl-*.txt
examples/jeopardy/results.txt examples/jeopardy/results.txt
koboldcpp.so
koboldcpp_noavx2.so

2
koboldcpp.py
View file

@ -134,6 +134,7 @@ def load_model(model_filename):
inputs.threads = args.threads inputs.threads = args.threads
inputs.f16_kv = True inputs.f16_kv = True
inputs.use_mmap = (not args.nommap) inputs.use_mmap = (not args.nommap)
inputs.use_mlock = args.mlock
if args.lora and args.lora!="": if args.lora and args.lora!="":
inputs.use_mmap = False inputs.use_mmap = False
inputs.use_smartcontext = args.smartcontext inputs.use_smartcontext = args.smartcontext
@ -589,6 +590,7 @@ if __name__ == '__main__':
parser.add_argument("--smartcontext", help="Reserving a portion of context to try processing less frequently.", action='store_true') parser.add_argument("--smartcontext", help="Reserving a portion of context to try processing less frequently.", action='store_true')
parser.add_argument("--unbantokens", help="Normally, KoboldAI prevents certain tokens such as EOS and Square Brackets. This flag unbans them.", action='store_true') parser.add_argument("--unbantokens", help="Normally, KoboldAI prevents certain tokens such as EOS and Square Brackets. This flag unbans them.", action='store_true')
parser.add_argument("--nommap", help="If set, do not use mmap to load newer models", action='store_true') parser.add_argument("--nommap", help="If set, do not use mmap to load newer models", action='store_true')
parser.add_argument("--mlock", help="Force system to keep model in RAM rather than swapping or compressing", action='store_true')
parser.add_argument("--noavx2", help="Do not use AVX2 instructions, a slower compatibility mode for older devices. Does not work with --clblast.", action='store_true') parser.add_argument("--noavx2", help="Do not use AVX2 instructions, a slower compatibility mode for older devices. Does not work with --clblast.", action='store_true')
parser.add_argument("--debugmode", help="Shows additional debug info in the terminal.", action='store_true') parser.add_argument("--debugmode", help="Shows additional debug info in the terminal.", action='store_true')
parser.add_argument("--skiplauncher", help="Doesn't display or use the new GUI launcher.", action='store_true') parser.add_argument("--skiplauncher", help="Doesn't display or use the new GUI launcher.", action='store_true')