vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

server: bench: enable flash_attn param

Browse source
This commit is contained in:
Pierrick HYMBERT 2024-04-17 23:26:29 +02:00
parent 405385726e
commit 5668c79ea0
1 changed files with 1 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

1
examples/server/bench/bench.py
View file

@ -268,6 +268,7 @@ def start_server_background(args):
server_args.extend(['--defrag-thold', "0.1"]) server_args.extend(['--defrag-thold', "0.1"])
server_args.append('--cont-batching') server_args.append('--cont-batching')
server_args.append('--metrics') server_args.append('--metrics')
server_args.append('--flash-attn')
server_args.extend(['--log-format', "text"]) server_args.extend(['--log-format', "text"])
args = [str(arg) for arg in [server_path, *server_args]] args = [str(arg) for arg in [server_path, *server_args]]
print(f"bench: starting server with: {' '.join(args)}") print(f"bench: starting server with: {' '.join(args)}")