increase default blas batch size
This commit is contained in:
parent
4d1825263b
commit
b91abc3316
3 changed files with 3 additions and 2 deletions
|
@ -209,7 +209,7 @@ generation_outputs gpttype_generate(const generation_inputs inputs, generation_o
|
|||
int original_threads = params.n_threads;
|
||||
if (blasmode)
|
||||
{
|
||||
params.n_batch = 512;
|
||||
params.n_batch = 1024;
|
||||
params.n_threads = 1;
|
||||
}
|
||||
|
||||
|
|
|
@ -377,6 +377,7 @@ if __name__ == '__main__':
|
|||
portgroup.add_argument("l_port", help="Port to listen on (deprecated)", default=defaultport, nargs="?", type=int, action='store')
|
||||
parser.add_argument("--host", help="Host IP to listen on. If empty, all routable interfaces are accepted.", default="")
|
||||
|
||||
#os.environ["OMP_NUM_THREADS"] = '12'
|
||||
# psutil.cpu_count(logical=False)
|
||||
physical_core_limit = 1
|
||||
if os.cpu_count()!=None and os.cpu_count()>1:
|
||||
|
|
|
@ -158,7 +158,7 @@ generation_outputs llama_generate(const generation_inputs inputs, generation_out
|
|||
int original_threads = params.n_threads;
|
||||
if (blasmode)
|
||||
{
|
||||
params.n_batch = 512;
|
||||
params.n_batch = 1024;
|
||||
params.n_threads = 1;
|
||||
}
|
||||
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue