vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

make

Browse source
This commit is contained in:
Robert Washbourne 2023-11-23 23:51:34 -05:00
parent 4b6e344bad
commit 22804439d2
2 changed files with 4 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

4
.devops/handler.py
View file

@ -1,8 +1,10 @@
import subprocess
import runpod
import os
import time
sleep_time = int(os.environ.get('SLEEP_TIME', 1))
llama_cmd = os.environ.get('LLAMA_CMD', "/server --host 0.0.0.0 --threads 8 -ngl 999 -np 8 -cb -m model.gguf -c 16384")
subprocess.Popen(llama_cmd.split(' '))
## load your model(s) into vram here

2
.devops/main-cuda.Dockerfile
View file

@ -27,7 +27,7 @@ ENV LLAMA_CUDA_MMV_Y=2
ENV LLAMA_CUDA_DMMV_X=64
ENV LLAMA_CUDA_F16=true
RUN make -j
RUN make
# Accept the build argument into an environment variable
ARG MODEL_URL