vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix handler

Browse source
This commit is contained in:
Robert Washbourne 2023-11-24 02:22:51 -05:00
parent 7e06600b38
commit 7e8607d097
2 changed files with 2 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

2
.devops/handler.py
View file

@ -26,7 +26,7 @@ async def handler(event):
print(event) print(event)
prompt = event["input"]["prompt"] prompt = event["input"]["prompt"]
async with aiohttp.ClientSession() as session: async with aiohttp.ClientSession() as session:
async with session.post(url, data=json.dumps(json_data = { async with session.post(url, data=json.dumps({
'stream': True, 'stream': True,
'n_predict': 2048, 'n_predict': 2048,
'temperature': 0.2, 'temperature': 0.2,

2
.devops/main-cuda.Dockerfile
View file

@ -51,4 +51,4 @@ COPY --from=build /app/models models
# CMD ["/bin/sh", "-c", "/server --model model.gguf --threads $(nproc) -ngl 99 -np $(nproc) -cb"] # CMD ["/bin/sh", "-c", "/server --model model.gguf --threads $(nproc) -ngl 99 -np $(nproc) -cb"]
# CMD ["/server --host 0.0.0.0 --threads 8 -ngl 999 -np 8 -cb -m model.gguf -c 16384"] # CMD ["/server --host 0.0.0.0 --threads 8 -ngl 999 -np 8 -cb -m model.gguf -c 16384"]
CMD [ "python", "-u", "/handler.py" ] CMD [ "python3", "-u", "/handler.py" ]