vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix bug & add truncation return

Browse source
This commit is contained in:
jwj7140 2023-07-05 01:02:05 +09:00
parent f713dd515d
commit 41f7a5004a
1 changed files with 12 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

13
examples/server/api_like_OAI.py
View file

@ -69,7 +69,10 @@ def make_postData(body, chat=False, stream=False):
if(is_present(body, "mirostat_eta")): postData["mirostat_eta"] = body["mirostat_eta"]
if(is_present(body, "seed")): postData["seed"] = body["seed"]
if(is_present(body, "logit_bias")): postData["logit_bias"] = [[int(token), body["logit_bias"][token]] for token in body["logit_bias"].keys()]
postData["stop"] = [args.stop]
if (args.stop != ""):
postData["stop"] = [args.stop]
else:
postData["stop"] = []
if(is_present(body, "stop")): postData["stop"] += body["stop"]
postData["n_keep"] = -1
postData["stream"] = stream
@ -81,6 +84,7 @@ def make_resData(data, chat=False, promptToken=[]):
"id": "chatcmpl" if (chat) else "cmpl",
"object": "chat.completion" if (chat) else "text_completion",
"created": int(time.time()),
"truncated": data["truncated"],
"model": "LLaMA_CPP",
"usage": {
"prompt_tokens": data["tokens_evaluated"],
@ -196,6 +200,7 @@ def completion():
if (not stream):
data = requests.request("POST", urllib.parse.urljoin(args.llama_api, "/completion"), data=json.dumps(postData))
print(data.json())
resData = make_resData(data.json(), chat=False, promptToken=promptToken)
return jsonify(resData)
else:
@ -210,5 +215,11 @@ def completion():
return Response(generate(), mimetype='text/event-stream')
if __name__ == '__main__':
app.run(args.host, port=args.port)