vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

OpenAI compat API adapter (#466)

Browse source 
* feat: oai-adapter

* simplify optional adapter for instruct start and end tags

---------

Co-authored-by: Concedo <39025047+LostRuins@users.noreply.github.com>
This commit is contained in:
Matěj Štágl 2023-10-09 17:24:48 +02:00 • committed by GitHub
parent 80e53af236
commit 96e9539f05
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
1 changed files with 23 additions and 5 deletions
Download patch file Download diff file Expand all files Collapse all files

28
koboldcpp.py
View file

@ -412,16 +412,34 @@ class ServerRequestHandler(http.server.SimpleHTTPRequestHandler):
elif api_format==4: elif api_format==4:
# translate openai chat completion messages format into one big string. # translate openai chat completion messages format into one big string.
messages_array = genparams.get('messages', []) messages_array = genparams.get('messages', [])
adapter_obj = genparams.get('adapter', {})
messages_string = "" messages_string = ""
system_message_start = adapter_obj.get("system_start", "\n### Instruction:\n")
system_message_end = adapter_obj.get("system_end", "")
user_message_start = adapter_obj.get("user_start", "\n### Instruction:\n")
user_message_end = adapter_obj.get("user_end", "")
assistant_message_start = adapter_obj.get("assistant_start", "\n### Response:\n")
assistant_message_end = adapter_obj.get("assistant_end", "")
for message in messages_array: for message in messages_array:
if message['role'] == "system": if message['role'] == "system":
messages_string+="\n### Instruction:\n" messages_string += system_message_start
elif message['role'] == "user": elif message['role'] == "user":
messages_string+="\n### Instruction:\n" messages_string += user_message_start
elif message['role'] == "assistant": elif message['role'] == "assistant":
messages_string+="\n### Response:\n" messages_string += assistant_message_start
messages_string+=message['content']
messages_string += "\n### Response:\n" messages_string += message['content']
if message['role'] == "system":
messages_string += system_message_end
elif message['role'] == "user":
messages_string += user_message_end
elif message['role'] == "assistant":
messages_string += assistant_message_end
messages_string += assistant_message_start
genparams["prompt"] = messages_string genparams["prompt"] = messages_string
frqp = genparams.get('frequency_penalty', 0.1) frqp = genparams.get('frequency_penalty', 0.1)
scaled_rep_pen = genparams.get('presence_penalty', frqp) + 1 scaled_rep_pen = genparams.get('presence_penalty', frqp) + 1