vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

added some checks to skip generation if busy

Browse source
This commit is contained in:
Concedo 2023-07-22 23:10:04 +08:00
parent 2807d98fd4
commit fa0270df7c
1 changed files with 24 additions and 11 deletions
Download patch file Download diff file Expand all files Collapse all files

35
koboldcpp.py
View file

@ -1374,7 +1374,8 @@ def run_horde_worker(args, api_key, worker_name):
current_id = None current_id = None
current_payload = None current_payload = None
print("Embedded Horde Worker '"+worker_name+"' Starting...\n(To use your own KAI Bridge/Scribe worker instead, don't set your API key)") current_generation = None
print("===\nEmbedded Horde Worker '"+worker_name+"' Starting...\n(To use your own KAI Bridge/Scribe worker instead, don't set your API key)")
BRIDGE_AGENT = f"KoboldCppEmbedWorker:1:https://github.com/LostRuins/koboldcpp" BRIDGE_AGENT = f"KoboldCppEmbedWorker:1:https://github.com/LostRuins/koboldcpp"
cluster = "https://horde.koboldai.net" cluster = "https://horde.koboldai.net"
while exitcounter < 10: while exitcounter < 10:
@ -1385,6 +1386,8 @@ def run_horde_worker(args, api_key, worker_name):
break break
while exitcounter < 10: while exitcounter < 10:
currentjob_attempts = 0
current_generation = None
#first, make sure we are not generating #first, make sure we are not generating
if modelbusy.locked(): if modelbusy.locked():
@ -1421,28 +1424,38 @@ def run_horde_worker(args, api_key, worker_name):
current_generation = make_url_request(f'http://localhost:{args.port}/api/v1/generate', current_payload) current_generation = make_url_request(f'http://localhost:{args.port}/api/v1/generate', current_payload)
if current_generation: if current_generation:
break break
else:
currentjob_attempts += 1
if currentjob_attempts>10:
break
print("Server Busy - Not ready to generate...") print("Server Busy - Not ready to generate...")
time.sleep(5) time.sleep(5)
#submit reply #submit reply
submit_dict = { if current_generation:
"id": current_id, submit_dict = {
"generation": current_generation["results"][0]["text"], "id": current_id,
"state": "ok" "generation": current_generation["results"][0]["text"],
} "state": "ok"
reply = make_url_request(cluster + '/api/v2/generate/text/submit', submit_dict) }
if not reply: reply = make_url_request(cluster + '/api/v2/generate/text/submit', submit_dict)
exitcounter += 1 if not reply:
print("\nError: Job submit failed.") exitcounter += 1
print("\nError: Job submit failed.")
else:
print(f'\nSubmitted generation to {cluster} with id {current_id} and contributed for {reply["reward"]}')
else: else:
print(f'\nSubmitted generation to {cluster} with id {current_id} and contributed for {reply["reward"]}') print("\nError: Abandoned current job due to errors. Getting new job.")
current_id = None current_id = None
current_payload = None current_payload = None
time.sleep(1) time.sleep(1)
if exitcounter<100: if exitcounter<100:
print("Horde Worker Shutdown - Too many errors.") print("Horde Worker Shutdown - Too many errors.")
time.sleep(2)
else: else:
print("Horde Worker Shutdown - Server Closing.") print("Horde Worker Shutdown - Server Closing.")
time.sleep(1)
sys.exit(2)
def main(args): def main(args):
embedded_kailite = None embedded_kailite = None