diff --git a/expose.h b/expose.h index 67f1e1937..24a7e4455 100644 --- a/expose.h +++ b/expose.h @@ -1,5 +1,6 @@ #pragma once +const int stop_token_max = 10; struct load_model_inputs { const int threads; @@ -24,6 +25,7 @@ struct generation_inputs const float top_p; const float rep_pen; const int rep_pen_range; + const char * stop_sequence[stop_token_max]; }; struct generation_outputs { diff --git a/gpttype_adapter.cpp b/gpttype_adapter.cpp index 11f15ead7..197828216 100644 --- a/gpttype_adapter.cpp +++ b/gpttype_adapter.cpp @@ -36,8 +36,8 @@ static std::vector last_n_tokens; static std::vector current_context_tokens; static size_t mem_per_token = 0; static std::vector logits; - static std::vector smartcontext; +static std::vector stop_sequence; inline bool IsNanCheck(float f) { @@ -154,6 +154,14 @@ ModelLoadResult gpttype_load_model(const load_model_inputs inputs, FileFormat in generation_outputs gpttype_generate(const generation_inputs inputs, generation_outputs &output) { + stop_sequence.clear(); + for(int x=0;x:root{--img_sword:url("data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAACAAAAAgBAMAAACBVGfHAAAAAXNSR0IB2cksfwAAAAlwSFlzAAAA7AAAAOwBeShxvQAAAB5QTFRFAAAA/rUT5uvzztXjq1kW5+r14ufw/8YF/8QHr1kWOCO8XQAAAAp0Uk5TAPr+/fwgpBqRPkYi9G8AAAC6SURBVHicjZCxDoIwEIZv0cLmryTiWl/AhOBOcgubcWAmDs5lglEWdWTwgT1MkGvj4A1N+/Xr3Z8S6VpcrXeurN1799baTIOzCMdQyANBg4+QHQIhq2fhMgqqZ/WfcAqE/LfQPR2REgzwoKUSIiB1uoMA3PWIEUCPMD1arHUG08YFvAz0Ymz0T8XMBWpPYMbWE7hs4L49+4R5aGalAbiU/OkEeiAZBGACst1RAFbjqp/IgA63CUQ6gtQbfGErFF7/nE4AAAAASUVORK5CYII=");--img_paper:url("data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAACAAAAAgBAMAAACBVGfHAAAAAXNSR0IB2cksfwAAAAlwSFlzAAAAnQAAAJ0Bj3LnbgAAAB5QTFRF+OmvAAAA89Ze14Rw2cCY1k8/8eGhmEQ/+uqj87Jse3RL9AAAAAp0Uk5T/wD49//9of8rH/vnQeUAAAEOSURBVHicXdG9asMwFAXggx1COmoJ8VgNptkKcmqyGaKSB0i127RkLiTgNV2CVxMo9G177rVSm2ow0se5Vz/Gbdub6YBz2//w0sV59s0wbs5X806X1j4JFN4Bx45La9eE7OM1PANIBHKCuW7CAYgkkIWgEaWeYN5DjHA0AthIZF8ILAgrpBJJ21N1hyGCXXvGA2EJibxJQaVwgUQKlNIkj5AePNKyrWAJtYS952dH6AlpJaQViW3AXc/shlnZFoRHAhd6hpkjrLGEl7lShFKuphXgA0B23WtF+UmwCjy1VijUw70H4iPeH0KaIMl/DKZjIf/lWo/QCJjVSAMYUoSvCH80gjHdEZibCQjJJuYXZ+xAP6Rjil4AAAAASUVORK5CYII=");--img_chat:url("data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAACAAAAAgBAMAAACBVGfHAAAAAXNSR0IB2cksfwAAAAlwSFlzAAAA7AAAAOwBeShxvQAAAB5QTFRFAAAA/tACTK/4OninExQMKU9s160EDQ4Jh28GBQcHB/ICrwAAAAp0Uk5TAP///fb//9b0XrugY20AAADDSURBVHicY2BgmOLi4mIMxJ4MEMAZpAQBKhMgAixKMFAAEzANBYFghIAxGBhRQYUxVVS4uLiDBYygLld2FJQAeYfTGea5RkExsK4pTsouIOAhKCjoADZlKlAxFEAE2IMiUAUYSkwy2jISO8TaYAIsLi0Rzh6uLSaCDVCBxBSPFjeXFA+EgGCimGCaYCJcSyKaoYQF2GECYlAPc3qkgYCgYAo0rhimlwNBSWJKJQMyYEn0ROEzVIgUoPA5PdAUcILDjwEAKyJGXpPDNnQAAAAASUVORK5CYII=")}body{background-color:#303030}.settinglabel input{width:5ch;background-color:inherit;border:none;outline:0}.settinglabel.mininiput{background-color:#fff;color:#555;border:0 solid #ccc;border-radius:4px;width:100%}.settinglabel.mininiput:focus{color:#555}.settingsmall{font-size:10px}.settinglabel input:focus{color:#cdf}#gametext,chunk,chunk *{outline:0 solid transparent}#topmenu{background-color:#757575;padding:8px;display:flex;line-height:normal}#topmenu.always-available,body.connected #topmenu{background-color:#337ab7}#menuitems{display:flex;width:100%}#navbar{margin:0}#navbar li{margin-right:5px;background-color:#828282;border-radius:5px}#navbar li.always-available,body.connected #navbar li{background-color:#4787be}#navbar li>a{color:#fff;font-weight:700}.settingsmenu{display:flex;flex-wrap:wrap;background-color:#4d4d4d;padding:10px}.settingsmenu.always-available,body.connected .settingsmenu{background-color:#295071}#formatmenu{display:none;background-color:#4d4d4d;padding:10px}#formatmenu.always-available,body.connected #formatmenu{background-color:#295071}#connectstatusdiv{display:flex;text-align:right;font-size:14px;width:120px}#gamescreen{overflow-x:hidden;height:66vh;display:flex;vertical-align:bottom;background-color:#262626;color:#fff;font-size:12pt;font-family:Helvetica}@media (max-width:720px){#gamescreen{height:58vh}}@media (max-width:406px){#gamescreen{height:52vh}}#gamescreen span{align-self:flex-end}#gametext{max-height:100%;width:100%;word-wrap:break-word;padding:10px;overflow-y:auto}#actionmenu{margin-top:6px}#actionmenuitems button{width:80px}#messagefield{margin-left:20px}#inputrow.show_mode{grid-template-columns:50px auto 64px}#inputrow{margin-top:10px;padding:0;width:100%;display:grid;grid-template-columns:0% auto 62px}.input_action{content:var(--img_sword)}.input_story{content:var(--img_paper)}#inputrowmode{position:relative;padding-right:0}#inputrowleft{padding-right:10px}#inputrowright{position:relative}#anotetext,#input_text,#memorytext{height:80px;resize:none;overflow:auto;background-color:#404040;color:#fff;resize:vertical}#btnmode{width:100%;height:100%;overflow:auto;overflow-x:hidden}#btnsend{width:100%;height:100%}#btnsend.wait{background-color:#6c6c6e}#btnsend.wait:hover{background-color:#98989a}#anoterowcontainer{display:none}#anoterow{margin-top:10px;padding:0;width:100%;display:grid;grid-template-columns:90% 10%}#anoterowleft{padding-right:10px}#anotetemplate{background-color:#404040;color:#fff;resize:none;overflow:auto}.anotetempbox{display:inline;width:calc(100% - 98px)}.anotetempscale{display:inline;width:94px;padding:6px 3px}#popuptitlebar{padding:10px;background-color:#757575}#popuptitlebar.always-available,body.connected #popuptitlebar{background-color:#337ab7}#popuptitletext{height:100%;display:flex;align-items:center;color:#fff;font-size:12pt}#popuplistheader{padding-left:10px;display:grid;grid-template-columns:28% 10% 60%;color:#737373}#popupcontent{height:325px;overflow-y:scroll}#popupfooter{width:100%;padding:10px;display:flex;justify-content:center;background-color:#4d4d4d}#popupfooter.always-available,body.connected #popupfooter{background-color:#295071}#popupfooter button{width:100px;margin-left:10px;margin-right:10px}#wimenu{padding-top:10px;max-height:100%;width:100%}#aidgpopup{width:350px;background-color:#262626;margin-top:100px}.loadpopup{width:600px;background-color:#262626;margin-top:150px}@media (max-width:768px){.loadpopup{width:100%;background-color:#262626;margin-top:150px}}.workerpopup{background-color:#262626;margin-top:170px}@media (max-width:768px){.workerpopup{width:100%;background-color:#262626;margin-top:170px}}.nspopup{background-color:#262626;margin-top:200px}.nspopup.moderate{margin-top:170px}.nspopup.higher{margin-top:120px}.nspopup.highest{margin-top:80px}.nspopup.fixsize{width:330px}.nspopup.flexsize{width:540px}@media (max-width:620px){.nspopup.flexsize{width:100%}}body:not(.connected) .btn-primary{background-color:#757575;border-color:#4a4a4a}.btn-primary.always-available{background-color:#337ab7;border-color:#2e6da4}body:not(.connected) .btn-primary.focus,body:not(.connected) .btn-primary:focus{background-color:#5c5c5c;border-color:#292929}.btn-primary.always-available:focus,.btn-primary.focus.always-available{background-color:#286090;border-color:#122b40}body:not(.connected) .btn-primary:hover{background-color:#5c5c5c;border-color:#4a4a4a}.btn-primary.always-available:hover{background-color:#286090;border-color:#204d74}body:not(.connected) a.dropdown-item:focus,body:not(.connected) a.dropdown-item:hover{color:#4f4f4f}a.dropdown-item.always-available:focus,a.dropdown-item.always-available:hover{color:#23527c!important}.aidgpopuplistheader{color:#737373;text-align:center}.anotelabel{font-size:10pt;color:#fff}.anotelabel:not(.no-padding){padding-top:10px}.airange{width:100px}.box{border-radius:5px;border:1px solid #646464;padding:4px;background:#373737}.box-label{color:#fff;padding-left:10px;padding-right:10px;padding-bottom:5px;padding-top:5px;display:inline-block;font-size:12px}.chunkhov:hover{color:#c0fc51;cursor:pointer}.chunkhov:hover>action{color:#00fa00}.colorfade,.colorfade *{-moz-transition:color 1s ease-in,text-shadow 1s ease-in;-o-transition:color 1s ease-in,text-shadow 1s ease-in;-webkit-transition:color 1s ease-in,text-shadow 1s ease-in;transition:color 1s ease-in,text-shadow 1s ease-in}.color_blueurl{color:#d3e7ff}.color_blueurl:hover{color:#fff}.color_blueurl:focus{color:#d3e7ff}.color_orange{color:#f7a223}.color_green{color:#3bf723}.color_darkgreen{color:#63975c}.bg_black{background-color:#202020}.bg_black:hover{background-color:#202020}.bg_black:focus{background-color:#202020}.bg_black:disabled{background-color:#202020}.bg_black:disabled:hover{background-color:#202020}.bg_green{background-color:#129c00}.bg_green:hover{background-color:#058105}.bg_green:focus{background-color:#058105}.bg_green:disabled{background-color:#8a8a8a}.bg_green:disabled:hover{background-color:#8a8a8a}.bg_red{background-color:#c40000}.bg_red:hover{background-color:#da0000}.bg_red:focus{background-color:#da0000}.bg_red:disabled{background-color:#8a8a8a}.bg_red:disabled:hover{background-color:#8a8a8a}.color_cyan{color:#7afaff}.color_gray{color:#9b9b9b}.color_red{color:#ff7967}.color_blue{color:#828eff}.color_yellow{color:#f1dd21}.color_pink{color:#ffbdbd}.dropdown-menu{background-color:#757575;width:200px}.dropdown-menu.always-available,body.connected .dropdown-menu{background-color:#337ab7}.dropdown-item{display:block;padding:10px;color:#fff;border-bottom:1px solid #4d4d4d}.dropdown-item.always-available,body.connected .dropdown-item{border-bottom:1px solid #295071}.dropdown-item:first-child{border-top:1px solid #4d4d4d}.dropdown-item:first-child.always-available,body.connected .dropdown-item:first-child{border-top:1px solid #295071}.dropdown-item:hover{background-color:#bababa;text-decoration:none}.dropdown-item.always-available:hover,body.connected .dropdown-item:hover{background-color:#98bcdb}.edit-flash,.edit-flash *{color:#3bf723!important}.status-flash{color:#fce94f!important;text-shadow:0 0 50px #fce94f,0 0 50px #fce94f,0 0 10px #fce94f,0 0 10px #fce94f,0 0 10px #fce94f,0 0 10px #fce94f,0 0 10px #fce94f}.flex{display:flex;align-items:center}.flex-row-container{display:flex;flex-flow:wrap}.flex-row{display:flex;flex-flow:row;flex-grow:1;width:100%}.flex-push-right{margin-left:auto}.formatcolumn{width:25%;padding-left:10px;padding-right:10px;display:inline-block}.formatcolumn>div:first-child{margin-bottom:5px}.formatlabel{color:#fff;padding-left:5px}.hidden{display:none}.heightfull{height:100%}.heighthalf{height:50%}.helpicon{display:inline-block;font-family:sans-serif;font-weight:700;text-align:center;width:2.2ex;height:2.4ex;font-size:1.4ex;line-height:1.8ex;border-radius:1.2ex;margin-right:4px;padding:1px;color:#295071;background:#fff;border:1px solid #fff;text-decoration:none}.statusicon{display:inline-block;font-weight:700;text-align:center;padding-left:8px;padding-right:8px;font-size:30px!important;font-weight:700;text-align:center;font-size:1.4ex;line-height:1.8ex;text-decoration:none;color:#9e9e9e}.statusicon.always-available,body.connected .statusicon{color:#68a2d4}.statusicon.active{color:#3bf723!important}.helpicon:hover,.statusicon:hover{cursor:pointer}.helpicon:hover .helptext,.statusicon.statustoggled .statustext,.statusicon:hover .statustext{display:inline-block;width:250px;background-color:#1f2931;color:#fff;font-size:11pt;font-weight:400;line-height:normal;border-radius:6px;padding:15px;margin-left:10px;border:1px solid #337ab7}.statusicon.statustoggled .statustext.statustext-wide,.statusicon:hover .statustext.statustext-wide{width:350px}.statusiconlabel{pointer-events:none;color:#757575;text-align:center;font-weight:700;font-size:13px}.statusiconlabel.always-available,body.connected .statusiconlabel{color:#337ab7}#usiconlabel{transform:translate(-3px,10px);-moz-transform:translate(-3px,10px);-webkit-transform:translate(-3px,10px);-ms-transform:translate(-3px,10px);-o-transform:translate(-3px,10px)}.status-container{z-index:1;text-shadow:none!important}.helptext,.statustext{display:none;font-family:sans-serif;position:absolute;z-index:1;text-shadow:none!important}.statustext{transform:translate(-105%,30px);-moz-transform:translate(-105%,30px);-webkit-transform:translate(-105%,30px);-ms-transform:translate(-105%,30px);-o-transform:translate(-105%,30px)}.statusheader{padding-bottom:10px}#stat-usactive{text-align:left;height:270px;overflow-y:scroll;position:relative;padding-left:20px}.justifyleft{text-align:left}.justifyright{text-align:right}.layer-container{display:grid}.layer-bottom{grid-area:1/1;z-index:0}.layer-top{grid-area:1/1;z-index:2}.icon-container{position:relative}hr{padding:0;margin:0}.navbar .navbar-nav .nav-link:hover{border-radius:5px;background-color:#bababa}.navbar .navbar-nav .nav-link.always-available:hover,body.connected .navbar .navbar-nav .nav-link:hover{background-color:#98bcdb}body .navbar .navbar-nav .dropdown-item.always-available{background-color:#337ab7}body .navbar .navbar-nav .dropdown-item.always-available:hover{background-color:#98bcdb}.navbar .navbar-nav .nav-link:focus{border-radius:5px;background-color:#bababa}.navbar .navbar-nav .nav-link.always-available:focus,body.connected .navbar .navbar-nav .nav-link:focus{background-color:#98bcdb}.navbar-toggler{background-color:#757575;border:1px solid #bababa;height:45px;width:60px;border-radius:6px}.navbar-toggler.always-available,body.connected .navbar-toggler{border:1px solid #98bcdb}body .navbar-toggler{background-color:#337ab7}.navbar-toggler:hover{background-color:#bababa}.navbar-togger.always-available:hover,body.connected .navbar-togger:hover{background-color:#98bcdb}@media (min-width:768px){.navbar-toggler{display:none}}@media (max-width:768px){.nav-item{margin-bottom:3px}}.navbar-button-bar{display:block;height:2px;width:42px;border:1px solid #fff}.navbar-button-bar+.navbar-button-bar{margin-top:4px}.navcontainer{width:100%}.nowrap{white-space:nowrap}.popupcontainer{position:absolute;top:0;left:0;z-index:3;width:100%;height:100%;flex-direction:column;align-items:center}.popupbg{position:fixed;top:0;bottom:0;left:0;right:0;z-index:-1;background-color:rgba(0,0,0,.5);flex-direction:column;align-items:center}.popuptitlebar{padding:10px;background-color:#757575}body.connected .popuptitlebar{background-color:#337ab7}.popuptitletext{display:flex;align-items:center;color:#fff;font-size:12pt}.popuperror{color:#ef2929;text-align:center}.popupfooter{width:100%;padding:10px;display:flex;justify-content:center;background-color:#4d4d4d}.popupfooter.always-available,body.connected .popupfooter{background-color:#295071}.popupfooter button{width:100px;margin-left:10px;margin-right:10px}.settingitem{width:50%;padding-left:10px;padding-right:10px;padding-bottom:5px;padding-top:5px;display:inline-block;border-bottom:1px solid #12324f}.settinglabel{color:#fff;display:flex;flex-flow:wrap}.settingminmax{display:grid;grid-template-columns:50% 50%}.settingminmax div{font-size:8pt;color:#fff}.spacer{display:inline-block;width:50px}@media only screen and (max-width:768px){.SideMenu.open{width:100%}}.tokens-in-box{position:relative}.token-budget{right:20px;bottom:3px;color:gray;position:absolute;font-size:8px;-webkit-user-select:none;-moz-user-select:none;-ms-user-select:none;user-select:none}.btn-secondary{padding:2px 6px}.maincontainer{padding-right:4px;padding-left:4px;margin-right:auto;margin-left:auto}.shareStory,.workerTableDiv{max-height:320px;overflow-y:auto;overflow-x:hidden}.workerTable{color:#fff;font-size:min(1.4vw,14px)}.workerTable>tbody>tr>td{padding:min(.4vw,5px)}.scenariopopup{width:600px;background-color:#262626;margin-top:60px}@media (max-width:768px){.scenariopopup{width:100%;background-color:#262626;margin-top:70px}}.scenariosearch{margin-top:8px;margin-left:8px;width:calc(100% - 16px);padding:4px}.scenariosearchbox1{display:inline;width:calc(100% - 98px)}.scenariosearchbox2{display:inline;width:94px;padding:6px 3px}.scenariogrid{height:330px;overflow-y:auto;margin-top:4px;padding:8px;display:grid;gap:8px;grid-template-columns:repeat(auto-fit,minmax(150px,1fr));grid-auto-rows:55px}.scenariodesc{padding:4px 12px;width:100%;height:120px;color:#b7e2ff;overflow-y:auto}.scenarioitem{font-size:15px;color:#fff;font-weight:500;font-family:'Segoe UI',Tahoma;background-repeat:no-repeat;background-position:top 4px left 4px,center;background-size:24px,100%;padding:2px 2px}.scenarioitem.blue{background-image:var(--img_paper),linear-gradient(to right,#63aae7,#337ab7)}.scenarioitem.blue:hover{background-image:var(--img_paper),linear-gradient(to right,#7ebbf0,#438ac7)}.scenarioitem.blue:focus{background-image:var(--img_paper),linear-gradient(to right,#4c7aa3,#4c7aa3)}.scenarioitem.green{background-image:var(--img_sword),linear-gradient(to right,#58db6e,#2ba04e)}.scenarioitem.green:hover{background-image:var(--img_sword),linear-gradient(to right,#68e47d,#37b85e)}.scenarioitem.green:focus{background-image:var(--img_sword),linear-gradient(to right,#53a34c,#4ca353)}.scenarioitem.red{background-image:var(--img_chat),linear-gradient(to right,#e76363,#b73333)}.scenarioitem.red:hover{background-image:var(--img_chat),linear-gradient(to right,#f07e7e,#c74343)}.scenarioitem.red:focus{background-image:var(--img_chat),linear-gradient(to right,#a34c4c,#a34c4c)}.scenarioitem.purple{background-image:none,linear-gradient(to right,#dc63e7,#ac33b7)}.scenarioitem.purple:hover{background-image:none,linear-gradient(to right,#f07ee6,#c743c7)}.scenarioitem.purple:focus{background-image:none,linear-gradient(to right,#a34c9c,#a34ca3)}.scenarioitem.yellow{background-image:none,linear-gradient(to right,#daae5d,#ad8823)}.scenarioitem.yellow:hover{background-image:none,linear-gradient(to right,#e0c56e,#bba632)}.scenarioitem.yellow:focus{background-image:none,linear-gradient(to right,#a38c4c,#a38c4c)}.widelbtn{font-size:12px;height:24px;padding:5px;margin:2px;font-weight:bolder}.wiinputkey{font-size:14px;height:24px;padding:2px;margin:0;width:20vw}.wiinputval{font-size:14px;height:24px;padding:2px;margin:0;width:60vw}.wilist{background-color:#434343;overflow-y:auto;max-height:250px;min-height:60px}.witoggleroff,.witoggleroff:focus,.witoggleroff:hover{color:transparent;text-shadow:0 0 0 gray;text-decoration:none}.witoggleron,.witoggleron:focus,.witoggleron:hover{color:transparent;text-shadow:0 0 0 #0cdb0c;text-decoration:none}.lastreq{font-size:9pt;padding-top:2px}.outerloader{display:flex;margin:auto;align-items:center;justify-content:center}.outerloadernum{position:absolute;color:#fff}.innerloader{width:32px;height:32px;border:6px solid #f3f3f3;border-top:6px solid #3498db;border-radius:50%;animation:spin 2s linear infinite}.innerloader.greenloader{border-top:6px solid #0dff35}.innerloader.redloader{border-top:6px solid #f7610a}.loader2{border:6px solid #8a8686;border-top:6px solid peru;border-radius:50%;width:32px;height:32px;display:flex;margin:auto;align-items:center;justify-content:center;animation:spin 2s linear infinite;top:0;bottom:0;left:0;right:0;position:absolute;margin:auto}.imagelabel{bottom:20%;left:0;right:0;position:absolute;margin:auto;text-align:center;color:peru;font-weight:700}.storyimgfloat{float:right;position:relative;padding:4px}.storyimg{text-align:center;position:relative;padding:4px;margin:0 auto}.zoomedimgdiv{text-align:center;position:relative;margin:0 auto;padding-top:6px;padding-bottom:4px}.zoomedimgdesc{max-height:120px;overflow-y:auto;overflow-x:hidden}.mdlpicker::-webkit-calendar-picker-indicator{opacity:100}@keyframes spin{0%{transform:rotate(0)}100%{transform:rotate(360deg)}}@media screen and (hover:hover) and (any-pointer:fine){::-webkit-scrollbar{width:5px}::-webkit-scrollbar-track{background:0 0}::-webkit-scrollbar-thumb{background-color:#9191915e;border-radius:10px;border:transparent}::-webkit-scrollbar-thumb:hover{background:#9494948a}}label.unstyled{font-weight:400;margin-bottom:0;display:block}.hlchunk{color:#cedaf0} - +
diff --git a/koboldcpp.py b/koboldcpp.py index 783303e35..737ed406c 100644 --- a/koboldcpp.py +++ b/koboldcpp.py @@ -8,6 +8,8 @@ import os import argparse import json, http.server, threading, socket, sys, time +stop_token_max = 10 + class load_model_inputs(ctypes.Structure): _fields_ = [("threads", ctypes.c_int), ("max_context_length", ctypes.c_int), @@ -29,7 +31,8 @@ class generation_inputs(ctypes.Structure): ("top_k", ctypes.c_int), ("top_p", ctypes.c_float), ("rep_pen", ctypes.c_float), - ("rep_pen_range", ctypes.c_int)] + ("rep_pen_range", ctypes.c_int), + ("stop_sequence", ctypes.c_char_p * stop_token_max)] class generation_outputs(ctypes.Structure): _fields_ = [("status", ctypes.c_int), @@ -85,7 +88,7 @@ def load_model(model_filename,batch_size=8,max_context_length=512,n_parts_overwr ret = handle.load_model(inputs) return ret -def generate(prompt,max_length=20, max_context_length=512,temperature=0.8,top_k=100,top_p=0.85,rep_pen=1.1,rep_pen_range=128,seed=-1): +def generate(prompt,max_length=20, max_context_length=512,temperature=0.8,top_k=100,top_p=0.85,rep_pen=1.1,rep_pen_range=128,seed=-1,stop_sequence=[]): inputs = generation_inputs() outputs = ctypes.create_unicode_buffer(ctypes.sizeof(generation_outputs)) inputs.prompt = prompt.encode("UTF-8") @@ -97,6 +100,11 @@ def generate(prompt,max_length=20, max_context_length=512,temperature=0.8,top_k= inputs.rep_pen = rep_pen inputs.rep_pen_range = rep_pen_range inputs.seed = seed + for n in range(0,stop_token_max): + if n >= len(stop_sequence): + inputs.stop_sequence[n] = "".encode("UTF-8") + else: + inputs.stop_sequence[n] = stop_sequence[n].encode("UTF-8") ret = handle.generate(inputs,outputs) if(ret.status==1): return ret.text.decode("UTF-8","ignore") @@ -170,6 +178,12 @@ class ServerRequestHandler(http.server.SimpleHTTPRequestHandler): self.end_headers() self.wfile.write(json.dumps({"values": []}).encode()) return + + if self.path.endswith(('/api/v1/info/version', '/api/latest/info/version')): + self.send_response(200) + self.end_headers() + self.wfile.write(json.dumps({"result":"1.2.2"}).encode()) + return self.send_response(404) self.end_headers() @@ -229,7 +243,8 @@ class ServerRequestHandler(http.server.SimpleHTTPRequestHandler): top_p=genparams.get('top_p', 0.85), rep_pen=genparams.get('rep_pen', 1.1), rep_pen_range=genparams.get('rep_pen_range', 128), - seed=-1 + seed=-1, + stop_sequence=genparams.get('stop_sequence', []) ) print("\nOutput: " + recvtxt) res = {"results": [{"text": recvtxt}]} @@ -242,7 +257,8 @@ class ServerRequestHandler(http.server.SimpleHTTPRequestHandler): top_p=genparams.get('top_p', 0.85), rep_pen=genparams.get('rep_pen', 1.1), rep_pen_range=genparams.get('rep_pen_range', 128), - seed=-1 + seed=-1, + stop_sequence=genparams.get('stop_sequence', []) ) print("\nOutput: " + recvtxt) res = {"data": {"seqs":[recvtxt]}} diff --git a/llama_adapter.cpp b/llama_adapter.cpp index fde59f5b3..6e0cf8581 100644 --- a/llama_adapter.cpp +++ b/llama_adapter.cpp @@ -34,6 +34,7 @@ static llama_context *ctx; static std::vector last_n_tokens; static std::vector current_context_tokens; static std::vector smartcontext; +static std::vector stop_sequence; bool llama_load_model(const load_model_inputs inputs, FileFormat in_file_format) { @@ -81,6 +82,15 @@ bool llama_load_model(const load_model_inputs inputs, FileFormat in_file_format) generation_outputs llama_generate(const generation_inputs inputs, generation_outputs &output) { + stop_sequence.clear(); + for(int x=0;x &embd) +{ + std::cout << "["; + bool first = true; + for (auto i : embd) + { + if (!first) + { + std::cout << ','; + } + first = false; + std::cout << i; + } + std::cout << "]\n"; +} void print_tok_vec(std::vector &embd) { std::cout << "["; diff --git a/model_adapter.h b/model_adapter.h index 81ac96b8e..c5ca98ab9 100644 --- a/model_adapter.h +++ b/model_adapter.h @@ -45,6 +45,7 @@ void timer_start(); double timer_check(); void print_tok_vec(std::vector &embd); void print_tok_vec(std::vector &embd); +void print_vec(std::vector &embd); std::vector LongestCommonSubseq(const std::vector x, const std::vector y); bool ArrStartWith(const std::vector targetArray, const std::vector searchSeq); int ArrFindIndexOf(const std::vector targetArray, const std::vector searchSeq);