vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

sync: minja (https://github.com/ggerganov/llama.cpp/pull/11774)

Browse source
This commit is contained in:
ochafik 2025-02-09 18:09:26 +00:00
parent 91542ca245
commit 8d82be902e
1 changed files with 24 additions and 9 deletions
Download patch file Download diff file Expand all files Collapse all files

33
common/minja.hpp
View file

@ -1385,6 +1385,13 @@ static std::string strip(const std::string & s) {
return s.substr(start, end - start + 1); return s.substr(start, end - start + 1);
} }
static std::string capitalize(const std::string & s) {
if (s.empty()) return s;
auto result = s;
result[0] = std::toupper(result[0]);
return result;
}
static std::string html_escape(const std::string & s) { static std::string html_escape(const std::string & s) {
std::string result; std::string result;
result.reserve(s.size()); result.reserve(s.size());
@ -1462,6 +1469,9 @@ public:
if (method->get_name() == "strip") { if (method->get_name() == "strip") {
vargs.expectArgs("strip method", {0, 0}, {0, 0}); vargs.expectArgs("strip method", {0, 0}, {0, 0});
return Value(strip(str)); return Value(strip(str));
} else if (method->get_name() == "capitalize") {
vargs.expectArgs("capitalize method", {0, 0}, {0, 0});
return Value(capitalize(str));
} else if (method->get_name() == "endswith") { } else if (method->get_name() == "endswith") {
vargs.expectArgs("endswith method", {1, 1}, {0, 0}); vargs.expectArgs("endswith method", {1, 1}, {0, 0});
auto suffix = vargs.args[0].get<std::string>(); auto suffix = vargs.args[0].get<std::string>();
@ -1792,7 +1802,7 @@ private:
auto left = parseStringConcat(); auto left = parseStringConcat();
if (!left) throw std::runtime_error("Expected left side of 'logical compare' expression"); if (!left) throw std::runtime_error("Expected left side of 'logical compare' expression");
static std::regex compare_tok(R"(==|!=|<=?|>=?|in\b|is\b|not[\r\n\s]+in\b)"); static std::regex compare_tok(R"(==|!=|<=?|>=?|in\b|is\b|not\s+in\b)");
static std::regex not_tok(R"(not\b)"); static std::regex not_tok(R"(not\b)");
std::string op_str; std::string op_str;
while (!(op_str = consumeToken(compare_tok)).empty()) { while (!(op_str = consumeToken(compare_tok)).empty()) {
@ -2171,7 +2181,7 @@ private:
using TemplateTokenIterator = TemplateTokenVector::const_iterator; using TemplateTokenIterator = TemplateTokenVector::const_iterator;
std::vector<std::string> parseVarNames() { std::vector<std::string> parseVarNames() {
static std::regex varnames_regex(R"(((?:\w+)(?:[\r\n\s]*,[\r\n\s]*(?:\w+))*)[\r\n\s]*)"); static std::regex varnames_regex(R"(((?:\w+)(?:\s*,\s*(?:\w+))*)\s*)");
std::vector<std::string> group; std::vector<std::string> group;
if ((group = consumeTokenGroups(varnames_regex)).empty()) throw std::runtime_error("Expected variable names"); if ((group = consumeTokenGroups(varnames_regex)).empty()) throw std::runtime_error("Expected variable names");
@ -2194,13 +2204,13 @@ private:
} }
TemplateTokenVector tokenize() { TemplateTokenVector tokenize() {
static std::regex comment_tok(R"(\{#([-~]?)([\s\S\r\n]*?)([-~]?)#\})"); static std::regex comment_tok(R"(\{#([-~]?)([\s\S]*?)([-~]?)#\})");
static std::regex expr_open_regex(R"(\{\{([-~])?)"); static std::regex expr_open_regex(R"(\{\{([-~])?)");
static std::regex block_open_regex(R"(^\{%([-~])?[\s\n\r]*)"); static std::regex block_open_regex(R"(^\{%([-~])?\s*)");
static std::regex block_keyword_tok(R"((if|else|elif|endif|for|endfor|generation|endgeneration|set|endset|block|endblock|macro|endmacro|filter|endfilter|break|continue)\b)"); static std::regex block_keyword_tok(R"((if|else|elif|endif|for|endfor|generation|endgeneration|set|endset|block|endblock|macro|endmacro|filter|endfilter|break|continue)\b)");
static std::regex non_text_open_regex(R"(\{\{|\{%|\{#)"); static std::regex non_text_open_regex(R"(\{\{|\{%|\{#)");
static std::regex expr_close_regex(R"([\s\n\r]*([-~])?\}\})"); static std::regex expr_close_regex(R"(\s*([-~])?\}\})");
static std::regex block_close_regex(R"([\s\n\r]*([-~])?%\})"); static std::regex block_close_regex(R"(\s*([-~])?%\})");
TemplateTokenVector tokens; TemplateTokenVector tokens;
std::vector<std::string> group; std::vector<std::string> group;
@ -2284,7 +2294,7 @@ private:
auto post_space = parseBlockClose(); auto post_space = parseBlockClose();
tokens.push_back(std::make_unique<EndGenerationTemplateToken>(location, pre_space, post_space)); tokens.push_back(std::make_unique<EndGenerationTemplateToken>(location, pre_space, post_space));
} else if (keyword == "set") { } else if (keyword == "set") {
static std::regex namespaced_var_regex(R"((\w+)[\s\n\r]*\.[\s\n\r]*(\w+))"); static std::regex namespaced_var_regex(R"((\w+)\s*\.\s*(\w+))");
std::string ns; std::string ns;
std::vector<std::string> var_names; std::vector<std::string> var_names;
@ -2336,6 +2346,11 @@ private:
throw std::runtime_error("Unexpected block: " + keyword); throw std::runtime_error("Unexpected block: " + keyword);
} }
} else if (std::regex_search(it, end, match, non_text_open_regex)) { } else if (std::regex_search(it, end, match, non_text_open_regex)) {
if (!match.position()) {
if (match[0] != "{#")
throw std::runtime_error("Internal error: Expected a comment");
throw std::runtime_error("Missing end of comment tag");
}
auto text_end = it + match.position(); auto text_end = it + match.position();
text = std::string(it, text_end); text = std::string(it, text_end);
it = text_end; it = text_end;
@ -2400,7 +2415,7 @@ private:
auto text = text_token->text; auto text = text_token->text;
if (post_space == SpaceHandling::Strip) { if (post_space == SpaceHandling::Strip) {
static std::regex trailing_space_regex(R"((\s|\r|\n)+$)"); static std::regex trailing_space_regex(R"(\s+$)");
text = std::regex_replace(text, trailing_space_regex, ""); text = std::regex_replace(text, trailing_space_regex, "");
} else if (options.lstrip_blocks && it != end) { } else if (options.lstrip_blocks && it != end) {
auto i = text.size(); auto i = text.size();
@ -2410,7 +2425,7 @@ private:
} }
} }
if (pre_space == SpaceHandling::Strip) { if (pre_space == SpaceHandling::Strip) {
static std::regex leading_space_regex(R"(^(\s|\r|\n)+)"); static std::regex leading_space_regex(R"(^\s+)");
text = std::regex_replace(text, leading_space_regex, ""); text = std::regex_replace(text, leading_space_regex, "");
} else if (options.trim_blocks && (it - 1) != begin && !dynamic_cast<ExpressionTemplateToken*>((*(it - 2)).get())) { } else if (options.trim_blocks && (it - 1) != begin && !dynamic_cast<ExpressionTemplateToken*>((*(it - 2)).get())) {
if (text.length() > 0 && text[0] == '\n') { if (text.length() > 0 && text[0] == '\n') {