vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Update minja.hpp 202aa2f3de

Browse source
This commit is contained in:
ochafik 2024-12-26 21:36:34 +00:00
parent f0bd69380b
commit f645887e0c
1 changed files with 60 additions and 28 deletions
Download patch file Download diff file Expand all files Collapse all files

88
common/minja.hpp
View file

@ -18,6 +18,12 @@
#include <unordered_set> #include <unordered_set>
#include <json.hpp> #include <json.hpp>
#ifdef _WIN32
#define ENDL "\r\n"
#else
#define ENDL "\n"
#endif
using json = nlohmann::ordered_json; using json = nlohmann::ordered_json;
namespace minja { namespace minja {
@ -32,6 +38,15 @@ struct Options {
struct ArgumentsValue; struct ArgumentsValue;
static std::string normalize_newlines(const std::string & s) {
#ifdef _WIN32
static const std::regex nl_regex("\r\n");
return std::regex_replace(s, nl_regex, "\n");
#else
return s;
#endif
}
/* Values that behave roughly like in Python. */ /* Values that behave roughly like in Python. */
class Value : public std::enable_shared_from_this<Value> { class Value : public std::enable_shared_from_this<Value> {
public: public:
@ -76,7 +91,7 @@ private:
void dump(std::ostringstream & out, int indent = -1, int level = 0, bool to_json = false) const { void dump(std::ostringstream & out, int indent = -1, int level = 0, bool to_json = false) const {
auto print_indent = [&](int level) { auto print_indent = [&](int level) {
if (indent > 0) { if (indent > 0) {
out << "\n"; out << ENDL;
for (int i = 0, n = level * indent; i < n; ++i) out << ' '; for (int i = 0, n = level * indent; i < n; ++i) out << ' ';
} }
}; };
@ -547,11 +562,11 @@ static std::string error_location_suffix(const std::string & source, size_t pos)
auto max_line = std::count(start, end, '\n') + 1; auto max_line = std::count(start, end, '\n') + 1;
auto col = pos - std::string(start, it).rfind('\n'); auto col = pos - std::string(start, it).rfind('\n');
std::ostringstream out; std::ostringstream out;
out << " at row " << line << ", column " << col << ":\n"; out << " at row " << line << ", column " << col << ":" ENDL;
if (line > 1) out << get_line(line - 1) << "\n"; if (line > 1) out << get_line(line - 1) << ENDL;
out << get_line(line) << "\n"; out << get_line(line) << ENDL;
out << std::string(col - 1, ' ') << "^" << "\n"; out << std::string(col - 1, ' ') << "^" << ENDL;
if (line < max_line) out << get_line(line + 1) << "\n"; if (line < max_line) out << get_line(line + 1) << ENDL;
return out.str(); return out.str();
} }
@ -786,7 +801,7 @@ public:
std::string render(const std::shared_ptr<Context> & context) const { std::string render(const std::shared_ptr<Context> & context) const {
std::ostringstream out; std::ostringstream out;
render(out, context); render(out, context);
return out.str(); return normalize_newlines(out.str());
} }
}; };
@ -1214,8 +1229,8 @@ public:
if (!l.to_bool()) return Value(false); if (!l.to_bool()) return Value(false);
return right->evaluate(context).to_bool(); return right->evaluate(context).to_bool();
} else if (op == Op::Or) { } else if (op == Op::Or) {
if (l.to_bool()) return Value(true); if (l.to_bool()) return l;
return right->evaluate(context).to_bool(); return right->evaluate(context);
} }
auto r = right->evaluate(context); auto r = right->evaluate(context);
@ -1292,6 +1307,10 @@ struct ArgumentsExpression {
static std::string strip(const std::string & s) { static std::string strip(const std::string & s) {
static std::regex trailing_spaces_regex("^\\s+|\\s+$"); static std::regex trailing_spaces_regex("^\\s+|\\s+$");
return std::regex_replace(s, trailing_spaces_regex, ""); return std::regex_replace(s, trailing_spaces_regex, "");
// auto start = s.find_first_not_of(" \t\n\r");
// if (start == std::string::npos) return "";
// auto end = s.find_last_not_of(" \t\n\r");
// return s.substr(start, end - start + 1);
} }
static std::string html_escape(const std::string & s) { static std::string html_escape(const std::string & s) {
@ -1302,7 +1321,7 @@ static std::string html_escape(const std::string & s) {
case '&': result += "&amp;"; break; case '&': result += "&amp;"; break;
case '<': result += "&lt;"; break; case '<': result += "&lt;"; break;
case '>': result += "&gt;"; break; case '>': result += "&gt;"; break;
case '"': result += "&quot;"; break; case '"': result += "&#34;"; break;
case '\'': result += "&apos;"; break; case '\'': result += "&apos;"; break;
default: result += c; break; default: result += c; break;
} }
@ -2101,13 +2120,14 @@ private:
static std::regex expr_open_regex(R"(\{\{([-~])?)"); static std::regex expr_open_regex(R"(\{\{([-~])?)");
static std::regex block_open_regex(R"(^\{%([-~])?[\s\n\r]*)"); static std::regex block_open_regex(R"(^\{%([-~])?[\s\n\r]*)");
static std::regex block_keyword_tok(R"((if|else|elif|endif|for|endfor|set|endset|block|endblock|macro|endmacro|filter|endfilter)\b)"); static std::regex block_keyword_tok(R"((if|else|elif|endif|for|endfor|set|endset|block|endblock|macro|endmacro|filter|endfilter)\b)");
static std::regex text_regex(R"([\s\S\n\r]*?($|(?=\{\{|\{%|\{#)))"); static std::regex non_text_open_regex(R"(\{\{|\{%|\{#)");
static std::regex expr_close_regex(R"([\s\n\r]*([-~])?\}\})"); static std::regex expr_close_regex(R"([\s\n\r]*([-~])?\}\})");
static std::regex block_close_regex(R"([\s\n\r]*([-~])?%\})"); static std::regex block_close_regex(R"([\s\n\r]*([-~])?%\})");
TemplateTokenVector tokens; TemplateTokenVector tokens;
std::vector<std::string> group; std::vector<std::string> group;
std::string text; std::string text;
std::smatch match;
try { try {
while (it != end) { while (it != end) {
@ -2228,10 +2248,15 @@ private:
} else { } else {
throw std::runtime_error("Unexpected block: " + keyword); throw std::runtime_error("Unexpected block: " + keyword);
} }
} else if (!(text = consumeToken(text_regex, SpaceHandling::Keep)).empty()) { } else if (std::regex_search(it, end, match, non_text_open_regex)) {
auto text_end = it + match.position();
text = std::string(it, text_end);
it = text_end;
tokens.push_back(std::make_unique<TextTemplateToken>(location, SpaceHandling::Keep, SpaceHandling::Keep, text)); tokens.push_back(std::make_unique<TextTemplateToken>(location, SpaceHandling::Keep, SpaceHandling::Keep, text));
} else { } else {
if (it != end) throw std::runtime_error("Unexpected character"); text = std::string(it, end);
it = end;
tokens.push_back(std::make_unique<TextTemplateToken>(location, SpaceHandling::Keep, SpaceHandling::Keep, text));
} }
} }
return tokens; return tokens;
@ -2280,24 +2305,31 @@ private:
SpaceHandling post_space = it != end ? (*it)->pre_space : SpaceHandling::Keep; SpaceHandling post_space = it != end ? (*it)->pre_space : SpaceHandling::Keep;
auto text = text_token->text; auto text = text_token->text;
if (pre_space == SpaceHandling::Strip) {
static std::regex leading_space_regex(R"(^(\s|\r|\n)+)");
text = std::regex_replace(text, leading_space_regex, "");
} else if (options.trim_blocks && (it - 1) != begin && !dynamic_cast<ExpressionTemplateToken*>((*(it - 2)).get())) {
static std::regex leading_line(R"(^[ \t]*\r?\n)");
text = std::regex_replace(text, leading_line, "");
}
if (post_space == SpaceHandling::Strip) { if (post_space == SpaceHandling::Strip) {
static std::regex trailing_space_regex(R"((\s|\r|\n)+$)"); static std::regex trailing_space_regex(R"((\s|\r|\n)+$)");
text = std::regex_replace(text, trailing_space_regex, ""); text = std::regex_replace(text, trailing_space_regex, "");
} else if (options.lstrip_blocks && it != end) { } else if (options.lstrip_blocks && it != end) {
static std::regex trailing_last_line_space_regex(R"((\r?\n)[ \t]*$)"); auto i = text.size();
text = std::regex_replace(text, trailing_last_line_space_regex, "$1"); while (i > 0 && (text[i - 1] == ' ' || text[i - 1] == '\t')) i--;
if ((i == 0 && (it - 1) == begin) || (i > 0 && text[i - 1] == '\n')) {
text.resize(i);
}
}
if (pre_space == SpaceHandling::Strip) {
static std::regex leading_space_regex(R"(^(\s|\r|\n)+)");
text = std::regex_replace(text, leading_space_regex, "");
} else if (options.trim_blocks && (it - 1) != begin && !dynamic_cast<ExpressionTemplateToken*>((*(it - 2)).get())) {
if (text.length() > 0 && text[0] == '\n') {
text.erase(0, 1);
}
} }
if (it == end && !options.keep_trailing_newline) { if (it == end && !options.keep_trailing_newline) {
static std::regex r(R"(\r?\n$)"); auto i = text.size();
text = std::regex_replace(text, r, ""); // Strip one trailing newline if (i > 0 && text[i - 1] == '\n') {
i--;
if (i > 0 && text[i - 1] == '\r') i--;
text.resize(i);
}
} }
children.emplace_back(std::make_shared<TextNode>(token->location, text)); children.emplace_back(std::make_shared<TextNode>(token->location, text));
} else if (auto expr_token = dynamic_cast<ExpressionTemplateToken*>(token.get())) { } else if (auto expr_token = dynamic_cast<ExpressionTemplateToken*>(token.get())) {
@ -2357,7 +2389,7 @@ private:
public: public:
static std::shared_ptr<TemplateNode> parse(const std::string& template_str, const Options & options) { static std::shared_ptr<TemplateNode> parse(const std::string& template_str, const Options & options) {
Parser parser(std::make_shared<std::string>(template_str), options); Parser parser(std::make_shared<std::string>(normalize_newlines(template_str)), options);
auto tokens = parser.tokenize(); auto tokens = parser.tokenize();
TemplateTokenIterator begin = tokens.begin(); TemplateTokenIterator begin = tokens.begin();
auto it = begin; auto it = begin;
@ -2627,11 +2659,11 @@ inline std::shared_ptr<Context> Context::builtins() {
while (std::getline(iss, line, '\n')) { while (std::getline(iss, line, '\n')) {
auto needs_indent = !is_first || first; auto needs_indent = !is_first || first;
if (is_first) is_first = false; if (is_first) is_first = false;
else out += "\n"; else out += ENDL;
if (needs_indent) out += indent; if (needs_indent) out += indent;
out += line; out += line;
} }
if (!text.empty() && text.back() == '\n') out += "\n"; if (!text.empty() && text.back() == '\n') out += ENDL;
return out; return out;
})); }));
globals.set("selectattr", Value::callable([=](const std::shared_ptr<Context> & context, ArgumentsValue & args) { globals.set("selectattr", Value::callable([=](const std::shared_ptr<Context> & context, ArgumentsValue & args) {