diff --git a/CMakeLists.txt b/CMakeLists.txt index 9309ca6bb..1e30fdaf9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -135,7 +135,7 @@ include(${CMAKE_CURRENT_SOURCE_DIR}/scripts/build-info.cmake) if (LLAMA_SYCL) set(CMAKE_CXX_STANDARD 17) else() - set(CMAKE_CXX_STANDARD 11) + set(CMAKE_CXX_STANDARD 17) endif() set(CMAKE_CXX_STANDARD_REQUIRED true) diff --git a/Makefile b/Makefile index d809b8b3b..9b6af7168 100644 --- a/Makefile +++ b/Makefile @@ -9,7 +9,8 @@ TEST_TARGETS = \ tests/test-llama-grammar tests/test-grammar-parser tests/test-double-float tests/test-grad0 tests/test-opt \ tests/test-quantize-fns tests/test-quantize-perf tests/test-sampling tests/test-tokenizer-0-llama \ tests/test-tokenizer-0-falcon tests/test-tokenizer-1-llama tests/test-tokenizer-1-bpe tests/test-rope \ - tests/test-backend-ops tests/test-model-load-cancel tests/test-autorelease + tests/test-backend-ops tests/test-model-load-cancel tests/test-autorelease \ + tests/test-json-schema-to-grammar # Code coverage output files COV_TARGETS = *.gcno tests/*.gcno *.gcda tests/*.gcda *.gcov tests/*.gcov lcov-report gcovr-report @@ -99,7 +100,7 @@ endif # keep standard at C11 and C++11 MK_CPPFLAGS = -I. -Icommon MK_CFLAGS = -std=c11 -fPIC -MK_CXXFLAGS = -std=c++11 -fPIC +MK_CXXFLAGS = -std=c++17 -fPIC MK_NVCCFLAGS = -std=c++11 # -Ofast tends to produce faster code, but may not be available for some compilers. @@ -653,6 +654,11 @@ console.o: common/console.cpp common/console.h grammar-parser.o: common/grammar-parser.cpp common/grammar-parser.h $(CXX) $(CXXFLAGS) -c $< -o $@ +json-schema-to-grammar.o: examples/server/json-schema-to-grammar.cpp examples/server/json-schema-to-grammar.h + $(CXX) $(CXXFLAGS) -c $< -o $@ + # $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) -DLLAMA_BUILD_JSON_SCHEMA_CONVERTER=1 + # $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) + train.o: common/train.cpp common/train.h $(CXX) $(CXXFLAGS) -c $< -o $@ @@ -728,7 +734,7 @@ save-load-state: examples/save-load-state/save-load-state.cpp ggml.o llama.o $(C $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) -server: examples/server/server.cpp examples/server/utils.hpp examples/server/httplib.h examples/server/json.hpp examples/server/index.html.hpp examples/server/index.js.hpp examples/server/completion.js.hpp common/stb_image.h ggml.o llama.o $(COMMON_DEPS) grammar-parser.o $(OBJS) +server: examples/server/server.cpp examples/server/utils.hpp examples/server/httplib.h examples/server/json.hpp examples/server/index.html.hpp examples/server/index.js.hpp examples/server/completion.js.hpp json-schema-to-grammar.o common/stb_image.h ggml.o llama.o $(COMMON_DEPS) grammar-parser.o $(OBJS) $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) $(CXX) $(CXXFLAGS) $(filter-out %.h %.hpp $<,$^) -Iexamples/server $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) $(LWINSOCK2) @@ -844,6 +850,10 @@ tests/test-double-float: tests/test-double-float.cpp ggml.o $(OBJS) $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) +tests/test-json-schema-to-grammar: tests/test-json-schema-to-grammar.cpp json-schema-to-grammar.o + $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) + $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) + tests/test-grad0: tests/test-grad0.cpp ggml.o $(OBJS) $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) diff --git a/examples/server/CMakeLists.txt b/examples/server/CMakeLists.txt index f94de1e99..1d42790b8 100644 --- a/examples/server/CMakeLists.txt +++ b/examples/server/CMakeLists.txt @@ -2,7 +2,14 @@ set(TARGET server) option(LLAMA_SERVER_VERBOSE "Build verbose logging option for Server" ON) option(LLAMA_SERVER_SSL "Build SSL support for the server" OFF) include_directories(${CMAKE_CURRENT_SOURCE_DIR}) -add_executable(${TARGET} server.cpp utils.hpp json.hpp httplib.h) +add_executable(${TARGET} + server.cpp + utils.hpp + json.hpp + httplib.h + json-schema-to-grammar.cpp + json-schema-to-grammar.h +) install(TARGETS ${TARGET} RUNTIME) target_compile_definitions(${TARGET} PRIVATE SERVER_VERBOSE=$ diff --git a/examples/server/json-schema-to-grammar.cpp b/examples/server/json-schema-to-grammar.cpp new file mode 100644 index 000000000..31f321b5d --- /dev/null +++ b/examples/server/json-schema-to-grammar.cpp @@ -0,0 +1,698 @@ +#include "json-schema-to-grammar.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using json = nlohmann::json; +using namespace std; + +const string SPACE_RULE = "\" \"?"; + +unordered_map PRIMITIVE_RULES = { + {"boolean", "(\"true\" | \"false\") space"}, + {"number", "(\"-\"? ([0-9] | [1-9] [0-9]*)) (\".\" [0-9]+)? ([eE] [-+]? [0-9]+)? space"}, + {"integer", "(\"-\"? ([0-9] | [1-9] [0-9]*)) space"}, + {"value", "object | array | string | number | boolean"}, + {"object", "\"{\" space ( string \":\" space value (\",\" space string \":\" space value)* )? \"}\" space"}, + {"array", "\"[\" space ( value (\",\" space value)* )? \"]\" space"}, + {"uuid", "\"\\\"\" [0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F] " + "\"-\" [0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F] " + "\"-\" [0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F] " + "\"-\" [0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F] " + "\"-\" [0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F] \"\\\"\" space"}, + {"string", " \"\\\"\" (\n" + " [^\"\\\\] |\n" + " \"\\\\\" ([\"\\\\/bfnrt] | \"u\" [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F])\n" + " )* \"\\\"\" space"}, + {"null", "\"null\" space"} +}; + +unordered_map DATE_RULES = { + {"date", "[0-9] [0-9] [0-9] [0-9] \"-\" ( \"0\" [1-9] | \"1\" [0-2] ) \"-\" ( [0-2] [0-9] | \"3\" [0-1] )"}, + {"time", "([01] [0-9] | \"2\" [0-3]) \":\" [0-5] [0-9] \":\" [0-5] [0-9] ( \".\" [0-9] [0-9] [0-9] )? ( \"Z\" | ( \"+\" | \"-\" ) ( [01] [0-9] | \"2\" [0-3] ) \":\" [0-5] [0-9] )"}, + {"date-time", "date \"T\" time"}, + {"date-string", "\"\\\"\" date \"\\\"\" space"}, + {"time-string", "\"\\\"\" time \"\\\"\" space"}, + {"date-time-string", "\"\\\"\" date-time \"\\\"\" space"} +}; + +regex INVALID_RULE_CHARS_RE("[^a-zA-Z0-9-]+"); +regex GRAMMAR_LITERAL_ESCAPE_RE("[\r\n\"]"); +regex GRAMMAR_RANGE_LITERAL_ESCAPE_RE("[\r\n\"\\]\\-\\\\]"); +unordered_map GRAMMAR_LITERAL_ESCAPES = { + {'\r', "\\r"}, {'\n', "\\n"}, {'"', "\\\""}, {'-', "\\-"}, {']', "\\]"} +}; + +unordered_set NON_LITERAL_SET = {'|', '.', '(', ')', '[', ']', '{', '}', '*', '+', '?'}; +unordered_set ESCAPED_IN_REGEXPS_BUT_NOT_IN_LITERALS = {'{', '*', '+', '?'}; + +template +string join(Iterator begin, Iterator end, const string& separator) { + ostringstream result; + if (begin != end) { + result << *begin; + for (Iterator it = begin + 1; it != end; ++it) { + result << separator << *it; + } + } + return result.str(); +} + +static std::vector split(const std::string& str, const std::string& delimiter) { + std::vector tokens; + size_t start = 0; + size_t end = str.find(delimiter); + + while (end != std::string::npos) { + tokens.push_back(str.substr(start, end - start)); + start = end + delimiter.length(); + end = str.find(delimiter, start); + } + + tokens.push_back(str.substr(start)); + + return tokens; +} + +static string repeat(const string& str, size_t n) { + if (n == 0) { + return ""; + } + + string result; + result.reserve(str.length() * n); + + for (size_t i = 0; i < n; ++i) { + result += str; + } + + return result; +} + +static std::string replacePattern(const std::string& input, const regex& regex, const function& replacement) { + std::smatch match; + std::string result; + + std::string::const_iterator searchStart(input.cbegin()); + std::string::const_iterator searchEnd(input.cend()); + + while (std::regex_search(searchStart, searchEnd, match, regex)) { + result.append(searchStart, searchStart + match.position()); + result.append(replacement(match)); + searchStart = match.suffix().first; + } + + result.append(searchStart, searchEnd); + + return result; +} + +static string _format_literal(const string& literal) { + string escaped = replacePattern(json(literal).dump(), GRAMMAR_LITERAL_ESCAPE_RE, [&](const auto& match) { + char c = match.str()[0]; + return GRAMMAR_LITERAL_ESCAPES.at(c); + }); + return "\"" + escaped + "\""; +} + +static string _format_range_char(const string& ch) { + return replacePattern(ch, GRAMMAR_RANGE_LITERAL_ESCAPE_RE, [&](const auto& match) { + char c = match.str()[0]; + return GRAMMAR_LITERAL_ESCAPES.at(c); + }); +} + + +class SchemaConverter { +private: + std::optional> _fetch_json; + bool _dotall; + map _rules; + unordered_map _refs; + unordered_set _refs_being_resolved; + vector _errors; + vector _warnings; + + string _add_rule(const string& name, const string& rule) { + string esc_name = regex_replace(name, INVALID_RULE_CHARS_RE, "-"); + if (_rules.find(esc_name) == _rules.end() || _rules[esc_name] == rule) { + _rules[esc_name] = rule; + return esc_name; + } else { + int i = 0; + while (_rules.find(esc_name + to_string(i)) != _rules.end() && _rules[esc_name + to_string(i)] != rule) { + i++; + } + string key = esc_name + to_string(i); + _rules[key] = rule; + return key; + } + } + + string _generate_union_rule(const string& name, const vector& alt_schemas) { + vector rules; + for (size_t i = 0; i < alt_schemas.size(); i++) { + rules.push_back(visit(alt_schemas[i], name + (name.empty() ? "" : "-") + to_string(i))); + } + return join(rules.begin(), rules.end(), " | "); + } + + string _visit_pattern(const string& pattern, const string& name) { + if (!(pattern.front() == '^' && pattern.back() == '$')) { + _errors.push_back("Pattern must start with '^' and end with '$'"); + return ""; + } + string sub_pattern = pattern.substr(1, pattern.length() - 2); + unordered_map sub_rule_ids; + + size_t i = 0; + size_t length = sub_pattern.length(); + + std::function()> transform = [&]() -> pair { + size_t start = i; + vector> seq; + + auto get_dot = [&]() { + string rule; + if (_dotall) { + rule = "[\\U00000000-\\U0010FFFF]"; + } else { + rule = "[\\U00000000-\\x09\\x0B\\x0C\\x0E-\\U0010FFFF]"; + } + return _add_rule("dot", rule); + }; + + // Joins the sequence, merging consecutive literals together. + auto join_seq = [&]() { + vector results; + + string literal; + auto flush_literal = [&]() { + if (literal.empty()) { + return false; + } + results.push_back("\"" + literal + "\""); + literal.clear(); + return true; + }; + + for (const auto& item : seq) { + if (item.second) { + literal += item.first.substr(1, item.first.length() - 2); + } else { + flush_literal(); + results.push_back(item.first); + } + } + if (flush_literal() && results.size() == 1) { + return make_pair(results[0], true); + } + return make_pair(join(results.begin(), results.end(), " "), false); + }; + + while (i < length) { + char c = sub_pattern[i]; + if (c == '.') { + seq.push_back(make_pair(get_dot(), false)); + i++; + } else if (c == '(') { + i++; + if (i < length) { + if (sub_pattern[i] == '?') { + _warnings.push_back("Unsupported pattern syntax"); + } + } + auto sub_result = transform(); + seq.push_back(make_pair("(" + sub_result.first + ")", false)); + } else if (c == ')') { + i++; + if (start > 0 && sub_pattern[start - 1] != '(') { + _errors.push_back("Unbalanced parentheses"); + } + return join_seq(); + } else if (c == '[') { + string square_brackets = string(1, c); + i++; + while (i < length && sub_pattern[i] != ']') { + if (sub_pattern[i] == '\\') { + square_brackets += sub_pattern.substr(i, 2); + i += 2; + } else { + square_brackets += sub_pattern[i]; + i++; + } + } + if (i >= length) { + _errors.push_back("Unbalanced square brackets"); + } + square_brackets += ']'; + i++; + seq.push_back(make_pair(square_brackets, false)); + } else if (c == '|') { + seq.push_back(make_pair("|", false)); + i++; + } else if (c == '*' || c == '+' || c == '?') { + seq.back().first += c; + i++; + } else if (c == '{') { + string curly_brackets = string(1, c); + i++; + while (i < length && sub_pattern[i] != '}') { + curly_brackets += sub_pattern[i]; + i++; + } + if (i >= length) { + _errors.push_back("Unbalanced curly brackets"); + } + curly_brackets += '}'; + i++; + auto nums = split(curly_brackets.substr(1, curly_brackets.length() - 2), ","); + int min_times = 0; + int max_times = numeric_limits::max(); + if (nums.size() == 1) { + min_times = max_times = stoi(nums[0]); + } else if (nums.size() != 2) { + _errors.push_back("Wrong number of values in curly brackets"); + } else { + if (!nums[0].empty()) { + min_times = stoi(nums[0]); + } + if (!nums[1].empty()) { + max_times = stoi(nums[1]); + } + } + auto [sub, sub_is_literal] = seq.back(); + if (min_times == 0 && max_times == numeric_limits::max()) { + seq.back().first = sub + "*"; + } else if (min_times == 0 && max_times == 1) { + seq.back().first = sub + "?"; + } else if (min_times == 1 && max_times == numeric_limits::max()) { + seq.back().first = sub + "+"; + } else { + if (!sub_is_literal) { + string& sub_id = sub_rule_ids[sub]; + if (sub_id.empty()) { + sub_id = _add_rule(name + "-" + to_string(sub_rule_ids.size()), sub); + } + sub = sub_id; + } + string result; + if (sub_is_literal && min_times > 0) { + result = "\"" + repeat(sub.substr(1, sub.length() - 2), min_times) + "\""; + } else { + for (int j = 0; j < min_times; j++) { + if (j > 0) { + result += " "; + } + result += sub; + } + } + if (min_times > 0 && min_times < max_times) { + result += " "; + } + if (max_times == numeric_limits::max()) { + result += sub + "*"; + } else { + for (int j = min_times; j < max_times; j++) { + if (j > min_times) { + result += " "; + } + result += sub + "?"; + } + } + seq.back().first = result; + seq.back().second = false; + } + } else { + string literal; + while (i < length && NON_LITERAL_SET.find(sub_pattern[i]) == NON_LITERAL_SET.end() && + (i == length - 1 || ESCAPED_IN_REGEXPS_BUT_NOT_IN_LITERALS.find(sub_pattern[i + 1]) == ESCAPED_IN_REGEXPS_BUT_NOT_IN_LITERALS.end())) { + if (sub_pattern[i] == '\\' && i < length - 1) { + i++; + if (NON_LITERAL_SET.find(sub_pattern[i]) != NON_LITERAL_SET.end()) { + literal += sub_pattern[i]; + } else { + literal += "\\" + string(1, sub_pattern[i]); + } + i++; + } else { + if (sub_pattern[i] == '"') { + literal += "\\"; + } + literal += sub_pattern[i]; + i++; + } + } + if (!literal.empty()) { + seq.push_back(make_pair("\"" + literal + "\"", true)); + } + if (i < length && NON_LITERAL_SET.find(sub_pattern[i]) == NON_LITERAL_SET.end()) { + seq.push_back(make_pair("\"" + string(1, sub_pattern[i]) + "\"", true)); + i++; + } + } + } + return join_seq(); + }; + return _add_rule(name, transform().first); + } + + string _resolve_ref(const string& ref) { + string ref_name = ref.substr(ref.find_last_of('/') + 1); + if (_rules.find(ref_name) == _rules.end() && _refs_being_resolved.find(ref) == _refs_being_resolved.end()) { + _refs_being_resolved.insert(ref); + json resolved = _refs[ref]; + ref_name = visit(resolved, ref_name); + _refs_being_resolved.erase(ref); + } + return ref_name; + } + + string _build_object_rule(const vector>& properties, const unordered_set& required, const string& name) { + vector required_props; + vector optional_props; + unordered_map prop_kv_rule_names; + for (const auto& [prop_name, prop_schema] : properties) { + string prop_rule_name = visit(prop_schema, name + (name.empty() ? "" : "-") + prop_name); + prop_kv_rule_names[prop_name] = _add_rule( + name + (name.empty() ? "" : "-") + prop_name + "-kv", + _format_literal(prop_name) + " space \":\" space " + prop_rule_name + ); + if (required.find(prop_name) != required.end()) { + required_props.push_back(prop_name); + } else { + optional_props.push_back(prop_name); + } + } + + string rule = "\"{\" space "; + for (size_t i = 0; i < required_props.size(); i++) { + if (i > 0) { + rule += " \",\" space "; + } + rule += prop_kv_rule_names[required_props[i]]; + } + + if (!optional_props.empty()) { + rule += " ("; + if (!required_props.empty()) { + rule += " \",\" space ( "; + } + + function&, bool)> get_recursive_refs = [&](const vector& ks, bool first_is_optional) { + string res; + if (ks.empty()) { + return res; + } + string k = ks[0]; + string kv_rule_name = prop_kv_rule_names[k]; + if (first_is_optional) { + res = "( \",\" space " + kv_rule_name + " )?"; + } else { + res = kv_rule_name; + } + if (ks.size() > 1) { + res += " " + _add_rule( + name + (name.empty() ? "" : "-") + k + "-rest", + get_recursive_refs(vector(ks.begin() + 1, ks.end()), true) + ); + } + return res; + }; + + for (size_t i = 0; i < optional_props.size(); i++) { + if (i > 0) { + rule += " | "; + } + rule += get_recursive_refs(vector(optional_props.begin() + i, optional_props.end()), false); + } + if (!required_props.empty()) { + rule += " )"; + } + rule += " )?"; + } + + rule += " \"}\" space "; + + return rule; + } + +public: + SchemaConverter( + const std::optional>& fetch_json, + bool dotall) + : _fetch_json(fetch_json), _dotall(dotall) + { + _rules["space"] = SPACE_RULE; + } + + void resolve_refs(nlohmann::json& schema, const std::string& url) { + /* + * Resolves all $ref fields in the given schema, fetching any remote schemas, + * replacing each $ref with absolute reference URL and populates _refs with the + * respective referenced (sub)schema dictionaries. + */ + function visit_refs = [&](json& n) { + if (n.is_array()) { + for (auto& x : n) { + visit_refs(x); + } + } else if (n.is_object()) { + if (n.contains("$ref")) { + string ref = n["$ref"]; + if (_refs.find(ref) == _refs.end()) { + json target; + if (ref.find("https://") == 0 && _fetch_json) { + string base_url = ref.substr(0, ref.find('#')); + auto it = _refs.find(base_url); + if (it != _refs.end()) { + target = it->second; + } else { + // Fetch the referenced schema and resolve its refs + auto referenced = _fetch_json.value()(ref); + resolve_refs(referenced, base_url); + _refs[base_url] = referenced; + } + if (ref.find('#') == string::npos || ref.substr(ref.find('#') + 1).empty()) { + return; + } + } else if (ref.find("#/") == 0) { + target = schema; + n["$ref"] = url + ref; + ref = url + ref; + } else { + _errors.push_back("Unsupported ref: " + ref); + return; + } + string pointer = ref.substr(ref.find('#') + 1); + vector tokens = split(pointer, "/"); + for (size_t i = 1; i < tokens.size(); ++i) { + string sel = tokens[i]; + if (target.is_null() || !target.contains(sel)) { + _errors.push_back("Error resolving ref " + ref + ": " + sel + " not in " + target.dump()); + return; + } + target = target[sel]; + } + _refs[ref] = target; + } + } else { + for (auto& [key, value] : n.items()) { + visit_refs(value); + } + } + } + }; + + visit_refs(schema); + } + + string visit(const json& schema, const string& name) { + json schema_type = schema.contains("type") ? schema["type"] : json(); + string schema_format = schema.contains("format") ? schema["format"].get() : ""; + string rule_name = name.empty() ? "root" : name; + + if (schema.contains("$ref")) { + return _resolve_ref(schema["$ref"]); + } else if (schema.contains("oneOf") || schema.contains("anyOf")) { + vector alt_schemas = schema.contains("oneOf") ? schema["oneOf"].get>() : schema["anyOf"].get>(); + return _add_rule(rule_name, _generate_union_rule(name, alt_schemas)); + } else if (schema_type.is_array()) { + vector schema_types; + for (const auto& t : schema_type) { + schema_types.push_back({{"type", t}}); + } + return _add_rule(rule_name, _generate_union_rule(name, schema_types)); + } else if (schema.contains("const")) { + return _add_rule(rule_name, _format_literal(schema["const"].dump())); + } else if (schema.contains("enum")) { + vector enum_values; + for (const auto& v : schema["enum"]) { + enum_values.push_back(_format_literal(v.dump())); + } + return _add_rule(rule_name, join(enum_values.begin(), enum_values.end(), " | ")); + } else if ((schema_type.is_null() || schema_type == "object") && schema.contains("properties")) { + unordered_set required; + if (schema.contains("required") && schema["required"].is_array()) { + for (const auto& item : schema["required"]) { + if (item.is_string()) { + required.insert(item.get()); + } + } + } + vector> properties; + for (const auto& prop : schema["properties"].items()) { + properties.emplace_back(prop.key(), prop.value()); + } + return _add_rule(rule_name, _build_object_rule(properties, required, name)); + } else if ((schema_type.is_null() || schema_type == "object") && schema.contains("allOf")) { + unordered_set required; + vector> properties; + string hybrid_name = name; + std::function add_component = [&](const json& comp_schema, bool is_required) { + if (comp_schema.contains("$ref")) { + add_component(_refs[comp_schema["$ref"]], is_required); + } else if (comp_schema.contains("properties")) { + for (const auto& prop : comp_schema["properties"].items()) { + properties.emplace_back(prop.key(), prop.value()); + if (is_required) { + required.insert(prop.key()); + } + } + } else { + // todo warning + } + }; + for (auto& t : schema["allOf"]) { + if (t.contains("anyOf")) { + for (auto& tt : t["anyOf"]) { + add_component(tt, false); + } + } else { + add_component(t, true); + } + } + return _add_rule(rule_name, _build_object_rule(properties, required, hybrid_name)); + } else if ((schema_type.is_null() || schema_type == "object") && schema.contains("additionalProperties")) { + json additional_properties = schema["additionalProperties"]; + if (!additional_properties.is_object()) { + additional_properties = json::object(); + } + string sub_name = name + (name.empty() ? "" : "-") + "additionalProperties"; + string value_rule = visit(additional_properties, sub_name + "-value"); + string kv_rule = _add_rule(sub_name + "-kv", "string \":\" space " + value_rule); + return _add_rule(rule_name, "( " + kv_rule + " ( \",\" space " + kv_rule + " )* )*"); + } else if ((schema_type.is_null() || schema_type == "array") && (schema.contains("items") || schema.contains("prefixItems"))) { + json items = schema.contains("items") ? schema["items"] : schema["prefixItems"]; + if (items.is_array()) { + string rule = "\"[\" space "; + for (size_t i = 0; i < items.size(); i++) { + if (i > 0) { + rule += " \",\" space "; + } + rule += visit(items[i], name + (name.empty() ? "" : "-") + "tuple-" + to_string(i)); + } + rule += " \"]\" space"; + return _add_rule(rule_name, rule); + } else { + string item_rule_name = visit(items, name + (name.empty() ? "" : "-") + "item"); + string list_item_operator = "( \",\" space " + item_rule_name + " )"; + string successive_items; + int min_items = schema.contains("minItems") ? schema["minItems"].get() : 0; + json max_items_json = schema.contains("maxItems") ? schema["maxItems"] : json(); + int max_items = max_items_json.is_number_integer() ? max_items_json.get() : -1; + if (min_items > 0) { + successive_items += repeat(list_item_operator, min_items - 1); + min_items--; + } + if (max_items >= 0 && max_items > min_items) { + successive_items += repeat(list_item_operator + "?", max_items - min_items - 1); + } else { + successive_items += list_item_operator + "*"; + } + string rule; + if (min_items == 0) { + rule = "\"[\" space ( " + item_rule_name + " " + successive_items + " )? \"]\" space"; + } else { + rule = "\"[\" space " + item_rule_name + " " + successive_items + " \"]\" space"; + } + return _add_rule(rule_name, rule); + } + } else if ((schema_type.is_null() || schema_type == "string") && schema.contains("pattern")) { + return _visit_pattern(schema["pattern"], rule_name); + } else if (schema.empty() || (schema.size() == 1 && schema_type == "object")) { + for (const auto& [t, r] : PRIMITIVE_RULES) { + _add_rule(t, r); + } + return "object"; + } else if ((schema_type.is_null() || schema_type == "string") && regex_match(schema_format, regex("^uuid[1-5]?$"))) { + return _add_rule(rule_name == "root" ? "root" : schema_format, PRIMITIVE_RULES.at("uuid")); + } else if ((schema_type.is_null() || schema_type == "string") && DATE_RULES.find(schema_format) != DATE_RULES.end()) { + for (const auto& [t, r] : DATE_RULES) { + _add_rule(t, r); + } + return schema_format + "-string"; + } else { + if (PRIMITIVE_RULES.find(schema_type.get()) == PRIMITIVE_RULES.end()) { + _errors.push_back("Unrecognized schema: " + schema.dump()); + return ""; + } + // TODO: support minimum, maximum, exclusiveMinimum, exclusiveMaximum at least for zero + return _add_rule(rule_name == "root" ? "root" : schema_type.get(), PRIMITIVE_RULES.at(schema_type.get())); + } + } + + void check_errors() { + if (!_errors.empty()) { + throw std::runtime_error("JSON schema conversion failed:\n" + join(_errors.begin(), _errors.end(), "\n")); + } + if (!_warnings.empty()) { + std::cerr << "WARNING: JSON schema conversion was incomplete: " + join(_warnings.begin(), _warnings.end(), "; ") << std::endl; + } + } + + string format_grammar() { + stringstream ss; + for (const auto& [name, rule] : _rules) { + ss << name << " ::= " << rule << endl; + } + return ss.str(); + } +}; + + +string json_schema_to_grammar(const json& schema) { + SchemaConverter converter(/* fetch_json= */ std::nullopt, /* dotall= */ false); + auto copy = schema; + converter.resolve_refs(copy, "input"); + converter.visit(copy, ""); + converter.check_errors(); + return converter.format_grammar(); +} + +#ifdef LLAMA_BUILD_JSON_SCHEMA_CONVERTER + +int main(int argc, const char** argv) { + if (argc != 2) { + cerr << "Expected only one argument" << endl; + return -1; + } + string file(argv[1]); + string schema; + if (file == "-") { + schema.append(istreambuf_iterator(cin), istreambuf_iterator()); + } else { + ifstream in(argv[1]); + schema.append(istreambuf_iterator(in), istreambuf_iterator()); + } + cout << json_schema_to_grammar(json::parse(schema)).c_str() << endl; +} + +#endif diff --git a/examples/server/json-schema-to-grammar.h b/examples/server/json-schema-to-grammar.h new file mode 100644 index 000000000..2e4c6ade3 --- /dev/null +++ b/examples/server/json-schema-to-grammar.h @@ -0,0 +1,4 @@ +#pragma once +#include "json.hpp" + +std::string json_schema_to_grammar(const nlohmann::json& schema); diff --git a/examples/server/server.cpp b/examples/server/server.cpp index c7d3ed01b..f45e9630a 100644 --- a/examples/server/server.cpp +++ b/examples/server/server.cpp @@ -1,6 +1,7 @@ #include "utils.hpp" #include "common.h" +#include "json-schema-to-grammar.h" #include "llama.h" #include "grammar-parser.h" @@ -178,6 +179,7 @@ struct server_slot { llama_token sampled; struct llama_sampling_params sparams; llama_sampling_context * ctx_sampling = nullptr; + json json_schema; int32_t ga_i = 0; // group-attention state int32_t ga_n = 1; // group-attention factor @@ -838,7 +840,12 @@ struct server_context { slot.sparams.penalize_nl = json_value(data, "penalize_nl", default_sparams.penalize_nl); slot.params.n_keep = json_value(data, "n_keep", slot.params.n_keep); slot.params.seed = json_value(data, "seed", default_params.seed); - slot.sparams.grammar = json_value(data, "grammar", default_sparams.grammar); + if (data.contains("json_schema") && !data.contains("grammar")) { + auto schema = json_value(data, "json_schema", json::object()); + slot.sparams.grammar = json_schema_to_grammar(schema); + } else { + slot.sparams.grammar = json_value(data, "grammar", default_sparams.grammar); + } slot.sparams.n_probs = json_value(data, "n_probs", default_sparams.n_probs); slot.sparams.min_keep = json_value(data, "min_keep", default_sparams.min_keep); diff --git a/examples/server/utils.hpp b/examples/server/utils.hpp index df0a27782..2aeb9ce02 100644 --- a/examples/server/utils.hpp +++ b/examples/server/utils.hpp @@ -361,10 +361,18 @@ static json oaicompat_completion_params_parse( llama_params["ignore_eos"] = json_value(body, "ignore_eos", false); llama_params["tfs_z"] = json_value(body, "tfs_z", default_sparams.tfs_z); - if (body.count("grammar") != 0) { + if (body.contains("grammar")) { llama_params["grammar"] = json_value(body, "grammar", json::object()); } + if (body.contains("response_format")) { + auto response_format = json_value(body, "response_format", json::object()); + if (response_format.contains("schema") && response_format["type"] == "json_object") { + llama_params["json_schema"] = json_value(response_format, "schema", json::object()); + std::cerr << "GOT " << llama_params["json_schema"] << std::endl; + } + } + // Handle 'stop' field if (body.contains("stop") && body["stop"].is_string()) { llama_params["stop"] = json::array({body["stop"].get()}); diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 10326d531..4389f4a16 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -59,6 +59,8 @@ llama_build_and_test_executable(test-backend-ops.cpp) llama_build_and_test_executable(test-rope.cpp) +llama_build_and_test_executable(test-json-schema-to-grammar.cpp) + llama_build_and_test_executable_with_label(test-model-load-cancel.cpp "model") llama_build_and_test_executable_with_label(test-autorelease.cpp "model") diff --git a/tests/run-json-schema-to-grammar.mjs b/tests/run-json-schema-to-grammar.mjs new file mode 100644 index 000000000..71bf62ed3 --- /dev/null +++ b/tests/run-json-schema-to-grammar.mjs @@ -0,0 +1,10 @@ +import { readFileSync } from "fs" +import { SchemaConverter } from "../examples/server/public/json-schema-to-grammar.mjs" + +const [, , file] = process.argv +const url = `file://${file}` +let schema = JSON.parse(readFileSync(file, "utf8")); +const converter = new SchemaConverter({}) +schema = await converter.resolveRefs(schema, url) +converter.visit(schema, '') +console.log(converter.formatGrammar()) diff --git a/tests/test-json-schema-to-grammar.cpp b/tests/test-json-schema-to-grammar.cpp new file mode 100755 index 000000000..c07a105ef --- /dev/null +++ b/tests/test-json-schema-to-grammar.cpp @@ -0,0 +1,307 @@ +#ifdef NDEBUG +#undef NDEBUG +#endif + +#include "llama.h" +#include "grammar-parser.h" +#include +#include +#include +#include +#include +#include +#include +#include "../examples/server/json-schema-to-grammar.h" +#include +#include + +using namespace std; +namespace fs = std::filesystem; + +string INPUT_NAME(tmpnam(nullptr)); +string OUT_NAME(tmpnam(nullptr)); + +static std::string trim(const std::string & source) { + std::string s(source); + s.erase(0,s.find_first_not_of(" \n\r\t")); + s.erase(s.find_last_not_of(" \n\r\t")+1); + return regex_replace(s, regex("(^|\n)[ \t]+"), "$1"); +} + +struct TestCase { + string name; + string schema; + string expected; + + void prepare() const { + ofstream f; + f.open(INPUT_NAME); + f << schema.c_str(); + f.close(); + } + + void read_and_verify(const string& series) const { + ostringstream actuals; + actuals << ifstream(OUT_NAME).rdbuf(); + auto actual = actuals.str(); + verify(series, actual); + } + + void verify(const string& series, const string& actual) const { + if (trim(actual) != trim(expected)) { + cerr << "#" << endl; + cerr << "# Test " << series.c_str() << " / " << name.c_str() << " failed." << endl; + cerr << "#" << endl; + cerr << schema.c_str() << endl; + cerr << "# EXPECTED:\n" << expected.c_str() << endl; + cerr << "# ACTUAL:\n" << actual.c_str() << endl; + assert(false); + } + } + +}; + + +static void run_py(const TestCase& tc) { + cerr << "# Running Python " << tc.name.c_str() << endl; + tc.prepare(); + std::system(("python ./examples/json-schema-to-grammar.py " + INPUT_NAME + " > " + OUT_NAME).c_str()); + tc.read_and_verify("Python"); +} +static void run_mjs(const TestCase& tc) { + cerr << "# Running MJS " << tc.name.c_str() << endl; + tc.prepare(); + std::system(("node ./tests/run-json-schema-to-grammar.mjs " + INPUT_NAME + " > " + OUT_NAME).c_str()); + tc.read_and_verify("JavaScript"); +} +static void run_cpp(const TestCase& tc) { + cerr << "# Running C++ " << tc.name.c_str() << endl; + auto actual = json_schema_to_grammar(nlohmann::json::parse(tc.schema)); + tc.verify("C++", actual); +} + +static void run_all(const TestCase& tc) { + run_py(tc); + run_mjs(tc); + run_cpp(tc); +} + +int main() { + run_all({ + .name = "exotic formats", + .schema = R"""({ + "prefixItems": [ + { "format": "date" }, + { "format": "uuid" }, + { "format": "time" }, + { "format": "date-time" } + ] + })""", + .expected = R"""( + date ::= [0-9] [0-9] [0-9] [0-9] "-" ( "0" [1-9] | "1" [0-2] ) "-" ( [0-2] [0-9] | "3" [0-1] ) + date-string ::= "\"" date "\"" space + date-time ::= date "T" time + date-time-string ::= "\"" date-time "\"" space + root ::= "[" space date-string "," space uuid "," space time-string "," space date-time-string "]" space + space ::= " "? + time ::= ([01] [0-9] | "2" [0-3]) ":" [0-5] [0-9] ":" [0-5] [0-9] ( "." [0-9] [0-9] [0-9] )? ( "Z" | ( "+" | "-" ) ( [01] [0-9] | "2" [0-3] ) ":" [0-5] [0-9] ) + time-string ::= "\"" time "\"" space + uuid ::= "\"" [0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F] "-" [0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F] "-" [0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F] "-" [0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F] "-" [0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F] "\"" space + )""" + }); + + run_all({ + .name = "string", + .schema = R"""({ + "type": "string" + })""", + .expected = R"""( + root ::= "\"" ( + [^"\\] | + "\\" (["\\/bfnrt] | "u" [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F]) + )* "\"" space + space ::= " "? + )""" + }); + + run_all({ + .name = "boolean", + .schema = R"""({ + "type": "boolean" + })""", + .expected = R"""( + root ::= ("true" | "false") space + space ::= " "? + )""" + }); + + run_all({ + .name = "integer", + .schema = R"""({ + "type": "integer" + })""", + .expected = R"""( + root ::= ("-"? ([0-9] | [1-9] [0-9]*)) space + space ::= " "? + )""" + }); + + run_all({ + .name = "number", + .schema = R"""({ + "type": "number" + })""", + .expected = R"""( + root ::= ("-"? ([0-9] | [1-9] [0-9]*)) ("." [0-9]+)? ([eE] [-+]? [0-9]+)? space + space ::= " "? + )""" + }); + + run_all({ + .name = "minItems", + .schema = R"""({ + "items": { + "type": "boolean" + }, + "minItems": 2 + })""", + .expected = R"""( + boolean ::= ("true" | "false") space + root ::= "[" space boolean ( "," space boolean )( "," space boolean )* "]" space + space ::= " "? + )""" + }); + + run_all({ + .name = "maxItems 1", + .schema = R"""({ + "items": { + "type": "boolean" + }, + "maxItems": 1 + })""", + .expected = R"""( + boolean ::= ("true" | "false") space + root ::= "[" space ( boolean )? "]" space + space ::= " "? + )""" + }); + + run_all({ + .name = "maxItems 2", + .schema = R"""({ + "items": { + "type": "boolean" + }, + "maxItems": 2 + })""", + .expected = R"""( + boolean ::= ("true" | "false") space + root ::= "[" space ( boolean ( "," space boolean )? )? "]" space + space ::= " "? + )""" + }); + + run_all({ + .name = "min + maxItems", + .schema = R"""({ + "items": { + "type": ["number", "integer"] + }, + "minItems": 3, + "maxItems": 5 + })""", + .expected = R"""( + integer ::= ("-"? ([0-9] | [1-9] [0-9]*)) space + item ::= number | integer + number ::= ("-"? ([0-9] | [1-9] [0-9]*)) ("." [0-9]+)? ([eE] [-+]? [0-9]+)? space + root ::= "[" space item ( "," space item )( "," space item )( "," space item )?( "," space item )? "]" space + space ::= " "? + )""" + }); + + run_all({ + .name = "regexp", + .schema = R"""({ + "type": "string", + "pattern": "^(\\([0-9]{1,3}\\))?[0-9]{3}-[0-9]{4} and...$" + })""", + .expected = R"""( + dot ::= [\U00000000-\x09\x0B\x0C\x0E-\U0010FFFF] + root ::= ("(" root-1 root-1? root-1? ")")? root-1 root-1 root-1 "-" root-1 root-1 root-1 root-1 " and" dot dot dot + root-1 ::= [0-9] + space ::= " "? + )""" + }); + + run_all({ + .name = "optionals", + .schema = R"""({ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "a": { + "type": "string" + }, + "b": { + "type": "string" + }, + "c": { + "type": [ + "number", + "string" + ] + }, + "d": { + "type": "string" + }, + "e": { + "type": "object", + "additionalProperties": { + "type": "array", + "items": { + "type": "array", + "minItems": 2, + "items": [ + { + "type": "string" + }, + { + "type": "number" + } + ], + "maxItems": 2 + } + } + } + }, + "required": [ + "a", + "b" + ], + "additionalProperties": false, + "definitions": {} + })""", + .expected = R"""( + a-kv ::= "\"a\"" space ":" space string + b-kv ::= "\"b\"" space ":" space string + c ::= number | string + c-kv ::= "\"c\"" space ":" space c + c-rest ::= ( "," space d-kv )? d-rest + d-kv ::= "\"d\"" space ":" space string + d-rest ::= ( "," space e-kv )? + e ::= ( e-additionalProperties-kv ( "," space e-additionalProperties-kv )* )* + e-additionalProperties-kv ::= string ":" space e-additionalProperties-value + e-additionalProperties-value ::= "[" space ( e-additionalProperties-value-item ( "," space e-additionalProperties-value-item )* )? "]" space + e-additionalProperties-value-item ::= "[" space string "," space number "]" space + e-kv ::= "\"e\"" space ":" space e + number ::= ("-"? ([0-9] | [1-9] [0-9]*)) ("." [0-9]+)? ([eE] [-+]? [0-9]+)? space + root ::= "{" space a-kv "," space b-kv ( "," space ( c-kv c-rest | d-kv d-rest | e-kv ) )? "}" space + space ::= " "? + string ::= "\"" ( + [^"\\] | + "\\" (["\\/bfnrt] | "u" [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F]) + )* "\"" space + )""" + }); +}