vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

json: prevent additional props to redefine a typed prop

Browse source
This commit is contained in:
ochafik 2024-06-11 00:16:12 +01:00
parent 43f74e0c1f
commit adca9af2f6
2 changed files with 81 additions and 17 deletions
Download patch file Download diff file Expand all files Collapse all files

65
common/json-schema-to-grammar.cpp
View file

@ -160,6 +160,63 @@ static std::string format_literal(const std::string & literal) {
return "\"" + escaped + "\"";
}
/*
Returns a rule that matches a JSON string that is none of the provided strings
not_strings({"and", "also"})
-> ["] ( [a] ([l] ([s] ([^"o]) | [^"s]) | [n] ([^"d]) | [^"ln]) | [^"a] ) char* ["]
*/
std::string not_strings(const std::vector<std::string> & strings) {
struct TrieNode {
std::map<char, TrieNode> children;
bool is_end_of_string;
void insert(const std::string & string) {
auto node = this;
for (char c : string) {
node = &node->children[c];
}
node->is_end_of_string = true;
}
};
TrieNode trie;
for (const auto & s : strings) {
trie.insert(s);
}
std::ostringstream out;
out << "[\"] ( ";
std::function<void(const TrieNode &)> visit = [&](const TrieNode & node) {
std::ostringstream rejects;
auto first = true;
for (const auto & kv : node.children) {
rejects << kv.first;
if (kv.second.is_end_of_string) {
continue;
}
if (first) {
first = false;
} else {
out << " | ";
}
out << "[" << kv.first << "] (";
visit(kv.second);
out << ")";
}
if (!node.children.empty()) {
if (!first) {
out << " | ";
}
out << "[^\"" << rejects.str() << "]";
}
};
visit(trie);
out << " ) char* [\"]";
return out.str();
}
class SchemaConverter {
private:
@ -408,6 +465,7 @@ private:
std::vector<std::string> required_props;
std::vector<std::string> optional_props;
std::unordered_map<std::string, std::string> prop_kv_rule_names;
std::vector<std::string> prop_names;
for (const auto & kv : properties) {
const auto &prop_name = kv.first;
const auto &prop_schema = kv.second;
@ -422,11 +480,16 @@ private:
} else {
optional_props.push_back(prop_name);
}
prop_names.push_back(prop_name);
}
if (additional_properties.is_object() || (additional_properties.is_boolean() && additional_properties.get<bool>())) {
std::string sub_name = name + (name.empty() ? "" : "-") + "additional";
std::string value_rule = visit(additional_properties.is_object() ? additional_properties : json::object(), sub_name + "-value");
std::string kv_rule = _add_rule(sub_name + "-kv", _add_primitive("string", PRIMITIVE_RULES.at("string")) + " \":\" space " + value_rule);
auto key_rule =
prop_names.empty() ? _add_primitive("string", PRIMITIVE_RULES.at("string"))
: _add_rule(sub_name + "-k", not_strings(prop_names));
std::string kv_rule = _add_rule(sub_name + "-kv", key_rule + " \":\" space " + value_rule);
prop_kv_rule_names["*"] = kv_rule;
optional_props.push_back("*");
}

33
tests/test-json-schema-to-grammar.cpp
View file

@ -634,7 +634,8 @@ static void test_all(const std::string & lang, std::function<void(const TestCase
})""",
R"""(
a-kv ::= "\"a\"" space ":" space number
additional-kv ::= string ":" space string
additional-k ::= ["] ( [^"a] ) char* ["]
additional-kv ::= additional-k ":" space string
char ::= [^"\\] | "\\" (["\\/bfnrt] | "u" [0-9a-fA-F]{4})
decimal-part ::= [0-9]{1,16}
integral-part ::= [0] | [1-9] [0-9]{0,15}
@ -658,14 +659,13 @@ static void test_all(const std::string & lang, std::function<void(const TestCase
R"""(
a-kv ::= "\"a\"" space ":" space number
a-rest ::= ( "," space additional-kv )*
additional-kv ::= string ":" space number
char ::= [^"\\] | "\\" (["\\/bfnrt] | "u" [0-9a-fA-F]{4})
additional-k ::= ["] ( [^"a] ) char* ["]
additional-kv ::= additional-k ":" space number
decimal-part ::= [0-9]{1,16}
integral-part ::= [0] | [1-9] [0-9]{0,15}
number ::= ("-"? integral-part) ("." decimal-part)? ([eE] [-+]? integral-part)? space
root ::= "{" space (a-kv a-rest | additional-kv ( "," space additional-kv )* )? "}" space
space ::= " "?
string ::= "\"" char* "\"" space
)"""
});
@ -675,24 +675,23 @@ static void test_all(const std::string & lang, std::function<void(const TestCase
R"""({
"type": "object",
"properties": {
"a": {"type": "number"},
"b": {"type": "number"}
"and": {"type": "number"},
"also": {"type": "number"}
},
"required": ["a"],
"required": ["and"],
"additionalProperties": {"type": "number"}
})""",
R"""(
a-kv ::= "\"a\"" space ":" space number
additional-kv ::= string ":" space number
b-kv ::= "\"b\"" space ":" space number
b-rest ::= ( "," space additional-kv )*
char ::= [^"\\] | "\\" (["\\/bfnrt] | "u" [0-9a-fA-F]{4})
additional-k ::= ["] ( [a] ([l] ([s] ([^"o]) | [^"s]) | [n] ([^"d]) | [^"ln]) | [^"a] ) char* ["]
additional-kv ::= additional-k ":" space number
also-kv ::= "\"also\"" space ":" space number
also-rest ::= ( "," space additional-kv )*
and-kv ::= "\"and\"" space ":" space number
decimal-part ::= [0-9]{1,16}
integral-part ::= [0] | [1-9] [0-9]{0,15}
number ::= ("-"? integral-part) ("." decimal-part)? ([eE] [-+]? integral-part)? space
root ::= "{" space a-kv ( "," space ( b-kv b-rest | additional-kv ( "," space additional-kv )* ) )? "}" space
root ::= "{" space and-kv ( "," space ( also-kv also-rest | additional-kv ( "," space additional-kv )* ) )? "}" space
space ::= " "?
string ::= "\"" char* "\"" space
)"""
});
@ -749,7 +748,8 @@ static void test_all(const std::string & lang, std::function<void(const TestCase
alternative-1 ::= bar
array ::= "[" space ( value ("," space value)* )? "]" space
bar ::= "{" space (bar-b-kv bar-b-rest | bar-additional-kv ( "," space bar-additional-kv )* )? "}" space
bar-additional-kv ::= string ":" space bar-additional-value
bar-additional-k ::= ["] ( [^"b] ) char* ["]
bar-additional-kv ::= bar-additional-k ":" space bar-additional-value
bar-additional-value ::= object
bar-b-kv ::= "\"b\"" space ":" space number
bar-b-rest ::= ( "," space bar-additional-kv )*
@ -759,7 +759,8 @@ static void test_all(const std::string & lang, std::function<void(const TestCase
foo ::= "{" space (foo-a-kv foo-a-rest | foo-additional-kv ( "," space foo-additional-kv )* )? "}" space
foo-a-kv ::= "\"a\"" space ":" space number
foo-a-rest ::= ( "," space foo-additional-kv )*
foo-additional-kv ::= string ":" space foo-additional-value
foo-additional-k ::= ["] ( [^"a] ) char* ["]
foo-additional-kv ::= foo-additional-k ":" space foo-additional-value
foo-additional-value ::= object
integral-part ::= [0] | [1-9] [0-9]{0,15}
null ::= "null" space