clang fmt
This commit is contained in:
parent
3675050804
commit
58006ddb13
1 changed files with 680 additions and 791 deletions
|
@ -1,5 +1,5 @@
|
||||||
#ifdef NDEBUG
|
#ifdef NDEBUG
|
||||||
#undef NDEBUG
|
# undef NDEBUG
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
#include "unicode.h"
|
#include "unicode.h"
|
||||||
|
@ -10,7 +10,6 @@
|
||||||
#include <string>
|
#include <string>
|
||||||
#include <vector>
|
#include <vector>
|
||||||
|
|
||||||
|
|
||||||
static bool test_build_grammar_fails(const std::string & grammar_str) {
|
static bool test_build_grammar_fails(const std::string & grammar_str) {
|
||||||
fprintf(stderr, "⚫ Testing failure for grammar: %s\n", grammar_str.c_str());
|
fprintf(stderr, "⚫ Testing failure for grammar: %s\n", grammar_str.c_str());
|
||||||
bool grammar_fails = false;
|
bool grammar_fails = false;
|
||||||
|
@ -48,7 +47,8 @@ static bool match_string(const std::string & input, llama_grammar * grammar) {
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
static void test(const std::string & test_desc, const std::string & grammar_str, const std::vector<std::string> & passing_strings, const std::vector<std::string> & failing_strings) {
|
static void test(const std::string & test_desc, const std::string & grammar_str,
|
||||||
|
const std::vector<std::string> & passing_strings, const std::vector<std::string> & failing_strings) {
|
||||||
fprintf(stderr, "⚫ Testing %s\n%s\n", test_desc.c_str(), grammar_str.c_str());
|
fprintf(stderr, "⚫ Testing %s\n%s\n", test_desc.c_str(), grammar_str.c_str());
|
||||||
fflush(stderr);
|
fflush(stderr);
|
||||||
|
|
||||||
|
@ -75,20 +75,23 @@ static void test(const std::string & test_desc, const std::string & grammar_str,
|
||||||
|
|
||||||
// DEBUG: Write strings to files so that we can analyze more easily with gbnf-validator program to see exactly where things failed.
|
// DEBUG: Write strings to files so that we can analyze more easily with gbnf-validator program to see exactly where things failed.
|
||||||
// DEBUG: Write the grammar_str to test-grammar-integration.grammar.gbnf
|
// DEBUG: Write the grammar_str to test-grammar-integration.grammar.gbnf
|
||||||
FILE* grammar_file = fopen("test-grammar-integration.grammar.gbnf", "w");
|
FILE * grammar_file = fopen("test-grammar-integration.grammar.gbnf", "w");
|
||||||
if (grammar_file) {
|
if (grammar_file) {
|
||||||
fprintf(grammar_file, "%s", grammar_str.c_str());
|
fprintf(grammar_file, "%s", grammar_str.c_str());
|
||||||
fclose(grammar_file);
|
fclose(grammar_file);
|
||||||
}
|
}
|
||||||
|
|
||||||
// DEBUG: Write the test string to test-grammar-integration.string.txt
|
// DEBUG: Write the test string to test-grammar-integration.string.txt
|
||||||
FILE* string_file = fopen("test-grammar-integration.string.txt", "w");
|
FILE * string_file = fopen("test-grammar-integration.string.txt", "w");
|
||||||
if (string_file) {
|
if (string_file) {
|
||||||
fprintf(string_file, "%s", test_string.c_str());
|
fprintf(string_file, "%s", test_string.c_str());
|
||||||
fclose(string_file);
|
fclose(string_file);
|
||||||
}
|
}
|
||||||
|
|
||||||
fprintf(stderr, "\n NOTE: Debug grammar file generated. To analyze this failure in detail, run the following command: ./llama-gbnf-validator test-grammar-integration.grammar.gbnf test-grammar-integration.string.txt\n\n");
|
fprintf(stderr,
|
||||||
|
"\n NOTE: Debug grammar file generated. To analyze this failure in detail, run the following "
|
||||||
|
"command: ./llama-gbnf-validator test-grammar-integration.grammar.gbnf "
|
||||||
|
"test-grammar-integration.string.txt\n\n");
|
||||||
} else {
|
} else {
|
||||||
fprintf(stdout, "✅︎\n");
|
fprintf(stdout, "✅︎\n");
|
||||||
}
|
}
|
||||||
|
@ -122,16 +125,22 @@ static void test(const std::string & test_desc, const std::string & grammar_str,
|
||||||
// Clean up allocated memory
|
// Clean up allocated memory
|
||||||
llama_grammar_free_impl(grammar);
|
llama_grammar_free_impl(grammar);
|
||||||
}
|
}
|
||||||
static void test_grammar(const std::string & test_desc, const std::string & grammar_str, const std::vector<std::string> & passing_strings, const std::vector<std::string> & failing_strings) {
|
|
||||||
|
static void test_grammar(const std::string & test_desc, const std::string & grammar_str,
|
||||||
|
const std::vector<std::string> & passing_strings,
|
||||||
|
const std::vector<std::string> & failing_strings) {
|
||||||
test(test_desc + ". Grammar: " + grammar_str, grammar_str, passing_strings, failing_strings);
|
test(test_desc + ". Grammar: " + grammar_str, grammar_str, passing_strings, failing_strings);
|
||||||
}
|
}
|
||||||
static void test_schema(const std::string & test_desc, const std::string & schema_str, const std::vector<std::string> & passing_strings, const std::vector<std::string> & failing_strings) {
|
|
||||||
test(test_desc + ". Schema: " + schema_str, json_schema_to_grammar(json::parse(schema_str), true), passing_strings, failing_strings);
|
static void test_schema(const std::string & test_desc, const std::string & schema_str,
|
||||||
|
const std::vector<std::string> & passing_strings,
|
||||||
|
const std::vector<std::string> & failing_strings) {
|
||||||
|
test(test_desc + ". Schema: " + schema_str, json_schema_to_grammar(json::parse(schema_str), true), passing_strings,
|
||||||
|
failing_strings);
|
||||||
}
|
}
|
||||||
|
|
||||||
static void test_simple_grammar() {
|
static void test_simple_grammar() {
|
||||||
test_schema(
|
test_schema("min 0",
|
||||||
"min 0",
|
|
||||||
R"""({
|
R"""({
|
||||||
"type": "integer",
|
"type": "integer",
|
||||||
"minimum": 0
|
"minimum": 0
|
||||||
|
@ -153,10 +162,8 @@ static void test_simple_grammar() {
|
||||||
"00",
|
"00",
|
||||||
"01",
|
"01",
|
||||||
"-0",
|
"-0",
|
||||||
}
|
});
|
||||||
);
|
test_schema("min 2",
|
||||||
test_schema(
|
|
||||||
"min 2",
|
|
||||||
// Schema
|
// Schema
|
||||||
R"""({
|
R"""({
|
||||||
"type": "integer",
|
"type": "integer",
|
||||||
|
@ -182,10 +189,8 @@ static void test_simple_grammar() {
|
||||||
"01",
|
"01",
|
||||||
"02",
|
"02",
|
||||||
"12345678900000000",
|
"12345678900000000",
|
||||||
}
|
});
|
||||||
);
|
test_schema("min 456",
|
||||||
test_schema(
|
|
||||||
"min 456",
|
|
||||||
R"""({
|
R"""({
|
||||||
"type": "integer",
|
"type": "integer",
|
||||||
"minimum": 456
|
"minimum": 456
|
||||||
|
@ -206,10 +211,8 @@ static void test_simple_grammar() {
|
||||||
"050",
|
"050",
|
||||||
"-1",
|
"-1",
|
||||||
"-456",
|
"-456",
|
||||||
}
|
});
|
||||||
);
|
test_schema("min -123",
|
||||||
test_schema(
|
|
||||||
"min -123",
|
|
||||||
R"""({
|
R"""({
|
||||||
"type": "integer",
|
"type": "integer",
|
||||||
"minimum": -123
|
"minimum": -123
|
||||||
|
@ -230,11 +233,9 @@ static void test_simple_grammar() {
|
||||||
{
|
{
|
||||||
"-1234",
|
"-1234",
|
||||||
"-124",
|
"-124",
|
||||||
}
|
});
|
||||||
);
|
|
||||||
|
|
||||||
test_schema(
|
test_schema("max 9999",
|
||||||
"max 9999",
|
|
||||||
// Schema
|
// Schema
|
||||||
R"""({
|
R"""({
|
||||||
"type": "integer",
|
"type": "integer",
|
||||||
|
@ -250,10 +251,8 @@ static void test_simple_grammar() {
|
||||||
{
|
{
|
||||||
"10000",
|
"10000",
|
||||||
"99991",
|
"99991",
|
||||||
}
|
});
|
||||||
);
|
test_schema("max -9999",
|
||||||
test_schema(
|
|
||||||
"max -9999",
|
|
||||||
// Schema
|
// Schema
|
||||||
R"""({
|
R"""({
|
||||||
"type": "integer",
|
"type": "integer",
|
||||||
|
@ -269,10 +268,8 @@ static void test_simple_grammar() {
|
||||||
"-9998",
|
"-9998",
|
||||||
"0",
|
"0",
|
||||||
"9999",
|
"9999",
|
||||||
}
|
});
|
||||||
);
|
test_schema("min 5 max 30",
|
||||||
test_schema(
|
|
||||||
"min 5 max 30",
|
|
||||||
// Schema
|
// Schema
|
||||||
R"""({
|
R"""({
|
||||||
"type": "integer",
|
"type": "integer",
|
||||||
|
@ -293,10 +290,8 @@ static void test_simple_grammar() {
|
||||||
"31",
|
"31",
|
||||||
"123",
|
"123",
|
||||||
"0123",
|
"0123",
|
||||||
}
|
});
|
||||||
);
|
test_schema("min -1 max 1",
|
||||||
test_schema(
|
|
||||||
"min -1 max 1",
|
|
||||||
R"""({
|
R"""({
|
||||||
"type": "integer",
|
"type": "integer",
|
||||||
"minimum": -1,
|
"minimum": -1,
|
||||||
|
@ -316,10 +311,8 @@ static void test_simple_grammar() {
|
||||||
"2",
|
"2",
|
||||||
"10",
|
"10",
|
||||||
"11",
|
"11",
|
||||||
}
|
});
|
||||||
);
|
test_schema("min -123 max 42",
|
||||||
test_schema(
|
|
||||||
"min -123 max 42",
|
|
||||||
R"""({
|
R"""({
|
||||||
"type": "integer",
|
"type": "integer",
|
||||||
"minimum": -123,
|
"minimum": -123,
|
||||||
|
@ -350,10 +343,8 @@ static void test_simple_grammar() {
|
||||||
"43",
|
"43",
|
||||||
"123",
|
"123",
|
||||||
"0123",
|
"0123",
|
||||||
}
|
});
|
||||||
);
|
test_schema("exclusive min / max",
|
||||||
test_schema(
|
|
||||||
"exclusive min / max",
|
|
||||||
// Schema
|
// Schema
|
||||||
R"""({
|
R"""({
|
||||||
"type": "integer",
|
"type": "integer",
|
||||||
|
@ -371,12 +362,10 @@ static void test_simple_grammar() {
|
||||||
"01",
|
"01",
|
||||||
"10000",
|
"10000",
|
||||||
"99999",
|
"99999",
|
||||||
}
|
});
|
||||||
);
|
|
||||||
|
|
||||||
// Test case for a simple grammar
|
// Test case for a simple grammar
|
||||||
test_grammar(
|
test_grammar("simple grammar",
|
||||||
"simple grammar",
|
|
||||||
R"""(
|
R"""(
|
||||||
root ::= expr
|
root ::= expr
|
||||||
expr ::= term ("+" term)*
|
expr ::= term ("+" term)*
|
||||||
|
@ -394,14 +383,12 @@ static void test_simple_grammar() {
|
||||||
"/ 3",
|
"/ 3",
|
||||||
"1+2+3+4+5+",
|
"1+2+3+4+5+",
|
||||||
"12a45",
|
"12a45",
|
||||||
}
|
});
|
||||||
);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
static void test_complex_grammar() {
|
static void test_complex_grammar() {
|
||||||
// Test case for a more complex grammar, with both failure strings and success strings
|
// Test case for a more complex grammar, with both failure strings and success strings
|
||||||
test_grammar(
|
test_grammar("medium complexity grammar",
|
||||||
"medium complexity grammar",
|
|
||||||
// Grammar
|
// Grammar
|
||||||
R"""(
|
R"""(
|
||||||
root ::= expression
|
root ::= expression
|
||||||
|
@ -413,8 +400,7 @@ static void test_complex_grammar() {
|
||||||
function-call ::= variable ws "(" (expression ("," ws expression)*)? ")"
|
function-call ::= variable ws "(" (expression ("," ws expression)*)? ")"
|
||||||
ws ::= [ \t\n\r]?)""",
|
ws ::= [ \t\n\r]?)""",
|
||||||
// Passing strings
|
// Passing strings
|
||||||
{
|
{ "42",
|
||||||
"42",
|
|
||||||
"1*2*3*4*5",
|
"1*2*3*4*5",
|
||||||
"x",
|
"x",
|
||||||
"x+10",
|
"x+10",
|
||||||
|
@ -433,8 +419,7 @@ static void test_complex_grammar() {
|
||||||
"f(g(x), h(y, z))",
|
"f(g(x), h(y, z))",
|
||||||
"123+456",
|
"123+456",
|
||||||
"123*456*789-123/456+789*123",
|
"123*456*789-123/456+789*123",
|
||||||
"123+456*789-123/456+789*123-456/789+123*456-789/123+456*789-123/456+789*123-456"
|
"123+456*789-123/456+789*123-456/789+123*456-789/123+456*789-123/456+789*123-456" },
|
||||||
},
|
|
||||||
// Failing strings
|
// Failing strings
|
||||||
{
|
{
|
||||||
"+",
|
"+",
|
||||||
|
@ -455,35 +440,22 @@ static void test_complex_grammar() {
|
||||||
"a * (b + c) - d /",
|
"a * (b + c) - d /",
|
||||||
"f(g(x), h(y, z)",
|
"f(g(x), h(y, z)",
|
||||||
"123+456*789-123/456+789*123-456/789+123*456-789/123+456*789-123/456+789*123-456/",
|
"123+456*789-123/456+789*123-456/789+123*456-789/123+456*789-123/456+789*123-456/",
|
||||||
}
|
});
|
||||||
);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
static void test_special_chars() {
|
static void test_special_chars() {
|
||||||
// A collection of tests to exercise special characters such as "."
|
// A collection of tests to exercise special characters such as "."
|
||||||
test_grammar(
|
test_grammar("special characters",
|
||||||
"special characters",
|
|
||||||
// Grammar
|
// Grammar
|
||||||
R"""(
|
R"""(
|
||||||
root ::= ... "abc" ...
|
root ::= ... "abc" ...
|
||||||
)""",
|
)""",
|
||||||
// Passing strings
|
// Passing strings
|
||||||
{
|
{ "abcabcabc", "aaaabcccc",
|
||||||
"abcabcabc",
|
|
||||||
"aaaabcccc",
|
|
||||||
// NOTE: Also ensures that multi-byte characters still count as a single character
|
// NOTE: Also ensures that multi-byte characters still count as a single character
|
||||||
"🔵🟠✅abc❌🟠🔵"
|
"🔵🟠✅abc❌🟠🔵" },
|
||||||
},
|
|
||||||
// Failing strings
|
// Failing strings
|
||||||
{
|
{ "aaabcccc", "aaaaabcccc", "aaaabccc", "aaaabccccc", "🔵🟠✅❌abc❌✅🟠🔵", "🔵🟠abc🟠🔵" });
|
||||||
"aaabcccc",
|
|
||||||
"aaaaabcccc",
|
|
||||||
"aaaabccc",
|
|
||||||
"aaaabccccc",
|
|
||||||
"🔵🟠✅❌abc❌✅🟠🔵",
|
|
||||||
"🔵🟠abc🟠🔵"
|
|
||||||
}
|
|
||||||
);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
static void test_quantifiers() {
|
static void test_quantifiers() {
|
||||||
|
@ -494,62 +466,30 @@ static void test_quantifiers() {
|
||||||
// Grammar
|
// Grammar
|
||||||
R"""(root ::= "a"*)""",
|
R"""(root ::= "a"*)""",
|
||||||
// Passing strings
|
// Passing strings
|
||||||
{
|
{ "", "a", "aaaaa", "aaaaaaaaaaaaaaaaaa", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" },
|
||||||
"",
|
|
||||||
"a",
|
|
||||||
"aaaaa",
|
|
||||||
"aaaaaaaaaaaaaaaaaa",
|
|
||||||
"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
|
|
||||||
},
|
|
||||||
// Failing strings
|
// Failing strings
|
||||||
{
|
{ "b", "ab", "aab", "ba", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaab" });
|
||||||
"b",
|
|
||||||
"ab",
|
|
||||||
"aab",
|
|
||||||
"ba",
|
|
||||||
"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaab"
|
|
||||||
}
|
|
||||||
);
|
|
||||||
test_grammar(
|
test_grammar(
|
||||||
"+ quantifier",
|
"+ quantifier",
|
||||||
// Grammar
|
// Grammar
|
||||||
R"""(root ::= "a"+)""",
|
R"""(root ::= "a"+)""",
|
||||||
// Passing strings
|
// Passing strings
|
||||||
{
|
{ "a", "aaaaa", "aaaaaaaaaaaaaaaaaa", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" },
|
||||||
"a",
|
|
||||||
"aaaaa",
|
|
||||||
"aaaaaaaaaaaaaaaaaa",
|
|
||||||
"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
|
|
||||||
},
|
|
||||||
// Failing strings
|
// Failing strings
|
||||||
{
|
{ "", "b", "ab", "aab", "ba", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaab" });
|
||||||
"",
|
test_grammar("? quantifier",
|
||||||
"b",
|
|
||||||
"ab",
|
|
||||||
"aab",
|
|
||||||
"ba",
|
|
||||||
"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaab"
|
|
||||||
}
|
|
||||||
);
|
|
||||||
test_grammar(
|
|
||||||
"? quantifier",
|
|
||||||
// Grammar
|
// Grammar
|
||||||
R"""(root ::= "a"?)""",
|
R"""(root ::= "a"?)""",
|
||||||
// Passing strings
|
// Passing strings
|
||||||
{
|
{ "", "a" },
|
||||||
"",
|
|
||||||
"a"
|
|
||||||
},
|
|
||||||
// Failing strings
|
// Failing strings
|
||||||
{
|
{
|
||||||
"b",
|
"b",
|
||||||
"ab",
|
"ab",
|
||||||
"aa",
|
"aa",
|
||||||
"ba",
|
"ba",
|
||||||
}
|
});
|
||||||
);
|
test_grammar("mixed quantifiers",
|
||||||
test_grammar(
|
|
||||||
"mixed quantifiers",
|
|
||||||
// Grammar
|
// Grammar
|
||||||
R"""(
|
R"""(
|
||||||
root ::= cons+ vowel* cons? (vowel cons)*
|
root ::= cons+ vowel* cons? (vowel cons)*
|
||||||
|
@ -571,10 +511,8 @@ static void test_quantifiers() {
|
||||||
"yesno",
|
"yesno",
|
||||||
"forty",
|
"forty",
|
||||||
"catyyy",
|
"catyyy",
|
||||||
}
|
});
|
||||||
);
|
test_grammar("simple exact repetition",
|
||||||
test_grammar(
|
|
||||||
"simple exact repetition",
|
|
||||||
// Grammar
|
// Grammar
|
||||||
R"""(
|
R"""(
|
||||||
root ::= [ab]{4}
|
root ::= [ab]{4}
|
||||||
|
@ -590,10 +528,8 @@ static void test_quantifiers() {
|
||||||
"a",
|
"a",
|
||||||
"b",
|
"b",
|
||||||
"aaaaa",
|
"aaaaa",
|
||||||
}
|
});
|
||||||
);
|
test_grammar("simple min repetition",
|
||||||
test_grammar(
|
|
||||||
"simple min repetition",
|
|
||||||
// Grammar
|
// Grammar
|
||||||
R"""(
|
R"""(
|
||||||
root ::= [ab]{4,}
|
root ::= [ab]{4,}
|
||||||
|
@ -609,10 +545,8 @@ static void test_quantifiers() {
|
||||||
{
|
{
|
||||||
"",
|
"",
|
||||||
"aba",
|
"aba",
|
||||||
}
|
});
|
||||||
);
|
test_grammar("simple max repetition",
|
||||||
test_grammar(
|
|
||||||
"simple max repetition",
|
|
||||||
// Grammar
|
// Grammar
|
||||||
R"""(
|
R"""(
|
||||||
root ::= [ab]{0,4}
|
root ::= [ab]{0,4}
|
||||||
|
@ -628,10 +562,8 @@ static void test_quantifiers() {
|
||||||
// Failing strings
|
// Failing strings
|
||||||
{
|
{
|
||||||
"aaaaa",
|
"aaaaa",
|
||||||
}
|
});
|
||||||
);
|
test_grammar("min / max repetition",
|
||||||
test_grammar(
|
|
||||||
"min / max repetition",
|
|
||||||
// Grammar
|
// Grammar
|
||||||
R"""(
|
R"""(
|
||||||
root ::= ("0x" [A-F0-9]{2} " "?){3,5}
|
root ::= ("0x" [A-F0-9]{2} " "?){3,5}
|
||||||
|
@ -647,8 +579,7 @@ static void test_quantifiers() {
|
||||||
"0xFF",
|
"0xFF",
|
||||||
"0xFF 0x12",
|
"0xFF 0x12",
|
||||||
"0xFF 0x12 0xAB 0x00 0x00 0x00",
|
"0xFF 0x12 0xAB 0x00 0x00 0x00",
|
||||||
}
|
});
|
||||||
);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
static void test_failure_missing_root() {
|
static void test_failure_missing_root() {
|
||||||
|
@ -730,8 +661,7 @@ static void test_json_schema() {
|
||||||
// but we convert each json schema to a grammar before parsing.
|
// but we convert each json schema to a grammar before parsing.
|
||||||
// Otherwise, this test structure is the same.
|
// Otherwise, this test structure is the same.
|
||||||
|
|
||||||
test_schema(
|
test_schema("empty schema (object)",
|
||||||
"empty schema (object)",
|
|
||||||
// Schema
|
// Schema
|
||||||
R"""(
|
R"""(
|
||||||
{}
|
{}
|
||||||
|
@ -748,8 +678,7 @@ static void test_json_schema() {
|
||||||
"null",
|
"null",
|
||||||
R"""("")""",
|
R"""("")""",
|
||||||
"true",
|
"true",
|
||||||
}
|
});
|
||||||
);
|
|
||||||
|
|
||||||
test_schema(
|
test_schema(
|
||||||
"exotic formats (list)",
|
"exotic formats (list)",
|
||||||
|
@ -774,11 +703,9 @@ static void test_json_schema() {
|
||||||
{
|
{
|
||||||
R"""(["foo", "bar"])""",
|
R"""(["foo", "bar"])""",
|
||||||
R"""(["12345678-1234-1234-1234-1234567890ab"])""",
|
R"""(["12345678-1234-1234-1234-1234567890ab"])""",
|
||||||
}
|
});
|
||||||
);
|
|
||||||
|
|
||||||
test_schema(
|
test_schema("string",
|
||||||
"string",
|
|
||||||
// Schema
|
// Schema
|
||||||
R"""({
|
R"""({
|
||||||
"type": "string"
|
"type": "string"
|
||||||
|
@ -793,11 +720,9 @@ static void test_json_schema() {
|
||||||
{
|
{
|
||||||
R"""({})""",
|
R"""({})""",
|
||||||
R"""("foo": "bar")""",
|
R"""("foo": "bar")""",
|
||||||
}
|
});
|
||||||
);
|
|
||||||
|
|
||||||
test_schema(
|
test_schema("string w/ min length 1",
|
||||||
"string w/ min length 1",
|
|
||||||
// Schema
|
// Schema
|
||||||
R"""({
|
R"""({
|
||||||
"type": "string",
|
"type": "string",
|
||||||
|
@ -813,11 +738,9 @@ static void test_json_schema() {
|
||||||
R"""("")""",
|
R"""("")""",
|
||||||
R"""({})""",
|
R"""({})""",
|
||||||
R"""("foo": "bar")""",
|
R"""("foo": "bar")""",
|
||||||
}
|
});
|
||||||
);
|
|
||||||
|
|
||||||
test_schema(
|
test_schema("string w/ min length 3",
|
||||||
"string w/ min length 3",
|
|
||||||
// Schema
|
// Schema
|
||||||
R"""({
|
R"""({
|
||||||
"type": "string",
|
"type": "string",
|
||||||
|
@ -834,11 +757,9 @@ static void test_json_schema() {
|
||||||
R"""("")""",
|
R"""("")""",
|
||||||
R"""("f")""",
|
R"""("f")""",
|
||||||
R"""("fo")""",
|
R"""("fo")""",
|
||||||
}
|
});
|
||||||
);
|
|
||||||
|
|
||||||
test_schema(
|
test_schema("string w/ max length",
|
||||||
"string w/ max length",
|
|
||||||
// Schema
|
// Schema
|
||||||
R"""({
|
R"""({
|
||||||
"type": "string",
|
"type": "string",
|
||||||
|
@ -855,11 +776,9 @@ static void test_json_schema() {
|
||||||
// Failing strings
|
// Failing strings
|
||||||
{
|
{
|
||||||
R"""("foobar")""",
|
R"""("foobar")""",
|
||||||
}
|
});
|
||||||
);
|
|
||||||
|
|
||||||
test_schema(
|
test_schema("string w/ min & max length",
|
||||||
"string w/ min & max length",
|
|
||||||
// Schema
|
// Schema
|
||||||
R"""({
|
R"""({
|
||||||
"type": "string",
|
"type": "string",
|
||||||
|
@ -878,11 +797,9 @@ static void test_json_schema() {
|
||||||
R"""("")""",
|
R"""("")""",
|
||||||
R"""("barfo")""",
|
R"""("barfo")""",
|
||||||
R"""("foobar")""",
|
R"""("foobar")""",
|
||||||
}
|
});
|
||||||
);
|
|
||||||
|
|
||||||
test_schema(
|
test_schema("boolean",
|
||||||
"boolean",
|
|
||||||
// Schema
|
// Schema
|
||||||
R"""({
|
R"""({
|
||||||
"type": "boolean"
|
"type": "boolean"
|
||||||
|
@ -898,11 +815,9 @@ static void test_json_schema() {
|
||||||
R"""("true")""",
|
R"""("true")""",
|
||||||
R"""(True)""",
|
R"""(True)""",
|
||||||
R"""(FALSE)""",
|
R"""(FALSE)""",
|
||||||
}
|
});
|
||||||
);
|
|
||||||
|
|
||||||
test_schema(
|
test_schema("integer",
|
||||||
"integer",
|
|
||||||
// Schema
|
// Schema
|
||||||
R"""({
|
R"""({
|
||||||
"type": "integer"
|
"type": "integer"
|
||||||
|
@ -919,11 +834,9 @@ static void test_json_schema() {
|
||||||
R"""(01)""",
|
R"""(01)""",
|
||||||
R"""(007)""",
|
R"""(007)""",
|
||||||
R"""(12345678901234567 )""",
|
R"""(12345678901234567 )""",
|
||||||
}
|
});
|
||||||
);
|
|
||||||
|
|
||||||
test_schema(
|
test_schema("string const",
|
||||||
"string const",
|
|
||||||
// Schema
|
// Schema
|
||||||
R"""({
|
R"""({
|
||||||
"const": "foo"
|
"const": "foo"
|
||||||
|
@ -936,11 +849,9 @@ static void test_json_schema() {
|
||||||
{
|
{
|
||||||
R"""(foo)""",
|
R"""(foo)""",
|
||||||
R"""("bar")""",
|
R"""("bar")""",
|
||||||
}
|
});
|
||||||
);
|
|
||||||
|
|
||||||
test_schema(
|
test_schema("non-string const",
|
||||||
"non-string const",
|
|
||||||
// Schema
|
// Schema
|
||||||
R"""({
|
R"""({
|
||||||
"const": true
|
"const": true
|
||||||
|
@ -954,11 +865,9 @@ static void test_json_schema() {
|
||||||
R"""()""",
|
R"""()""",
|
||||||
R"""(foo)""",
|
R"""(foo)""",
|
||||||
R"""("true")""",
|
R"""("true")""",
|
||||||
}
|
});
|
||||||
);
|
|
||||||
|
|
||||||
test_schema(
|
test_schema("non-string const",
|
||||||
"non-string const",
|
|
||||||
// Schema
|
// Schema
|
||||||
R"""({
|
R"""({
|
||||||
"enum": ["red", "amber", "green", null, 42, ["foo"]]
|
"enum": ["red", "amber", "green", null, 42, ["foo"]]
|
||||||
|
@ -976,11 +885,9 @@ static void test_json_schema() {
|
||||||
R"""(420)""",
|
R"""(420)""",
|
||||||
R"""(true)""",
|
R"""(true)""",
|
||||||
R"""(foo)""",
|
R"""(foo)""",
|
||||||
}
|
});
|
||||||
);
|
|
||||||
|
|
||||||
test_schema(
|
test_schema("simple pattern",
|
||||||
"simple pattern",
|
|
||||||
// Schema
|
// Schema
|
||||||
R"""({
|
R"""({
|
||||||
"pattern": "^[a-zA-Z0-9_-]*$"
|
"pattern": "^[a-zA-Z0-9_-]*$"
|
||||||
|
@ -994,11 +901,9 @@ static void test_json_schema() {
|
||||||
{
|
{
|
||||||
R"""("!")""",
|
R"""("!")""",
|
||||||
R"""("Hello World")""",
|
R"""("Hello World")""",
|
||||||
}
|
});
|
||||||
);
|
|
||||||
|
|
||||||
test_schema(
|
test_schema("pattern with escapes",
|
||||||
"pattern with escapes",
|
|
||||||
// Schema
|
// Schema
|
||||||
R"""({
|
R"""({
|
||||||
"pattern": "^a\\^\\$\\.\\[\\]\\(\\)\\|\\{\\}\\*\\+\\?b$"
|
"pattern": "^a\\^\\$\\.\\[\\]\\(\\)\\|\\{\\}\\*\\+\\?b$"
|
||||||
|
@ -1010,11 +915,9 @@ static void test_json_schema() {
|
||||||
// Failing strings
|
// Failing strings
|
||||||
{
|
{
|
||||||
R"""("ab")""",
|
R"""("ab")""",
|
||||||
}
|
});
|
||||||
);
|
|
||||||
|
|
||||||
test_schema(
|
test_schema("",
|
||||||
"",
|
|
||||||
// Schema
|
// Schema
|
||||||
R"""(
|
R"""(
|
||||||
{
|
{
|
||||||
|
@ -1035,11 +938,9 @@ static void test_json_schema() {
|
||||||
"[123]",
|
"[123]",
|
||||||
"\"foo\"",
|
"\"foo\"",
|
||||||
"[\"foo\", 42]",
|
"[\"foo\", 42]",
|
||||||
}
|
});
|
||||||
);
|
|
||||||
|
|
||||||
test_schema(
|
test_schema("min+max items",
|
||||||
"min+max items",
|
|
||||||
// Schema
|
// Schema
|
||||||
R"""({
|
R"""({
|
||||||
"items": {
|
"items": {
|
||||||
|
@ -1059,12 +960,10 @@ static void test_json_schema() {
|
||||||
R"""([1, 2])""",
|
R"""([1, 2])""",
|
||||||
R"""([1, 2, 3, 4, 5, 6])""",
|
R"""([1, 2, 3, 4, 5, 6])""",
|
||||||
R"""(1)""",
|
R"""(1)""",
|
||||||
}
|
});
|
||||||
);
|
|
||||||
|
|
||||||
// Properties (from: https://json-schema.org/understanding-json-schema/reference/object#properties)
|
// Properties (from: https://json-schema.org/understanding-json-schema/reference/object#properties)
|
||||||
test_schema(
|
test_schema("object properties",
|
||||||
"object properties",
|
|
||||||
// Schema
|
// Schema
|
||||||
R"""({
|
R"""({
|
||||||
"type": "object",
|
"type": "object",
|
||||||
|
@ -1095,11 +994,9 @@ static void test_json_schema() {
|
||||||
// "Additional properties default to false for generation, even though the spec says true.
|
// "Additional properties default to false for generation, even though the spec says true.
|
||||||
R"""({ "number": 1600, "street_name": "Pennsylvania", "street_type":"Avenue", "direction":"NW"})""",
|
R"""({ "number": 1600, "street_name": "Pennsylvania", "street_type":"Avenue", "direction":"NW"})""",
|
||||||
|
|
||||||
}
|
});
|
||||||
);
|
|
||||||
|
|
||||||
test_schema(
|
test_schema("additional properties can't override other properties",
|
||||||
"additional properties can't override other properties",
|
|
||||||
R"""({
|
R"""({
|
||||||
"properties": {
|
"properties": {
|
||||||
"a": {"type": "integer"},
|
"a": {"type": "integer"},
|
||||||
|
@ -1119,12 +1016,10 @@ static void test_json_schema() {
|
||||||
R"""()""",
|
R"""()""",
|
||||||
R"""({"a": ""})""",
|
R"""({"a": ""})""",
|
||||||
R"""({"a": "", "b": ""})""",
|
R"""({"a": "", "b": ""})""",
|
||||||
}
|
});
|
||||||
);
|
|
||||||
|
|
||||||
// Properties (from: https://json-schema.org/understanding-json-schema/reference/object#properties)
|
// Properties (from: https://json-schema.org/understanding-json-schema/reference/object#properties)
|
||||||
test_schema(
|
test_schema("object properties, additionalProperties: true",
|
||||||
"object properties, additionalProperties: true",
|
|
||||||
// Schema
|
// Schema
|
||||||
R"""({
|
R"""({
|
||||||
"type": "object",
|
"type": "object",
|
||||||
|
@ -1153,8 +1048,7 @@ static void test_json_schema() {
|
||||||
R"""({ "number": "1600", "street_name": "Pennsylvania", "street_type":"Avenue"})""",
|
R"""({ "number": "1600", "street_name": "Pennsylvania", "street_type":"Avenue"})""",
|
||||||
// Reorder properties
|
// Reorder properties
|
||||||
R"""({ "street_name": "Pennsylvania", "number": 1600, "street_type":"Avenue"})""",
|
R"""({ "street_name": "Pennsylvania", "number": 1600, "street_type":"Avenue"})""",
|
||||||
}
|
});
|
||||||
);
|
|
||||||
|
|
||||||
// Additional properties: false
|
// Additional properties: false
|
||||||
test_schema(
|
test_schema(
|
||||||
|
@ -1184,11 +1078,9 @@ static void test_json_schema() {
|
||||||
R"""({ "street_type": "Avenue", "number": 1600 })""",
|
R"""({ "street_type": "Avenue", "number": 1600 })""",
|
||||||
// Add "direction"
|
// Add "direction"
|
||||||
R"""({ "number": 1600, "street_name": "Pennsylvania", "street_type": "Avenue", "direction": "NW" })""",
|
R"""({ "number": 1600, "street_name": "Pennsylvania", "street_type": "Avenue", "direction": "NW" })""",
|
||||||
}
|
});
|
||||||
);
|
|
||||||
|
|
||||||
test_schema(
|
test_schema("required + optional props each in original order",
|
||||||
"required + optional props each in original order",
|
|
||||||
// Schema
|
// Schema
|
||||||
R"""({
|
R"""({
|
||||||
"properties": {
|
"properties": {
|
||||||
|
@ -1212,8 +1104,7 @@ static void test_json_schema() {
|
||||||
R"""({"b": "bar"})""",
|
R"""({"b": "bar"})""",
|
||||||
R"""({"a": "foo", "c": "baz"})""",
|
R"""({"a": "foo", "c": "baz"})""",
|
||||||
R"""({"a":"foo", "b":"bar", "c":"baz", "d":"qux"})""",
|
R"""({"a":"foo", "b":"bar", "c":"baz", "d":"qux"})""",
|
||||||
}
|
});
|
||||||
);
|
|
||||||
|
|
||||||
// NOTE: Example from https://json-schema.org/learn/getting-started-step-by-step#define-required-properties
|
// NOTE: Example from https://json-schema.org/learn/getting-started-step-by-step#define-required-properties
|
||||||
test_schema(
|
test_schema(
|
||||||
|
@ -1285,8 +1176,7 @@ static void test_json_schema() {
|
||||||
R"""({"productId": 1, "productName": "A green door", "price": 12.50, "dimensions": {"length": 785, "width": 250.5, "height": -0.359}, "tags": ["home", "green"]})""", // Tags and dimensions are out of order
|
R"""({"productId": 1, "productName": "A green door", "price": 12.50, "dimensions": {"length": 785, "width": 250.5, "height": -0.359}, "tags": ["home", "green"]})""", // Tags and dimensions are out of order
|
||||||
// TODO: The following line should fail, but currently it passes. `uniqueItems` is not supported, as it would likely be too difficult to implement.
|
// TODO: The following line should fail, but currently it passes. `uniqueItems` is not supported, as it would likely be too difficult to implement.
|
||||||
// R"""({"productId": 1, "productName": "A green door", "price": 12.50, "tags": ["home", "green", "home"]})""",
|
// R"""({"productId": 1, "productName": "A green door", "price": 12.50, "tags": ["home", "green", "home"]})""",
|
||||||
}
|
});
|
||||||
);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
int main(int argc, const char ** argv) {
|
int main(int argc, const char ** argv) {
|
||||||
|
@ -1301,7 +1191,6 @@ int main(int argc, const char ** argv) {
|
||||||
|
|
||||||
fprintf(stderr, "reading vocab from: '%s'\n", vocab_file);
|
fprintf(stderr, "reading vocab from: '%s'\n", vocab_file);
|
||||||
|
|
||||||
|
|
||||||
test_simple_grammar();
|
test_simple_grammar();
|
||||||
test_complex_grammar();
|
test_complex_grammar();
|
||||||
test_special_chars();
|
test_special_chars();
|
||||||
|
|
Loading…
Add table
Add a link
Reference in a new issue