vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Separating test calls to give more helpful stack traces on failure. Adding verbose messages to give visibility for what is being tested.

Browse source
This commit is contained in:
Clint Herron 2024-04-27 09:38:45 -04:00
parent 7b04c215eb
commit b7f24dba00
1 changed files with 187 additions and 212 deletions
Download patch file Download diff file Expand all files Collapse all files

399
tests/test-grammar-integration.cpp
View file

@ -52,6 +52,49 @@ static bool match_string(const std::string & input, llama_grammar* grammar) {
return false; return false;
} }
static void test_grammar(const std::string & grammar_str, const std::vector<std::string> & passing_strings, const std::vector<std::string> & failing_strings) {
fprintf(stderr, "🟢 Testing grammar: %s\n", grammar_str.c_str());
auto grammar = build_grammar(grammar_str);
// Save the original grammar stacks so that we can reset after every new string we want to test
auto original_stacks = grammar->stacks;
// Passing strings
for (const auto & test_string : passing_strings) {
bool matched = match_string(test_string, grammar);
if (!matched) {
fprintf(stderr, " ❌ Failed to match string: %s\n", test_string.c_str());
} else {
fprintf(stdout, " ✅︎ Matched string: %s\n", test_string.c_str());
}
assert(matched);
// Reset the grammar stacks
grammar->stacks = original_stacks;
}
// Failing strings
for (const auto & test_string : failing_strings) {
bool matched = match_string(test_string, grammar);
if (matched) {
fprintf(stderr, " ❌ Improperly matched string: %s\n", test_string.c_str());
} else {
fprintf(stdout, " ✅︎ Correctly did not match string: %s\n", test_string.c_str());
}
assert(!matched);
// Reset the grammar stacks
grammar->stacks = original_stacks;
}
// Clean up allocated memory
llama_grammar_free(grammar);
}
static void test_simple_grammar() { static void test_simple_grammar() {
// Test case for a simple grammar // Test case for a simple grammar
const std::string grammar_str = R"""(root ::= expr const std::string grammar_str = R"""(root ::= expr
@ -71,225 +114,151 @@ number ::= [0-9]+)""";
static void test_complex_grammar() { static void test_complex_grammar() {
// Test case for a more complex grammar, with both failure strings and success strings // Test case for a more complex grammar, with both failure strings and success strings
const std::string grammar_str = R"""(root ::= expression test_grammar(
expression ::= term ws (("+"|"-") ws term)* // Grammar
term ::= factor ws (("*"|"/") ws factor)* R"""(
factor ::= number | variable | "(" expression ")" | function-call root ::= expression
number ::= [0-9]+ expression ::= term ws (("+"|"-") ws term)*
variable ::= [a-zA-Z_][a-zA-Z0-9_]* term ::= factor ws (("*"|"/") ws factor)*
function-call ::= variable ws "(" (expression ("," ws expression)*)? ")" factor ::= number | variable | "(" expression ")" | function-call
ws ::= [ \t\n\r]?)"""; number ::= [0-9]+
variable ::= [a-zA-Z_][a-zA-Z0-9_]*
auto grammar = build_grammar(grammar_str); function-call ::= variable ws "(" (expression ("," ws expression)*)? ")"
ws ::= [ \t\n\r]?)""",
// Save the original grammar stacks so that we can reset after every new string we want to test // Passing strings
auto original_stacks = grammar->stacks; {
"42",
// Test a few strings "1*2*3*4*5",
std::vector<std::string> test_strings_pass = { "x",
"42", "x+10",
"1*2*3*4*5", "x1+y2",
"x", "(a+b)*(c-d)",
"x+10", "func()",
"x1+y2", "func(x,y+2)",
"(a+b)*(c-d)", "a*(b+c)-d/e",
"func()", "f(g(x),h(y,z))",
"func(x,y+2)", "x + 10",
"a*(b+c)-d/e", "x1 + y2",
"f(g(x),h(y,z))", "(a + b) * (c - d)",
"x + 10", "func()",
"x1 + y2", "func(x, y + 2)",
"(a + b) * (c - d)", "a * (b + c) - d / e",
"func()", "f(g(x), h(y, z))",
"func(x, y + 2)", "123+456",
"a * (b + c) - d / e", "123*456*789-123/456+789*123",
"f(g(x), h(y, z))", "123+456*789-123/456+789*123-456/789+123*456-789/123+456*789-123/456+789*123-456"
"123+456", },
"123*456*789-123/456+789*123", // Failing strings
"123+456*789-123/456+789*123-456/789+123*456-789/123+456*789-123/456+789*123-456" {
}; "+",
"/ 3x",
std::vector<std::string> test_strings_fail = { "x + + y",
"+", "a * / b",
"/ 3x", "func(,)",
"x + + y", "func(x y)",
"a * / b", "(a + b",
"func(,)", "x + y)",
"func(x y)", "a + b * (c - d",
"(a + b", "42 +",
"x + y)", "x +",
"a + b * (c - d", "x + 10 +",
"42 +", "(a + b) * (c - d",
"x +", "func(",
"x + 10 +", "func(x, y + 2",
"(a + b) * (c - d", "a * (b + c) - d /",
"func(", "f(g(x), h(y, z)",
"func(x, y + 2", "123+456*789-123/456+789*123-456/789+123*456-789/123+456*789-123/456+789*123-456/",
"a * (b + c) - d /", }
"f(g(x), h(y, z)", );
"123+456*789-123/456+789*123-456/789+123*456-789/123+456*789-123/456+789*123-456/",
};
// Passing strings
for (const auto & test_string : test_strings_pass) {
bool matched = match_string(test_string, grammar);
assert(matched);
// Reset the grammar stacks
grammar->stacks = original_stacks;
}
// Failing strings
for (const auto & test_string : test_strings_fail) {
bool matched = match_string(test_string, grammar);
assert(!matched);
// Reset the grammar stacks
grammar->stacks = original_stacks;
}
// Clean up allocated memory
llama_grammar_free(grammar);
} }
static void test_quantifiers() { static void test_quantifiers() {
// Populate test data with grammar strings and their associated collections of expected passing and failing strings // A collection of tests to exercise * + and ? quantifiers
const std::vector<
std::tuple<
std::string,
std::vector<std::string>,
std::vector<std::string>>>
test_data = {
{
// Grammar
R"""(root ::= "a"*)""",
// Passing strings
{
"",
"a",
"aaaaa",
"aaaaaaaaaaaaaaaaaa",
"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
},
// Failing strings
{
"b",
"ab",
"aab",
"ba",
"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaab"
}
},
{
// Grammar
R"""(root ::= "a"+)""",
// Passing strings
{
"a",
"aaaaa",
"aaaaaaaaaaaaaaaaaa",
"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
},
// Failing strings
{
"",
"b",
"ab",
"aab",
"ba",
"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaab"
}
},
{
// Grammar
R"""(root ::= "a"?)""",
// Passing strings
{
"",
"a"
},
// Failing strings
{
"b",
"ab",
"aa",
"ba",
}
},
{
// Grammar
R"""(
root ::= cons+ vowel* cons? (vowel cons)*
vowel ::= [aeiouy]
cons ::= [bcdfghjklmnpqrstvwxyz]
)""",
// Passing strings
{
"yes",
"no",
"noyes",
"crwth",
"four",
"bryyyy",
},
// Failing strings
{
"yess",
"yesno",
"forty",
"catyyy",
}
}
};
for (const auto & test_datum : test_data) {
const auto & grammar_str = std::get<0>(test_datum);
const auto & passing_strings = std::get<1>(test_datum);
const auto & failing_strings = std::get<2>(test_datum);
auto grammar = build_grammar(grammar_str);
// Save the original grammar stacks so that we can reset after every new string we want to test
auto original_stacks = grammar->stacks;
test_grammar(
// Grammar
R"""(root ::= "a"*)""",
// Passing strings // Passing strings
for (const auto & test_string : passing_strings) { {
bool matched = match_string(test_string, grammar); "",
"a",
if (!matched) { "aaaaa",
fprintf(stderr, "Against grammar: %s\n", grammar_str.c_str()); "aaaaaaaaaaaaaaaaaa",
fprintf(stderr, "Failed to match string: %s\n", test_string.c_str()); "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
} },
assert(matched);
// Reset the grammar stacks
grammar->stacks = original_stacks;
}
// Failing strings // Failing strings
for (const auto & test_string : failing_strings) { {
bool matched = match_string(test_string, grammar); "b",
"ab",
if (matched) { "aab",
fprintf(stderr, "Against grammar: %s\n", grammar_str.c_str()); "ba",
fprintf(stderr, "Improperly matched string: %s\n", test_string.c_str()); "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaab"
}
assert(!matched);
// Reset the grammar stacks
grammar->stacks = original_stacks;
} }
);
// Clean up allocated memory test_grammar(
llama_grammar_free(grammar); // Grammar
} R"""(root ::= "a"+)""",
// Passing strings
{
"a",
"aaaaa",
"aaaaaaaaaaaaaaaaaa",
"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
},
// Failing strings
{
"",
"b",
"ab",
"aab",
"ba",
"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaab"
}
);
test_grammar(
// Grammar
R"""(root ::= "a"?)""",
// Passing strings
{
"",
"a"
},
// Failing strings
{
"b",
"ab",
"aa",
"ba",
}
);
test_grammar(
// Grammar
R"""(
root ::= cons+ vowel* cons? (vowel cons)*
vowel ::= [aeiouy]
cons ::= [bcdfghjklmnpqrstvwxyz]
)""",
// Passing strings
{
"yes",
"no",
"noyes",
"crwth",
"four",
"bryyyy",
},
// Failing strings
{
"yess",
"yesno",
"forty",
"catyyy",
}
);
} }
static void test_failure_missing_root() { static void test_failure_missing_root() {
fprintf(stderr, "🟢 Testing for missing root node:\n");
// Test case for a grammar that is missing a root rule // Test case for a grammar that is missing a root rule
const std::string grammar_str = R"""(rot ::= expr const std::string grammar_str = R"""(rot ::= expr
expr ::= term ("+" term)* expr ::= term ("+" term)*
@ -303,26 +272,32 @@ number ::= [0-9]+)""";
// Ensure we do NOT have a root node // Ensure we do NOT have a root node
assert(parsed_grammar.symbol_ids.find("root") == parsed_grammar.symbol_ids.end()); assert(parsed_grammar.symbol_ids.find("root") == parsed_grammar.symbol_ids.end());
fprintf(stderr, " ✅︎ Passed\n");
} }
static void test_failure_missing_reference() { static void test_failure_missing_reference() {
fprintf(stderr, "🟢 Testing for missing reference node:\n");
// Test case for a grammar that is missing a referenced rule // Test case for a grammar that is missing a referenced rule
const std::string grammar_str = R"""(root ::= expr const std::string grammar_str =
R"""(root ::= expr
expr ::= term ("+" term)* expr ::= term ("+" term)*
term ::= numero term ::= numero
number ::= [0-9]+)"""; number ::= [0-9]+)""";
fprintf(stderr, "Expected error: "); fprintf(stderr, " Expected error: ");
grammar_parser::parse_state parsed_grammar = grammar_parser::parse(grammar_str.c_str()); grammar_parser::parse_state parsed_grammar = grammar_parser::parse(grammar_str.c_str());
// Ensure we did NOT parsed correctly // Ensure we did NOT parsed correctly
assert(parsed_grammar.rules.empty()); assert(parsed_grammar.rules.empty());
fprintf(stderr, "End of expected error.\n"); fprintf(stderr, " End of expected error.\n");
fprintf(stderr, " ✅︎ Passed\n");
} }
int main() { int main() {
fprintf(stdout, "Running grammar integration tests...\n");
test_simple_grammar(); test_simple_grammar();
test_complex_grammar(); test_complex_grammar();
test_quantifiers(); test_quantifiers();