vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Separating test calls to give more helpful stack traces on failure. Adding verbose messages to give visibility for what is being tested.

Browse source
This commit is contained in:
Clint Herron 2024-04-27 09:38:45 -04:00
parent 7b04c215eb
commit b7f24dba00
1 changed files with 187 additions and 212 deletions
Download patch file Download diff file Expand all files Collapse all files

169
tests/test-grammar-integration.cpp
View file

@ -52,6 +52,49 @@ static bool match_string(const std::string & input, llama_grammar* grammar) {
return false; return false;
} }
static void test_grammar(const std::string & grammar_str, const std::vector<std::string> & passing_strings, const std::vector<std::string> & failing_strings) {
fprintf(stderr, "🟢 Testing grammar: %s\n", grammar_str.c_str());
auto grammar = build_grammar(grammar_str);
// Save the original grammar stacks so that we can reset after every new string we want to test
auto original_stacks = grammar->stacks;
// Passing strings
for (const auto & test_string : passing_strings) {
bool matched = match_string(test_string, grammar);
if (!matched) {
fprintf(stderr, " ❌ Failed to match string: %s\n", test_string.c_str());
} else {
fprintf(stdout, " ✅︎ Matched string: %s\n", test_string.c_str());
}
assert(matched);
// Reset the grammar stacks
grammar->stacks = original_stacks;
}
// Failing strings
for (const auto & test_string : failing_strings) {
bool matched = match_string(test_string, grammar);
if (matched) {
fprintf(stderr, " ❌ Improperly matched string: %s\n", test_string.c_str());
} else {
fprintf(stdout, " ✅︎ Correctly did not match string: %s\n", test_string.c_str());
}
assert(!matched);
// Reset the grammar stacks
grammar->stacks = original_stacks;
}
// Clean up allocated memory
llama_grammar_free(grammar);
}
static void test_simple_grammar() { static void test_simple_grammar() {
// Test case for a simple grammar // Test case for a simple grammar
const std::string grammar_str = R"""(root ::= expr const std::string grammar_str = R"""(root ::= expr
@ -71,22 +114,19 @@ number ::= [0-9]+)""";
static void test_complex_grammar() { static void test_complex_grammar() {
// Test case for a more complex grammar, with both failure strings and success strings // Test case for a more complex grammar, with both failure strings and success strings
const std::string grammar_str = R"""(root ::= expression test_grammar(
// Grammar
R"""(
root ::= expression
expression ::= term ws (("+"|"-") ws term)* expression ::= term ws (("+"|"-") ws term)*
term ::= factor ws (("*"|"/") ws factor)* term ::= factor ws (("*"|"/") ws factor)*
factor ::= number | variable | "(" expression ")" | function-call factor ::= number | variable | "(" expression ")" | function-call
number ::= [0-9]+ number ::= [0-9]+
variable ::= [a-zA-Z_][a-zA-Z0-9_]* variable ::= [a-zA-Z_][a-zA-Z0-9_]*
function-call ::= variable ws "(" (expression ("," ws expression)*)? ")" function-call ::= variable ws "(" (expression ("," ws expression)*)? ")"
ws ::= [ \t\n\r]?)"""; ws ::= [ \t\n\r]?)""",
// Passing strings
auto grammar = build_grammar(grammar_str); {
// Save the original grammar stacks so that we can reset after every new string we want to test
auto original_stacks = grammar->stacks;
// Test a few strings
std::vector<std::string> test_strings_pass = {
"42", "42",
"1*2*3*4*5", "1*2*3*4*5",
"x", "x",
@ -107,9 +147,9 @@ ws ::= [ \t\n\r]?)""";
"123+456", "123+456",
"123*456*789-123/456+789*123", "123*456*789-123/456+789*123",
"123+456*789-123/456+789*123-456/789+123*456-789/123+456*789-123/456+789*123-456" "123+456*789-123/456+789*123-456/789+123*456-789/123+456*789-123/456+789*123-456"
}; },
// Failing strings
std::vector<std::string> test_strings_fail = { {
"+", "+",
"/ 3x", "/ 3x",
"x + + y", "x + + y",
@ -128,41 +168,14 @@ ws ::= [ \t\n\r]?)""";
"a * (b + c) - d /", "a * (b + c) - d /",
"f(g(x), h(y, z)", "f(g(x), h(y, z)",
"123+456*789-123/456+789*123-456/789+123*456-789/123+456*789-123/456+789*123-456/", "123+456*789-123/456+789*123-456/789+123*456-789/123+456*789-123/456+789*123-456/",
};
// Passing strings
for (const auto & test_string : test_strings_pass) {
bool matched = match_string(test_string, grammar);
assert(matched);
// Reset the grammar stacks
grammar->stacks = original_stacks;
} }
);
// Failing strings
for (const auto & test_string : test_strings_fail) {
bool matched = match_string(test_string, grammar);
assert(!matched);
// Reset the grammar stacks
grammar->stacks = original_stacks;
}
// Clean up allocated memory
llama_grammar_free(grammar);
} }
static void test_quantifiers() { static void test_quantifiers() {
// Populate test data with grammar strings and their associated collections of expected passing and failing strings // A collection of tests to exercise * + and ? quantifiers
const std::vector<
std::tuple< test_grammar(
std::string,
std::vector<std::string>,
std::vector<std::string>>>
test_data = {
{
// Grammar // Grammar
R"""(root ::= "a"*)""", R"""(root ::= "a"*)""",
// Passing strings // Passing strings
@ -181,8 +194,8 @@ static void test_quantifiers() {
"ba", "ba",
"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaab" "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaab"
} }
}, );
{ test_grammar(
// Grammar // Grammar
R"""(root ::= "a"+)""", R"""(root ::= "a"+)""",
// Passing strings // Passing strings
@ -201,8 +214,8 @@ static void test_quantifiers() {
"ba", "ba",
"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaab" "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaab"
} }
}, );
{ test_grammar(
// Grammar // Grammar
R"""(root ::= "a"?)""", R"""(root ::= "a"?)""",
// Passing strings // Passing strings
@ -217,8 +230,8 @@ static void test_quantifiers() {
"aa", "aa",
"ba", "ba",
} }
}, );
{ test_grammar(
// Grammar // Grammar
R"""( R"""(
root ::= cons+ vowel* cons? (vowel cons)* root ::= cons+ vowel* cons? (vowel cons)*
@ -241,55 +254,11 @@ cons ::= [bcdfghjklmnpqrstvwxyz]
"forty", "forty",
"catyyy", "catyyy",
} }
} );
};
for (const auto & test_datum : test_data) {
const auto & grammar_str = std::get<0>(test_datum);
const auto & passing_strings = std::get<1>(test_datum);
const auto & failing_strings = std::get<2>(test_datum);
auto grammar = build_grammar(grammar_str);
// Save the original grammar stacks so that we can reset after every new string we want to test
auto original_stacks = grammar->stacks;
// Passing strings
for (const auto & test_string : passing_strings) {
bool matched = match_string(test_string, grammar);
if (!matched) {
fprintf(stderr, "Against grammar: %s\n", grammar_str.c_str());
fprintf(stderr, "Failed to match string: %s\n", test_string.c_str());
}
assert(matched);
// Reset the grammar stacks
grammar->stacks = original_stacks;
}
// Failing strings
for (const auto & test_string : failing_strings) {
bool matched = match_string(test_string, grammar);
if (matched) {
fprintf(stderr, "Against grammar: %s\n", grammar_str.c_str());
fprintf(stderr, "Improperly matched string: %s\n", test_string.c_str());
}
assert(!matched);
// Reset the grammar stacks
grammar->stacks = original_stacks;
}
// Clean up allocated memory
llama_grammar_free(grammar);
}
} }
static void test_failure_missing_root() { static void test_failure_missing_root() {
fprintf(stderr, "🟢 Testing for missing root node:\n");
// Test case for a grammar that is missing a root rule // Test case for a grammar that is missing a root rule
const std::string grammar_str = R"""(rot ::= expr const std::string grammar_str = R"""(rot ::= expr
expr ::= term ("+" term)* expr ::= term ("+" term)*
@ -303,11 +272,15 @@ number ::= [0-9]+)""";
// Ensure we do NOT have a root node // Ensure we do NOT have a root node
assert(parsed_grammar.symbol_ids.find("root") == parsed_grammar.symbol_ids.end()); assert(parsed_grammar.symbol_ids.find("root") == parsed_grammar.symbol_ids.end());
fprintf(stderr, " ✅︎ Passed\n");
} }
static void test_failure_missing_reference() { static void test_failure_missing_reference() {
fprintf(stderr, "🟢 Testing for missing reference node:\n");
// Test case for a grammar that is missing a referenced rule // Test case for a grammar that is missing a referenced rule
const std::string grammar_str = R"""(root ::= expr const std::string grammar_str =
R"""(root ::= expr
expr ::= term ("+" term)* expr ::= term ("+" term)*
term ::= numero term ::= numero
number ::= [0-9]+)"""; number ::= [0-9]+)""";
@ -320,9 +293,11 @@ number ::= [0-9]+)""";
assert(parsed_grammar.rules.empty()); assert(parsed_grammar.rules.empty());
fprintf(stderr, " End of expected error.\n"); fprintf(stderr, " End of expected error.\n");
fprintf(stderr, " ✅︎ Passed\n");
} }
int main() { int main() {
fprintf(stdout, "Running grammar integration tests...\n");
test_simple_grammar(); test_simple_grammar();
test_complex_grammar(); test_complex_grammar();
test_quantifiers(); test_quantifiers();