vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Added support for . (any character) token in grammar engine. (#6467)

Browse source 
* Added support for . (any characer) token in grammar engine.

* Add integration tests for any-character symbol.
This commit is contained in:
Clint Herron 2024-06-06 06:08:52 -07:00 • committed by GitHub
parent a143c04375
commit ad675e1c67
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
4 changed files with 52 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

28
tests/test-grammar-integration.cpp
View file

@ -205,6 +205,33 @@ static void test_complex_grammar() {
);
}
static void test_special_chars() {
// A collection of tests to exercise special characters such as "."
test_grammar(
"special characters",
// Grammar
R"""(
root ::= ... "abc" ...
)""",
// Passing strings
{
"abcabcabc",
"aaaabcccc",
// NOTE: Also ensures that multi-byte characters still count as a single character
"🔵🟠✅abc❌🟠🔵"
},
// Failing strings
{
"aaabcccc",
"aaaaabcccc",
"aaaabccc",
"aaaabccccc",
"🔵🟠✅❌abc❌✅🟠🔵"
"🔵🟠abc🟠🔵"
}
);
}
static void test_quantifiers() {
// A collection of tests to exercise * + and ? quantifiers
@ -445,6 +472,7 @@ int main() {
fprintf(stdout, "Running grammar integration tests...\n");
test_simple_grammar();
test_complex_grammar();
test_special_chars();
test_quantifiers();
test_failure_missing_root();
test_failure_missing_reference();