vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

build and run test

Browse source
This commit is contained in:
Michal Moskal 2025-01-26 08:49:05 -08:00
parent 036b91fbc3
commit f245ca26f5
2 changed files with 73 additions and 128 deletions
Download patch file Download diff file Expand all files Collapse all files

1
tests/CMakeLists.txt
View file

@ -86,6 +86,7 @@ llama_test(test-tokenizer-0 NAME test-tokenizer-0-qwen2 ARGS ${CMAKE
llama_test(test-tokenizer-0 NAME test-tokenizer-0-refact ARGS ${CMAKE_CURRENT_SOURCE_DIR}/../models/ggml-vocab-refact.gguf) llama_test(test-tokenizer-0 NAME test-tokenizer-0-refact ARGS ${CMAKE_CURRENT_SOURCE_DIR}/../models/ggml-vocab-refact.gguf)
llama_test(test-tokenizer-0 NAME test-tokenizer-0-starcoder ARGS ${CMAKE_CURRENT_SOURCE_DIR}/../models/ggml-vocab-starcoder.gguf) llama_test(test-tokenizer-0 NAME test-tokenizer-0-starcoder ARGS ${CMAKE_CURRENT_SOURCE_DIR}/../models/ggml-vocab-starcoder.gguf)
llama_target_and_test(test-llguidance.cpp ARGS ${CMAKE_CURRENT_SOURCE_DIR}/../models/ggml-vocab-llama-bpe.gguf)
if (NOT WIN32) if (NOT WIN32)
# these tests are disabled on Windows because they use internal functions not exported with LLAMA_API # these tests are disabled on Windows because they use internal functions not exported with LLAMA_API

200
tests/test-llguidance.cpp
View file

@ -4,47 +4,47 @@
#include "unicode.h" #include "unicode.h"
#include "sampling.h" #include "sampling.h"
#include "json-schema-to-grammar.h"
#include <cassert> #include <cassert>
#include <string> #include <string>
#include <vector> #include <vector>
static bool test_build_grammar_fails(const std::string & grammar_str) { static const llama_vocab * vocab;
fprintf(stderr, "⚫ Testing failure for grammar: %s\n", grammar_str.c_str());
bool grammar_fails = false; static bool match_string(const std::string & input, llama_sampler * grammar) {
llama_grammar * grammar = build_grammar(grammar_str); llama_sampler_reset(grammar);
if (grammar != nullptr) { auto tokens = common_tokenize(vocab, input, false, false);
fprintf(stderr, " ❌ Expected build failure, but succeeded\n");
} else { auto n_vocab = llama_vocab_n_tokens(vocab);
grammar_fails = true; std::vector<llama_token_data> cur;
fprintf(stdout, " ✅︎\n"); cur.reserve(n_vocab);
for (llama_token token_id = 0; token_id < (llama_token) n_vocab; token_id++) {
cur.emplace_back(llama_token_data{ token_id, 0.0f, 0.0f });
} }
return grammar_fails; auto tok_arr = llama_token_data_array{ cur.data(), cur.size(), -1, false };
}
static bool match_string(const std::string & input, llama_grammar * grammar) { for (const auto token : tokens) {
const auto cpts = unicode_cpts_from_utf8(input); for (llama_token token_id = 0; token_id < (llama_token) n_vocab; token_id++) {
cur[token_id].logit = 0.0f;
auto & stacks_cur = llama_grammar_get_stacks(grammar); }
llama_sampler_apply(grammar, &tok_arr);
for (const auto & cpt : cpts) { if (cur[token].logit < 0.0f) {
llama_grammar_accept(grammar, cpt);
if (stacks_cur.empty()) {
// no stacks means that the grammar failed to match at this point
return false; return false;
} }
llama_sampler_accept(grammar, token);
} }
for (const auto & stack : stacks_cur) { // do we allow EOS at the end? if so the grammar is accepting
if (stack.empty()) {
// An empty stack means that the grammar has been completed auto tok_eos = llama_vocab_eot(vocab);
return true; if (tok_eos == LLAMA_TOKEN_NULL) {
} tok_eos = llama_vocab_eos(vocab);
} }
return false; cur[tok_eos].logit = 0.0f;
llama_sampler_apply(grammar, &tok_arr);
return cur[tok_eos].logit >= 0.0f;
} }
static void test(const std::string & test_desc, const std::string & grammar_str, static void test(const std::string & test_desc, const std::string & grammar_str,
@ -52,14 +52,7 @@ static void test(const std::string & test_desc, const std::string & grammar_str,
fprintf(stderr, "⚫ Testing %s\n%s\n", test_desc.c_str(), grammar_str.c_str()); fprintf(stderr, "⚫ Testing %s\n%s\n", test_desc.c_str(), grammar_str.c_str());
fflush(stderr); fflush(stderr);
auto * sampler = llama_sampler_init_llg(); auto * grammar = llama_sampler_init_llg(vocab, "lark", grammar_str.c_str());
auto * grammar = build_grammar(grammar_str);
// Save the original grammar stacks so that we can reset after every new string we want to test
const llama_grammar_stacks stacks_org = llama_grammar_get_stacks(grammar); // copy
llama_grammar_stacks & stacks_cur = llama_grammar_get_stacks(grammar);
fprintf(stderr, " 🔵 Valid strings:\n"); fprintf(stderr, " 🔵 Valid strings:\n");
@ -97,9 +90,6 @@ static void test(const std::string & test_desc, const std::string & grammar_str,
} }
assert(matched); assert(matched);
// Reset the grammar stacks
stacks_cur = stacks_org;
} }
fprintf(stderr, " 🟠 Invalid strings:\n"); fprintf(stderr, " 🟠 Invalid strings:\n");
@ -117,13 +107,9 @@ static void test(const std::string & test_desc, const std::string & grammar_str,
fprintf(stdout, "✅︎\n"); fprintf(stdout, "✅︎\n");
} }
assert(!matched); assert(!matched);
// Reset the grammar stacks
stacks_cur = stacks_org;
} }
// Clean up allocated memory llama_sampler_free(grammar);
llama_grammar_free_impl(grammar);
} }
static void test_grammar(const std::string & test_desc, const std::string & grammar_str, static void test_grammar(const std::string & test_desc, const std::string & grammar_str,
@ -135,7 +121,7 @@ static void test_grammar(const std::string & test_desc, const std::string & gram
static void test_schema(const std::string & test_desc, const std::string & schema_str, static void test_schema(const std::string & test_desc, const std::string & schema_str,
const std::vector<std::string> & passing_strings, const std::vector<std::string> & passing_strings,
const std::vector<std::string> & failing_strings) { const std::vector<std::string> & failing_strings) {
test(test_desc + ". Schema: " + schema_str, json_schema_to_grammar(json::parse(schema_str), true), passing_strings, test(test_desc + ". Schema: " + schema_str, "%llguidance {}\nstart: %json " + schema_str, passing_strings,
failing_strings); failing_strings);
} }
@ -158,7 +144,7 @@ static void test_simple_grammar() {
"-10", "-10",
"-10000", "-10000",
"-100000000000000000000000000000000", "-100000000000000000000000000000000",
"100000000000000000000000000000000", // "100000000000000000000000000000000",
"00", "00",
"01", "01",
"-0", "-0",
@ -188,7 +174,7 @@ static void test_simple_grammar() {
"1", "1",
"01", "01",
"02", "02",
"12345678900000000", // "12345678900000000",
}); });
test_schema("min 456", test_schema("min 456",
R"""({ R"""({
@ -582,80 +568,6 @@ static void test_quantifiers() {
}); });
} }
static void test_failure_missing_root() {
fprintf(stderr, "⚫ Testing missing root node:\n");
// Test case for a grammar that is missing a root rule
const std::string grammar_str = R"""(
rot ::= expr
expr ::= term ("+" term)*
term ::= number
number ::= [0-9]+)""";
llama_grammar_parser parsed_grammar;
parsed_grammar.parse(grammar_str.c_str());
// Ensure we parsed correctly
assert(!parsed_grammar.rules.empty());
// Ensure we do NOT have a root node
assert(parsed_grammar.symbol_ids.find("root") == parsed_grammar.symbol_ids.end());
fprintf(stderr, " ✅︎ Passed\n");
}
static void test_failure_missing_reference() {
fprintf(stderr, "⚫ Testing missing reference node:\n");
// Test case for a grammar that is missing a referenced rule
const std::string grammar_str =
R"""(root ::= expr
expr ::= term ("+" term)*
term ::= numero
number ::= [0-9]+)""";
fprintf(stderr, " Expected error: ");
llama_grammar_parser parsed_grammar;
parsed_grammar.parse(grammar_str.c_str());
// Ensure we did NOT parsed correctly
assert(parsed_grammar.rules.empty());
fprintf(stderr, " End of expected error.\n");
fprintf(stderr, " ✅︎ Passed\n");
}
static void test_failure_left_recursion() {
fprintf(stderr, "⚫ Testing left recursion detection:\n");
// Test simple left recursion detection
const std::string simple_str = R"""(root ::= "a" | root "a")""";
assert(test_build_grammar_fails(simple_str));
// Test more complicated left recursion detection
const std::string medium_str = R"""(
root ::= asdf
asdf ::= "a" | asdf "a"
)""";
assert(test_build_grammar_fails(medium_str));
// Test even more complicated left recursion detection
const std::string hard_str = R"""(
root ::= asdf
asdf ::= "a" | foo "b"
foo ::= "c" | asdf "d" | "e")""";
assert(test_build_grammar_fails(hard_str));
// Test yet even more complicated left recursion detection
const std::string hardest_str = R"""(
root ::= asdf
asdf ::= "a" | foo "b"
foo ::= "c" | empty asdf "d" | "e"
empty ::= "blah" | )""";
assert(test_build_grammar_fails(hardest_str));
fprintf(stderr, " ✅︎ Passed\n");
}
static void test_json_schema() { static void test_json_schema() {
// Note that this is similar to the regular grammar tests, // Note that this is similar to the regular grammar tests,
// but we convert each json schema to a grammar before parsing. // but we convert each json schema to a grammar before parsing.
@ -1191,14 +1103,46 @@ int main(int argc, const char ** argv) {
fprintf(stderr, "reading vocab from: '%s'\n", vocab_file); fprintf(stderr, "reading vocab from: '%s'\n", vocab_file);
llama_model * model;
llama_context * ctx;
llama_backend_init();
// load the vocab
{
auto mparams = llama_model_default_params();
mparams.vocab_only = true;
model = llama_model_load_from_file(vocab_file, mparams);
if (model == NULL) {
fprintf(stderr, "%s: error: failed to load vocab '%s'\n", __func__, vocab_file);
return 1;
}
// needed?
auto cparams = llama_context_default_params();
ctx = llama_init_from_model(model, cparams);
if (ctx == NULL) {
fprintf(stderr, "%s: error: failed to load vocab '%s'\n", __func__, vocab_file);
llama_model_free(model);
return 1;
}
}
vocab = llama_model_get_vocab(model);
test_simple_grammar(); test_simple_grammar();
test_complex_grammar(); // test_complex_grammar();
test_special_chars(); // test_special_chars();
test_quantifiers(); // test_quantifiers();
test_failure_missing_root(); // test_failure_missing_root();
test_failure_missing_reference(); // test_failure_missing_reference();
test_failure_left_recursion(); // test_failure_left_recursion();
test_json_schema(); // test_json_schema();
fprintf(stdout, "All tests passed.\n"); fprintf(stdout, "All tests passed.\n");
return 0; return 0;
} }