json: test C++, JS & Python versions
This commit is contained in:
ochafik
parent
8fee84b45c
commit
192a58a5d3
6 changed files with 332 additions and 5 deletions
|
|
@ -135,7 +135,7 @@ include(${CMAKE_CURRENT_SOURCE_DIR}/scripts/build-info.cmake)
|
|||
if (LLAMA_SYCL)
|
||||
set(CMAKE_CXX_STANDARD 17)
|
||||
else()
|
||||
set(CMAKE_CXX_STANDARD 11)
|
||||
set(CMAKE_CXX_STANDARD 17)
|
||||
endif()
|
||||
|
||||
set(CMAKE_CXX_STANDARD_REQUIRED true)
|
||||
|
|
|
|||
14
Makefile
14
Makefile
|
|
@ -9,7 +9,8 @@ TEST_TARGETS = \
|
|||
tests/test-llama-grammar tests/test-grammar-parser tests/test-double-float tests/test-grad0 tests/test-opt \
|
||||
tests/test-quantize-fns tests/test-quantize-perf tests/test-sampling tests/test-tokenizer-0-llama \
|
||||
tests/test-tokenizer-0-falcon tests/test-tokenizer-1-llama tests/test-tokenizer-1-bpe tests/test-rope \
|
||||
tests/test-backend-ops tests/test-model-load-cancel tests/test-autorelease
|
||||
tests/test-backend-ops tests/test-model-load-cancel tests/test-autorelease \
|
||||
tests/test-json-schema-to-grammar
|
||||
|
||||
# Code coverage output files
|
||||
COV_TARGETS = *.gcno tests/*.gcno *.gcda tests/*.gcda *.gcov tests/*.gcov lcov-report gcovr-report
|
||||
|
|
@ -653,6 +654,11 @@ console.o: common/console.cpp common/console.h
|
|||
grammar-parser.o: common/grammar-parser.cpp common/grammar-parser.h
|
||||
$(CXX) $(CXXFLAGS) -c $< -o $@
|
||||
|
||||
json-schema-to-grammar.o: examples/server/json-schema-to-grammar.cpp examples/server/json-schema-to-grammar.h
|
||||
$(CXX) $(CXXFLAGS) -c $< -o $@
|
||||
# $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) -DLLAMA_BUILD_JSON_SCHEMA_CONVERTER=1
|
||||
# $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS)
|
||||
|
||||
train.o: common/train.cpp common/train.h
|
||||
$(CXX) $(CXXFLAGS) -c $< -o $@
|
||||
|
||||
|
|
@ -728,7 +734,7 @@ save-load-state: examples/save-load-state/save-load-state.cpp ggml.o llama.o $(C
|
|||
$(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<)
|
||||
$(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS)
|
||||
|
||||
server: examples/server/server.cpp examples/server/utils.hpp examples/server/httplib.h examples/server/json.hpp examples/server/index.html.hpp examples/server/index.js.hpp examples/server/completion.js.hpp examples/server/json-schema-to-grammar.cpp examples/server/json-schema-to-grammar.h common/stb_image.h ggml.o llama.o $(COMMON_DEPS) grammar-parser.o $(OBJS)
|
||||
server: examples/server/server.cpp examples/server/utils.hpp examples/server/httplib.h examples/server/json.hpp examples/server/index.html.hpp examples/server/index.js.hpp examples/server/completion.js.hpp json-schema-to-grammar.o common/stb_image.h ggml.o llama.o $(COMMON_DEPS) grammar-parser.o $(OBJS)
|
||||
$(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<)
|
||||
$(CXX) $(CXXFLAGS) $(filter-out %.h %.hpp $<,$^) -Iexamples/server $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) $(LWINSOCK2)
|
||||
|
||||
|
|
@ -844,6 +850,10 @@ tests/test-double-float: tests/test-double-float.cpp ggml.o $(OBJS)
|
|||
$(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<)
|
||||
$(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS)
|
||||
|
||||
tests/test-json-schema-to-grammar: tests/test-json-schema-to-grammar.cpp json-schema-to-grammar.o
|
||||
$(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<)
|
||||
$(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS)
|
||||
|
||||
tests/test-grad0: tests/test-grad0.cpp ggml.o $(OBJS)
|
||||
$(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<)
|
||||
$(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS)
|
||||
|
|
|
|||
|
|
@ -60,8 +60,6 @@ add_library(${TARGET} STATIC
|
|||
console.cpp
|
||||
grammar-parser.h
|
||||
grammar-parser.cpp
|
||||
json-schema-to-grammar.h
|
||||
json-schema-to-grammar.cpp
|
||||
train.h
|
||||
train.cpp
|
||||
)
|
||||
|
|
|
|||
|
|
@ -59,6 +59,8 @@ llama_build_and_test_executable(test-backend-ops.cpp)
|
|||
|
||||
llama_build_and_test_executable(test-rope.cpp)
|
||||
|
||||
llama_build_and_test_executable(test-json-schema-to-grammar.cpp)
|
||||
|
||||
llama_build_and_test_executable_with_label(test-model-load-cancel.cpp "model")
|
||||
llama_build_and_test_executable_with_label(test-autorelease.cpp "model")
|
||||
|
||||
|
|
|
|||
10
tests/run-json-schema-to-grammar.mjs
Normal file
10
tests/run-json-schema-to-grammar.mjs
Normal file
|
|
@ -0,0 +1,10 @@
|
|||
import { readFileSync } from "fs"
|
||||
import { SchemaConverter } from "../examples/server/public/json-schema-to-grammar.mjs"
|
||||
|
||||
const [, , file] = process.argv
|
||||
const url = `file://${file}`
|
||||
let schema = JSON.parse(readFileSync(file, "utf8"));
|
||||
const converter = new SchemaConverter({})
|
||||
schema = await converter.resolveRefs(schema, url)
|
||||
converter.visit(schema, '')
|
||||
console.log(converter.formatGrammar())
|
||||
307
tests/test-json-schema-to-grammar.cpp
Executable file
307
tests/test-json-schema-to-grammar.cpp
Executable file
|
|
@ -0,0 +1,307 @@
|
|||
#ifdef NDEBUG
|
||||
#undef NDEBUG
|
||||
#endif
|
||||
|
||||
#include "llama.h"
|
||||
#include "grammar-parser.h"
|
||||
#include <cstdio>
|
||||
#include <cstdlib>
|
||||
#include <filesystem>
|
||||
#include <iostream>
|
||||
#include <istream>
|
||||
#include <fstream>
|
||||
#include <sstream>
|
||||
#include "../examples/server/json-schema-to-grammar.h"
|
||||
#include <cassert>
|
||||
#include <regex>
|
||||
|
||||
using namespace std;
|
||||
namespace fs = std::filesystem;
|
||||
|
||||
string INPUT_NAME(tmpnam(nullptr));
|
||||
string OUT_NAME(tmpnam(nullptr));
|
||||
|
||||
static std::string trim(const std::string & source) {
|
||||
std::string s(source);
|
||||
s.erase(0,s.find_first_not_of(" \n\r\t"));
|
||||
s.erase(s.find_last_not_of(" \n\r\t")+1);
|
||||
return regex_replace(s, regex("(^|\n)[ \t]+"), "$1");
|
||||
}
|
||||
|
||||
struct TestCase {
|
||||
string name;
|
||||
string schema;
|
||||
string expected;
|
||||
|
||||
void prepare() const {
|
||||
ofstream f;
|
||||
f.open(INPUT_NAME);
|
||||
f << schema.c_str();
|
||||
f.close();
|
||||
}
|
||||
|
||||
void read_and_verify(const string& series) const {
|
||||
ostringstream actuals;
|
||||
actuals << ifstream(OUT_NAME).rdbuf();
|
||||
auto actual = actuals.str();
|
||||
verify(series, actual);
|
||||
}
|
||||
|
||||
void verify(const string& series, const string& actual) const {
|
||||
if (trim(actual) != trim(expected)) {
|
||||
cerr << "#" << endl;
|
||||
cerr << "# Test " << series.c_str() << " / " << name.c_str() << " failed." << endl;
|
||||
cerr << "#" << endl;
|
||||
cerr << schema.c_str() << endl;
|
||||
cerr << "# EXPECTED:\n" << expected.c_str() << endl;
|
||||
cerr << "# ACTUAL:\n" << actual.c_str() << endl;
|
||||
assert(false);
|
||||
}
|
||||
}
|
||||
|
||||
};
|
||||
|
||||
|
||||
static void run_py(const TestCase& tc) {
|
||||
cerr << "# Running Python " << tc.name.c_str() << endl;
|
||||
tc.prepare();
|
||||
std::system(("python ./examples/json-schema-to-grammar.py " + INPUT_NAME + " > " + OUT_NAME).c_str());
|
||||
tc.read_and_verify("Python");
|
||||
}
|
||||
static void run_mjs(const TestCase& tc) {
|
||||
cerr << "# Running MJS " << tc.name.c_str() << endl;
|
||||
tc.prepare();
|
||||
std::system(("node ./tests/run-json-schema-to-grammar.mjs " + INPUT_NAME + " > " + OUT_NAME).c_str());
|
||||
tc.read_and_verify("JavaScript");
|
||||
}
|
||||
static void run_cpp(const TestCase& tc) {
|
||||
cerr << "# Running C++ " << tc.name.c_str() << endl;
|
||||
auto actual = json_schema_to_grammar(nlohmann::json::parse(tc.schema));
|
||||
tc.verify("C++", actual);
|
||||
}
|
||||
|
||||
static void run_all(const TestCase& tc) {
|
||||
run_py(tc);
|
||||
run_mjs(tc);
|
||||
run_cpp(tc);
|
||||
}
|
||||
|
||||
int main() {
|
||||
run_all({
|
||||
.name = "exotic formats",
|
||||
.schema = R"""({
|
||||
"prefixItems": [
|
||||
{ "format": "date" },
|
||||
{ "format": "uuid" },
|
||||
{ "format": "time" },
|
||||
{ "format": "date-time" }
|
||||
]
|
||||
})""",
|
||||
.expected = R"""(
|
||||
date ::= [0-9] [0-9] [0-9] [0-9] "-" ( "0" [1-9] | "1" [0-2] ) "-" ( [0-2] [0-9] | "3" [0-1] )
|
||||
date-string ::= "\"" date "\"" space
|
||||
date-time ::= date "T" time
|
||||
date-time-string ::= "\"" date-time "\"" space
|
||||
root ::= "[" space date-string "," space uuid "," space time-string "," space date-time-string "]" space
|
||||
space ::= " "?
|
||||
time ::= ([01] [0-9] | "2" [0-3]) ":" [0-5] [0-9] ":" [0-5] [0-9] ( "." [0-9] [0-9] [0-9] )? ( "Z" | ( "+" | "-" ) ( [01] [0-9] | "2" [0-3] ) ":" [0-5] [0-9] )
|
||||
time-string ::= "\"" time "\"" space
|
||||
uuid ::= "\"" [0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F] "-" [0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F] "-" [0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F] "-" [0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F] "-" [0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F] "\"" space
|
||||
)"""
|
||||
});
|
||||
|
||||
run_all({
|
||||
.name = "string",
|
||||
.schema = R"""({
|
||||
"type": "string"
|
||||
})""",
|
||||
.expected = R"""(
|
||||
root ::= "\"" (
|
||||
[^"\\] |
|
||||
"\\" (["\\/bfnrt] | "u" [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F])
|
||||
)* "\"" space
|
||||
space ::= " "?
|
||||
)"""
|
||||
});
|
||||
|
||||
run_all({
|
||||
.name = "boolean",
|
||||
.schema = R"""({
|
||||
"type": "boolean"
|
||||
})""",
|
||||
.expected = R"""(
|
||||
root ::= ("true" | "false") space
|
||||
space ::= " "?
|
||||
)"""
|
||||
});
|
||||
|
||||
run_all({
|
||||
.name = "integer",
|
||||
.schema = R"""({
|
||||
"type": "integer"
|
||||
})""",
|
||||
.expected = R"""(
|
||||
root ::= ("-"? ([0-9] | [1-9] [0-9]*)) space
|
||||
space ::= " "?
|
||||
)"""
|
||||
});
|
||||
|
||||
run_all({
|
||||
.name = "number",
|
||||
.schema = R"""({
|
||||
"type": "number"
|
||||
})""",
|
||||
.expected = R"""(
|
||||
root ::= ("-"? ([0-9] | [1-9] [0-9]*)) ("." [0-9]+)? ([eE] [-+]? [0-9]+)? space
|
||||
space ::= " "?
|
||||
)"""
|
||||
});
|
||||
|
||||
run_all({
|
||||
.name = "minItems",
|
||||
.schema = R"""({
|
||||
"items": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"minItems": 2
|
||||
})""",
|
||||
.expected = R"""(
|
||||
boolean ::= ("true" | "false") space
|
||||
root ::= "[" space boolean ( "," space boolean )( "," space boolean )* "]" space
|
||||
space ::= " "?
|
||||
)"""
|
||||
});
|
||||
|
||||
run_all({
|
||||
.name = "maxItems 1",
|
||||
.schema = R"""({
|
||||
"items": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"maxItems": 1
|
||||
})""",
|
||||
.expected = R"""(
|
||||
boolean ::= ("true" | "false") space
|
||||
root ::= "[" space ( boolean )? "]" space
|
||||
space ::= " "?
|
||||
)"""
|
||||
});
|
||||
|
||||
run_all({
|
||||
.name = "maxItems 2",
|
||||
.schema = R"""({
|
||||
"items": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"maxItems": 2
|
||||
})""",
|
||||
.expected = R"""(
|
||||
boolean ::= ("true" | "false") space
|
||||
root ::= "[" space ( boolean ( "," space boolean )? )? "]" space
|
||||
space ::= " "?
|
||||
)"""
|
||||
});
|
||||
|
||||
run_all({
|
||||
.name = "min + maxItems",
|
||||
.schema = R"""({
|
||||
"items": {
|
||||
"type": ["number", "integer"]
|
||||
},
|
||||
"minItems": 3,
|
||||
"maxItems": 5
|
||||
})""",
|
||||
.expected = R"""(
|
||||
integer ::= ("-"? ([0-9] | [1-9] [0-9]*)) space
|
||||
item ::= number | integer
|
||||
number ::= ("-"? ([0-9] | [1-9] [0-9]*)) ("." [0-9]+)? ([eE] [-+]? [0-9]+)? space
|
||||
root ::= "[" space item ( "," space item )( "," space item )( "," space item )?( "," space item )? "]" space
|
||||
space ::= " "?
|
||||
)"""
|
||||
});
|
||||
|
||||
run_all({
|
||||
.name = "regexp",
|
||||
.schema = R"""({
|
||||
"type": "string",
|
||||
"pattern": "^(\\([0-9]{1,3}\\))?[0-9]{3}-[0-9]{4} and...$"
|
||||
})""",
|
||||
.expected = R"""(
|
||||
dot ::= [\U00000000-\x09\x0B\x0C\x0E-\U0010FFFF]
|
||||
root ::= ("(" root-1 root-1? root-1? ")")? root-1 root-1 root-1 "-" root-1 root-1 root-1 root-1 " and" dot dot dot
|
||||
root-1 ::= [0-9]
|
||||
space ::= " "?
|
||||
)"""
|
||||
});
|
||||
|
||||
run_all({
|
||||
.name = "optionals",
|
||||
.schema = R"""({
|
||||
"$schema": "http://json-schema.org/draft-07/schema#",
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"a": {
|
||||
"type": "string"
|
||||
},
|
||||
"b": {
|
||||
"type": "string"
|
||||
},
|
||||
"c": {
|
||||
"type": [
|
||||
"number",
|
||||
"string"
|
||||
]
|
||||
},
|
||||
"d": {
|
||||
"type": "string"
|
||||
},
|
||||
"e": {
|
||||
"type": "object",
|
||||
"additionalProperties": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "array",
|
||||
"minItems": 2,
|
||||
"items": [
|
||||
{
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"type": "number"
|
||||
}
|
||||
],
|
||||
"maxItems": 2
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"a",
|
||||
"b"
|
||||
],
|
||||
"additionalProperties": false,
|
||||
"definitions": {}
|
||||
})""",
|
||||
.expected = R"""(
|
||||
a-kv ::= "\"a\"" space ":" space string
|
||||
b-kv ::= "\"b\"" space ":" space string
|
||||
c ::= number | string
|
||||
c-kv ::= "\"c\"" space ":" space c
|
||||
c-rest ::= ( "," space d-kv )? d-rest
|
||||
d-kv ::= "\"d\"" space ":" space string
|
||||
d-rest ::= ( "," space e-kv )?
|
||||
e ::= ( e-additionalProperties-kv ( "," space e-additionalProperties-kv )* )*
|
||||
e-additionalProperties-kv ::= string ":" space e-additionalProperties-value
|
||||
e-additionalProperties-value ::= "[" space ( e-additionalProperties-value-item ( "," space e-additionalProperties-value-item )* )? "]" space
|
||||
e-additionalProperties-value-item ::= "[" space string "," space number "]" space
|
||||
e-kv ::= "\"e\"" space ":" space e
|
||||
number ::= ("-"? ([0-9] | [1-9] [0-9]*)) ("." [0-9]+)? ([eE] [-+]? [0-9]+)? space
|
||||
root ::= "{" space a-kv "," space b-kv ( "," space ( c-kv c-rest | d-kv d-rest | e-kv ) )? "}" space
|
||||
space ::= " "?
|
||||
string ::= "\"" (
|
||||
[^"\\] |
|
||||
"\\" (["\\/bfnrt] | "u" [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F])
|
||||
)* "\"" space
|
||||
)"""
|
||||
});
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue