From 192a58a5d32291981d34d194b843ff99ab13bd97 Mon Sep 17 00:00:00 2001 From: ochafik Date: Tue, 12 Mar 2024 03:46:55 +0000 Subject: [PATCH] json: test C++, JS & Python versions --- CMakeLists.txt | 2 +- Makefile | 14 +- common/CMakeLists.txt | 2 - tests/CMakeLists.txt | 2 + tests/run-json-schema-to-grammar.mjs | 10 + tests/test-json-schema-to-grammar.cpp | 307 ++++++++++++++++++++++++++ 6 files changed, 332 insertions(+), 5 deletions(-) create mode 100644 tests/run-json-schema-to-grammar.mjs create mode 100755 tests/test-json-schema-to-grammar.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 9309ca6bb..1e30fdaf9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -135,7 +135,7 @@ include(${CMAKE_CURRENT_SOURCE_DIR}/scripts/build-info.cmake) if (LLAMA_SYCL) set(CMAKE_CXX_STANDARD 17) else() - set(CMAKE_CXX_STANDARD 11) + set(CMAKE_CXX_STANDARD 17) endif() set(CMAKE_CXX_STANDARD_REQUIRED true) diff --git a/Makefile b/Makefile index c2d6c58e0..9b6af7168 100644 --- a/Makefile +++ b/Makefile @@ -9,7 +9,8 @@ TEST_TARGETS = \ tests/test-llama-grammar tests/test-grammar-parser tests/test-double-float tests/test-grad0 tests/test-opt \ tests/test-quantize-fns tests/test-quantize-perf tests/test-sampling tests/test-tokenizer-0-llama \ tests/test-tokenizer-0-falcon tests/test-tokenizer-1-llama tests/test-tokenizer-1-bpe tests/test-rope \ - tests/test-backend-ops tests/test-model-load-cancel tests/test-autorelease + tests/test-backend-ops tests/test-model-load-cancel tests/test-autorelease \ + tests/test-json-schema-to-grammar # Code coverage output files COV_TARGETS = *.gcno tests/*.gcno *.gcda tests/*.gcda *.gcov tests/*.gcov lcov-report gcovr-report @@ -653,6 +654,11 @@ console.o: common/console.cpp common/console.h grammar-parser.o: common/grammar-parser.cpp common/grammar-parser.h $(CXX) $(CXXFLAGS) -c $< -o $@ +json-schema-to-grammar.o: examples/server/json-schema-to-grammar.cpp examples/server/json-schema-to-grammar.h + $(CXX) $(CXXFLAGS) -c $< -o $@ + # $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) -DLLAMA_BUILD_JSON_SCHEMA_CONVERTER=1 + # $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) + train.o: common/train.cpp common/train.h $(CXX) $(CXXFLAGS) -c $< -o $@ @@ -728,7 +734,7 @@ save-load-state: examples/save-load-state/save-load-state.cpp ggml.o llama.o $(C $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) -server: examples/server/server.cpp examples/server/utils.hpp examples/server/httplib.h examples/server/json.hpp examples/server/index.html.hpp examples/server/index.js.hpp examples/server/completion.js.hpp examples/server/json-schema-to-grammar.cpp examples/server/json-schema-to-grammar.h common/stb_image.h ggml.o llama.o $(COMMON_DEPS) grammar-parser.o $(OBJS) +server: examples/server/server.cpp examples/server/utils.hpp examples/server/httplib.h examples/server/json.hpp examples/server/index.html.hpp examples/server/index.js.hpp examples/server/completion.js.hpp json-schema-to-grammar.o common/stb_image.h ggml.o llama.o $(COMMON_DEPS) grammar-parser.o $(OBJS) $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) $(CXX) $(CXXFLAGS) $(filter-out %.h %.hpp $<,$^) -Iexamples/server $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) $(LWINSOCK2) @@ -844,6 +850,10 @@ tests/test-double-float: tests/test-double-float.cpp ggml.o $(OBJS) $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) +tests/test-json-schema-to-grammar: tests/test-json-schema-to-grammar.cpp json-schema-to-grammar.o + $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) + $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) + tests/test-grad0: tests/test-grad0.cpp ggml.o $(OBJS) $(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<) $(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) diff --git a/common/CMakeLists.txt b/common/CMakeLists.txt index 9fc0a7805..350bbdf7f 100644 --- a/common/CMakeLists.txt +++ b/common/CMakeLists.txt @@ -60,8 +60,6 @@ add_library(${TARGET} STATIC console.cpp grammar-parser.h grammar-parser.cpp - json-schema-to-grammar.h - json-schema-to-grammar.cpp train.h train.cpp ) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 10326d531..4389f4a16 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -59,6 +59,8 @@ llama_build_and_test_executable(test-backend-ops.cpp) llama_build_and_test_executable(test-rope.cpp) +llama_build_and_test_executable(test-json-schema-to-grammar.cpp) + llama_build_and_test_executable_with_label(test-model-load-cancel.cpp "model") llama_build_and_test_executable_with_label(test-autorelease.cpp "model") diff --git a/tests/run-json-schema-to-grammar.mjs b/tests/run-json-schema-to-grammar.mjs new file mode 100644 index 000000000..71bf62ed3 --- /dev/null +++ b/tests/run-json-schema-to-grammar.mjs @@ -0,0 +1,10 @@ +import { readFileSync } from "fs" +import { SchemaConverter } from "../examples/server/public/json-schema-to-grammar.mjs" + +const [, , file] = process.argv +const url = `file://${file}` +let schema = JSON.parse(readFileSync(file, "utf8")); +const converter = new SchemaConverter({}) +schema = await converter.resolveRefs(schema, url) +converter.visit(schema, '') +console.log(converter.formatGrammar()) diff --git a/tests/test-json-schema-to-grammar.cpp b/tests/test-json-schema-to-grammar.cpp new file mode 100755 index 000000000..c07a105ef --- /dev/null +++ b/tests/test-json-schema-to-grammar.cpp @@ -0,0 +1,307 @@ +#ifdef NDEBUG +#undef NDEBUG +#endif + +#include "llama.h" +#include "grammar-parser.h" +#include +#include +#include +#include +#include +#include +#include +#include "../examples/server/json-schema-to-grammar.h" +#include +#include + +using namespace std; +namespace fs = std::filesystem; + +string INPUT_NAME(tmpnam(nullptr)); +string OUT_NAME(tmpnam(nullptr)); + +static std::string trim(const std::string & source) { + std::string s(source); + s.erase(0,s.find_first_not_of(" \n\r\t")); + s.erase(s.find_last_not_of(" \n\r\t")+1); + return regex_replace(s, regex("(^|\n)[ \t]+"), "$1"); +} + +struct TestCase { + string name; + string schema; + string expected; + + void prepare() const { + ofstream f; + f.open(INPUT_NAME); + f << schema.c_str(); + f.close(); + } + + void read_and_verify(const string& series) const { + ostringstream actuals; + actuals << ifstream(OUT_NAME).rdbuf(); + auto actual = actuals.str(); + verify(series, actual); + } + + void verify(const string& series, const string& actual) const { + if (trim(actual) != trim(expected)) { + cerr << "#" << endl; + cerr << "# Test " << series.c_str() << " / " << name.c_str() << " failed." << endl; + cerr << "#" << endl; + cerr << schema.c_str() << endl; + cerr << "# EXPECTED:\n" << expected.c_str() << endl; + cerr << "# ACTUAL:\n" << actual.c_str() << endl; + assert(false); + } + } + +}; + + +static void run_py(const TestCase& tc) { + cerr << "# Running Python " << tc.name.c_str() << endl; + tc.prepare(); + std::system(("python ./examples/json-schema-to-grammar.py " + INPUT_NAME + " > " + OUT_NAME).c_str()); + tc.read_and_verify("Python"); +} +static void run_mjs(const TestCase& tc) { + cerr << "# Running MJS " << tc.name.c_str() << endl; + tc.prepare(); + std::system(("node ./tests/run-json-schema-to-grammar.mjs " + INPUT_NAME + " > " + OUT_NAME).c_str()); + tc.read_and_verify("JavaScript"); +} +static void run_cpp(const TestCase& tc) { + cerr << "# Running C++ " << tc.name.c_str() << endl; + auto actual = json_schema_to_grammar(nlohmann::json::parse(tc.schema)); + tc.verify("C++", actual); +} + +static void run_all(const TestCase& tc) { + run_py(tc); + run_mjs(tc); + run_cpp(tc); +} + +int main() { + run_all({ + .name = "exotic formats", + .schema = R"""({ + "prefixItems": [ + { "format": "date" }, + { "format": "uuid" }, + { "format": "time" }, + { "format": "date-time" } + ] + })""", + .expected = R"""( + date ::= [0-9] [0-9] [0-9] [0-9] "-" ( "0" [1-9] | "1" [0-2] ) "-" ( [0-2] [0-9] | "3" [0-1] ) + date-string ::= "\"" date "\"" space + date-time ::= date "T" time + date-time-string ::= "\"" date-time "\"" space + root ::= "[" space date-string "," space uuid "," space time-string "," space date-time-string "]" space + space ::= " "? + time ::= ([01] [0-9] | "2" [0-3]) ":" [0-5] [0-9] ":" [0-5] [0-9] ( "." [0-9] [0-9] [0-9] )? ( "Z" | ( "+" | "-" ) ( [01] [0-9] | "2" [0-3] ) ":" [0-5] [0-9] ) + time-string ::= "\"" time "\"" space + uuid ::= "\"" [0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F] "-" [0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F] "-" [0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F] "-" [0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F] "-" [0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F] "\"" space + )""" + }); + + run_all({ + .name = "string", + .schema = R"""({ + "type": "string" + })""", + .expected = R"""( + root ::= "\"" ( + [^"\\] | + "\\" (["\\/bfnrt] | "u" [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F]) + )* "\"" space + space ::= " "? + )""" + }); + + run_all({ + .name = "boolean", + .schema = R"""({ + "type": "boolean" + })""", + .expected = R"""( + root ::= ("true" | "false") space + space ::= " "? + )""" + }); + + run_all({ + .name = "integer", + .schema = R"""({ + "type": "integer" + })""", + .expected = R"""( + root ::= ("-"? ([0-9] | [1-9] [0-9]*)) space + space ::= " "? + )""" + }); + + run_all({ + .name = "number", + .schema = R"""({ + "type": "number" + })""", + .expected = R"""( + root ::= ("-"? ([0-9] | [1-9] [0-9]*)) ("." [0-9]+)? ([eE] [-+]? [0-9]+)? space + space ::= " "? + )""" + }); + + run_all({ + .name = "minItems", + .schema = R"""({ + "items": { + "type": "boolean" + }, + "minItems": 2 + })""", + .expected = R"""( + boolean ::= ("true" | "false") space + root ::= "[" space boolean ( "," space boolean )( "," space boolean )* "]" space + space ::= " "? + )""" + }); + + run_all({ + .name = "maxItems 1", + .schema = R"""({ + "items": { + "type": "boolean" + }, + "maxItems": 1 + })""", + .expected = R"""( + boolean ::= ("true" | "false") space + root ::= "[" space ( boolean )? "]" space + space ::= " "? + )""" + }); + + run_all({ + .name = "maxItems 2", + .schema = R"""({ + "items": { + "type": "boolean" + }, + "maxItems": 2 + })""", + .expected = R"""( + boolean ::= ("true" | "false") space + root ::= "[" space ( boolean ( "," space boolean )? )? "]" space + space ::= " "? + )""" + }); + + run_all({ + .name = "min + maxItems", + .schema = R"""({ + "items": { + "type": ["number", "integer"] + }, + "minItems": 3, + "maxItems": 5 + })""", + .expected = R"""( + integer ::= ("-"? ([0-9] | [1-9] [0-9]*)) space + item ::= number | integer + number ::= ("-"? ([0-9] | [1-9] [0-9]*)) ("." [0-9]+)? ([eE] [-+]? [0-9]+)? space + root ::= "[" space item ( "," space item )( "," space item )( "," space item )?( "," space item )? "]" space + space ::= " "? + )""" + }); + + run_all({ + .name = "regexp", + .schema = R"""({ + "type": "string", + "pattern": "^(\\([0-9]{1,3}\\))?[0-9]{3}-[0-9]{4} and...$" + })""", + .expected = R"""( + dot ::= [\U00000000-\x09\x0B\x0C\x0E-\U0010FFFF] + root ::= ("(" root-1 root-1? root-1? ")")? root-1 root-1 root-1 "-" root-1 root-1 root-1 root-1 " and" dot dot dot + root-1 ::= [0-9] + space ::= " "? + )""" + }); + + run_all({ + .name = "optionals", + .schema = R"""({ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "a": { + "type": "string" + }, + "b": { + "type": "string" + }, + "c": { + "type": [ + "number", + "string" + ] + }, + "d": { + "type": "string" + }, + "e": { + "type": "object", + "additionalProperties": { + "type": "array", + "items": { + "type": "array", + "minItems": 2, + "items": [ + { + "type": "string" + }, + { + "type": "number" + } + ], + "maxItems": 2 + } + } + } + }, + "required": [ + "a", + "b" + ], + "additionalProperties": false, + "definitions": {} + })""", + .expected = R"""( + a-kv ::= "\"a\"" space ":" space string + b-kv ::= "\"b\"" space ":" space string + c ::= number | string + c-kv ::= "\"c\"" space ":" space c + c-rest ::= ( "," space d-kv )? d-rest + d-kv ::= "\"d\"" space ":" space string + d-rest ::= ( "," space e-kv )? + e ::= ( e-additionalProperties-kv ( "," space e-additionalProperties-kv )* )* + e-additionalProperties-kv ::= string ":" space e-additionalProperties-value + e-additionalProperties-value ::= "[" space ( e-additionalProperties-value-item ( "," space e-additionalProperties-value-item )* )? "]" space + e-additionalProperties-value-item ::= "[" space string "," space number "]" space + e-kv ::= "\"e\"" space ":" space e + number ::= ("-"? ([0-9] | [1-9] [0-9]*)) ("." [0-9]+)? ([eE] [-+]? [0-9]+)? space + root ::= "{" space a-kv "," space b-kv ( "," space ( c-kv c-rest | d-kv d-rest | e-kv ) )? "}" space + space ::= " "? + string ::= "\"" ( + [^"\\] | + "\\" (["\\/bfnrt] | "u" [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F]) + )* "\"" space + )""" + }); +}