From 886ffadbaa24a5d937a19383c7fe4598f9c3d51c Mon Sep 17 00:00:00 2001
From: ochafik <ochafik@google.com>
Date: Fri, 28 Jun 2024 21:55:28 +0100
Subject: [PATCH] json: bring JS & Python clis closer in space and spirit

---
 examples/json_schema_to_grammar.mjs   | 27 +++++++++++++++++++++++++++
 examples/json_schema_to_grammar.py    | 13 +++++++++++++
 grammars/README.md                    |  4 ++--
 tests/run-json-schema-to-grammar.mjs  | 13 -------------
 tests/test-json-schema-to-grammar.cpp |  2 +-
 5 files changed, 43 insertions(+), 16 deletions(-)
 create mode 100644 examples/json_schema_to_grammar.mjs
 delete mode 100644 tests/run-json-schema-to-grammar.mjs

diff --git a/examples/json_schema_to_grammar.mjs b/examples/json_schema_to_grammar.mjs
new file mode 100644
index 000000000..892e3a1cd
--- /dev/null
+++ b/examples/json_schema_to_grammar.mjs
@@ -0,0 +1,27 @@
+/*
+  JSON Schema to Grammar converter (JavaScript version)
+
+  There are C++ and a Python converters w/ the same features.
+  (More flags are currently exposed for the Python version)
+
+  Usage:
+    node examples/json_schema_to_grammar.mjs schema.json
+    node examples/json_schema_to_grammar.mjs https://json.schemastore.org/tsconfig.json
+    echo '{"type": "object"}' | node examples/json_schema_to_grammar.mjs -
+*/
+import { readFileSync } from "fs"
+import { SchemaConverter } from "./server/public/json-schema-to-grammar.mjs"
+import fs from 'fs'
+
+const [, , file] = process.argv
+let schema;
+if (file === '-') {
+  schema = JSON.parse(fs.readFileSync(0, 'utf8'))
+} else if (file.startsWith('https://')) {
+  schema = await (await fetch(file)).json()
+} else {
+  schema = JSON.parse(readFileSync(file, "utf8"));
+}
+const converter = new SchemaConverter({})
+converter.visit(schema, '')
+console.log(converter.formatGrammar())
diff --git a/examples/json_schema_to_grammar.py b/examples/json_schema_to_grammar.py
index 9a407910a..0805aea67 100755
--- a/examples/json_schema_to_grammar.py
+++ b/examples/json_schema_to_grammar.py
@@ -1,4 +1,17 @@
 #!/usr/bin/env python3
+'''
+  JSON Schema to Grammar conversion
+
+  There are C++ and JavaScript converters w/ the same features.
+
+  Usage:
+    python examples/json_schema_to_grammar.py schema.json
+    python examples/json_schema_to_grammar.py https://json.schemastore.org/tsconfig.json
+    echo '{"type": "object"}' | python examples/json_schema_to_grammar.py -
+
+  Also see https://github.com/ggerganov/llama.cpp/tree/master/grammars
+'''
+
 import argparse
 import itertools
 import json
diff --git a/grammars/README.md b/grammars/README.md
index 652aeeb09..3ca71086b 100644
--- a/grammars/README.md
+++ b/grammars/README.md
@@ -123,8 +123,8 @@ You can use GBNF grammars:
     - For the `/chat/completions` endpoint, passed inside the `result_format` body field (e.g. `{"type", "json_object", "schema": {"items": {}}}`)
 - In [llama-cli](../examples/main), passed as the `--json` / `-j` flag
 - To convert to a grammar ahead of time:
-    - in CLI, with [examples/json_schema_to_grammar.py](../examples/json_schema_to_grammar.py)
-    - in JavaScript with [json-schema-to-grammar.mjs](../examples/server/public/json-schema-to-grammar.mjs) (this is used by the [server](../examples/server)'s Web UI)
+    - in CLI, with [`node examples/json_schema_to_grammar.py schema.json`](../examples/json_schema_to_grammar.py)
+    - in JavaScript with [`node examples/json_schema_to_grammar.mjs schema.json`](../examples/json_schema_to_grammar.mjs) (uses same lib as the [server](../examples/server)'s Web UI)
 
 Take a look at [tests](../tests/test-json-schema-to-grammar.cpp) to see which features are likely supported (you'll also find usage examples in https://github.com/ggerganov/llama.cpp/pull/5978, https://github.com/ggerganov/llama.cpp/pull/6659 & https://github.com/ggerganov/llama.cpp/pull/6555).
 
diff --git a/tests/run-json-schema-to-grammar.mjs b/tests/run-json-schema-to-grammar.mjs
deleted file mode 100644
index a50e3e683..000000000
--- a/tests/run-json-schema-to-grammar.mjs
+++ /dev/null
@@ -1,13 +0,0 @@
-import { readFileSync } from "fs"
-import { SchemaConverter } from "../examples/server/public/json-schema-to-grammar.mjs"
-
-const [, , file] = process.argv
-let schema;
-if (file.startsWith('https://')) {
-  schema = await (await fetch(file)).json()
-} else {
-  schema = JSON.parse(readFileSync(file, "utf8"));
-}
-const converter = new SchemaConverter({})
-converter.visit(schema, '')
-console.log(converter.formatGrammar())
diff --git a/tests/test-json-schema-to-grammar.cpp b/tests/test-json-schema-to-grammar.cpp
index 7e547698b..0d7579f34 100755
--- a/tests/test-json-schema-to-grammar.cpp
+++ b/tests/test-json-schema-to-grammar.cpp
@@ -1299,7 +1299,7 @@ int main() {
             test_all("JavaScript", [](const TestCase & tc) {
                 write("test-json-schema-input.tmp", tc.schema);
                 tc.verify_status(std::system(
-                    "node ./tests/run-json-schema-to-grammar.mjs test-json-schema-input.tmp > test-grammar-output.tmp") == 0 ? SUCCESS : FAILURE);
+                    "node ./examples/json_schema_to_grammar.mjs test-json-schema-input.tmp > test-grammar-output.tmp") == 0 ? SUCCESS : FAILURE);
                 tc.verify(read("test-grammar-output.tmp"));
             });
         } else {