json: restrict external refs to https, remove allowFetch options

common/json-schema-to-grammar.cpp

@@ -838,10 +838,13 @@ public:
             auto it = _external_refs.find(url);
             if (it != _external_refs.end()) {
                 target = it->second;
-            } else {
+            } else if (url.rfind("https://", 0) == 0) {
                 // Fetch the referenced schema and resolve its refs
                 target = _fetch_json(url);
                 _external_refs[url] = target;
+            } else {
+                _errors.push_back("Error resolving ref " + ref + ": unsupported url scheme");
+                return {json(), "", false};
             }
         }
         if (parts.size() == 1) {

examples/json_schema_to_grammar.py

@@ -235,9 +235,8 @@ ESCAPED_IN_REGEXPS_BUT_NOT_IN_LITERALS = set('^$.[]()|{}*+?')
 
 
 class SchemaConverter:
-    def __init__(self, *, prop_order, allow_fetch, dotall, raw_pattern):
+    def __init__(self, *, prop_order, dotall, raw_pattern):
         self._prop_order = prop_order
-        self._allow_fetch = allow_fetch
         self._dotall = dotall
         self._raw_pattern = raw_pattern
         self._rules = {
@@ -507,7 +506,7 @@ class SchemaConverter:
 
     def _resolve_ref(self, ref: str):
         parts = ref.split('#')
-        assert len(parts) == 2, f'Unsupported ref: {ref}'
+        assert len(parts) <= 2, f'Unsupported ref: {ref}'
         url = parts[0]
         target = None
         is_local = not url
@@ -518,13 +517,18 @@ class SchemaConverter:
             target = self._external_refs.get(url)
             if target is None:
                 # Fetch the referenced schema and resolve its refs
-                target = self._fetch_json(url)
+                assert url.startswith("https://"), f"Error resolving ref {ref}: unsupported url scheme"
+                import requests
+                target = requests.get(url).json()
                 self._external_refs[url] = target
 
-        tokens = parts[1].split('/')
+        if len(parts) == 1:
-        for sel in tokens[1:]:
+            return self.ResolvedRef(target, '', is_local)
-            assert target is not None and sel in target, f'Error resolving ref {ref}: {sel} not in {target}'
+        else:
-            target = target[sel]
+            tokens = parts[1].split('/')
+            for sel in tokens[1:]:
+                assert target is not None and sel in target, f'Error resolving ref {ref}: {sel} not in {target}'
+                target = target[sel]
 
         return self.ResolvedRef(target, tokens[-1] if tokens else '', is_local)
 
@@ -760,11 +764,6 @@ def main(args_in = None):
             given precedence over optional properties.
         '''
     )
-    parser.add_argument(
-        '--allow-fetch',
-        action='store_true',
-        default=False,
-        help='Whether to allow fetching referenced schemas over HTTPS')
     parser.add_argument(
         '--dotall',
         action='store_true',
@@ -792,7 +791,6 @@ def main(args_in = None):
             schema = json.load(f)
     converter = SchemaConverter(
         prop_order={name: idx for idx, name in enumerate(args.prop_order)},
-        allow_fetch=args.allow_fetch,
         dotall=args.dotall,
         raw_pattern=args.raw_pattern)
     converter.visit(schema, '')

examples/server/public/json-schema-to-grammar.mjs

@@ -264,7 +264,6 @@ const ESCAPED_IN_REGEXPS_BUT_NOT_IN_LITERALS = new Set('^$.[]()|{}*+?');
 export class SchemaConverter {
   constructor(options) {
     this._propOrder = options.prop_order || {};
-    this._allowFetch = options.allow_fetch || false;
     this._dotall = options.dotall || false;
     this._rules = {'space': SPACE_RULE};
     this._refs = {};
@@ -558,6 +557,9 @@ export class SchemaConverter {
       target = this._externalRefs.get(url);
       if (target === undefined) {
         // Fetch the referenced schema and resolve its refs
+        if (!url.startsWith('https://')) {
+          throw new Error(`Error resolving ref ${ref}: unsupported url scheme`);
+        }
         target = this._fetchJson(url);
         this._externalRefs.set(url, target);
       }

tests/run-json-schema-to-grammar.mjs

@@ -2,8 +2,12 @@ import { readFileSync } from "fs"
 import { SchemaConverter } from "../examples/server/public/json-schema-to-grammar.mjs"
 
 const [, , file] = process.argv
-const url = `file://${file}`
+let schema;
-const schema = JSON.parse(readFileSync(file, "utf8"));
+if (file.startsWith('https://')) {
+  schema = await (await fetch(file)).json()
+} else {
+  schema = JSON.parse(readFileSync(file, "utf8"));
+}
 const converter = new SchemaConverter({})
 converter.visit(schema, '')
 console.log(converter.formatGrammar())