Integrate official Draft 2020-12 Basic output tests and preserve annotation value shapes

ports/javascript/index.d.mts

@@ -22,7 +22,7 @@ export interface StandardOutputAnnotationEntry {
   keywordLocation: string;
   absoluteKeywordLocation: string;
   instanceLocation: string;
-  annotation: unknown[];
+  annotation: unknown;
 }
 
 export type StandardOutputFlagResult = { valid: boolean };

ports/javascript/index.mjs

@@ -4118,17 +4118,17 @@ class SimpleOutput {
           keywordLocation: evaluatePath,
           absoluteKeywordLocation: instruction[3],
           instanceLocation,
-          annotation: [ annotation ]
+          values: [ annotation ]
         };
         this.annotations.set(annotationKey, bucket);
       } else {
-        const last = bucket.annotation[bucket.annotation.length - 1];
+        const last = bucket.values[bucket.values.length - 1];
         let isSame = last === annotation;
         if (!isSame && Array.isArray(last) && Array.isArray(annotation) &&
             last.length === annotation.length) {
           isSame = last.every((value, index) => value === annotation[index]);
         }
-        if (!isSame) bucket.annotation.push(annotation);
+        if (!isSame) bucket.values.push(annotation);
       }
       return;
     }
@@ -4201,14 +4201,30 @@ class SimpleOutput {
     if (valid) {
       const result = { valid: true };
       if (this.annotations.size > 0) {
-        result.annotations = [ ...this.annotations.values() ];
+        result.annotations = [ ...this.annotations.values() ].map(entry => ({
+          keywordLocation: entry.keywordLocation,
+          absoluteKeywordLocation: entry.absoluteKeywordLocation,
+          instanceLocation: entry.instanceLocation,
+          annotation: isArrayAggregatedKeyword(entry.keywordLocation)
+            ? entry.values
+            : entry.values[entry.values.length - 1]
+        }));
       }
       return result;
     }
     return { valid: false, errors: this.errors };
   }
 }
 
+function isArrayAggregatedKeyword(evaluatePath) {
+  const lastSlash = evaluatePath.lastIndexOf('/');
+  const keyword =
+      lastSlash === -1 ? evaluatePath : evaluatePath.slice(lastSlash + 1);
+  return keyword === 'properties' || keyword === 'patternProperties' ||
+         keyword === 'additionalProperties' ||
+         keyword === 'unevaluatedProperties' || keyword === 'contains';
+}
+
 function runStandard(evaluator, instance, format) {
   if (format === 'flag') {
     return { valid: evaluator.validate(instance) };

src/output/include/sourcemeta/blaze/output_standard.h

@@ -26,9 +26,6 @@ enum class StandardOutput : std::uint8_t {
   // TODO: Implement the "detailed" and "verbose" output formats
 };
 
-// TODO: Integrate with
-// https://github.com/json-schema-org/JSON-Schema-Test-Suite/tree/main/output-tests
-
 /// @ingroup output
 /// Perform JSON Schema evaluation using Standard Output formats. For example:
 ///

src/output/output_standard.cc

@@ -29,6 +29,17 @@ struct AnnotationLocation {
   std::reference_wrapper<const std::string> schema_location;
 };
 
+auto is_array_aggregated_keyword(
+    const sourcemeta::core::WeakPointer &evaluate_path) noexcept -> bool {
+  if (evaluate_path.empty() || !evaluate_path.back().is_property()) {
+    return false;
+  }
+  const auto &keyword{evaluate_path.back().to_property()};
+  return keyword == "properties" || keyword == "patternProperties" ||
+         keyword == "additionalProperties" ||
+         keyword == "unevaluatedProperties" || keyword == "contains";
+}
+
 auto group_annotations(const SimpleOutput &output)
     -> std::map<AnnotationLocation, std::vector<sourcemeta::core::JSON>> {
   std::map<AnnotationLocation, std::vector<sourcemeta::core::JSON>> result;
@@ -87,8 +98,14 @@ auto handle_standard(Evaluator &evaluator, const Template &schema,
           }
         }
 
-        unit.assign_assume_new("annotation",
-                               sourcemeta::core::to_json(annotation.second));
+        if (is_array_aggregated_keyword(annotation.first.evaluate_path)) {
+          unit.assign_assume_new("annotation",
+                                 sourcemeta::core::to_json(annotation.second));
+        } else {
+          assert(!annotation.second.empty());
+          unit.assign_assume_new(
+              "annotation", sourcemeta::core::JSON{annotation.second.back()});
+        }
         annotations.push_back(std::move(unit));
       }
 

test/output/CMakeLists.txt

@@ -51,3 +51,19 @@ target_link_libraries(sourcemeta_blaze_output_standard_flag_suite_unit
   PRIVATE sourcemeta::blaze::evaluator)
 target_link_libraries(sourcemeta_blaze_output_standard_flag_suite_unit
   PRIVATE sourcemeta::blaze::compiler)
+
+sourcemeta_test(NAMESPACE sourcemeta PROJECT blaze
+  NAME output_official_suite
+  SOURCES output_official_suite.cc)
+target_compile_definitions(sourcemeta_blaze_output_official_suite_unit
+  PRIVATE OFFICIAL_OUTPUT_SUITE_PATH="${PROJECT_SOURCE_DIR}/vendor/jsonschema-test-suite/output-tests")
+target_link_libraries(sourcemeta_blaze_output_official_suite_unit
+  PRIVATE sourcemeta::core::json)
+target_link_libraries(sourcemeta_blaze_output_official_suite_unit
+  PRIVATE sourcemeta::blaze::foundation)
+target_link_libraries(sourcemeta_blaze_output_official_suite_unit
+  PRIVATE sourcemeta::blaze::output)
+target_link_libraries(sourcemeta_blaze_output_official_suite_unit
+  PRIVATE sourcemeta::blaze::evaluator)
+target_link_libraries(sourcemeta_blaze_output_official_suite_unit
+  PRIVATE sourcemeta::blaze::compiler)

test/output/output_official_suite.cc

@@ -0,0 +1,157 @@
+#include <sourcemeta/core/test.h>
+
+#include <sourcemeta/blaze/compiler.h>
+#include <sourcemeta/blaze/evaluator.h>
+#include <sourcemeta/blaze/output.h>
+
+#include <sourcemeta/blaze/foundation.h>
+#include <sourcemeta/core/json.h>
+
+#include <cassert>
+#include <cctype>
+#include <cstdio>
+#include <cstdlib>
+#include <filesystem>
+#include <optional>
+#include <string>
+#include <string_view>
+#include <utility>
+#include <vector>
+
+namespace {
+
+auto output_schema_resolver(std::string_view identifier)
+    -> std::optional<sourcemeta::core::JSON> {
+  const std::filesystem::path suite_path{OFFICIAL_OUTPUT_SUITE_PATH};
+
+  if (identifier == "https://json-schema.org/draft/2020-12/output/schema" ||
+      identifier == "/draft/2020-12/output/schema") {
+    return sourcemeta::core::read_json(suite_path / "draft2020-12" /
+                                       "output-schema.json");
+  }
+
+  if (identifier == "https://json-schema.org/draft/2019-09/output/schema" ||
+      identifier == "/draft/2019-09/output/schema") {
+    return sourcemeta::core::read_json(suite_path / "draft2019-09" /
+                                       "output-schema.json");
+  }
+
+  return sourcemeta::blaze::schema_resolver(identifier);
+}
+
+auto run_official_output_test(const sourcemeta::core::JSON &input_schema,
+                              const sourcemeta::core::JSON &data,
+                              const sourcemeta::core::JSON &output_basic_schema,
+                              const sourcemeta::blaze::Mode mode,
+                              const std::string &default_dialect) -> void {
+  const auto input_template{sourcemeta::blaze::compile(
+      input_schema, sourcemeta::blaze::schema_walker, output_schema_resolver,
+      sourcemeta::blaze::default_schema_compiler, mode, default_dialect)};
+
+  sourcemeta::blaze::Evaluator evaluator;
+  const auto blaze_output{
+      sourcemeta::blaze::standard(evaluator, input_template, data,
+                                  sourcemeta::blaze::StandardOutput::Basic)};
+
+  const auto output_template{sourcemeta::blaze::compile(
+      output_basic_schema, sourcemeta::blaze::schema_walker,
+      output_schema_resolver, sourcemeta::blaze::default_schema_compiler, mode,
+      default_dialect)};
+
+  const auto valid{evaluator.validate(output_template, blaze_output)};
+  EXPECT_TRUE(valid);
+}
+
+auto slugify(const std::string &input) -> std::string {
+  std::string result;
+  result.reserve(input.size());
+  for (const auto character : input) {
+    result.push_back(std::isalnum(static_cast<unsigned char>(character)) != 0
+                         ? character
+                         : '_');
+  }
+  return result;
+}
+
+auto register_tests(const std::filesystem::path &path,
+                    const std::string &suite_name,
+                    const std::string &default_dialect) -> void {
+  std::fprintf(stderr, "-- Parsing: %s\n", path.string().c_str());
+  const auto suite{sourcemeta::core::read_json(path)};
+  assert(suite.is_array());
+
+  const auto file_stem{path.stem().string()};
+
+  for (const auto &case_entry : suite.as_array()) {
+    assert(case_entry.is_object());
+    assert(case_entry.defines("description"));
+    assert(case_entry.defines("schema"));
+    assert(case_entry.defines("tests"));
+
+    const auto &case_description{case_entry.at("description").to_string()};
+    const auto &input_schema{case_entry.at("schema")};
+
+    for (const auto &test_entry : case_entry.at("tests").as_array()) {
+      assert(test_entry.is_object());
+      assert(test_entry.defines("description"));
+      assert(test_entry.defines("data"));
+      assert(test_entry.defines("output"));
+
+      const auto &test_description{test_entry.at("description").to_string()};
+      const auto &data{test_entry.at("data")};
+      const auto &output_def{test_entry.at("output")};
+
+      assert(output_def.defines("basic"));
+      const auto &output_basic_schema{output_def.at("basic")};
+
+      const auto title{file_stem + "_" + slugify(case_description) + "_" +
+                       slugify(test_description)};
+
+      sourcemeta::core::test_register(
+          suite_name, title, __FILE__, __LINE__,
+          [input_schema, data, output_basic_schema, default_dialect]() -> void {
+            run_official_output_test(input_schema, data, output_basic_schema,
+                                     sourcemeta::blaze::Mode::Exhaustive,
+                                     default_dialect);
+          });
+    }
+  }
+}
+
+} // namespace
+
+auto main(int argc, char **argv) -> int {
+  try {
+    const std::filesystem::path suite_path{OFFICIAL_OUTPUT_SUITE_PATH};
+
+    const std::vector<std::string> test_files{
+        "escape.json",
+        "general.json",
+        "readOnly.json",
+        "type.json",
+    };
+
+    // 2020-12
+    const std::filesystem::path content_2020_12_dir{suite_path /
+                                                    "draft2020-12" / "content"};
+    for (const auto &file : test_files) {
+      register_tests(content_2020_12_dir / file,
+                     "Output_official_2020_12_suite",
+                     "https://json-schema.org/draft/2020-12/schema");
+    }
+
+    // 2019-09
+    const std::filesystem::path content_2019_09_dir{suite_path /
+                                                    "draft2019-09" / "content"};
+    for (const auto &file : test_files) {
+      register_tests(content_2019_09_dir / file,
+                     "Output_official_2019_09_suite",
+                     "https://json-schema.org/draft/2019-09/schema");
+    }
+  } catch (const std::exception &error) {
+    std::fprintf(stderr, "Error: %s\n", error.what());
+    return EXIT_FAILURE;
+  }
+
+  return sourcemeta::core::test_run(argc, argv);
+}

test/output/output_standard_basic.json

@@ -20,6 +20,30 @@
       ]
     }
   },
+  {
+    "description": "success_properties_multiple",
+    "schema": {
+      "$schema": "https://json-schema.org/draft/2020-12/schema",
+      "properties": {
+        "foo": { "type": "string" },
+        "bar": { "type": "string" }
+      }
+    },
+    "instance": { "foo": "a", "bar": "b" },
+    "valid": true,
+    "fast": { "valid": true },
+    "exhaustive": {
+      "valid": true,
+      "annotations": [
+        {
+          "keywordLocation": "/properties",
+          "absoluteKeywordLocation": "#/properties",
+          "instanceLocation": "",
+          "annotation": [ "bar", "foo" ]
+        }
+      ]
+    }
+  },
   {
     "description": "success_2",
     "schema": {
@@ -39,7 +63,7 @@
           "keywordLocation": "/anyOf/1/title",
           "absoluteKeywordLocation": "#/anyOf/1/title",
           "instanceLocation": "",
-          "annotation": [ "#2" ]
+          "annotation": "#2"
         }
       ]
     }
@@ -85,7 +109,7 @@
           "keywordLocation": "/oneOf/1/title",
           "absoluteKeywordLocation": "#/oneOf/1/title",
           "instanceLocation": "",
-          "annotation": [ "Second" ]
+          "annotation": "Second"
         }
       ]
     }
@@ -142,7 +166,7 @@
           "keywordLocation": "/anyOf/1/title",
           "absoluteKeywordLocation": "#/anyOf/1/title",
           "instanceLocation": "",
-          "annotation": [ "#2" ]
+          "annotation": "#2"
         }
       ]
     }
@@ -176,7 +200,7 @@
           "keywordLocation": "/then/title",
           "absoluteKeywordLocation": "#/then/title",
           "instanceLocation": "",
-          "annotation": [ "Then Title" ]
+          "annotation": "Then Title"
         }
       ]
     }
@@ -210,7 +234,7 @@
           "keywordLocation": "/else/title",
           "absoluteKeywordLocation": "#/else/title",
           "instanceLocation": "",
-          "annotation": [ "Else Title" ]
+          "annotation": "Else Title"
         }
       ]
     }