From 653dbf14cc3e00e0bbf0a1a1063369087c7d2e4b Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Fri, 17 Jul 2026 10:55:55 +0800 Subject: [PATCH] feat(java): add configurable JSON benchmark reports --- benchmarks/java/plot_json_benchmark.py | 345 ++++++++++++++++++ benchmarks/java/run_json.sh | 174 +++++++++ .../benchmark/JsonSerializationSuite.java | 88 ++++- 3 files changed, 600 insertions(+), 7 deletions(-) create mode 100755 benchmarks/java/plot_json_benchmark.py create mode 100755 benchmarks/java/run_json.sh diff --git a/benchmarks/java/plot_json_benchmark.py b/benchmarks/java/plot_json_benchmark.py new file mode 100755 index 0000000000..5d5125f5f4 --- /dev/null +++ b/benchmarks/java/plot_json_benchmark.py @@ -0,0 +1,345 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Generate the Java JSON benchmark chart and Markdown report from JMH JSON.""" + +from __future__ import annotations + +import argparse +import json +import math +import re +import sys +from pathlib import Path +from typing import Any + +import matplotlib.pyplot as plt +import numpy as np +from matplotlib.ticker import FuncFormatter + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from plot_style import ( # noqa: E402 + BAR_EDGE_COLOR, + apply_benchmark_style, + format_markdown_with_prettier, + format_throughput_label, + format_throughput_tick, + save_benchmark_figure, + style_throughput_axis, +) + +apply_benchmark_style(plt) + +DEFAULT_LIBS = "fory-json,jackson,gson" +LIB_ALIASES = { + "fory": "fory", + "fory-json": "fory", + "jackson": "jackson", + "gson": "gson", + "fastjson2": "fastjson2", +} +SERIALIZER_LABELS = { + "fory": "fory-json", + "jackson": "jackson", + "gson": "gson", + "fastjson2": "fastjson2", +} +COLORS = { + "fory": "#FF6F01", + "jackson": "#55BCC2", + "gson": "#8C6D8A", + "fastjson2": (0.90, 0.43, 0.5), +} +OPERATIONS = ("to", "from") +REPRESENTATIONS = ("string", "bytes") +CASE_LABELS = { + "to": "Serialize", + "from": "Deserialize", +} +BENCHMARK_PATTERN = re.compile( + r"(?:^|[.])(?Pfory|jackson|gson|fastjson2)" + r"(?PTo|From)Json(?PBytes|String)$" +) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Generate a Java JSON benchmark chart and Markdown report" + ) + parser.add_argument( + "--json-file", + default="reports/json/benchmark_results.json", + help="Raw JMH JSON result file", + ) + parser.add_argument( + "--output-dir", + default="reports/json", + help="Directory for throughput.png and README.md", + ) + parser.add_argument( + "--libs", + default=DEFAULT_LIBS, + help="Comma-separated libraries in bar and table order", + ) + parser.add_argument( + "--print-lib-config", + action="store_true", + help=argparse.SUPPRESS, + ) + return parser.parse_args() + + +def parse_libs(value: str) -> tuple[str, ...]: + requested = [name.strip().lower() for name in value.split(",")] + if not requested or any(not name for name in requested): + raise ValueError("--libs requires one or more comma-separated library names") + serializers = [] + for name in requested: + serializer = LIB_ALIASES.get(name) + if serializer is None: + choices = ", ".join(("fory-json", "jackson", "gson", "fastjson2")) + raise ValueError(f"Unknown JSON library: {name}. Available: {choices}") + if serializer in serializers: + raise ValueError(f"Duplicate JSON library: {SERIALIZER_LABELS[serializer]}") + serializers.append(serializer) + return tuple(serializers) + + +def load_json(path: Path) -> list[dict[str, Any]]: + with path.open("r", encoding="utf-8") as source: + payload = json.load(source) + benchmarks = payload if isinstance(payload, list) else payload.get("benchmarks", []) + if not isinstance(benchmarks, list): + raise ValueError(f"Expected a JMH benchmark list in {path}") + return benchmarks + + +def ops_per_second(value: float, unit: str) -> float: + multipliers = { + "ops/s": 1, + "ops/ms": 1_000, + "ops/us": 1_000_000, + "ops/ns": 1_000_000_000, + } + if unit not in multipliers: + raise ValueError(f"Unsupported JMH throughput unit: {unit}") + return value * multipliers[unit] + + +def collect_results( + benchmarks: list[dict[str, Any]], + serializers: tuple[str, ...], +) -> dict[tuple[str, str], dict[str, tuple[float, float]]]: + results: dict[tuple[str, str], dict[str, tuple[float, float]]] = { + (operation, representation): {} + for representation in REPRESENTATIONS + for operation in OPERATIONS + } + for benchmark in benchmarks: + name = benchmark.get("benchmark", "") + match = BENCHMARK_PATTERN.search(name) + if match is None: + continue + serializer = match.group("serializer") + case = ( + match.group("operation").lower(), + match.group("representation").lower(), + ) + metric = benchmark.get("primaryMetric", {}) + unit = metric.get("scoreUnit", "ops/s") + score = ops_per_second(float(metric["score"]), unit) + error = ops_per_second(float(metric.get("scoreError", 0.0)), unit) + if not math.isfinite(error): + error = 0.0 + results[case][serializer] = (score, error) + + missing = [ + f"{serializer}{operation.title()}Json{representation.title()}" + for representation in REPRESENTATIONS + for operation in OPERATIONS + for serializer in serializers + if serializer not in results[(operation, representation)] + ] + if missing: + raise ValueError("Missing JMH benchmark results: " + ", ".join(missing)) + return results + + +def render_plot( + results: dict[tuple[str, str], dict[str, tuple[float, float]]], + serializers: tuple[str, ...], + representation: str, + output: Path, +) -> None: + figure, axes = plt.subplots(1, 2, figsize=(11.5, 5.2)) + x = np.arange(len(serializers), dtype=float) + + for axis, operation in zip(axes, OPERATIONS): + case = (operation, representation) + values = [results[case][serializer][0] for serializer in serializers] + errors = [results[case][serializer][1] for serializer in serializers] + bars = axis.bar( + x, + values, + width=0.62, + yerr=errors, + capsize=2.5, + color=[COLORS[serializer] for serializer in serializers], + edgecolor=BAR_EDGE_COLOR, + linewidth=0.8, + ) + axis.bar_label( + bars, + labels=[format_throughput_label(value) for value in values], + padding=3, + fontsize=8, + ) + highest = max(value + error for value, error in zip(values, errors)) + axis.set_ylim(0, highest * 1.18) + axis.set_xticks(x) + axis.set_xticklabels([SERIALIZER_LABELS[name] for name in serializers]) + axis.set_title(CASE_LABELS[operation], pad=10) + axis.yaxis.set_major_formatter(FuncFormatter(format_throughput_tick)) + style_throughput_axis(axis) + + axes[0].set_ylabel("Throughput (ops/sec)") + representation_title = "String" if representation == "string" else "UTF-8 Bytes" + figure.suptitle( + f"Java JSON {representation_title} Serialization and Deserialization Throughput", + y=0.98, + ) + figure.tight_layout(rect=[0, 0, 1, 0.95], w_pad=2.4) + save_benchmark_figure(figure, output) + plt.close(figure) + + +def format_score(value: float) -> str: + return f"{value:,.0f}" + + +def benchmark_metadata(benchmarks: list[dict[str, Any]]) -> list[str]: + if not benchmarks: + return [] + first = benchmarks[0] + items = [] + if first.get("jdkVersion"): + items.append(f"JDK: `{first['jdkVersion']}`") + if first.get("vmName"): + items.append(f"VM: `{first['vmName']}`") + if first.get("jmhVersion"): + items.append(f"JMH: `{first['jmhVersion']}`") + if first.get("warmupIterations") is not None and first.get("warmupTime"): + items.append( + f"Warmup: {first['warmupIterations']} iterations × `{first['warmupTime']}`" + ) + if first.get("measurementIterations") is not None and first.get("measurementTime"): + items.append( + "Measurement: " + f"{first['measurementIterations']} iterations × `{first['measurementTime']}`" + ) + if first.get("forks") is not None and first.get("threads") is not None: + items.append(f"Forks: {first['forks']}; threads: {first['threads']}") + items.append("Mode: throughput; higher is better") + return items + + +def render_report( + benchmarks: list[dict[str, Any]], + results: dict[tuple[str, str], dict[str, tuple[float, float]]], + serializers: tuple[str, ...], + output: Path, +) -> None: + library_names = ", ".join(SERIALIZER_LABELS[name] for name in serializers) + lines = [ + "# Java JSON Benchmark Report\n\n", + f"The benchmark compares {library_names} using the same MediaContent data. " + "The String group excludes UTF-8 conversion. The UTF-8 bytes group uses " + "direct byte-array APIs when available. " + + ( + "Gson includes String-to-UTF-8 encoding and UTF-8-to-String decoding.\n\n" + if "gson" in serializers + else "\n\n" + ), + "```bash\n", + "cd benchmarks/java\n", + "./run_json.sh --libs " + + ",".join(SERIALIZER_LABELS[name] for name in serializers) + + "\n", + "```\n\n", + ] + metadata = benchmark_metadata(benchmarks) + if metadata: + lines.extend(f"- {item}\n" for item in metadata) + lines.append("\n") + lines.extend( + [ + "## String\n\n", + "![Java JSON String benchmark throughput](string_throughput.png)\n\n", + "## UTF-8 Bytes\n\n", + "![Java JSON UTF-8 bytes benchmark throughput](utf8_bytes_throughput.png)\n\n", + "## Results\n\n", + "| Representation | Operation | " + + " | ".join(f"{SERIALIZER_LABELS[name]} ops/sec" for name in serializers) + + " | Fastest |\n", + "| --- | --- | " + " | ".join("---:" for _ in serializers) + " | --- |\n", + ] + ) + for representation in REPRESENTATIONS: + for operation in OPERATIONS: + values = results[(operation, representation)] + fastest = max(serializers, key=lambda serializer: values[serializer][0]) + lines.append( + f"| {'String' if representation == 'string' else 'UTF-8 bytes'} | " + f"{'Serialize' if operation == 'to' else 'Deserialize'} | " + + " | ".join( + format_score(values[serializer][0]) for serializer in serializers + ) + + f" | {SERIALIZER_LABELS[fastest]} |\n" + ) + output.write_text("".join(lines), encoding="utf-8") + format_markdown_with_prettier(output) + + +def main() -> None: + args = parse_args() + try: + serializers = parse_libs(args.libs) + except ValueError as error: + raise SystemExit(str(error)) from error + if args.print_lib_config: + labels = ",".join(SERIALIZER_LABELS[name] for name in serializers) + print(labels + "\t" + "|".join(serializers)) + return + json_path = Path(args.json_file) + output_dir = Path(args.output_dir) + output_dir.mkdir(parents=True, exist_ok=True) + benchmarks = load_json(json_path) + results = collect_results(benchmarks, serializers) + string_chart_path = output_dir / "string_throughput.png" + bytes_chart_path = output_dir / "utf8_bytes_throughput.png" + report_path = output_dir / "README.md" + render_plot(results, serializers, "string", string_chart_path) + render_plot(results, serializers, "bytes", bytes_chart_path) + render_report(benchmarks, results, serializers, report_path) + print(f"Generated {string_chart_path}") + print(f"Generated {bytes_chart_path}") + print(f"Generated {report_path}") + + +if __name__ == "__main__": + main() diff --git a/benchmarks/java/run_json.sh b/benchmarks/java/run_json.sh new file mode 100755 index 0000000000..d074d6e145 --- /dev/null +++ b/benchmarks/java/run_json.sh @@ -0,0 +1,174 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPORT_DIR="${SCRIPT_DIR}/reports/json" +DURATION_SECONDS="2" +WARMUP_ITERATIONS="3" +MEASUREMENT_ITERATIONS="5" +LIBS="fory-json,jackson,gson" +SKIP_BUILD="false" + +usage() { + cat <<'EOF' +Usage: ./run_json.sh [OPTIONS] + +Run selected Java JSON serialization and deserialization benchmarks, then +generate charts and a Markdown report. + +Options: + --duration Warmup and measurement time per iteration (default: 2) + --warmup-iterations Number of JMH warmup iterations (default: 3) + --iterations Number of JMH measurement iterations (default: 5) + --libs Comma-separated libraries in chart order + (default: fory-json,jackson,gson) + Available: fory-json, jackson, gson, fastjson2 + --reports-dir Output directory (default: reports/json) + --skip-build Reuse target/benchmarks.jar + --help Show this help +EOF +} + +require_value() { + if [[ $# -lt 2 || -z "$2" ]]; then + echo "Missing value for $1" >&2 + exit 1 + fi +} + +jmh_time() { + case "$1" in + *[!0-9.]*) + printf '%s' "$1" + ;; + *.*) + python3 - "$1" <<'PY' +import sys + +millis = max(1, round(float(sys.argv[1]) * 1000)) +print(f"{millis}ms") +PY + ;; + *) + printf '%ss' "$1" + ;; + esac +} + +while [[ $# -gt 0 ]]; do + case "$1" in + --duration) + require_value "$@" + DURATION_SECONDS="$2" + shift 2 + ;; + --warmup-iterations) + require_value "$@" + WARMUP_ITERATIONS="$2" + shift 2 + ;; + --iterations) + require_value "$@" + MEASUREMENT_ITERATIONS="$2" + shift 2 + ;; + --libs) + require_value "$@" + LIBS="$2" + shift 2 + ;; + --reports-dir) + require_value "$@" + REPORT_DIR="$2" + shift 2 + ;; + --skip-build) + SKIP_BUILD="true" + shift + ;; + --help|-h) + usage + exit 0 + ;; + *) + echo "Unknown option: $1" >&2 + usage >&2 + exit 1 + ;; + esac +done + +case "${REPORT_DIR}" in + /*) ;; + *) REPORT_DIR="${PWD}/${REPORT_DIR}" ;; +esac + +if ! python3 -c "import matplotlib, numpy" >/dev/null 2>&1; then + echo "Missing plotting dependencies. Install them with: pip install matplotlib numpy" >&2 + exit 1 +fi + +LIB_CONFIG="$(python3 "${SCRIPT_DIR}/plot_json_benchmark.py" \ + --libs "${LIBS}" --print-lib-config 2>&1)" || { + echo "${LIB_CONFIG}" >&2 + exit 1 +} +IFS=$'\t' read -r NORMALIZED_LIBS SERIALIZER_REGEX <<<"${LIB_CONFIG}" + +JMH_DURATION="$(jmh_time "${DURATION_SECONDS}")" +RESULT_JSON="${REPORT_DIR}/benchmark_results.json" +BENCHMARK_REGEX="org\\.apache\\.fory\\.benchmark\\.JsonSerializationSuite\\.(${SERIALIZER_REGEX})(To|From)Json(Bytes|String)$" + +mkdir -p "${REPORT_DIR}" +cd "${SCRIPT_DIR}" + +if [[ "${SKIP_BUILD}" != "true" ]]; then + mvn -Pjmh -DskipTests package +fi + +if [[ ! -f target/benchmarks.jar ]]; then + echo "Missing target/benchmarks.jar; rerun without --skip-build." >&2 + exit 1 +fi + +ENABLE_FORY_DEBUG_OUTPUT=0 java \ + --add-opens=java.base/java.lang.invoke=ALL-UNNAMED \ + -jar target/benchmarks.jar "${BENCHMARK_REGEX}" \ + -f 1 \ + -wi "${WARMUP_ITERATIONS}" \ + -i "${MEASUREMENT_ITERATIONS}" \ + -t 1 \ + -w "${JMH_DURATION}" \ + -r "${JMH_DURATION}" \ + -bm thrpt \ + -tu s \ + -rf json \ + -rff "${RESULT_JSON}" + +python3 plot_json_benchmark.py \ + --json-file "${RESULT_JSON}" \ + --output-dir "${REPORT_DIR}" \ + --libs "${NORMALIZED_LIBS}" + +echo "JSON benchmark report: ${REPORT_DIR}/README.md" +echo "JSON String benchmark chart: ${REPORT_DIR}/string_throughput.png" +echo "JSON UTF-8 bytes benchmark chart: ${REPORT_DIR}/utf8_bytes_throughput.png" +echo "Raw JMH results: ${RESULT_JSON}" diff --git a/benchmarks/java/src/main/java/org/apache/fory/benchmark/JsonSerializationSuite.java b/benchmarks/java/src/main/java/org/apache/fory/benchmark/JsonSerializationSuite.java index 52f31985e7..725575c610 100644 --- a/benchmarks/java/src/main/java/org/apache/fory/benchmark/JsonSerializationSuite.java +++ b/benchmarks/java/src/main/java/org/apache/fory/benchmark/JsonSerializationSuite.java @@ -20,6 +20,8 @@ package org.apache.fory.benchmark; import com.alibaba.fastjson2.JSON; +import com.alibaba.fastjson2.JSONFactory; +import com.alibaba.fastjson2.JSONReader; import com.alibaba.fastjson2.JSONWriter; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.gson.Gson; @@ -52,29 +54,59 @@ public class JsonSerializationSuite { @State(Scope.Thread) public static class JsonState { ForyJson foryJson; - JSONWriter.Context fastjson2Context; + JSONReader.Context fastjson2ReadContext; + JSONWriter.Context fastjson2WriteContext; ObjectMapper mapper; Gson gson; MediaContent mediaContent; + byte[] jsonBytes; + String jsonString; @Setup public void setup() { foryJson = ForyJson.builder().build(); - fastjson2Context = new JSONWriter.Context(); + fastjson2ReadContext = JSONFactory.createReadContext(); + fastjson2WriteContext = new JSONWriter.Context(); mapper = new ObjectMapper(); gson = new Gson(); - mediaContent = JSON.parseObject(readResource(), MediaContent.class); + jsonString = readResource(); + jsonBytes = jsonString.getBytes(StandardCharsets.UTF_8); + mediaContent = JSON.parseObject(jsonString, MediaContent.class); byte[] foryBytes = foryJson.toJsonBytes(mediaContent); byte[] fastjsonBytes = - JSON.toJSONBytes(mediaContent, StandardCharsets.UTF_8, fastjson2Context); + JSON.toJSONBytes(mediaContent, StandardCharsets.UTF_8, fastjson2WriteContext); if (!JSON.parseObject(foryBytes).equals(JSON.parseObject(fastjsonBytes))) { throw new IllegalStateException("Fory JSON and fastjson2 produce different JSON objects"); } String foryString = foryJson.toJson(mediaContent); - String fastjsonString = JSON.toJSONString(mediaContent, fastjson2Context); + String fastjsonString = JSON.toJSONString(mediaContent, fastjson2WriteContext); if (!JSON.parseObject(foryString).equals(JSON.parseObject(fastjsonString))) { throw new IllegalStateException("Fory JSON and fastjson2 produce different JSON strings"); } + try { + verifyDecoded("Fory JSON bytes", foryJson.fromJson(jsonBytes, MediaContent.class)); + verifyDecoded( + "Fastjson2 bytes", + JSON.parseObject(jsonBytes, MediaContent.class, fastjson2ReadContext)); + verifyDecoded("Jackson bytes", mapper.readValue(jsonBytes, MediaContent.class)); + verifyDecoded( + "Gson bytes", + gson.fromJson(new String(jsonBytes, StandardCharsets.UTF_8), MediaContent.class)); + verifyDecoded("Fory JSON string", foryJson.fromJson(jsonString, MediaContent.class)); + verifyDecoded( + "Fastjson2 string", + JSON.parseObject(jsonString, MediaContent.class, fastjson2ReadContext)); + verifyDecoded("Jackson string", mapper.readValue(jsonString, MediaContent.class)); + verifyDecoded("Gson string", gson.fromJson(jsonString, MediaContent.class)); + } catch (IOException e) { + throw new IllegalStateException("Unable to verify JSON deserialization", e); + } + } + + private void verifyDecoded(String serializer, MediaContent decoded) { + if (!mediaContent.equals(decoded)) { + throw new IllegalStateException(serializer + " produced different MediaContent"); + } } private static String readResource() { @@ -105,7 +137,8 @@ public byte[] foryToJsonBytes(JsonState state) { @Benchmark public byte[] fastjson2ToJsonBytes(JsonState state) { - return JSON.toJSONBytes(state.mediaContent, StandardCharsets.UTF_8, state.fastjson2Context); + return JSON.toJSONBytes( + state.mediaContent, StandardCharsets.UTF_8, state.fastjson2WriteContext); } @Benchmark @@ -125,7 +158,7 @@ public String foryToJsonString(JsonState state) { @Benchmark public String fastjson2ToJsonString(JsonState state) { - return JSON.toJSONString(state.mediaContent, state.fastjson2Context); + return JSON.toJSONString(state.mediaContent, state.fastjson2WriteContext); } @Benchmark @@ -137,4 +170,45 @@ public String jacksonToJsonString(JsonState state) throws IOException { public String gsonToJsonString(JsonState state) { return state.gson.toJson(state.mediaContent); } + + @Benchmark + public MediaContent foryFromJsonBytes(JsonState state) { + return state.foryJson.fromJson(state.jsonBytes, MediaContent.class); + } + + @Benchmark + public MediaContent fastjson2FromJsonBytes(JsonState state) { + return JSON.parseObject(state.jsonBytes, MediaContent.class, state.fastjson2ReadContext); + } + + @Benchmark + public MediaContent jacksonFromJsonBytes(JsonState state) throws IOException { + return state.mapper.readValue(state.jsonBytes, MediaContent.class); + } + + @Benchmark + public MediaContent gsonFromJsonBytes(JsonState state) { + return state.gson.fromJson( + new String(state.jsonBytes, StandardCharsets.UTF_8), MediaContent.class); + } + + @Benchmark + public MediaContent foryFromJsonString(JsonState state) { + return state.foryJson.fromJson(state.jsonString, MediaContent.class); + } + + @Benchmark + public MediaContent fastjson2FromJsonString(JsonState state) { + return JSON.parseObject(state.jsonString, MediaContent.class, state.fastjson2ReadContext); + } + + @Benchmark + public MediaContent jacksonFromJsonString(JsonState state) throws IOException { + return state.mapper.readValue(state.jsonString, MediaContent.class); + } + + @Benchmark + public MediaContent gsonFromJsonString(JsonState state) { + return state.gson.fromJson(state.jsonString, MediaContent.class); + } }