From 88a0bb72b95beec1ba441098dc2727a43e2a2255 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sat, 30 May 2026 21:42:58 +0800 Subject: [PATCH 1/6] fix(java): use util List in lazy array data --- .../main/java/org/apache/fory/format/encoder/LazyArrayData.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java/fory-format/src/main/java/org/apache/fory/format/encoder/LazyArrayData.java b/java/fory-format/src/main/java/org/apache/fory/format/encoder/LazyArrayData.java index 38a9b92b57..c75f5b222d 100644 --- a/java/fory-format/src/main/java/org/apache/fory/format/encoder/LazyArrayData.java +++ b/java/fory-format/src/main/java/org/apache/fory/format/encoder/LazyArrayData.java @@ -21,7 +21,7 @@ import static org.apache.fory.type.TypeUtils.getRawType; -import java.awt.List; +import java.util.List; import org.apache.fory.annotation.Internal; import org.apache.fory.codegen.ClosureVisitable; import org.apache.fory.codegen.Code; From 7712638afd93996d3c1007129d4d4faf4bfec964 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sun, 31 May 2026 00:14:35 +0800 Subject: [PATCH 2/6] feat(java): add JPMS module descriptors --- ci/run_ci.sh | 1 + .../jpms_tests/run_jlink_smoke.sh | 201 +++++++++++++ java/fory-core/pom.xml | 176 ++++++++++++ .../CompressedArraySerializers.java | 2 +- .../fory/util/ArrayCompressionUtils.java | 176 ++++++++++++ .../util/PrimitiveArrayCompressionType.java | 51 +--- .../src/main/java16/module-info.java | 59 ++++ .../fory/util/VectorArrayCompression.java | 88 ++++++ .../fory-core/src/main/java9/module-info.java | 58 ++++ .../fory/serializer/ArrayCompressionTest.java | 0 .../serializer/ArrayCompressionUtilsTest.java | 0 java/fory-format/pom.xml | 64 +++++ .../src/main/java11/module-info.java | 33 +++ java/fory-simd/pom.xml | 94 ------ .../fory/util/ArrayCompressionUtils.java | 268 ------------------ java/pom.xml | 1 - 16 files changed, 863 insertions(+), 409 deletions(-) create mode 100755 integration_tests/jpms_tests/run_jlink_smoke.sh rename java/{fory-simd => fory-core}/src/main/java/org/apache/fory/serializer/CompressedArraySerializers.java (99%) create mode 100644 java/fory-core/src/main/java/org/apache/fory/util/ArrayCompressionUtils.java rename java/{fory-simd => fory-core}/src/main/java/org/apache/fory/util/PrimitiveArrayCompressionType.java (59%) create mode 100644 java/fory-core/src/main/java16/module-info.java create mode 100644 java/fory-core/src/main/java16/org/apache/fory/util/VectorArrayCompression.java create mode 100644 java/fory-core/src/main/java9/module-info.java rename java/{fory-simd => fory-core}/src/test/java/org/apache/fory/serializer/ArrayCompressionTest.java (100%) rename java/{fory-simd => fory-core}/src/test/java/org/apache/fory/serializer/ArrayCompressionUtilsTest.java (100%) create mode 100644 java/fory-format/src/main/java11/module-info.java delete mode 100644 java/fory-simd/pom.xml delete mode 100644 java/fory-simd/src/main/java/org/apache/fory/util/ArrayCompressionUtils.java diff --git a/ci/run_ci.sh b/ci/run_ci.sh index 45e096a45e..fafe3185de 100755 --- a/ci/run_ci.sh +++ b/ci/run_ci.sh @@ -98,6 +98,7 @@ integration_tests() { echo "Start JPMS tests" cd "$ROOT"/integration_tests/jpms_tests mvn -T10 -B --no-transfer-progress clean compile + ./run_jlink_smoke.sh echo "Start jdk compatibility tests" cd "$ROOT"/integration_tests/jdk_compatibility_tests mvn -T10 -B --no-transfer-progress clean test diff --git a/integration_tests/jpms_tests/run_jlink_smoke.sh b/integration_tests/jpms_tests/run_jlink_smoke.sh new file mode 100755 index 0000000000..a1c70edb39 --- /dev/null +++ b/integration_tests/jpms_tests/run_jlink_smoke.sh @@ -0,0 +1,201 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -euo pipefail + +ROOT="$(git rev-parse --show-toplevel)" +JAVA_ROOT="$ROOT/java" +VERSION="$(mvn -q -B -f "$JAVA_ROOT/pom.xml" help:evaluate -Dexpression=project.version -DforceStdout)" +JAVA_HOME="${JAVA_HOME:-$( + java -XshowSettings:properties -version 2>&1 \ + | awk -F= '/java.home =/ { gsub(/^ +| +$/, "", $2); print $2; exit }' +)}" +JAVA_MAJOR="$( + java -version 2>&1 \ + | awk -F '"' '/version/ {print $2; exit}' \ + | awk -F. '{ if ($1 == "1") print $2; else print $1 }' +)" + +if [[ "$JAVA_MAJOR" -lt 11 ]]; then + echo "Skipping jlink smoke test on JDK $JAVA_MAJOR; fory-format is Java 11+." + exit 0 +fi +if [[ ! -d "$JAVA_HOME/jmods" ]]; then + echo "Cannot find JDK modules under JAVA_HOME=$JAVA_HOME; jlink smoke requires a JDK." >&2 + exit 1 +fi + +artifact_jar() { + local artifact="$1" + local module="$2" + local target_jar="$JAVA_ROOT/$module/target/$artifact-$VERSION.jar" + local repo_jar="$HOME/.m2/repository/org/apache/fory/$artifact/$VERSION/$artifact-$VERSION.jar" + if [[ -f "$target_jar" ]]; then + echo "$target_jar" + elif [[ -f "$repo_jar" ]]; then + echo "$repo_jar" + else + echo "Cannot find $artifact jar; run java/fory-core and java/fory-format package/install first." >&2 + exit 1 + fi +} + +CORE_JAR="$(artifact_jar fory-core fory-core)" +FORMAT_JAR="$(artifact_jar fory-format fory-format)" +WORK_DIR="$(mktemp -d "${TMPDIR:-/tmp}/fory-jpms-jlink.XXXXXX")" +trap 'rm -rf "$WORK_DIR"' EXIT + +require_jar_entry() { + local jar_file="$1" + local entry="$2" + if ! jar tf "$jar_file" | grep -qx "$entry"; then + echo "Missing $entry in $jar_file" >&2 + exit 1 + fi +} + +reject_jar_entry() { + local jar_file="$1" + local entry="$2" + if jar tf "$jar_file" | grep -qx "$entry"; then + echo "Unexpected root $entry in $jar_file" >&2 + exit 1 + fi +} + +require_jar_entry "$CORE_JAR" "META-INF/versions/9/module-info.class" +reject_jar_entry "$CORE_JAR" "module-info.class" +require_jar_entry "$FORMAT_JAR" "META-INF/versions/11/module-info.class" +reject_jar_entry "$FORMAT_JAR" "module-info.class" + +if [[ "$JAVA_MAJOR" -ge 16 ]] \ + && jar tf "$CORE_JAR" | grep -qx "META-INF/versions/16/module-info.class"; then + require_jar_entry "$CORE_JAR" "META-INF/versions/16/org/apache/fory/util/VectorArrayCompression.class" + jar --file "$CORE_JAR" --describe-module --release 16 | grep -q "requires jdk.incubator.vector static" +fi + +jar --file "$CORE_JAR" --describe-module --release 9 | grep -q "requires java.sql static" +jar --file "$CORE_JAR" --describe-module --release 9 | grep -q "requires com.google.common static" +jar --file "$FORMAT_JAR" --describe-module --release 11 | grep -q "requires java.sql static" +jar --file "$FORMAT_JAR" --describe-module --release 11 | grep -q "requires org.apache.arrow.vector static transitive" +jar --file "$FORMAT_JAR" --describe-module --release 11 | grep -q "requires org.apache.arrow.memory.core static transitive" + +DEPS_FILE="$WORK_DIR/format-deps.txt" +mvn -q -B -f "$JAVA_ROOT/pom.xml" -pl fory-format dependency:list \ + -DincludeScope=compile \ + -DoutputAbsoluteArtifactFilename=true \ + -DexcludeTransitive=false \ + -DoutputFile="$DEPS_FILE" \ + -Dstyle.color=never + +COMPILE_MODULE_PATH="$WORK_DIR/compile-module-path.txt" +awk -F: '/:jar:/ { + path = $NF + sub(/ .*/, "", path) + if (path ~ /^\// && path ~ /\.jar$/ && path !~ /\/org\/apache\/fory\/fory-core\//) { + print path + } +}' "$DEPS_FILE" > "$COMPILE_MODULE_PATH" + +mkdir -p "$WORK_DIR/src/jpms.smoke/org/apache/fory/jpms" "$WORK_DIR/mods" +cat > "$WORK_DIR/src/jpms.smoke/module-info.java" <<'EOF' +module jpms.smoke { + requires org.apache.fory.core; + requires org.apache.fory.format; +} +EOF +cat > "$WORK_DIR/src/jpms.smoke/org/apache/fory/jpms/Smoke.java" <<'EOF' +package org.apache.fory.jpms; + +import org.apache.fory.Fory; + +public final class Smoke { + public static void main(String[] args) throws Exception { + Fory.builder().build(); + Class.forName("org.apache.fory.format.encoder.Encoders"); + System.out.println("ok"); + } +} +EOF + +JOINED_COMPILE_MODULE_PATH="$CORE_JAR:$FORMAT_JAR" +if [[ -s "$COMPILE_MODULE_PATH" ]]; then + JOINED_COMPILE_MODULE_PATH="$JOINED_COMPILE_MODULE_PATH:$(paste -sd: "$COMPILE_MODULE_PATH")" +fi + +javac \ + --module-path "$JOINED_COMPILE_MODULE_PATH" \ + -d "$WORK_DIR/mods" \ + --module-source-path "$WORK_DIR/src" \ + -m jpms.smoke + +jlink \ + --module-path "$JAVA_HOME/jmods:$CORE_JAR:$FORMAT_JAR:$WORK_DIR/mods" \ + --add-modules jpms.smoke \ + --output "$WORK_DIR/image" + +"$WORK_DIR/image/bin/java" -m jpms.smoke/org.apache.fory.jpms.Smoke | grep -qx "ok" + +IMAGE_MODULES="$("$WORK_DIR/image/bin/java" --list-modules)" +echo "$IMAGE_MODULES" | grep -q "^org.apache.fory.core" +echo "$IMAGE_MODULES" | grep -q "^org.apache.fory.format" +if echo "$IMAGE_MODULES" \ + | grep -Eq "^(java\.sql|com\.google|jsr305|org\.apache\.arrow|org\.slf4j|jdk\.incubator\.vector)"; then + echo "Optional static modules leaked into the minimal jlink image:" >&2 + echo "$IMAGE_MODULES" \ + | grep -E "^(java\.sql|com\.google|jsr305|org\.apache\.arrow|org\.slf4j|jdk\.incubator\.vector)" >&2 + exit 1 +fi + +if [[ "$JAVA_MAJOR" -ge 16 ]] \ + && jar tf "$CORE_JAR" | grep -qx "META-INF/versions/16/org/apache/fory/util/VectorArrayCompression.class"; then + mkdir -p "$WORK_DIR/vector-classes" + cat > "$WORK_DIR/VectorSmoke.java" <<'EOF' +import java.lang.reflect.Method; +import org.apache.fory.util.ArrayCompressionUtils; +import org.apache.fory.util.PrimitiveArrayCompressionType; + +public final class VectorSmoke { + public static void main(String[] args) throws Exception { + Method method = ArrayCompressionUtils.class.getDeclaredMethod("isVectorCompressionEnabled"); + method.setAccessible(true); + if (!Boolean.TRUE.equals(method.invoke(null))) { + throw new AssertionError("Vector compression support was not enabled"); + } + int[] values = new int[1024]; + for (int i = 0; i < values.length; i++) { + values[i] = (i & 0xff) - 128; + } + if (ArrayCompressionUtils.determineIntCompressionType(values) + != PrimitiveArrayCompressionType.INT_TO_BYTE) { + throw new AssertionError("Vector compression returned the wrong range"); + } + System.out.println("vector-ok"); + } +} +EOF + javac -cp "$CORE_JAR" -d "$WORK_DIR/vector-classes" "$WORK_DIR/VectorSmoke.java" + java \ + --add-modules jdk.incubator.vector \ + -cp "$CORE_JAR:$WORK_DIR/vector-classes" \ + VectorSmoke \ + | grep -qx "vector-ok" +fi + +echo "JPMS jlink smoke test passed with minimal optional dependencies." diff --git a/java/fory-core/pom.xml b/java/fory-core/pom.xml index 4176d74184..f6d6cd5d97 100644 --- a/java/fory-core/pom.xml +++ b/java/fory-core/pom.xml @@ -78,6 +78,7 @@ org.apache.fory.core + true @@ -119,6 +120,7 @@ org.apache.fory.core + true @@ -135,6 +137,180 @@ + + jpms-java9 + + [9,) + + + + + org.apache.maven.plugins + maven-antrun-plugin + 3.1.0 + + + compile-java9-module-info + process-classes + + run + + + + + + + + + + + + + + + + + + + inject-java9-module-info + package + + run + + + + + + + + + + + + + org.apache.maven.plugins + maven-resources-plugin + 3.3.1 + + + copy-java9-module-info + prepare-package + + copy-resources + + + ${project.build.outputDirectory}/META-INF/versions/9 + + + ${project.build.directory}/jpms-classes/java9 + + module-info.class + + + + + + + + + + + + jpms-java16 + + [16,) + + + + + org.apache.maven.plugins + maven-antrun-plugin + 3.1.0 + + + compile-java16-sources + process-classes + + run + + + + + + + + + + + + + + + + + + + + + inject-java16-classes + package + + run + + + + + + + + + + + + + org.apache.maven.plugins + maven-resources-plugin + 3.3.1 + + + copy-java16-classes + prepare-package + + copy-resources + + + ${project.build.outputDirectory}/META-INF/versions/16 + + + ${project.build.directory}/jpms-classes/java16 + + **/*.class + + + + + + + + + + xlang-parallel diff --git a/java/fory-simd/src/main/java/org/apache/fory/serializer/CompressedArraySerializers.java b/java/fory-core/src/main/java/org/apache/fory/serializer/CompressedArraySerializers.java similarity index 99% rename from java/fory-simd/src/main/java/org/apache/fory/serializer/CompressedArraySerializers.java rename to java/fory-core/src/main/java/org/apache/fory/serializer/CompressedArraySerializers.java index ba4f2e247a..9f4eee99cd 100644 --- a/java/fory-simd/src/main/java/org/apache/fory/serializer/CompressedArraySerializers.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/CompressedArraySerializers.java @@ -36,7 +36,7 @@ import org.apache.fory.util.PrimitiveArrayCompressionType; /** - * Compressed array serializers using Java 16+ Vector API for SIMD acceleration. + * Compressed array serializers with optional Java 16+ Vector API acceleration. * *

To use these serializers, simply call {@code CompressedArraySerializers.register(fory)} on * your Fory instance. These will override the default array serializers for {@code int[]} and diff --git a/java/fory-core/src/main/java/org/apache/fory/util/ArrayCompressionUtils.java b/java/fory-core/src/main/java/org/apache/fory/util/ArrayCompressionUtils.java new file mode 100644 index 0000000000..9a0187c85b --- /dev/null +++ b/java/fory-core/src/main/java/org/apache/fory/util/ArrayCompressionUtils.java @@ -0,0 +1,176 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fory.util; + +/** Utility methods for optional primitive array compression. */ +public final class ArrayCompressionUtils { + static final int MIN_COMPRESSION_SIZE = 1 << 9; + private static final CompressionSupport SCALAR_SUPPORT = new ScalarCompressionSupport(); + private static final CompressionSupport COMPRESSION_SUPPORT = loadCompressionSupport(); + + private ArrayCompressionUtils() {} + + public static PrimitiveArrayCompressionType determineIntCompressionType(int[] array) { + if (array == null) { + throw new NullPointerException("Input array cannot be null"); + } + if (array.length < MIN_COMPRESSION_SIZE) { + return PrimitiveArrayCompressionType.NONE; + } + return COMPRESSION_SUPPORT.determineIntCompressionType(array); + } + + public static PrimitiveArrayCompressionType determineLongCompressionType(long[] array) { + if (array == null) { + throw new NullPointerException("Input array cannot be null"); + } + if (array.length < MIN_COMPRESSION_SIZE) { + return PrimitiveArrayCompressionType.NONE; + } + return COMPRESSION_SUPPORT.determineLongCompressionType(array); + } + + public static byte[] compressToBytes(int[] array) { + if (array == null) { + throw new NullPointerException("Array cannot be null"); + } + byte[] compressed = new byte[array.length]; + for (int i = 0; i < array.length; i++) { + compressed[i] = (byte) array[i]; + } + return compressed; + } + + public static short[] compressToShorts(int[] array) { + if (array == null) { + throw new NullPointerException("Array cannot be null"); + } + short[] compressed = new short[array.length]; + for (int i = 0; i < array.length; i++) { + compressed[i] = (short) array[i]; + } + return compressed; + } + + public static int[] compressToInts(long[] array) { + if (array == null) { + throw new NullPointerException("Array cannot be null"); + } + int[] compressed = new int[array.length]; + for (int i = 0; i < array.length; i++) { + compressed[i] = (int) array[i]; + } + return compressed; + } + + public static int[] decompressFromBytes(byte[] array) { + if (array == null) { + throw new NullPointerException("Array cannot be null"); + } + int[] decompressed = new int[array.length]; + for (int i = 0; i < array.length; i++) { + decompressed[i] = array[i]; + } + return decompressed; + } + + public static int[] decompressFromShorts(short[] array) { + if (array == null) { + throw new NullPointerException("Array cannot be null"); + } + int[] decompressed = new int[array.length]; + for (int i = 0; i < array.length; i++) { + decompressed[i] = array[i]; + } + return decompressed; + } + + public static long[] decompressFromInts(int[] array) { + if (array == null) { + throw new NullPointerException("Array cannot be null"); + } + long[] decompressed = new long[array.length]; + for (int i = 0; i < array.length; i++) { + decompressed[i] = array[i]; + } + return decompressed; + } + + static boolean isVectorCompressionEnabled() { + return COMPRESSION_SUPPORT != SCALAR_SUPPORT; + } + + interface CompressionSupport { + PrimitiveArrayCompressionType determineIntCompressionType(int[] array); + + PrimitiveArrayCompressionType determineLongCompressionType(long[] array); + } + + private static CompressionSupport loadCompressionSupport() { + try { + // The Vector implementation lives in the Java 16 multi-release tree and links to the + // optional incubator module. Missing classes or read edges must fall back to scalar code so + // minimal jlink images can start without resolving jdk.incubator.vector. + Class cls = + Class.forName( + "org.apache.fory.util.VectorArrayCompression", + true, + ArrayCompressionUtils.class.getClassLoader()); + return (CompressionSupport) cls.getDeclaredConstructor().newInstance(); + } catch (ClassCastException | LinkageError | ReflectiveOperationException e) { + return SCALAR_SUPPORT; + } + } + + private static final class ScalarCompressionSupport implements CompressionSupport { + @Override + public PrimitiveArrayCompressionType determineIntCompressionType(int[] array) { + boolean canCompressToByte = true; + boolean canCompressToShort = true; + for (int value : array) { + if (canCompressToByte && (value < Byte.MIN_VALUE || value > Byte.MAX_VALUE)) { + canCompressToByte = false; + } + if (value < Short.MIN_VALUE || value > Short.MAX_VALUE) { + canCompressToShort = false; + } + if (!canCompressToByte && !canCompressToShort) { + return PrimitiveArrayCompressionType.NONE; + } + } + if (canCompressToByte) { + return PrimitiveArrayCompressionType.INT_TO_BYTE; + } + return canCompressToShort + ? PrimitiveArrayCompressionType.INT_TO_SHORT + : PrimitiveArrayCompressionType.NONE; + } + + @Override + public PrimitiveArrayCompressionType determineLongCompressionType(long[] array) { + for (long value : array) { + if (value > Integer.MAX_VALUE || value < Integer.MIN_VALUE) { + return PrimitiveArrayCompressionType.NONE; + } + } + return PrimitiveArrayCompressionType.LONG_TO_INT; + } + } +} diff --git a/java/fory-simd/src/main/java/org/apache/fory/util/PrimitiveArrayCompressionType.java b/java/fory-core/src/main/java/org/apache/fory/util/PrimitiveArrayCompressionType.java similarity index 59% rename from java/fory-simd/src/main/java/org/apache/fory/util/PrimitiveArrayCompressionType.java rename to java/fory-core/src/main/java/org/apache/fory/util/PrimitiveArrayCompressionType.java index 61e7c6fe62..3bca1b8a57 100644 --- a/java/fory-simd/src/main/java/org/apache/fory/util/PrimitiveArrayCompressionType.java +++ b/java/fory-core/src/main/java/org/apache/fory/util/PrimitiveArrayCompressionType.java @@ -22,19 +22,13 @@ /** * Compression types for primitive arrays. * - *

Defines the available compression strategies for reducing the size of primitive arrays by - * detecting when values can fit in smaller data types. + *

These types are used by the optional compressed array serializers when array values can be + * stored with a narrower primitive type. */ public enum PrimitiveArrayCompressionType { - // No compression applied NONE(0), - - // Compression for int arrays: - // int[] → byte[] compression (75% size reduction) INT_TO_BYTE(1), - // int[] → short[] compression (50% size reduction) INT_TO_SHORT(2), - // Compression for long arrays: long[] → int[] compression (50% size reduction) LONG_TO_INT(3); private final int value; @@ -43,22 +37,10 @@ public enum PrimitiveArrayCompressionType { this.value = value; } - /** - * Gets the numeric value for this compression type. - * - * @return the numeric value - */ public int getValue() { return value; } - /** - * Gets the compression type from its numeric value. - * - * @param value the numeric value - * @return the corresponding compression type - * @throws IllegalArgumentException if the value is not valid - */ public static PrimitiveArrayCompressionType fromValue(int value) { switch (value) { case 0: @@ -74,46 +56,25 @@ public static PrimitiveArrayCompressionType fromValue(int value) { } } - /** Compression utilities for int arrays. Supports compression to byte[] and short[] formats. */ public static final class IntArrayCompression { - /** - * Determines the best compression type for the given int array. - * - * @param array the array to analyze - * @return the optimal compression type - */ + private IntArrayCompression() {} + public static PrimitiveArrayCompressionType determine(int[] array) { return ArrayCompressionUtils.determineIntCompressionType(array); } - /** - * Checks if the compression type is supported for int arrays. - * - * @param type the compression type to check - * @return true if supported - */ public static boolean isSupported(PrimitiveArrayCompressionType type) { return type == NONE || type == INT_TO_BYTE || type == INT_TO_SHORT; } } public static final class LongArrayCompression { - /** - * Determines the best compression type for the given long array. - * - * @param array the array to analyze - * @return the optimal compression type - */ + private LongArrayCompression() {} + public static PrimitiveArrayCompressionType determine(long[] array) { return ArrayCompressionUtils.determineLongCompressionType(array); } - /** - * Checks if the compression type is supported for long arrays. - * - * @param type the compression type to check - * @return true if supported - */ public static boolean isSupported(PrimitiveArrayCompressionType type) { return type == NONE || type == LONG_TO_INT; } diff --git a/java/fory-core/src/main/java16/module-info.java b/java/fory-core/src/main/java16/module-info.java new file mode 100644 index 0000000000..40c773b6ac --- /dev/null +++ b/java/fory-core/src/main/java16/module-info.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +module org.apache.fory.core { + requires java.logging; + requires jdk.unsupported; + + requires static java.sql; + requires static com.google.common; + requires static org.slf4j; + requires static jsr305; + requires static jdk.incubator.vector; + + exports org.apache.fory; + exports org.apache.fory.annotation; + exports org.apache.fory.builder; + exports org.apache.fory.codegen; + exports org.apache.fory.collection; + exports org.apache.fory.config; + exports org.apache.fory.context; + exports org.apache.fory.exception; + exports org.apache.fory.io; + exports org.apache.fory.logging; + exports org.apache.fory.memory; + exports org.apache.fory.meta; + exports org.apache.fory.platform; + exports org.apache.fory.pool; + exports org.apache.fory.reflect; + exports org.apache.fory.resolver; + exports org.apache.fory.serializer; + exports org.apache.fory.serializer.collection; + exports org.apache.fory.serializer.converter; + exports org.apache.fory.serializer.scala; + exports org.apache.fory.serializer.struct; + exports org.apache.fory.type; + exports org.apache.fory.type.union; + exports org.apache.fory.type.unsigned; + exports org.apache.fory.util; + exports org.apache.fory.util.function; + exports org.apache.fory.util.record; + exports org.apache.fory.util.unsafe to + org.apache.fory.format; +} diff --git a/java/fory-core/src/main/java16/org/apache/fory/util/VectorArrayCompression.java b/java/fory-core/src/main/java16/org/apache/fory/util/VectorArrayCompression.java new file mode 100644 index 0000000000..5798ca2bfd --- /dev/null +++ b/java/fory-core/src/main/java16/org/apache/fory/util/VectorArrayCompression.java @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fory.util; + +import jdk.incubator.vector.IntVector; +import jdk.incubator.vector.LongVector; +import jdk.incubator.vector.VectorOperators; +import jdk.incubator.vector.VectorSpecies; + +final class VectorArrayCompression implements ArrayCompressionUtils.CompressionSupport { + private static final VectorSpecies INT_SPECIES = IntVector.SPECIES_PREFERRED; + private static final VectorSpecies LONG_SPECIES = LongVector.SPECIES_PREFERRED; + + @Override + public PrimitiveArrayCompressionType determineIntCompressionType(int[] array) { + boolean canCompressToByte = true; + boolean canCompressToShort = true; + int i = 0; + int upperBound = INT_SPECIES.loopBound(array.length); + for (; i < upperBound && (canCompressToByte || canCompressToShort); i += INT_SPECIES.length()) { + IntVector vector = IntVector.fromArray(INT_SPECIES, array, i); + if (canCompressToByte) { + if (vector.compare(VectorOperators.GT, Byte.MAX_VALUE).anyTrue() + || vector.compare(VectorOperators.LT, Byte.MIN_VALUE).anyTrue()) { + canCompressToByte = false; + } + } + if (canCompressToShort) { + if (vector.compare(VectorOperators.GT, Short.MAX_VALUE).anyTrue() + || vector.compare(VectorOperators.LT, Short.MIN_VALUE).anyTrue()) { + canCompressToShort = false; + } + } + } + for (; i < array.length && (canCompressToByte || canCompressToShort); i++) { + int value = array[i]; + if (canCompressToByte && (value < Byte.MIN_VALUE || value > Byte.MAX_VALUE)) { + canCompressToByte = false; + } + if (value < Short.MIN_VALUE || value > Short.MAX_VALUE) { + canCompressToShort = false; + } + } + if (canCompressToByte) { + return PrimitiveArrayCompressionType.INT_TO_BYTE; + } + return canCompressToShort + ? PrimitiveArrayCompressionType.INT_TO_SHORT + : PrimitiveArrayCompressionType.NONE; + } + + @Override + public PrimitiveArrayCompressionType determineLongCompressionType(long[] array) { + int i = 0; + int upperBound = LONG_SPECIES.loopBound(array.length); + for (; i < upperBound; i += LONG_SPECIES.length()) { + LongVector vector = LongVector.fromArray(LONG_SPECIES, array, i); + if (vector.compare(VectorOperators.GT, Integer.MAX_VALUE).anyTrue() + || vector.compare(VectorOperators.LT, Integer.MIN_VALUE).anyTrue()) { + return PrimitiveArrayCompressionType.NONE; + } + } + for (; i < array.length; i++) { + long value = array[i]; + if (value > Integer.MAX_VALUE || value < Integer.MIN_VALUE) { + return PrimitiveArrayCompressionType.NONE; + } + } + return PrimitiveArrayCompressionType.LONG_TO_INT; + } +} diff --git a/java/fory-core/src/main/java9/module-info.java b/java/fory-core/src/main/java9/module-info.java new file mode 100644 index 0000000000..e381802cae --- /dev/null +++ b/java/fory-core/src/main/java9/module-info.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +module org.apache.fory.core { + requires java.logging; + requires jdk.unsupported; + + requires static java.sql; + requires static com.google.common; + requires static org.slf4j; + requires static jsr305; + + exports org.apache.fory; + exports org.apache.fory.annotation; + exports org.apache.fory.builder; + exports org.apache.fory.codegen; + exports org.apache.fory.collection; + exports org.apache.fory.config; + exports org.apache.fory.context; + exports org.apache.fory.exception; + exports org.apache.fory.io; + exports org.apache.fory.logging; + exports org.apache.fory.memory; + exports org.apache.fory.meta; + exports org.apache.fory.platform; + exports org.apache.fory.pool; + exports org.apache.fory.reflect; + exports org.apache.fory.resolver; + exports org.apache.fory.serializer; + exports org.apache.fory.serializer.collection; + exports org.apache.fory.serializer.converter; + exports org.apache.fory.serializer.scala; + exports org.apache.fory.serializer.struct; + exports org.apache.fory.type; + exports org.apache.fory.type.union; + exports org.apache.fory.type.unsigned; + exports org.apache.fory.util; + exports org.apache.fory.util.function; + exports org.apache.fory.util.record; + exports org.apache.fory.util.unsafe to + org.apache.fory.format; +} diff --git a/java/fory-simd/src/test/java/org/apache/fory/serializer/ArrayCompressionTest.java b/java/fory-core/src/test/java/org/apache/fory/serializer/ArrayCompressionTest.java similarity index 100% rename from java/fory-simd/src/test/java/org/apache/fory/serializer/ArrayCompressionTest.java rename to java/fory-core/src/test/java/org/apache/fory/serializer/ArrayCompressionTest.java diff --git a/java/fory-simd/src/test/java/org/apache/fory/serializer/ArrayCompressionUtilsTest.java b/java/fory-core/src/test/java/org/apache/fory/serializer/ArrayCompressionUtilsTest.java similarity index 100% rename from java/fory-simd/src/test/java/org/apache/fory/serializer/ArrayCompressionUtilsTest.java rename to java/fory-core/src/test/java/org/apache/fory/serializer/ArrayCompressionUtilsTest.java diff --git a/java/fory-format/pom.xml b/java/fory-format/pom.xml index 01cdd773ec..4a0a6da6c5 100644 --- a/java/fory-format/pom.xml +++ b/java/fory-format/pom.xml @@ -106,6 +106,7 @@ org.apache.fory.format + true @@ -121,6 +122,67 @@ + + jpms-java11 + + [11,) + + + + + org.apache.maven.plugins + maven-antrun-plugin + 3.1.0 + + + compile-java11-module-info + process-classes + + run + + + + + + + + + + + + + + + + + + + inject-java11-module-info + package + + run + + + + + + + + + + + + + + releaseShade @@ -170,6 +232,8 @@ *:* + module-info.class + META-INF/versions/**/module-info.class META-INF/*.SF META-INF/*.DSA META-INF/*.RSA diff --git a/java/fory-format/src/main/java11/module-info.java b/java/fory-format/src/main/java11/module-info.java new file mode 100644 index 0000000000..0f6064b8e9 --- /dev/null +++ b/java/fory-format/src/main/java11/module-info.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +module org.apache.fory.format { + requires transitive org.apache.fory.core; + + requires static java.sql; + requires static transitive org.apache.arrow.memory.core; + requires static transitive org.apache.arrow.vector; + + exports org.apache.fory.format.encoder; + exports org.apache.fory.format.row; + exports org.apache.fory.format.row.binary; + exports org.apache.fory.format.row.binary.writer; + exports org.apache.fory.format.type; + exports org.apache.fory.format.vectorized; +} diff --git a/java/fory-simd/pom.xml b/java/fory-simd/pom.xml deleted file mode 100644 index 675e183748..0000000000 --- a/java/fory-simd/pom.xml +++ /dev/null @@ -1,94 +0,0 @@ - - - - - org.apache.fory - fory-parent - 1.1.0-SNAPSHOT - - 4.0.0 - - fory-simd - - - Apache Fory™ is a blazingly fast multi-language serialization framework powered by jit and zero-copy. - - - - 17 - 17 - ${basedir}/.. - - - - - org.apache.fory - fory-core - ${project.version} - - - - - org.testng - testng - test - - - - - - - org.apache.maven.plugins - maven-compiler-plugin - - 16 - 16 - - --add-modules=jdk.incubator.vector - - - - - - org.apache.maven.plugins - maven-surefire-plugin - - --add-modules=jdk.incubator.vector - - - - - org.apache.maven.plugins - maven-javadoc-plugin - - - false - - --add-modules=jdk.incubator.vector - - - - - - diff --git a/java/fory-simd/src/main/java/org/apache/fory/util/ArrayCompressionUtils.java b/java/fory-simd/src/main/java/org/apache/fory/util/ArrayCompressionUtils.java deleted file mode 100644 index a55a53da8b..0000000000 --- a/java/fory-simd/src/main/java/org/apache/fory/util/ArrayCompressionUtils.java +++ /dev/null @@ -1,268 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.fory.util; - -import jdk.incubator.vector.IntVector; -import jdk.incubator.vector.LongVector; -import jdk.incubator.vector.VectorOperators; -import jdk.incubator.vector.VectorSpecies; - -/** - * Utility class for primitive array compression operations. It uses SIMD-accelerated array - * compression using Java 16+ Vector API. - * - *

This utility provides compression for primitive arrays by detecting when values can fit in - * smaller data types: - * - *

    - *
  • int[] → byte[] when all values are in [-128, 127] range (75% size reduction) - *
  • int[] → short[] when all values are in [-32768, 32767] range (50% size reduction) - *
  • long[] → int[] when all values fit in integer range (50% size reduction) - *
- * - *

Uses the best available compression provider via ServiceLoader pattern. SIMD optimizations are - * used when fory-simd module is available. When no compression provider is available, compression - * is disabled entirely. - */ -public final class ArrayCompressionUtils { - private static final VectorSpecies INT_SPECIES = IntVector.SPECIES_PREFERRED; - private static final VectorSpecies LONG_SPECIES = LongVector.SPECIES_PREFERRED; - - // Minimum array size to justify compression overhead - private static final int MIN_COMPRESSION_SIZE = 1 << 9; // 512 elements - - /** - * Determine the best compression type for int array. - * - * @param array the int array to analyze - * @return compression type (NONE, INT_TO_BYTE, or INT_TO_SHORT) - * @throws NullPointerException if array is null - */ - public static PrimitiveArrayCompressionType determineIntCompressionType(int[] array) { - if (array == null) { - throw new NullPointerException("Input array cannot be null"); - } - if (array.length < MIN_COMPRESSION_SIZE) { - // No compression for empty or too small arrays - return PrimitiveArrayCompressionType.NONE; - } - - boolean canCompressToByte = true; - boolean canCompressToShort = true; - - int i = 0; - final int upperBound = INT_SPECIES.loopBound(array.length); - - // SIMD loop - for (; i < upperBound && (canCompressToByte || canCompressToShort); i += INT_SPECIES.length()) { - IntVector vector = IntVector.fromArray(INT_SPECIES, array, i); - - // Check byte compression using mask operations - if (canCompressToByte) { - var byteMaxMask = vector.compare(VectorOperators.GT, Byte.MAX_VALUE); - var byteMinMask = vector.compare(VectorOperators.LT, Byte.MIN_VALUE); - if (byteMaxMask.anyTrue() || byteMinMask.anyTrue()) { - canCompressToByte = false; - } - } - - // Check short compression using mask operations - if (canCompressToShort) { - var shortMaxMask = vector.compare(VectorOperators.GT, Short.MAX_VALUE); - var shortMinMask = vector.compare(VectorOperators.LT, Short.MIN_VALUE); - if (shortMaxMask.anyTrue() || shortMinMask.anyTrue()) { - canCompressToShort = false; - } - } - } - - // Handle remaining elements with scalar code - for (; i < array.length && (canCompressToByte || canCompressToShort); i++) { - int value = array[i]; - if (canCompressToByte && (value < Byte.MIN_VALUE || value > Byte.MAX_VALUE)) { - canCompressToByte = false; - } - if (value < Short.MIN_VALUE || value > Short.MAX_VALUE) { - canCompressToShort = false; - } - } - - if (canCompressToByte) { - return PrimitiveArrayCompressionType.INT_TO_BYTE; - } else if (canCompressToShort) { - return PrimitiveArrayCompressionType.INT_TO_SHORT; - } else { - return PrimitiveArrayCompressionType.NONE; - } - } - - /** - * Determine the best compression type for long array. - * - * @param array the long array to analyze - * @return compression type (NONE or LONG_TO_INT) - * @throws NullPointerException if array is null - */ - public static PrimitiveArrayCompressionType determineLongCompressionType(long[] array) { - if (array == null) { - throw new NullPointerException("Input array cannot be null"); - } - if (array.length < MIN_COMPRESSION_SIZE) { - return PrimitiveArrayCompressionType.NONE; - } - boolean canCompressToInt = true; - - int i = 0; - int upperBound = LONG_SPECIES.loopBound(array.length); - - // SIMD loop - for (; i < upperBound && canCompressToInt; i += LONG_SPECIES.length()) { - LongVector vector = LongVector.fromArray(LONG_SPECIES, array, i); - - // Check int compression using mask operations - var maxMask = vector.compare(VectorOperators.GT, Integer.MAX_VALUE); - var minMask = vector.compare(VectorOperators.LT, Integer.MIN_VALUE); - if (maxMask.anyTrue() || minMask.anyTrue()) { - canCompressToInt = false; - } - } - - // Handle remaining elements - for (; i < array.length && canCompressToInt; i++) { - long value = array[i]; - if (value > Integer.MAX_VALUE || value < Integer.MIN_VALUE) { - canCompressToInt = false; - } - } - - return canCompressToInt - ? PrimitiveArrayCompressionType.LONG_TO_INT - : PrimitiveArrayCompressionType.NONE; - } - - /** - * Compress int array to byte array. - * - * @param array the int array to compress - * @return compressed byte array - * @throws NullPointerException if array is null - */ - public static byte[] compressToBytes(int[] array) { - if (array == null) { - throw new NullPointerException("Array cannot be null"); - } - byte[] compressed = new byte[array.length]; - for (int i = 0; i < array.length; i++) { - compressed[i] = (byte) array[i]; - } - return compressed; - } - - /** - * Compress int array to short array. - * - * @param array the int array to compress (values must be in short range) - * @return compressed short array - * @throws NullPointerException if array is null - */ - public static short[] compressToShorts(int[] array) { - if (array == null) { - throw new NullPointerException("Array cannot be null"); - } - short[] compressed = new short[array.length]; - for (int i = 0; i < array.length; i++) { - compressed[i] = (short) array[i]; - } - return compressed; - } - - /** - * Compress long array to int array. - * - * @param array the long array to compress (values must be in int range) - * @return compressed int array - * @throws NullPointerException if array is null - */ - public static int[] compressToInts(long[] array) { - if (array == null) { - throw new NullPointerException("Array cannot be null"); - } - int[] compressed = new int[array.length]; - for (int i = 0; i < array.length; i++) { - compressed[i] = (int) array[i]; - } - return compressed; - } - - /** - * Decompress byte array to int array. - * - * @param array the byte array to decompress - * @return decompressed int array - * @throws NullPointerException if array is null - */ - public static int[] decompressFromBytes(byte[] array) { - if (array == null) { - throw new NullPointerException("Array cannot be null"); - } - int[] decompressed = new int[array.length]; - for (int i = 0; i < array.length; i++) { - decompressed[i] = array[i]; - } - return decompressed; - } - - /** - * Decompress short array to int array. - * - * @param array the short array to decompress - * @return decompressed int array - * @throws NullPointerException if array is null - */ - public static int[] decompressFromShorts(short[] array) { - if (array == null) { - throw new NullPointerException("Array cannot be null"); - } - int[] decompressed = new int[array.length]; - for (int i = 0; i < array.length; i++) { - decompressed[i] = array[i]; - } - return decompressed; - } - - /** - * Decompress int array to long array. - * - * @param array the int array to decompress - * @return decompressed long array - * @throws NullPointerException if array is null - */ - public static long[] decompressFromInts(int[] array) { - if (array == null) { - throw new NullPointerException("Array cannot be null"); - } - - long[] decompressed = new long[array.length]; - for (int i = 0; i < array.length; i++) { - decompressed[i] = array[i]; - } - return decompressed; - } -} diff --git a/java/pom.xml b/java/pom.xml index 4e1d14ae43..473b2c6a9e 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -105,7 +105,6 @@ [21,] - fory-simd fory-latest-jdk-tests From a0b5d4388e52ab909ad2275a50105fedae8260e4 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sun, 31 May 2026 00:57:44 +0800 Subject: [PATCH 3/6] fix(java): keep SIMD compression APIs Java 16-only --- .../jpms_tests/run_jlink_smoke.sh | 6 + .../CompressedArraySerializers.java | 0 .../fory/util/ArrayCompressionUtils.java | 76 ++++++++- .../util/PrimitiveArrayCompressionType.java | 49 +++++- .../fory/util/VectorArrayCompression.java | 10 ++ .../fory/serializer/ArrayCompressionTest.java | 34 +++- .../serializer/ArrayCompressionUtilsTest.java | 149 +++++++++--------- .../serializer/Java16CompressionSupport.java | 88 +++++++++++ 8 files changed, 330 insertions(+), 82 deletions(-) rename java/fory-core/src/main/{java => java16}/org/apache/fory/serializer/CompressedArraySerializers.java (100%) rename java/fory-core/src/main/{java => java16}/org/apache/fory/util/ArrayCompressionUtils.java (68%) rename java/fory-core/src/main/{java => java16}/org/apache/fory/util/PrimitiveArrayCompressionType.java (56%) rename java/{fory-core => fory-testsuite}/src/test/java/org/apache/fory/serializer/ArrayCompressionTest.java (86%) rename java/{fory-core => fory-testsuite}/src/test/java/org/apache/fory/serializer/ArrayCompressionUtilsTest.java (59%) create mode 100644 java/fory-testsuite/src/test/java/org/apache/fory/serializer/Java16CompressionSupport.java diff --git a/integration_tests/jpms_tests/run_jlink_smoke.sh b/integration_tests/jpms_tests/run_jlink_smoke.sh index a1c70edb39..e03f32f5ff 100755 --- a/integration_tests/jpms_tests/run_jlink_smoke.sh +++ b/integration_tests/jpms_tests/run_jlink_smoke.sh @@ -86,7 +86,13 @@ reject_jar_entry "$FORMAT_JAR" "module-info.class" if [[ "$JAVA_MAJOR" -ge 16 ]] \ && jar tf "$CORE_JAR" | grep -qx "META-INF/versions/16/module-info.class"; then + require_jar_entry "$CORE_JAR" "META-INF/versions/16/org/apache/fory/serializer/CompressedArraySerializers.class" + require_jar_entry "$CORE_JAR" "META-INF/versions/16/org/apache/fory/util/ArrayCompressionUtils.class" + require_jar_entry "$CORE_JAR" "META-INF/versions/16/org/apache/fory/util/PrimitiveArrayCompressionType.class" require_jar_entry "$CORE_JAR" "META-INF/versions/16/org/apache/fory/util/VectorArrayCompression.class" + reject_jar_entry "$CORE_JAR" "org/apache/fory/serializer/CompressedArraySerializers.class" + reject_jar_entry "$CORE_JAR" "org/apache/fory/util/ArrayCompressionUtils.class" + reject_jar_entry "$CORE_JAR" "org/apache/fory/util/PrimitiveArrayCompressionType.class" jar --file "$CORE_JAR" --describe-module --release 16 | grep -q "requires jdk.incubator.vector static" fi diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/CompressedArraySerializers.java b/java/fory-core/src/main/java16/org/apache/fory/serializer/CompressedArraySerializers.java similarity index 100% rename from java/fory-core/src/main/java/org/apache/fory/serializer/CompressedArraySerializers.java rename to java/fory-core/src/main/java16/org/apache/fory/serializer/CompressedArraySerializers.java diff --git a/java/fory-core/src/main/java/org/apache/fory/util/ArrayCompressionUtils.java b/java/fory-core/src/main/java16/org/apache/fory/util/ArrayCompressionUtils.java similarity index 68% rename from java/fory-core/src/main/java/org/apache/fory/util/ArrayCompressionUtils.java rename to java/fory-core/src/main/java16/org/apache/fory/util/ArrayCompressionUtils.java index 9a0187c85b..66745d7362 100644 --- a/java/fory-core/src/main/java/org/apache/fory/util/ArrayCompressionUtils.java +++ b/java/fory-core/src/main/java16/org/apache/fory/util/ArrayCompressionUtils.java @@ -19,14 +19,38 @@ package org.apache.fory.util; -/** Utility methods for optional primitive array compression. */ +/** + * Utility methods for optional primitive array compression. + * + *

The compressed array serializers use these helpers when every value in a primitive array fits + * in a narrower primitive type: + * + *

    + *
  • {@code int[]} to {@code byte[]} when all values are in byte range. + *
  • {@code int[]} to {@code short[]} when all values are in short range. + *
  • {@code long[]} to {@code int[]} when all values are in int range. + *
+ * + *

Java 16+ runtimes can load the Vector API implementation from the multi-release tree. Older + * runtimes, or minimal module images without {@code jdk.incubator.vector}, use the scalar + * implementation. + */ public final class ArrayCompressionUtils { + // Minimum array size to justify compression analysis and the compressed payload marker overhead. static final int MIN_COMPRESSION_SIZE = 1 << 9; private static final CompressionSupport SCALAR_SUPPORT = new ScalarCompressionSupport(); private static final CompressionSupport COMPRESSION_SUPPORT = loadCompressionSupport(); private ArrayCompressionUtils() {} + /** + * Determines the best compression type for an int array. + * + * @param array the array to analyze + * @return {@link PrimitiveArrayCompressionType#INT_TO_BYTE}, {@link + * PrimitiveArrayCompressionType#INT_TO_SHORT}, or {@link PrimitiveArrayCompressionType#NONE} + * @throws NullPointerException if {@code array} is null + */ public static PrimitiveArrayCompressionType determineIntCompressionType(int[] array) { if (array == null) { throw new NullPointerException("Input array cannot be null"); @@ -37,6 +61,14 @@ public static PrimitiveArrayCompressionType determineIntCompressionType(int[] ar return COMPRESSION_SUPPORT.determineIntCompressionType(array); } + /** + * Determines the best compression type for a long array. + * + * @param array the array to analyze + * @return {@link PrimitiveArrayCompressionType#LONG_TO_INT} or {@link + * PrimitiveArrayCompressionType#NONE} + * @throws NullPointerException if {@code array} is null + */ public static PrimitiveArrayCompressionType determineLongCompressionType(long[] array) { if (array == null) { throw new NullPointerException("Input array cannot be null"); @@ -47,6 +79,13 @@ public static PrimitiveArrayCompressionType determineLongCompressionType(long[] return COMPRESSION_SUPPORT.determineLongCompressionType(array); } + /** + * Compresses an int array to a byte array. + * + * @param array the int array to compress; values must be in byte range + * @return compressed byte array + * @throws NullPointerException if {@code array} is null + */ public static byte[] compressToBytes(int[] array) { if (array == null) { throw new NullPointerException("Array cannot be null"); @@ -58,6 +97,13 @@ public static byte[] compressToBytes(int[] array) { return compressed; } + /** + * Compresses an int array to a short array. + * + * @param array the int array to compress; values must be in short range + * @return compressed short array + * @throws NullPointerException if {@code array} is null + */ public static short[] compressToShorts(int[] array) { if (array == null) { throw new NullPointerException("Array cannot be null"); @@ -69,6 +115,13 @@ public static short[] compressToShorts(int[] array) { return compressed; } + /** + * Compresses a long array to an int array. + * + * @param array the long array to compress; values must be in int range + * @return compressed int array + * @throws NullPointerException if {@code array} is null + */ public static int[] compressToInts(long[] array) { if (array == null) { throw new NullPointerException("Array cannot be null"); @@ -80,6 +133,13 @@ public static int[] compressToInts(long[] array) { return compressed; } + /** + * Decompresses a byte array to an int array. + * + * @param array the byte array to decompress + * @return decompressed int array + * @throws NullPointerException if {@code array} is null + */ public static int[] decompressFromBytes(byte[] array) { if (array == null) { throw new NullPointerException("Array cannot be null"); @@ -91,6 +151,13 @@ public static int[] decompressFromBytes(byte[] array) { return decompressed; } + /** + * Decompresses a short array to an int array. + * + * @param array the short array to decompress + * @return decompressed int array + * @throws NullPointerException if {@code array} is null + */ public static int[] decompressFromShorts(short[] array) { if (array == null) { throw new NullPointerException("Array cannot be null"); @@ -102,6 +169,13 @@ public static int[] decompressFromShorts(short[] array) { return decompressed; } + /** + * Decompresses an int array to a long array. + * + * @param array the int array to decompress + * @return decompressed long array + * @throws NullPointerException if {@code array} is null + */ public static long[] decompressFromInts(int[] array) { if (array == null) { throw new NullPointerException("Array cannot be null"); diff --git a/java/fory-core/src/main/java/org/apache/fory/util/PrimitiveArrayCompressionType.java b/java/fory-core/src/main/java16/org/apache/fory/util/PrimitiveArrayCompressionType.java similarity index 56% rename from java/fory-core/src/main/java/org/apache/fory/util/PrimitiveArrayCompressionType.java rename to java/fory-core/src/main/java16/org/apache/fory/util/PrimitiveArrayCompressionType.java index 3bca1b8a57..477b2037ba 100644 --- a/java/fory-core/src/main/java/org/apache/fory/util/PrimitiveArrayCompressionType.java +++ b/java/fory-core/src/main/java16/org/apache/fory/util/PrimitiveArrayCompressionType.java @@ -22,13 +22,20 @@ /** * Compression types for primitive arrays. * - *

These types are used by the optional compressed array serializers when array values can be - * stored with a narrower primitive type. + *

Defines the available compression strategies for reducing the size of primitive arrays by + * detecting when values can be stored with a narrower primitive type. */ public enum PrimitiveArrayCompressionType { + /** No compression applied. */ NONE(0), + + /** Compresses {@code int[]} values to {@code byte[]} when every value fits in byte range. */ INT_TO_BYTE(1), + + /** Compresses {@code int[]} values to {@code short[]} when every value fits in short range. */ INT_TO_SHORT(2), + + /** Compresses {@code long[]} values to {@code int[]} when every value fits in int range. */ LONG_TO_INT(3); private final int value; @@ -37,10 +44,22 @@ public enum PrimitiveArrayCompressionType { this.value = value; } + /** + * Gets the numeric value for this compression type. + * + * @return the numeric value written to the serialized payload + */ public int getValue() { return value; } + /** + * Gets the compression type from its numeric value. + * + * @param value the numeric value read from the serialized payload + * @return the corresponding compression type + * @throws IllegalArgumentException if the value is not valid + */ public static PrimitiveArrayCompressionType fromValue(int value) { switch (value) { case 0: @@ -56,25 +75,51 @@ public static PrimitiveArrayCompressionType fromValue(int value) { } } + /** Compression utilities for int arrays. Supports compression to byte[] and short[] formats. */ public static final class IntArrayCompression { private IntArrayCompression() {} + /** + * Determines the best compression type for the given int array. + * + * @param array the array to analyze + * @return the optimal compression type + */ public static PrimitiveArrayCompressionType determine(int[] array) { return ArrayCompressionUtils.determineIntCompressionType(array); } + /** + * Checks if the compression type is supported for int arrays. + * + * @param type the compression type to check + * @return true if supported + */ public static boolean isSupported(PrimitiveArrayCompressionType type) { return type == NONE || type == INT_TO_BYTE || type == INT_TO_SHORT; } } + /** Compression utilities for long arrays. Supports compression to int[] format. */ public static final class LongArrayCompression { private LongArrayCompression() {} + /** + * Determines the best compression type for the given long array. + * + * @param array the array to analyze + * @return the optimal compression type + */ public static PrimitiveArrayCompressionType determine(long[] array) { return ArrayCompressionUtils.determineLongCompressionType(array); } + /** + * Checks if the compression type is supported for long arrays. + * + * @param type the compression type to check + * @return true if supported + */ public static boolean isSupported(PrimitiveArrayCompressionType type) { return type == NONE || type == LONG_TO_INT; } diff --git a/java/fory-core/src/main/java16/org/apache/fory/util/VectorArrayCompression.java b/java/fory-core/src/main/java16/org/apache/fory/util/VectorArrayCompression.java index 5798ca2bfd..d5c1c85a4e 100644 --- a/java/fory-core/src/main/java16/org/apache/fory/util/VectorArrayCompression.java +++ b/java/fory-core/src/main/java16/org/apache/fory/util/VectorArrayCompression.java @@ -24,6 +24,7 @@ import jdk.incubator.vector.VectorOperators; import jdk.incubator.vector.VectorSpecies; +/** Java 16+ Vector API implementation for primitive array compression analysis. */ final class VectorArrayCompression implements ArrayCompressionUtils.CompressionSupport { private static final VectorSpecies INT_SPECIES = IntVector.SPECIES_PREFERRED; private static final VectorSpecies LONG_SPECIES = LongVector.SPECIES_PREFERRED; @@ -34,6 +35,9 @@ public PrimitiveArrayCompressionType determineIntCompressionType(int[] array) { boolean canCompressToShort = true; int i = 0; int upperBound = INT_SPECIES.loopBound(array.length); + + // Vector loop: test each lane against the target primitive ranges and stop checking a narrower + // representation once any lane exceeds its range. for (; i < upperBound && (canCompressToByte || canCompressToShort); i += INT_SPECIES.length()) { IntVector vector = IntVector.fromArray(INT_SPECIES, array, i); if (canCompressToByte) { @@ -49,6 +53,8 @@ public PrimitiveArrayCompressionType determineIntCompressionType(int[] array) { } } } + + // Scalar tail for elements that do not fill a complete vector. for (; i < array.length && (canCompressToByte || canCompressToShort); i++) { int value = array[i]; if (canCompressToByte && (value < Byte.MIN_VALUE || value > Byte.MAX_VALUE)) { @@ -70,6 +76,8 @@ public PrimitiveArrayCompressionType determineIntCompressionType(int[] array) { public PrimitiveArrayCompressionType determineLongCompressionType(long[] array) { int i = 0; int upperBound = LONG_SPECIES.loopBound(array.length); + + // Vector loop: any lane outside int range means long-to-int compression is not safe. for (; i < upperBound; i += LONG_SPECIES.length()) { LongVector vector = LongVector.fromArray(LONG_SPECIES, array, i); if (vector.compare(VectorOperators.GT, Integer.MAX_VALUE).anyTrue() @@ -77,6 +85,8 @@ public PrimitiveArrayCompressionType determineLongCompressionType(long[] array) return PrimitiveArrayCompressionType.NONE; } } + + // Scalar tail for elements that do not fill a complete vector. for (; i < array.length; i++) { long value = array[i]; if (value > Integer.MAX_VALUE || value < Integer.MIN_VALUE) { diff --git a/java/fory-core/src/test/java/org/apache/fory/serializer/ArrayCompressionTest.java b/java/fory-testsuite/src/test/java/org/apache/fory/serializer/ArrayCompressionTest.java similarity index 86% rename from java/fory-core/src/test/java/org/apache/fory/serializer/ArrayCompressionTest.java rename to java/fory-testsuite/src/test/java/org/apache/fory/serializer/ArrayCompressionTest.java index 6650264107..e4e48ce915 100644 --- a/java/fory-core/src/test/java/org/apache/fory/serializer/ArrayCompressionTest.java +++ b/java/fory-testsuite/src/test/java/org/apache/fory/serializer/ArrayCompressionTest.java @@ -22,6 +22,8 @@ import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertTrue; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; import java.util.Random; import org.apache.fory.Fory; import org.apache.fory.config.ForyBuilder; @@ -29,7 +31,6 @@ import org.testng.annotations.Test; public class ArrayCompressionTest { - @DataProvider(name = "intArrayData") public Object[][] intArrayData() { return new Object[][] { @@ -58,7 +59,7 @@ public Object[][] longArrayData() { public void testIntArrayCompressionRoundTrip(String description, int[] originalArray) { Fory foryWithCompression = new ForyBuilder().withXlang(false).withIntArrayCompressed(true).build(); - CompressedArraySerializers.registerSerializers(foryWithCompression); + registerSerializers(foryWithCompression); byte[] serializedWithCompression = foryWithCompression.serialize(originalArray); int[] deserializedWithCompression = (int[]) foryWithCompression.deserialize(serializedWithCompression); @@ -72,7 +73,7 @@ public void testIntArrayCompressionRoundTrip(String description, int[] originalA public void testLongArrayCompressionRoundTrip(String description, long[] originalArray) { Fory foryWithCompression = new ForyBuilder().withXlang(false).withLongArrayCompressed(true).build(); - CompressedArraySerializers.registerSerializers(foryWithCompression); + registerSerializers(foryWithCompression); byte[] serializedWithCompression = foryWithCompression.serialize(originalArray); long[] deserializedWithCompression = (long[]) foryWithCompression.deserialize(serializedWithCompression); @@ -90,7 +91,7 @@ public void testCompressionRatios() { .withIntArrayCompressed(true) .withLongArrayCompressed(true) .build(); - CompressedArraySerializers.registerSerializers(foryWithCompression); + registerSerializers(foryWithCompression); Fory foryWithoutCompression = new ForyBuilder() @@ -138,7 +139,7 @@ public void testLargeArrays() { .withIntArrayCompressed(true) .withLongArrayCompressed(true) .build(); - CompressedArraySerializers.registerSerializers(fory); + registerSerializers(fory); // Test very large compressible arrays int[] largeIntArray = createByteRangeArray(100_000); @@ -210,4 +211,27 @@ private long[] createLargeLongValueArray(int size) { } return array; } + + private static void registerSerializers(Fory fory) { + try { + Method method = + Java16CompressionSupport.loadClass( + "org.apache.fory.serializer.CompressedArraySerializers") + .getMethod("registerSerializers", Fory.class); + method.invoke(null, fory); + } catch (IllegalAccessException e) { + throw new AssertionError("Cannot access CompressedArraySerializers.registerSerializers", e); + } catch (InvocationTargetException e) { + Throwable cause = e.getCause(); + if (cause instanceof RuntimeException) { + throw (RuntimeException) cause; + } + if (cause instanceof Error) { + throw (Error) cause; + } + throw new AssertionError("Unexpected checked exception from registerSerializers", cause); + } catch (NoSuchMethodException e) { + throw new AssertionError("Missing CompressedArraySerializers.registerSerializers", e); + } + } } diff --git a/java/fory-core/src/test/java/org/apache/fory/serializer/ArrayCompressionUtilsTest.java b/java/fory-testsuite/src/test/java/org/apache/fory/serializer/ArrayCompressionUtilsTest.java similarity index 59% rename from java/fory-core/src/test/java/org/apache/fory/serializer/ArrayCompressionUtilsTest.java rename to java/fory-testsuite/src/test/java/org/apache/fory/serializer/ArrayCompressionUtilsTest.java index 29145b7a7a..a7fbbb4f2b 100644 --- a/java/fory-core/src/test/java/org/apache/fory/serializer/ArrayCompressionUtilsTest.java +++ b/java/fory-testsuite/src/test/java/org/apache/fory/serializer/ArrayCompressionUtilsTest.java @@ -22,13 +22,12 @@ import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertTrue; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; import java.util.Random; -import org.apache.fory.util.ArrayCompressionUtils; -import org.apache.fory.util.PrimitiveArrayCompressionType; import org.testng.annotations.Test; public class ArrayCompressionUtilsTest { - @Test public void testIntArrayCompressionDetection() { // Test byte range compression - make array size >= 512 @@ -36,33 +35,25 @@ public void testIntArrayCompressionDetection() { for (int i = 0; i < 1024; i++) { byteRangeArray[i] = (i % 256) - 128; // byte range values } - assertEquals( - ArrayCompressionUtils.determineIntCompressionType(byteRangeArray), - PrimitiveArrayCompressionType.INT_TO_BYTE); + assertEquals(determineIntCompressionType(byteRangeArray), compressionType("INT_TO_BYTE")); // Test short range compression - make array size >= 512 int[] shortRangeArray = new int[1024]; for (int i = 0; i < 1024; i++) { shortRangeArray[i] = (i % 65536) - 32768; // short range values } - assertEquals( - ArrayCompressionUtils.determineIntCompressionType(shortRangeArray), - PrimitiveArrayCompressionType.INT_TO_SHORT); + assertEquals(determineIntCompressionType(shortRangeArray), compressionType("INT_TO_SHORT")); // Test no compression for large values - make array size >= 512 int[] largeArray = new int[1024]; for (int i = 0; i < 1024; i++) { largeArray[i] = i % 2 == 0 ? Integer.MIN_VALUE : Integer.MAX_VALUE; } - assertEquals( - ArrayCompressionUtils.determineIntCompressionType(largeArray), - PrimitiveArrayCompressionType.NONE); + assertEquals(determineIntCompressionType(largeArray), compressionType("NONE")); // Test small arrays don't compress int[] smallArray = {1, 2, 3}; - assertEquals( - ArrayCompressionUtils.determineIntCompressionType(smallArray), - PrimitiveArrayCompressionType.NONE); + assertEquals(determineIntCompressionType(smallArray), compressionType("NONE")); } @Test @@ -72,24 +63,18 @@ public void testLongArrayCompressionDetection() { for (int i = 0; i < 1024; i++) { intRangeArray[i] = i % 2 == 0 ? Integer.MIN_VALUE : Integer.MAX_VALUE; } - assertEquals( - ArrayCompressionUtils.determineLongCompressionType(intRangeArray), - PrimitiveArrayCompressionType.LONG_TO_INT); + assertEquals(determineLongCompressionType(intRangeArray), compressionType("LONG_TO_INT")); // Test no compression for large values - make array size >= 512 long[] largeArray = new long[1024]; for (int i = 0; i < 1024; i++) { largeArray[i] = i % 2 == 0 ? Long.MIN_VALUE : Long.MAX_VALUE; } - assertEquals( - ArrayCompressionUtils.determineLongCompressionType(largeArray), - PrimitiveArrayCompressionType.NONE); + assertEquals(determineLongCompressionType(largeArray), compressionType("NONE")); // Test small arrays don't compress long[] smallArray = {1L, 2L, 3L}; - assertEquals( - ArrayCompressionUtils.determineLongCompressionType(smallArray), - PrimitiveArrayCompressionType.NONE); + assertEquals(determineLongCompressionType(smallArray), compressionType("NONE")); } @Test @@ -99,8 +84,8 @@ public void testCompressionRoundTrip() { for (int i = 0; i < 1024; i++) { originalInts[i] = (i % 201) - 100; // byte range values } - byte[] compressed = ArrayCompressionUtils.compressToBytes(originalInts); - int[] decompressed = ArrayCompressionUtils.decompressFromBytes(compressed); + byte[] compressed = (byte[]) invokeUtils("compressToBytes", int[].class, originalInts); + int[] decompressed = (int[]) invokeUtils("decompressFromBytes", byte[].class, compressed); assertEquals(decompressed, originalInts); // Test int to short compression @@ -108,8 +93,10 @@ public void testCompressionRoundTrip() { for (int i = 0; i < 1024; i++) { originalShorts[i] = (i % 102401) - 30000; // short range values } - short[] compressedShorts = ArrayCompressionUtils.compressToShorts(originalShorts); - int[] decompressedShorts = ArrayCompressionUtils.decompressFromShorts(compressedShorts); + short[] compressedShorts = + (short[]) invokeUtils("compressToShorts", int[].class, originalShorts); + int[] decompressedShorts = + (int[]) invokeUtils("decompressFromShorts", short[].class, compressedShorts); assertEquals(decompressedShorts, originalShorts); // Test long to int compression @@ -117,8 +104,9 @@ public void testCompressionRoundTrip() { for (int i = 0; i < 1024; i++) { originalLongs[i] = i % 2 == 0 ? -2000000000L : 2000000000L; // int range values } - int[] compressedInts = ArrayCompressionUtils.compressToInts(originalLongs); - long[] decompressedLongs = ArrayCompressionUtils.decompressFromInts(compressedInts); + int[] compressedInts = (int[]) invokeUtils("compressToInts", long[].class, originalLongs); + long[] decompressedLongs = + (long[]) invokeUtils("decompressFromInts", int[].class, compressedInts); assertEquals(decompressedLongs, originalLongs); } @@ -131,27 +119,21 @@ public void testLargeArraysWithSIMD() { for (int i = 0; i < largeByteArray.length; i++) { largeByteArray[i] = random.nextInt(256) - 128; // byte range } - assertEquals( - ArrayCompressionUtils.determineIntCompressionType(largeByteArray), - PrimitiveArrayCompressionType.INT_TO_BYTE); + assertEquals(determineIntCompressionType(largeByteArray), compressionType("INT_TO_BYTE")); // Test large array that compresses to shorts int[] largeShortArray = new int[10000]; for (int i = 0; i < largeShortArray.length; i++) { largeShortArray[i] = random.nextInt(65536) - 32768; // short range } - assertEquals( - ArrayCompressionUtils.determineIntCompressionType(largeShortArray), - PrimitiveArrayCompressionType.INT_TO_SHORT); + assertEquals(determineIntCompressionType(largeShortArray), compressionType("INT_TO_SHORT")); // Test large array that doesn't compress int[] largeArray = new int[10000]; for (int i = 0; i < largeArray.length; i++) { largeArray[i] = random.nextInt(); } - assertEquals( - ArrayCompressionUtils.determineIntCompressionType(largeArray), - PrimitiveArrayCompressionType.NONE); + assertEquals(determineIntCompressionType(largeArray), compressionType("NONE")); } @Test @@ -163,82 +145,62 @@ public void testLargeLongArraysWithSIMD() { for (int i = 0; i < largeIntArray.length; i++) { largeIntArray[i] = random.nextInt(); // int range } - assertEquals( - ArrayCompressionUtils.determineLongCompressionType(largeIntArray), - PrimitiveArrayCompressionType.LONG_TO_INT); + assertEquals(determineLongCompressionType(largeIntArray), compressionType("LONG_TO_INT")); // Test large array that doesn't compress long[] largeArray = new long[10000]; for (int i = 0; i < largeArray.length; i++) { largeArray[i] = random.nextLong(); } - assertEquals( - ArrayCompressionUtils.determineLongCompressionType(largeArray), - PrimitiveArrayCompressionType.NONE); + assertEquals(determineLongCompressionType(largeArray), compressionType("NONE")); } @Test public void testEdgeCases() { // Test empty arrays int[] emptyIntArray = {}; - assertEquals( - ArrayCompressionUtils.determineIntCompressionType(emptyIntArray), - PrimitiveArrayCompressionType.NONE); + assertEquals(determineIntCompressionType(emptyIntArray), compressionType("NONE")); long[] emptyLongArray = {}; - assertEquals( - ArrayCompressionUtils.determineLongCompressionType(emptyLongArray), - PrimitiveArrayCompressionType.NONE); + assertEquals(determineLongCompressionType(emptyLongArray), compressionType("NONE")); // Test boundary values - use arrays >= 512 int[] byteBoundary = new int[1024]; for (int i = 0; i < 1024; i++) { byteBoundary[i] = i % 2 == 0 ? Byte.MIN_VALUE : Byte.MAX_VALUE; } - assertEquals( - ArrayCompressionUtils.determineIntCompressionType(byteBoundary), - PrimitiveArrayCompressionType.INT_TO_BYTE); + assertEquals(determineIntCompressionType(byteBoundary), compressionType("INT_TO_BYTE")); int[] shortBoundary = new int[1024]; for (int i = 0; i < 1024; i++) { shortBoundary[i] = i % 2 == 0 ? Short.MIN_VALUE : Short.MAX_VALUE; } - assertEquals( - ArrayCompressionUtils.determineIntCompressionType(shortBoundary), - PrimitiveArrayCompressionType.INT_TO_SHORT); + assertEquals(determineIntCompressionType(shortBoundary), compressionType("INT_TO_SHORT")); long[] intBoundary = new long[1024]; for (int i = 0; i < 1024; i++) { intBoundary[i] = i % 2 == 0 ? Integer.MIN_VALUE : Integer.MAX_VALUE; } - assertEquals( - ArrayCompressionUtils.determineLongCompressionType(intBoundary), - PrimitiveArrayCompressionType.LONG_TO_INT); + assertEquals(determineLongCompressionType(intBoundary), compressionType("LONG_TO_INT")); // Test just outside boundaries int[] outsideByte = new int[1024]; for (int i = 0; i < 1024; i++) { outsideByte[i] = i % 2 == 0 ? Byte.MIN_VALUE - 1 : Byte.MAX_VALUE + 1; } - assertEquals( - ArrayCompressionUtils.determineIntCompressionType(outsideByte), - PrimitiveArrayCompressionType.INT_TO_SHORT); + assertEquals(determineIntCompressionType(outsideByte), compressionType("INT_TO_SHORT")); int[] outsideShort = new int[1024]; for (int i = 0; i < 1024; i++) { outsideShort[i] = i % 2 == 0 ? Short.MIN_VALUE - 1 : Short.MAX_VALUE + 1; } - assertEquals( - ArrayCompressionUtils.determineIntCompressionType(outsideShort), - PrimitiveArrayCompressionType.NONE); + assertEquals(determineIntCompressionType(outsideShort), compressionType("NONE")); long[] outsideInt = new long[1024]; for (int i = 0; i < 1024; i++) { outsideInt[i] = i % 2 == 0 ? Integer.MIN_VALUE - 1L : Integer.MAX_VALUE + 1L; } - assertEquals( - ArrayCompressionUtils.determineLongCompressionType(outsideInt), - PrimitiveArrayCompressionType.NONE); + assertEquals(determineLongCompressionType(outsideInt), compressionType("NONE")); } @Test @@ -249,12 +211,12 @@ public void testCompressionEffectiveness() { byteRangeArray[i] = i % 200 - 100; // byte range } - byte[] compressed = ArrayCompressionUtils.compressToBytes(byteRangeArray); + byte[] compressed = (byte[]) invokeUtils("compressToBytes", int[].class, byteRangeArray); assertEquals(compressed.length, byteRangeArray.length); // 4x smaller assertTrue(compressed.length < byteRangeArray.length * 4); // Verify decompression - int[] decompressed = ArrayCompressionUtils.decompressFromBytes(compressed); + int[] decompressed = (int[]) invokeUtils("decompressFromBytes", byte[].class, compressed); assertEquals(decompressed, byteRangeArray); } @@ -271,8 +233,47 @@ public void testMixedValues() { mixedArray[i] = Integer.MAX_VALUE - i; } - assertEquals( - ArrayCompressionUtils.determineIntCompressionType(mixedArray), - PrimitiveArrayCompressionType.NONE); + assertEquals(determineIntCompressionType(mixedArray), compressionType("NONE")); + } + + private static Object determineIntCompressionType(int[] array) { + return invokeUtils("determineIntCompressionType", int[].class, array); + } + + private static Object determineLongCompressionType(long[] array) { + return invokeUtils("determineLongCompressionType", long[].class, array); + } + + @SuppressWarnings({"unchecked", "rawtypes"}) + private static Object compressionType(String name) { + return Enum.valueOf((Class) primitiveArrayCompressionTypeClass(), name); + } + + private static Object invokeUtils(String name, Class paramType, Object arg) { + try { + Method method = arrayCompressionUtilsClass().getMethod(name, paramType); + return method.invoke(null, arg); + } catch (IllegalAccessException e) { + throw new AssertionError("Cannot access " + name, e); + } catch (InvocationTargetException e) { + Throwable cause = e.getCause(); + if (cause instanceof RuntimeException) { + throw (RuntimeException) cause; + } + if (cause instanceof Error) { + throw (Error) cause; + } + throw new AssertionError("Unexpected checked exception from " + name, cause); + } catch (NoSuchMethodException e) { + throw new AssertionError("Missing method " + name, e); + } + } + + private static Class arrayCompressionUtilsClass() { + return Java16CompressionSupport.loadClass("org.apache.fory.util.ArrayCompressionUtils"); + } + + private static Class primitiveArrayCompressionTypeClass() { + return Java16CompressionSupport.loadClass("org.apache.fory.util.PrimitiveArrayCompressionType"); } } diff --git a/java/fory-testsuite/src/test/java/org/apache/fory/serializer/Java16CompressionSupport.java b/java/fory-testsuite/src/test/java/org/apache/fory/serializer/Java16CompressionSupport.java new file mode 100644 index 0000000000..7f406df646 --- /dev/null +++ b/java/fory-testsuite/src/test/java/org/apache/fory/serializer/Java16CompressionSupport.java @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fory.serializer; + +import java.net.MalformedURLException; +import java.net.URL; +import java.net.URLClassLoader; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import org.testng.SkipException; + +final class Java16CompressionSupport { + private static volatile ClassLoader loader; + + private Java16CompressionSupport() {} + + static Class loadClass(String name) { + if (javaMajorVersion() < 16) { + throw new SkipException("Compressed array serializers are Java 16+ multi-release classes."); + } + try { + return Class.forName(name); + } catch (ClassNotFoundException ignored) { + // Reactor test classpaths use fory-core/target/classes, which does not expose versioned + // entries as MR-JAR classes. Load the compiled Java 16 output directly instead. + } + try { + return Class.forName(name, true, getLoader()); + } catch (ClassNotFoundException e) { + throw new AssertionError("Missing Java 16 compression class: " + name, e); + } + } + + private static ClassLoader getLoader() { + ClassLoader current = loader; + if (current != null) { + return current; + } + Path java16Classes = + Paths.get(System.getProperty("user.dir")) + .getParent() + .resolve("fory-core") + .resolve("target") + .resolve("jpms-classes") + .resolve("java16"); + if (!Files.isDirectory(java16Classes)) { + throw new AssertionError( + "Missing " + java16Classes + "; run fory-testsuite with -am or build fory-core first."); + } + try { + current = + new URLClassLoader( + new URL[] {java16Classes.toUri().toURL()}, + Thread.currentThread().getContextClassLoader()); + } catch (MalformedURLException e) { + throw new AssertionError("Invalid Java 16 class output path: " + java16Classes, e); + } + loader = current; + return current; + } + + private static int javaMajorVersion() { + String version = System.getProperty("java.specification.version"); + if (version.startsWith("1.")) { + return Integer.parseInt(version.substring(2)); + } + int dot = version.indexOf('.'); + return Integer.parseInt(dot < 0 ? version : version.substring(0, dot)); + } +} From 9037ec745ef3bcbd3ea40bc29b73e6e27d495a06 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sun, 31 May 2026 01:12:20 +0800 Subject: [PATCH 4/6] test(java): run Java16 compression tests directly --- java/fory-testsuite/pom.xml | 57 +++++++ .../serializer/Java16CompressionSupport.java | 88 ----------- .../fory/serializer/ArrayCompressionTest.java | 33 +--- .../serializer/ArrayCompressionUtilsTest.java | 148 +++++++++--------- 4 files changed, 134 insertions(+), 192 deletions(-) delete mode 100644 java/fory-testsuite/src/test/java/org/apache/fory/serializer/Java16CompressionSupport.java rename java/fory-testsuite/src/test/{java => java16}/org/apache/fory/serializer/ArrayCompressionTest.java (86%) rename java/fory-testsuite/src/test/{java => java16}/org/apache/fory/serializer/ArrayCompressionUtilsTest.java (59%) diff --git a/java/fory-testsuite/pom.xml b/java/fory-testsuite/pom.xml index 7831d56dc9..c08a9b8023 100644 --- a/java/fory-testsuite/pom.xml +++ b/java/fory-testsuite/pom.xml @@ -142,4 +142,61 @@ + + + java16-compression-tests + + [16,) + + + + + org.apache.maven.plugins + maven-antrun-plugin + 3.1.0 + + + compile-java16-compression-tests + test-compile + + run + + + + + + + + + + + + + + + + + + + org.apache.maven.plugins + maven-surefire-plugin + + --add-modules=jdk.incubator.vector + + ${project.basedir}/../fory-core/target/jpms-classes/java16 + + + + + + + + diff --git a/java/fory-testsuite/src/test/java/org/apache/fory/serializer/Java16CompressionSupport.java b/java/fory-testsuite/src/test/java/org/apache/fory/serializer/Java16CompressionSupport.java deleted file mode 100644 index 7f406df646..0000000000 --- a/java/fory-testsuite/src/test/java/org/apache/fory/serializer/Java16CompressionSupport.java +++ /dev/null @@ -1,88 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.fory.serializer; - -import java.net.MalformedURLException; -import java.net.URL; -import java.net.URLClassLoader; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import org.testng.SkipException; - -final class Java16CompressionSupport { - private static volatile ClassLoader loader; - - private Java16CompressionSupport() {} - - static Class loadClass(String name) { - if (javaMajorVersion() < 16) { - throw new SkipException("Compressed array serializers are Java 16+ multi-release classes."); - } - try { - return Class.forName(name); - } catch (ClassNotFoundException ignored) { - // Reactor test classpaths use fory-core/target/classes, which does not expose versioned - // entries as MR-JAR classes. Load the compiled Java 16 output directly instead. - } - try { - return Class.forName(name, true, getLoader()); - } catch (ClassNotFoundException e) { - throw new AssertionError("Missing Java 16 compression class: " + name, e); - } - } - - private static ClassLoader getLoader() { - ClassLoader current = loader; - if (current != null) { - return current; - } - Path java16Classes = - Paths.get(System.getProperty("user.dir")) - .getParent() - .resolve("fory-core") - .resolve("target") - .resolve("jpms-classes") - .resolve("java16"); - if (!Files.isDirectory(java16Classes)) { - throw new AssertionError( - "Missing " + java16Classes + "; run fory-testsuite with -am or build fory-core first."); - } - try { - current = - new URLClassLoader( - new URL[] {java16Classes.toUri().toURL()}, - Thread.currentThread().getContextClassLoader()); - } catch (MalformedURLException e) { - throw new AssertionError("Invalid Java 16 class output path: " + java16Classes, e); - } - loader = current; - return current; - } - - private static int javaMajorVersion() { - String version = System.getProperty("java.specification.version"); - if (version.startsWith("1.")) { - return Integer.parseInt(version.substring(2)); - } - int dot = version.indexOf('.'); - return Integer.parseInt(dot < 0 ? version : version.substring(0, dot)); - } -} diff --git a/java/fory-testsuite/src/test/java/org/apache/fory/serializer/ArrayCompressionTest.java b/java/fory-testsuite/src/test/java16/org/apache/fory/serializer/ArrayCompressionTest.java similarity index 86% rename from java/fory-testsuite/src/test/java/org/apache/fory/serializer/ArrayCompressionTest.java rename to java/fory-testsuite/src/test/java16/org/apache/fory/serializer/ArrayCompressionTest.java index e4e48ce915..a248b84c34 100644 --- a/java/fory-testsuite/src/test/java/org/apache/fory/serializer/ArrayCompressionTest.java +++ b/java/fory-testsuite/src/test/java16/org/apache/fory/serializer/ArrayCompressionTest.java @@ -22,8 +22,6 @@ import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertTrue; -import java.lang.reflect.InvocationTargetException; -import java.lang.reflect.Method; import java.util.Random; import org.apache.fory.Fory; import org.apache.fory.config.ForyBuilder; @@ -59,7 +57,7 @@ public Object[][] longArrayData() { public void testIntArrayCompressionRoundTrip(String description, int[] originalArray) { Fory foryWithCompression = new ForyBuilder().withXlang(false).withIntArrayCompressed(true).build(); - registerSerializers(foryWithCompression); + CompressedArraySerializers.registerSerializers(foryWithCompression); byte[] serializedWithCompression = foryWithCompression.serialize(originalArray); int[] deserializedWithCompression = (int[]) foryWithCompression.deserialize(serializedWithCompression); @@ -73,7 +71,7 @@ public void testIntArrayCompressionRoundTrip(String description, int[] originalA public void testLongArrayCompressionRoundTrip(String description, long[] originalArray) { Fory foryWithCompression = new ForyBuilder().withXlang(false).withLongArrayCompressed(true).build(); - registerSerializers(foryWithCompression); + CompressedArraySerializers.registerSerializers(foryWithCompression); byte[] serializedWithCompression = foryWithCompression.serialize(originalArray); long[] deserializedWithCompression = (long[]) foryWithCompression.deserialize(serializedWithCompression); @@ -91,7 +89,7 @@ public void testCompressionRatios() { .withIntArrayCompressed(true) .withLongArrayCompressed(true) .build(); - registerSerializers(foryWithCompression); + CompressedArraySerializers.registerSerializers(foryWithCompression); Fory foryWithoutCompression = new ForyBuilder() @@ -139,7 +137,7 @@ public void testLargeArrays() { .withIntArrayCompressed(true) .withLongArrayCompressed(true) .build(); - registerSerializers(fory); + CompressedArraySerializers.registerSerializers(fory); // Test very large compressible arrays int[] largeIntArray = createByteRangeArray(100_000); @@ -211,27 +209,4 @@ private long[] createLargeLongValueArray(int size) { } return array; } - - private static void registerSerializers(Fory fory) { - try { - Method method = - Java16CompressionSupport.loadClass( - "org.apache.fory.serializer.CompressedArraySerializers") - .getMethod("registerSerializers", Fory.class); - method.invoke(null, fory); - } catch (IllegalAccessException e) { - throw new AssertionError("Cannot access CompressedArraySerializers.registerSerializers", e); - } catch (InvocationTargetException e) { - Throwable cause = e.getCause(); - if (cause instanceof RuntimeException) { - throw (RuntimeException) cause; - } - if (cause instanceof Error) { - throw (Error) cause; - } - throw new AssertionError("Unexpected checked exception from registerSerializers", cause); - } catch (NoSuchMethodException e) { - throw new AssertionError("Missing CompressedArraySerializers.registerSerializers", e); - } - } } diff --git a/java/fory-testsuite/src/test/java/org/apache/fory/serializer/ArrayCompressionUtilsTest.java b/java/fory-testsuite/src/test/java16/org/apache/fory/serializer/ArrayCompressionUtilsTest.java similarity index 59% rename from java/fory-testsuite/src/test/java/org/apache/fory/serializer/ArrayCompressionUtilsTest.java rename to java/fory-testsuite/src/test/java16/org/apache/fory/serializer/ArrayCompressionUtilsTest.java index a7fbbb4f2b..6c3b64a0ad 100644 --- a/java/fory-testsuite/src/test/java/org/apache/fory/serializer/ArrayCompressionUtilsTest.java +++ b/java/fory-testsuite/src/test/java16/org/apache/fory/serializer/ArrayCompressionUtilsTest.java @@ -22,9 +22,9 @@ import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertTrue; -import java.lang.reflect.InvocationTargetException; -import java.lang.reflect.Method; import java.util.Random; +import org.apache.fory.util.ArrayCompressionUtils; +import org.apache.fory.util.PrimitiveArrayCompressionType; import org.testng.annotations.Test; public class ArrayCompressionUtilsTest { @@ -35,25 +35,33 @@ public void testIntArrayCompressionDetection() { for (int i = 0; i < 1024; i++) { byteRangeArray[i] = (i % 256) - 128; // byte range values } - assertEquals(determineIntCompressionType(byteRangeArray), compressionType("INT_TO_BYTE")); + assertEquals( + ArrayCompressionUtils.determineIntCompressionType(byteRangeArray), + PrimitiveArrayCompressionType.INT_TO_BYTE); // Test short range compression - make array size >= 512 int[] shortRangeArray = new int[1024]; for (int i = 0; i < 1024; i++) { shortRangeArray[i] = (i % 65536) - 32768; // short range values } - assertEquals(determineIntCompressionType(shortRangeArray), compressionType("INT_TO_SHORT")); + assertEquals( + ArrayCompressionUtils.determineIntCompressionType(shortRangeArray), + PrimitiveArrayCompressionType.INT_TO_SHORT); // Test no compression for large values - make array size >= 512 int[] largeArray = new int[1024]; for (int i = 0; i < 1024; i++) { largeArray[i] = i % 2 == 0 ? Integer.MIN_VALUE : Integer.MAX_VALUE; } - assertEquals(determineIntCompressionType(largeArray), compressionType("NONE")); + assertEquals( + ArrayCompressionUtils.determineIntCompressionType(largeArray), + PrimitiveArrayCompressionType.NONE); // Test small arrays don't compress int[] smallArray = {1, 2, 3}; - assertEquals(determineIntCompressionType(smallArray), compressionType("NONE")); + assertEquals( + ArrayCompressionUtils.determineIntCompressionType(smallArray), + PrimitiveArrayCompressionType.NONE); } @Test @@ -63,18 +71,24 @@ public void testLongArrayCompressionDetection() { for (int i = 0; i < 1024; i++) { intRangeArray[i] = i % 2 == 0 ? Integer.MIN_VALUE : Integer.MAX_VALUE; } - assertEquals(determineLongCompressionType(intRangeArray), compressionType("LONG_TO_INT")); + assertEquals( + ArrayCompressionUtils.determineLongCompressionType(intRangeArray), + PrimitiveArrayCompressionType.LONG_TO_INT); // Test no compression for large values - make array size >= 512 long[] largeArray = new long[1024]; for (int i = 0; i < 1024; i++) { largeArray[i] = i % 2 == 0 ? Long.MIN_VALUE : Long.MAX_VALUE; } - assertEquals(determineLongCompressionType(largeArray), compressionType("NONE")); + assertEquals( + ArrayCompressionUtils.determineLongCompressionType(largeArray), + PrimitiveArrayCompressionType.NONE); // Test small arrays don't compress long[] smallArray = {1L, 2L, 3L}; - assertEquals(determineLongCompressionType(smallArray), compressionType("NONE")); + assertEquals( + ArrayCompressionUtils.determineLongCompressionType(smallArray), + PrimitiveArrayCompressionType.NONE); } @Test @@ -84,8 +98,8 @@ public void testCompressionRoundTrip() { for (int i = 0; i < 1024; i++) { originalInts[i] = (i % 201) - 100; // byte range values } - byte[] compressed = (byte[]) invokeUtils("compressToBytes", int[].class, originalInts); - int[] decompressed = (int[]) invokeUtils("decompressFromBytes", byte[].class, compressed); + byte[] compressed = ArrayCompressionUtils.compressToBytes(originalInts); + int[] decompressed = ArrayCompressionUtils.decompressFromBytes(compressed); assertEquals(decompressed, originalInts); // Test int to short compression @@ -93,10 +107,8 @@ public void testCompressionRoundTrip() { for (int i = 0; i < 1024; i++) { originalShorts[i] = (i % 102401) - 30000; // short range values } - short[] compressedShorts = - (short[]) invokeUtils("compressToShorts", int[].class, originalShorts); - int[] decompressedShorts = - (int[]) invokeUtils("decompressFromShorts", short[].class, compressedShorts); + short[] compressedShorts = ArrayCompressionUtils.compressToShorts(originalShorts); + int[] decompressedShorts = ArrayCompressionUtils.decompressFromShorts(compressedShorts); assertEquals(decompressedShorts, originalShorts); // Test long to int compression @@ -104,9 +116,8 @@ public void testCompressionRoundTrip() { for (int i = 0; i < 1024; i++) { originalLongs[i] = i % 2 == 0 ? -2000000000L : 2000000000L; // int range values } - int[] compressedInts = (int[]) invokeUtils("compressToInts", long[].class, originalLongs); - long[] decompressedLongs = - (long[]) invokeUtils("decompressFromInts", int[].class, compressedInts); + int[] compressedInts = ArrayCompressionUtils.compressToInts(originalLongs); + long[] decompressedLongs = ArrayCompressionUtils.decompressFromInts(compressedInts); assertEquals(decompressedLongs, originalLongs); } @@ -119,21 +130,27 @@ public void testLargeArraysWithSIMD() { for (int i = 0; i < largeByteArray.length; i++) { largeByteArray[i] = random.nextInt(256) - 128; // byte range } - assertEquals(determineIntCompressionType(largeByteArray), compressionType("INT_TO_BYTE")); + assertEquals( + ArrayCompressionUtils.determineIntCompressionType(largeByteArray), + PrimitiveArrayCompressionType.INT_TO_BYTE); // Test large array that compresses to shorts int[] largeShortArray = new int[10000]; for (int i = 0; i < largeShortArray.length; i++) { largeShortArray[i] = random.nextInt(65536) - 32768; // short range } - assertEquals(determineIntCompressionType(largeShortArray), compressionType("INT_TO_SHORT")); + assertEquals( + ArrayCompressionUtils.determineIntCompressionType(largeShortArray), + PrimitiveArrayCompressionType.INT_TO_SHORT); // Test large array that doesn't compress int[] largeArray = new int[10000]; for (int i = 0; i < largeArray.length; i++) { largeArray[i] = random.nextInt(); } - assertEquals(determineIntCompressionType(largeArray), compressionType("NONE")); + assertEquals( + ArrayCompressionUtils.determineIntCompressionType(largeArray), + PrimitiveArrayCompressionType.NONE); } @Test @@ -145,62 +162,82 @@ public void testLargeLongArraysWithSIMD() { for (int i = 0; i < largeIntArray.length; i++) { largeIntArray[i] = random.nextInt(); // int range } - assertEquals(determineLongCompressionType(largeIntArray), compressionType("LONG_TO_INT")); + assertEquals( + ArrayCompressionUtils.determineLongCompressionType(largeIntArray), + PrimitiveArrayCompressionType.LONG_TO_INT); // Test large array that doesn't compress long[] largeArray = new long[10000]; for (int i = 0; i < largeArray.length; i++) { largeArray[i] = random.nextLong(); } - assertEquals(determineLongCompressionType(largeArray), compressionType("NONE")); + assertEquals( + ArrayCompressionUtils.determineLongCompressionType(largeArray), + PrimitiveArrayCompressionType.NONE); } @Test public void testEdgeCases() { // Test empty arrays int[] emptyIntArray = {}; - assertEquals(determineIntCompressionType(emptyIntArray), compressionType("NONE")); + assertEquals( + ArrayCompressionUtils.determineIntCompressionType(emptyIntArray), + PrimitiveArrayCompressionType.NONE); long[] emptyLongArray = {}; - assertEquals(determineLongCompressionType(emptyLongArray), compressionType("NONE")); + assertEquals( + ArrayCompressionUtils.determineLongCompressionType(emptyLongArray), + PrimitiveArrayCompressionType.NONE); // Test boundary values - use arrays >= 512 int[] byteBoundary = new int[1024]; for (int i = 0; i < 1024; i++) { byteBoundary[i] = i % 2 == 0 ? Byte.MIN_VALUE : Byte.MAX_VALUE; } - assertEquals(determineIntCompressionType(byteBoundary), compressionType("INT_TO_BYTE")); + assertEquals( + ArrayCompressionUtils.determineIntCompressionType(byteBoundary), + PrimitiveArrayCompressionType.INT_TO_BYTE); int[] shortBoundary = new int[1024]; for (int i = 0; i < 1024; i++) { shortBoundary[i] = i % 2 == 0 ? Short.MIN_VALUE : Short.MAX_VALUE; } - assertEquals(determineIntCompressionType(shortBoundary), compressionType("INT_TO_SHORT")); + assertEquals( + ArrayCompressionUtils.determineIntCompressionType(shortBoundary), + PrimitiveArrayCompressionType.INT_TO_SHORT); long[] intBoundary = new long[1024]; for (int i = 0; i < 1024; i++) { intBoundary[i] = i % 2 == 0 ? Integer.MIN_VALUE : Integer.MAX_VALUE; } - assertEquals(determineLongCompressionType(intBoundary), compressionType("LONG_TO_INT")); + assertEquals( + ArrayCompressionUtils.determineLongCompressionType(intBoundary), + PrimitiveArrayCompressionType.LONG_TO_INT); // Test just outside boundaries int[] outsideByte = new int[1024]; for (int i = 0; i < 1024; i++) { outsideByte[i] = i % 2 == 0 ? Byte.MIN_VALUE - 1 : Byte.MAX_VALUE + 1; } - assertEquals(determineIntCompressionType(outsideByte), compressionType("INT_TO_SHORT")); + assertEquals( + ArrayCompressionUtils.determineIntCompressionType(outsideByte), + PrimitiveArrayCompressionType.INT_TO_SHORT); int[] outsideShort = new int[1024]; for (int i = 0; i < 1024; i++) { outsideShort[i] = i % 2 == 0 ? Short.MIN_VALUE - 1 : Short.MAX_VALUE + 1; } - assertEquals(determineIntCompressionType(outsideShort), compressionType("NONE")); + assertEquals( + ArrayCompressionUtils.determineIntCompressionType(outsideShort), + PrimitiveArrayCompressionType.NONE); long[] outsideInt = new long[1024]; for (int i = 0; i < 1024; i++) { outsideInt[i] = i % 2 == 0 ? Integer.MIN_VALUE - 1L : Integer.MAX_VALUE + 1L; } - assertEquals(determineLongCompressionType(outsideInt), compressionType("NONE")); + assertEquals( + ArrayCompressionUtils.determineLongCompressionType(outsideInt), + PrimitiveArrayCompressionType.NONE); } @Test @@ -211,12 +248,12 @@ public void testCompressionEffectiveness() { byteRangeArray[i] = i % 200 - 100; // byte range } - byte[] compressed = (byte[]) invokeUtils("compressToBytes", int[].class, byteRangeArray); + byte[] compressed = ArrayCompressionUtils.compressToBytes(byteRangeArray); assertEquals(compressed.length, byteRangeArray.length); // 4x smaller assertTrue(compressed.length < byteRangeArray.length * 4); // Verify decompression - int[] decompressed = (int[]) invokeUtils("decompressFromBytes", byte[].class, compressed); + int[] decompressed = ArrayCompressionUtils.decompressFromBytes(compressed); assertEquals(decompressed, byteRangeArray); } @@ -233,47 +270,8 @@ public void testMixedValues() { mixedArray[i] = Integer.MAX_VALUE - i; } - assertEquals(determineIntCompressionType(mixedArray), compressionType("NONE")); - } - - private static Object determineIntCompressionType(int[] array) { - return invokeUtils("determineIntCompressionType", int[].class, array); - } - - private static Object determineLongCompressionType(long[] array) { - return invokeUtils("determineLongCompressionType", long[].class, array); - } - - @SuppressWarnings({"unchecked", "rawtypes"}) - private static Object compressionType(String name) { - return Enum.valueOf((Class) primitiveArrayCompressionTypeClass(), name); - } - - private static Object invokeUtils(String name, Class paramType, Object arg) { - try { - Method method = arrayCompressionUtilsClass().getMethod(name, paramType); - return method.invoke(null, arg); - } catch (IllegalAccessException e) { - throw new AssertionError("Cannot access " + name, e); - } catch (InvocationTargetException e) { - Throwable cause = e.getCause(); - if (cause instanceof RuntimeException) { - throw (RuntimeException) cause; - } - if (cause instanceof Error) { - throw (Error) cause; - } - throw new AssertionError("Unexpected checked exception from " + name, cause); - } catch (NoSuchMethodException e) { - throw new AssertionError("Missing method " + name, e); - } - } - - private static Class arrayCompressionUtilsClass() { - return Java16CompressionSupport.loadClass("org.apache.fory.util.ArrayCompressionUtils"); - } - - private static Class primitiveArrayCompressionTypeClass() { - return Java16CompressionSupport.loadClass("org.apache.fory.util.PrimitiveArrayCompressionType"); + assertEquals( + ArrayCompressionUtils.determineIntCompressionType(mixedArray), + PrimitiveArrayCompressionType.NONE); } } From 9f748b008cbb725b09b387c5aa0af3e23d896185 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sun, 31 May 2026 01:31:16 +0800 Subject: [PATCH 5/6] fix(java): remove compression fallback scaffolding --- .../jpms_tests/run_jlink_smoke.sh | 9 +- java/fory-core/pom.xml | 13 ++ .../fory/util/ArrayCompressionUtils.java | 135 +++++++++--------- .../fory/util/VectorArrayCompression.java | 98 ------------- java/fory-format/pom.xml | 6 +- 5 files changed, 85 insertions(+), 176 deletions(-) delete mode 100644 java/fory-core/src/main/java16/org/apache/fory/util/VectorArrayCompression.java diff --git a/integration_tests/jpms_tests/run_jlink_smoke.sh b/integration_tests/jpms_tests/run_jlink_smoke.sh index e03f32f5ff..08364dc486 100755 --- a/integration_tests/jpms_tests/run_jlink_smoke.sh +++ b/integration_tests/jpms_tests/run_jlink_smoke.sh @@ -89,7 +89,6 @@ if [[ "$JAVA_MAJOR" -ge 16 ]] \ require_jar_entry "$CORE_JAR" "META-INF/versions/16/org/apache/fory/serializer/CompressedArraySerializers.class" require_jar_entry "$CORE_JAR" "META-INF/versions/16/org/apache/fory/util/ArrayCompressionUtils.class" require_jar_entry "$CORE_JAR" "META-INF/versions/16/org/apache/fory/util/PrimitiveArrayCompressionType.class" - require_jar_entry "$CORE_JAR" "META-INF/versions/16/org/apache/fory/util/VectorArrayCompression.class" reject_jar_entry "$CORE_JAR" "org/apache/fory/serializer/CompressedArraySerializers.class" reject_jar_entry "$CORE_JAR" "org/apache/fory/util/ArrayCompressionUtils.class" reject_jar_entry "$CORE_JAR" "org/apache/fory/util/PrimitiveArrayCompressionType.class" @@ -170,20 +169,14 @@ if echo "$IMAGE_MODULES" \ fi if [[ "$JAVA_MAJOR" -ge 16 ]] \ - && jar tf "$CORE_JAR" | grep -qx "META-INF/versions/16/org/apache/fory/util/VectorArrayCompression.class"; then + && jar tf "$CORE_JAR" | grep -qx "META-INF/versions/16/org/apache/fory/util/ArrayCompressionUtils.class"; then mkdir -p "$WORK_DIR/vector-classes" cat > "$WORK_DIR/VectorSmoke.java" <<'EOF' -import java.lang.reflect.Method; import org.apache.fory.util.ArrayCompressionUtils; import org.apache.fory.util.PrimitiveArrayCompressionType; public final class VectorSmoke { public static void main(String[] args) throws Exception { - Method method = ArrayCompressionUtils.class.getDeclaredMethod("isVectorCompressionEnabled"); - method.setAccessible(true); - if (!Boolean.TRUE.equals(method.invoke(null))) { - throw new AssertionError("Vector compression support was not enabled"); - } int[] values = new int[1024]; for (int i = 0; i < values.length; i++) { values[i] = (i & 0xff) - 128; diff --git a/java/fory-core/pom.xml b/java/fory-core/pom.xml index f6d6cd5d97..3acb8eef41 100644 --- a/java/fory-core/pom.xml +++ b/java/fory-core/pom.xml @@ -243,6 +243,7 @@ + @@ -265,6 +266,18 @@ + + clean-java16-package-classes + prepare-package + + run + + + + + + + inject-java16-classes package diff --git a/java/fory-core/src/main/java16/org/apache/fory/util/ArrayCompressionUtils.java b/java/fory-core/src/main/java16/org/apache/fory/util/ArrayCompressionUtils.java index 66745d7362..08d2111dfa 100644 --- a/java/fory-core/src/main/java16/org/apache/fory/util/ArrayCompressionUtils.java +++ b/java/fory-core/src/main/java16/org/apache/fory/util/ArrayCompressionUtils.java @@ -19,6 +19,11 @@ package org.apache.fory.util; +import jdk.incubator.vector.IntVector; +import jdk.incubator.vector.LongVector; +import jdk.incubator.vector.VectorOperators; +import jdk.incubator.vector.VectorSpecies; + /** * Utility methods for optional primitive array compression. * @@ -30,16 +35,12 @@ *

  • {@code int[]} to {@code short[]} when all values are in short range. *
  • {@code long[]} to {@code int[]} when all values are in int range. * - * - *

    Java 16+ runtimes can load the Vector API implementation from the multi-release tree. Older - * runtimes, or minimal module images without {@code jdk.incubator.vector}, use the scalar - * implementation. */ public final class ArrayCompressionUtils { // Minimum array size to justify compression analysis and the compressed payload marker overhead. static final int MIN_COMPRESSION_SIZE = 1 << 9; - private static final CompressionSupport SCALAR_SUPPORT = new ScalarCompressionSupport(); - private static final CompressionSupport COMPRESSION_SUPPORT = loadCompressionSupport(); + private static final VectorSpecies INT_SPECIES = IntVector.SPECIES_PREFERRED; + private static final VectorSpecies LONG_SPECIES = LongVector.SPECIES_PREFERRED; private ArrayCompressionUtils() {} @@ -58,7 +59,45 @@ public static PrimitiveArrayCompressionType determineIntCompressionType(int[] ar if (array.length < MIN_COMPRESSION_SIZE) { return PrimitiveArrayCompressionType.NONE; } - return COMPRESSION_SUPPORT.determineIntCompressionType(array); + boolean canCompressToByte = true; + boolean canCompressToShort = true; + int i = 0; + int upperBound = INT_SPECIES.loopBound(array.length); + + // Vector loop: test each lane against the target primitive ranges and stop checking a narrower + // representation once any lane exceeds its range. + for (; i < upperBound && (canCompressToByte || canCompressToShort); i += INT_SPECIES.length()) { + IntVector vector = IntVector.fromArray(INT_SPECIES, array, i); + if (canCompressToByte) { + if (vector.compare(VectorOperators.GT, Byte.MAX_VALUE).anyTrue() + || vector.compare(VectorOperators.LT, Byte.MIN_VALUE).anyTrue()) { + canCompressToByte = false; + } + } + if (canCompressToShort) { + if (vector.compare(VectorOperators.GT, Short.MAX_VALUE).anyTrue() + || vector.compare(VectorOperators.LT, Short.MIN_VALUE).anyTrue()) { + canCompressToShort = false; + } + } + } + + // Scalar tail for elements that do not fill a complete vector. + for (; i < array.length && (canCompressToByte || canCompressToShort); i++) { + int value = array[i]; + if (canCompressToByte && (value < Byte.MIN_VALUE || value > Byte.MAX_VALUE)) { + canCompressToByte = false; + } + if (value < Short.MIN_VALUE || value > Short.MAX_VALUE) { + canCompressToShort = false; + } + } + if (canCompressToByte) { + return PrimitiveArrayCompressionType.INT_TO_BYTE; + } + return canCompressToShort + ? PrimitiveArrayCompressionType.INT_TO_SHORT + : PrimitiveArrayCompressionType.NONE; } /** @@ -76,7 +115,26 @@ public static PrimitiveArrayCompressionType determineLongCompressionType(long[] if (array.length < MIN_COMPRESSION_SIZE) { return PrimitiveArrayCompressionType.NONE; } - return COMPRESSION_SUPPORT.determineLongCompressionType(array); + int i = 0; + int upperBound = LONG_SPECIES.loopBound(array.length); + + // Vector loop: any lane outside int range means long-to-int compression is not safe. + for (; i < upperBound; i += LONG_SPECIES.length()) { + LongVector vector = LongVector.fromArray(LONG_SPECIES, array, i); + if (vector.compare(VectorOperators.GT, Integer.MAX_VALUE).anyTrue() + || vector.compare(VectorOperators.LT, Integer.MIN_VALUE).anyTrue()) { + return PrimitiveArrayCompressionType.NONE; + } + } + + // Scalar tail for elements that do not fill a complete vector. + for (; i < array.length; i++) { + long value = array[i]; + if (value > Integer.MAX_VALUE || value < Integer.MIN_VALUE) { + return PrimitiveArrayCompressionType.NONE; + } + } + return PrimitiveArrayCompressionType.LONG_TO_INT; } /** @@ -186,65 +244,4 @@ public static long[] decompressFromInts(int[] array) { } return decompressed; } - - static boolean isVectorCompressionEnabled() { - return COMPRESSION_SUPPORT != SCALAR_SUPPORT; - } - - interface CompressionSupport { - PrimitiveArrayCompressionType determineIntCompressionType(int[] array); - - PrimitiveArrayCompressionType determineLongCompressionType(long[] array); - } - - private static CompressionSupport loadCompressionSupport() { - try { - // The Vector implementation lives in the Java 16 multi-release tree and links to the - // optional incubator module. Missing classes or read edges must fall back to scalar code so - // minimal jlink images can start without resolving jdk.incubator.vector. - Class cls = - Class.forName( - "org.apache.fory.util.VectorArrayCompression", - true, - ArrayCompressionUtils.class.getClassLoader()); - return (CompressionSupport) cls.getDeclaredConstructor().newInstance(); - } catch (ClassCastException | LinkageError | ReflectiveOperationException e) { - return SCALAR_SUPPORT; - } - } - - private static final class ScalarCompressionSupport implements CompressionSupport { - @Override - public PrimitiveArrayCompressionType determineIntCompressionType(int[] array) { - boolean canCompressToByte = true; - boolean canCompressToShort = true; - for (int value : array) { - if (canCompressToByte && (value < Byte.MIN_VALUE || value > Byte.MAX_VALUE)) { - canCompressToByte = false; - } - if (value < Short.MIN_VALUE || value > Short.MAX_VALUE) { - canCompressToShort = false; - } - if (!canCompressToByte && !canCompressToShort) { - return PrimitiveArrayCompressionType.NONE; - } - } - if (canCompressToByte) { - return PrimitiveArrayCompressionType.INT_TO_BYTE; - } - return canCompressToShort - ? PrimitiveArrayCompressionType.INT_TO_SHORT - : PrimitiveArrayCompressionType.NONE; - } - - @Override - public PrimitiveArrayCompressionType determineLongCompressionType(long[] array) { - for (long value : array) { - if (value > Integer.MAX_VALUE || value < Integer.MIN_VALUE) { - return PrimitiveArrayCompressionType.NONE; - } - } - return PrimitiveArrayCompressionType.LONG_TO_INT; - } - } } diff --git a/java/fory-core/src/main/java16/org/apache/fory/util/VectorArrayCompression.java b/java/fory-core/src/main/java16/org/apache/fory/util/VectorArrayCompression.java deleted file mode 100644 index d5c1c85a4e..0000000000 --- a/java/fory-core/src/main/java16/org/apache/fory/util/VectorArrayCompression.java +++ /dev/null @@ -1,98 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.fory.util; - -import jdk.incubator.vector.IntVector; -import jdk.incubator.vector.LongVector; -import jdk.incubator.vector.VectorOperators; -import jdk.incubator.vector.VectorSpecies; - -/** Java 16+ Vector API implementation for primitive array compression analysis. */ -final class VectorArrayCompression implements ArrayCompressionUtils.CompressionSupport { - private static final VectorSpecies INT_SPECIES = IntVector.SPECIES_PREFERRED; - private static final VectorSpecies LONG_SPECIES = LongVector.SPECIES_PREFERRED; - - @Override - public PrimitiveArrayCompressionType determineIntCompressionType(int[] array) { - boolean canCompressToByte = true; - boolean canCompressToShort = true; - int i = 0; - int upperBound = INT_SPECIES.loopBound(array.length); - - // Vector loop: test each lane against the target primitive ranges and stop checking a narrower - // representation once any lane exceeds its range. - for (; i < upperBound && (canCompressToByte || canCompressToShort); i += INT_SPECIES.length()) { - IntVector vector = IntVector.fromArray(INT_SPECIES, array, i); - if (canCompressToByte) { - if (vector.compare(VectorOperators.GT, Byte.MAX_VALUE).anyTrue() - || vector.compare(VectorOperators.LT, Byte.MIN_VALUE).anyTrue()) { - canCompressToByte = false; - } - } - if (canCompressToShort) { - if (vector.compare(VectorOperators.GT, Short.MAX_VALUE).anyTrue() - || vector.compare(VectorOperators.LT, Short.MIN_VALUE).anyTrue()) { - canCompressToShort = false; - } - } - } - - // Scalar tail for elements that do not fill a complete vector. - for (; i < array.length && (canCompressToByte || canCompressToShort); i++) { - int value = array[i]; - if (canCompressToByte && (value < Byte.MIN_VALUE || value > Byte.MAX_VALUE)) { - canCompressToByte = false; - } - if (value < Short.MIN_VALUE || value > Short.MAX_VALUE) { - canCompressToShort = false; - } - } - if (canCompressToByte) { - return PrimitiveArrayCompressionType.INT_TO_BYTE; - } - return canCompressToShort - ? PrimitiveArrayCompressionType.INT_TO_SHORT - : PrimitiveArrayCompressionType.NONE; - } - - @Override - public PrimitiveArrayCompressionType determineLongCompressionType(long[] array) { - int i = 0; - int upperBound = LONG_SPECIES.loopBound(array.length); - - // Vector loop: any lane outside int range means long-to-int compression is not safe. - for (; i < upperBound; i += LONG_SPECIES.length()) { - LongVector vector = LongVector.fromArray(LONG_SPECIES, array, i); - if (vector.compare(VectorOperators.GT, Integer.MAX_VALUE).anyTrue() - || vector.compare(VectorOperators.LT, Integer.MIN_VALUE).anyTrue()) { - return PrimitiveArrayCompressionType.NONE; - } - } - - // Scalar tail for elements that do not fill a complete vector. - for (; i < array.length; i++) { - long value = array[i]; - if (value > Integer.MAX_VALUE || value < Integer.MIN_VALUE) { - return PrimitiveArrayCompressionType.NONE; - } - } - return PrimitiveArrayCompressionType.LONG_TO_INT; - } -} diff --git a/java/fory-format/pom.xml b/java/fory-format/pom.xml index 4a0a6da6c5..a1fecb2ef1 100644 --- a/java/fory-format/pom.xml +++ b/java/fory-format/pom.xml @@ -143,7 +143,11 @@ - + + + + Date: Sun, 31 May 2026 01:42:25 +0800 Subject: [PATCH 6/6] fix(java): compile vector MR sources without release symbols --- java/fory-core/pom.xml | 4 +++- java/fory-testsuite/pom.xml | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/java/fory-core/pom.xml b/java/fory-core/pom.xml index 3acb8eef41..23d05bd6a8 100644 --- a/java/fory-core/pom.xml +++ b/java/fory-core/pom.xml @@ -254,7 +254,9 @@ executable="${java.home}/bin/javac" debug="true"> - + + + diff --git a/java/fory-testsuite/pom.xml b/java/fory-testsuite/pom.xml index c08a9b8023..7363073431 100644 --- a/java/fory-testsuite/pom.xml +++ b/java/fory-testsuite/pom.xml @@ -174,7 +174,9 @@ executable="${java.home}/bin/javac" debug="true"> - + + +