A JNI provider, so that JDK 17 can call the engine

.github/workflows/ci.yml

@@ -89,6 +89,12 @@ jobs:
           echo "engine $engine_abi, client $client_abi"
           test "$engine_abi" = "$client_abi"
 
+      # The shim is written against this header and opens a libzu at run
+      # time, so a copy that has drifted compiles clean and then reads a
+      # struct that moved. This is the step that stops it drifting.
+      - name: The vendored header is the engine's header
+        run: diff -u engine/crates/zu-capi/include/zu.h zudb-jni/src/main/c/zu.h
+
       - name: Build libzu
         working-directory: engine
         run: cargo build --release -p zu-capi
@@ -100,6 +106,12 @@ jobs:
           test -n "$lib"
           echo "ZU_LIBRARY=$GITHUB_WORKSPACE/$lib" >> "$GITHUB_ENV"
 
+      # One platform, this runner's, which is all a runner can build and
+      # all the suite here needs. The other six are built in the release,
+      # on runners of their own.
+      - name: Build the JNI shim
+        run: ./scripts/build-shim.sh
+
       - run: mvn $MAVEN_ARGS test
 
       # The suite again with assertions on everywhere, including the ones
@@ -115,6 +127,66 @@ jobs:
 
       - run: java -jar zudb-bench/target/benchmarks.jar -f 1 -wi 1 -i 1 -r 1s -w 1s
 
+  # The JNI provider on the JDKs it exists for. Panama is not there on
+  # 17 or 21, so on those two this is the only way to call the engine at
+  # all, and a client that claims 17 and is only ever tested on 25 is a
+  # client that claims 17.
+  #
+  # It runs the same cases the Panama provider runs, out of the same
+  # artifact, so a difference between the two providers is a red job here
+  # rather than something a user finds.
+  jni:
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, macos-latest]
+        java: ["17", "21"]
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@v5
+
+      - uses: actions/checkout@v5
+        with:
+          repository: tamnd/zu
+          path: engine
+
+      - uses: actions/setup-java@v5
+        with:
+          distribution: temurin
+          java-version: ${{ matrix.java }}
+          cache: maven
+
+      - uses: Swatinem/rust-cache@v2
+        with:
+          workspaces: engine
+
+      - name: The vendored header is the engine's header
+        run: diff -u engine/crates/zu-capi/include/zu.h zudb-jni/src/main/c/zu.h
+
+      - name: Build libzu
+        working-directory: engine
+        run: cargo build --release -p zu-capi
+
+      - name: Where the library landed
+        run: |
+          set -eu
+          lib="$(ls engine/target/release/libzu.dylib engine/target/release/libzu.so 2>/dev/null | head -1)"
+          test -n "$lib"
+          echo "ZU_LIBRARY=$GITHUB_WORKSPACE/$lib" >> "$GITHUB_ENV"
+
+      - name: Build the JNI shim
+        run: ./scripts/build-shim.sh
+
+      # Named modules rather than the whole reactor, because the Panama
+      # provider compiles to release 25 and this JDK cannot be asked for
+      # that. What is left is exactly what a caller on 17 gets.
+      - run: mvn $MAVEN_ARGS -pl zudb,zudb-tck,zudb-jni -am test
+
+      # Again with every assertion on, including the JDK's own. A wrong
+      # length handed to NewDirectByteBuffer is the difference between a
+      # test that fails and a test that reads somebody else's memory.
+      - run: mvn $MAVEN_ARGS -pl zudb,zudb-tck,zudb-jni -am test -Dzu.test.args="-ea -esa"
+
   # The claim the zudb-native artifact makes is that a user who added a
   # dependency and installed nothing has an engine. Nothing in the test
   # suite can check that, because the suite is told where the library is

.github/workflows/release.yml

@@ -33,7 +33,82 @@ env:
   MAVEN_ARGS: -B -ntp
 
 jobs:
+  # The JNI shim, one build a platform, because a C toolchain is a thing
+  # a runner is rather than a thing a runner can be asked for. Seven
+  # jobs of about a minute each, and the release job below collects what
+  # they made into one jar.
+  #
+  # The shim links against nothing: it opens libzu at run time and
+  # resolves what it calls, so none of this needs Rust and none of it
+  # needs the engine.
+  shims:
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - platform: linux-amd64
+            runs-on: ubuntu-latest
+          - platform: linux-arm64
+            runs-on: ubuntu-24.04-arm
+          - platform: linux-amd64-musl
+            runs-on: ubuntu-latest
+            container: alpine:3.21
+          - platform: linux-arm64-musl
+            runs-on: ubuntu-24.04-arm
+            container: alpine:3.21
+          - platform: darwin-amd64
+            runs-on: macos-latest
+            # An arm runner building for intel, because the shim is one
+            # file with no dependencies and clang takes an -arch.
+            cflags: -arch x86_64
+          - platform: darwin-arm64
+            runs-on: macos-latest
+          - platform: windows-amd64
+            runs-on: windows-latest
+    runs-on: ${{ matrix.runs-on }}
+    container: ${{ matrix.container }}
+    steps:
+      - uses: actions/checkout@v5
+
+      # The musl builds run in a container that has neither a compiler
+      # nor a JDK, and setup-java has no musl build to give it, so both
+      # come from the distribution's own packages.
+      - name: A compiler and the JDK headers
+        if: matrix.container
+        run: apk add --no-cache bash build-base openjdk21
+
+      - uses: actions/setup-java@v5
+        if: ${{ !matrix.container }}
+        with:
+          distribution: temurin
+          java-version: "21"
+
+      - name: Build the shim
+        shell: bash
+        env:
+          CFLAGS: ${{ matrix.cflags }}
+        run: |
+          set -eu
+          if [ -n "${{ matrix.container }}" ]; then
+            export JAVA_HOME=/usr/lib/jvm/java-21-openjdk
+          fi
+          ./scripts/build-shim.sh
+          # The script names the directory after the machine it ran on,
+          # which is right everywhere but the one cross build, so that
+          # one is put where it belongs.
+          if [ ! -d "zudb-jni/shim/${{ matrix.platform }}" ]; then
+            mv zudb-jni/shim/* "zudb-jni/shim/${{ matrix.platform }}"
+          fi
+          ls -l "zudb-jni/shim/${{ matrix.platform }}"
+
+      - uses: actions/upload-artifact@v4
+        with:
+          name: shim-${{ matrix.platform }}
+          path: zudb-jni/shim/${{ matrix.platform }}
+          if-no-files-found: error
+
   release:
+    needs: shims
     runs-on: ubuntu-latest
     permissions:
       contents: read
@@ -75,6 +150,23 @@ jobs:
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: ./scripts/stage-natives.sh "$TAG"
 
+      # The seven shims the jobs above built, laid out the way
+      # build-shim.sh would have laid them out if one machine could have
+      # built them all.
+      - uses: actions/download-artifact@v4
+        with:
+          pattern: shim-*
+          path: zudb-jni/shim
+
+      - name: The shims are seven and are named right
+        run: |
+          set -eu
+          for dir in zudb-jni/shim/shim-*; do
+            mv "$dir" "zudb-jni/shim/$(basename "$dir" | sed 's/^shim-//')"
+          done
+          chmod -R a+rX zudb-jni/shim
+          ls -lR zudb-jni/shim
+
       # The suite needs an engine, and one of the seven that was just
       # staged is the platform this runner is, so it runs against the
       # library that is about to be published rather than against a build
@@ -84,22 +176,25 @@ jobs:
           set -eu
           echo "ZU_LIBRARY=$PWD/zudb-native/lib/linux-amd64/libzu.so" >> "$GITHUB_ENV"
 
-      - run: mvn $MAVEN_ARGS -Pnatives test
+      # The shim for this runner is one of the seven already downloaded,
+      # so the suite runs against the same file that is about to be
+      # published rather than against one built here.
+      - run: mvn $MAVEN_ARGS -Pnatives,shims test
 
       - name: Build, sign and upload
         if: github.event_name == 'push' || inputs.publish
         env:
           CENTRAL_USERNAME: ${{ secrets.CENTRAL_USERNAME }}
           CENTRAL_PASSWORD: ${{ secrets.CENTRAL_PASSWORD }}
           GPG_PASSPHRASE: ${{ secrets.GPG_PASSPHRASE }}
-        run: mvn $MAVEN_ARGS -Prelease,natives -DskipTests deploy
+        run: mvn $MAVEN_ARGS -Prelease,natives,shims -DskipTests deploy
 
       # What a dispatch with publish off is for: everything up to the
       # upload, so that a change to the packaging can be checked without
       # a version being spent on it.
       - name: Build what would have been uploaded
         if: github.event_name != 'push' && !inputs.publish
-        run: mvn $MAVEN_ARGS -Prelease,natives -DskipTests -Dgpg.skip=true package
+        run: mvn $MAVEN_ARGS -Prelease,natives,shims -DskipTests -Dgpg.skip=true package
 
       - uses: actions/upload-artifact@v4
         with:

.gitignore

@@ -4,3 +4,7 @@ target/
 # What scripts/stage-natives.sh downloads. A build of the engine is a
 # thing a release fetches, not a thing a git history carries.
 zudb-native/lib/
+
+# What scripts/build-shim.sh compiles. One platform a run, so a checked
+# in copy would be a copy of whichever machine ran it last.
+zudb-jni/shim/

README.md

@@ -261,21 +261,22 @@ On the module path the artifact needs `--add-modules dev.zudb.natives`. Nothing
 
 ## How it binds
 
-The Foreign Function and Memory API is the primary path. The downcall handles are written by hand against `zu.h` rather than generated with `jextract`, because the C ABI here is around seventy functions with a stable shape, and a hand-written layer is where the interesting decisions live: which calls are `Linker.Option.critical` because they are short pure accessors, where the out-parameter scratch space comes from so that a query does not allocate, and how a `zu_error` becomes a typed Java exception exactly once. There is no native code in this repository beyond `libzu` itself.
+The Foreign Function and Memory API is the primary path. The downcall handles are written by hand against `zu.h` rather than generated with `jextract`, because the C ABI here is around seventy functions with a stable shape, and a hand-written layer is where the interesting decisions live: which calls are `Linker.Option.critical` because they are short pure accessors, where the out-parameter scratch space comes from so that a query does not allocate, and how a `zu_error` becomes a typed Java exception exactly once.
+
+An SDK that requires a recent JDK in 2026 excludes a large part of the enterprise ecosystem, so there is a JNI provider too. It is a small C shim, one build a platform, and it is the only native code in this repository that is ours. Four decisions in it are worth knowing about. It does not link against `libzu`, it opens one at run time and resolves what it calls, so building it needs a C compiler and the JDK headers and nothing else, and a shim built today opens a `libzu` built next year at the same ABI. Every string crosses as a `byte[]` rather than as a `jstring`, because JNI's own conversions speak modified UTF-8 and the engine validates the real thing, so an emoji handed to `NewStringUTF` is a byte sequence the engine refuses. One symbol is exported under the name JNI derives, and it registers the other hundred and two from a static table, because `JNI_OnLoad` cannot find a class that a module path or an application server's class loader holds. And a failure is built on the Java side, so the two providers cannot come to disagree about which exception a GQLSTATUS class names.
 
-An SDK that requires a recent JDK in 2026 excludes a large part of the enterprise ecosystem, so there is a JNI provider too:
 
 | Artifact | Baseline | Role |
 |---|---|---|
 | `dev.zudb:zudb` | Java 17 | the API, no native code, no FFM types in the public surface |
 | `dev.zudb:zudb-ffm` | Java 25 | the FFM provider, selected automatically |
-| `dev.zudb:zudb-jni` | Java 17 | the fallback provider |
+| `dev.zudb:zudb-jni` | Java 17 | the JNI provider, with the shim for all seven platforms |
 | `dev.zudb:zudb-arrow` | Java 17 | the Arrow reader, the only artifact that names arrow-java |
 | `dev.zudb:zudb-native` | | the `libzu` binaries, all platforms or one by classifier |
 
-A `ServiceLoader` picks the provider at run time and application code never names one. The FFM artifact targets Java 25 rather than the Java 22 that finalised the API, because 22 has been out of support since September 2024 and shipping against an unsupported release only moves the problem. CI runs 17, 21, 25, and 26.
+A `ServiceLoader` picks the provider at run time and application code never names one, and the same suite of cases runs against both every build, so a difference between the two is a red job rather than something you find. Add `zudb-jni` beside `zudb` on 17 through 21, add `zudb-ffm` on 25 and later, or add both and let the loader pick. The FFM artifact targets Java 25 rather than the Java 22 that finalised the API, because 22 has been out of support since September 2024 and shipping against an unsupported release only moves the problem. CI runs 17, 21, 25, and 26.
 
-One thing to know before your first run: from JDK 24, native access must be granted explicitly. The jars carry `Enable-Native-Access: ALL-UNNAMED` for the class path case, the module path case wants `--enable-native-access=dev.zudb.ffm`, and the provider checks `Module::isNativeAccessEnabled` before the first downcall so that the failure is an exception naming the flag rather than a JVM warning on stderr three frames from any of our code.
+One thing to know before your first run: from JDK 24, native access must be granted explicitly. The jars carry `Enable-Native-Access: ALL-UNNAMED` for the class path case, the module path case wants `--enable-native-access=dev.zudb.ffm` or `--enable-native-access=dev.zudb.jni` for whichever provider is in play, and the FFM provider checks `Module::isNativeAccessEnabled` before the first downcall so that the failure is an exception naming the flag rather than a JVM warning on stderr three frames from any of our code.
 
 ## Errors
 
@@ -298,9 +299,12 @@ The engine has no DDL yet, so there is no `CREATE NODE TABLE` and no statement i
 ## Building
 
 ```sh
+scripts/build-shim.sh
 mvn test -Dzu.library=/path/to/libzu.dylib
 ```
 
+The first line builds the JNI shim for this machine, and it is needed once rather than once a build, since the output lives outside `target/`. It needs a C compiler and a `JAVA_HOME` with headers in it. A checkout with no engine build beside it is still green, because the engine is a different repository and its absence is an ordinary state to be in, but a checkout with no shim in it is red, because the shim is ours and a suite that quietly skipped half its providers would be worth nothing.
+
 The library is looked for in four places, in order: `-Dzu.library`, then `ZU_LIBRARY`, then a `zudb-native` artifact on the class path, then the platform's own search. A named path is first because a bisect and a bug report both start by pointing this at a build, and the platform's search is last because it is the one that can pick up a library nobody in the process chose. `Zu.library()` and `Zu.source()` say which file was loaded and which of the four it came from, and a failure to bind lists what was ruled out on the way. The tests skip rather than fail when no `libzu` is reachable, so a checkout with no engine build beside it is still green.
 
 The `zudb-native` module is not built unless it is asked for, because what it packages is downloaded rather than compiled:

pom.xml

@@ -49,7 +49,9 @@
 
   <modules>
     <module>zudb</module>
+    <module>zudb-tck</module>
     <module>zudb-ffm</module>
+    <module>zudb-jni</module>
     <module>zudb-arrow</module>
     <module>zudb-bench</module>
   </modules>
@@ -103,6 +105,11 @@
         <artifactId>zudb</artifactId>
         <version>${project.version}</version>
       </dependency>
+      <dependency>
+        <groupId>dev.zudb</groupId>
+        <artifactId>zudb-tck</artifactId>
+        <version>${project.version}</version>
+      </dependency>
       <dependency>
         <groupId>org.apache.arrow</groupId>
         <artifactId>arrow-bom</artifactId>
@@ -223,6 +230,7 @@
               <waitUntil>validated</waitUntil>
               <excludeArtifacts>
                 <excludeArtifact>zudb-bench</excludeArtifact>
+                <excludeArtifact>zudb-tck</excludeArtifact>
               </excludeArtifacts>
             </configuration>
           </plugin>