From 570d23377387e0465fca6664e551bc417ee8cfbb Mon Sep 17 00:00:00 2001 From: Tam Nguyen Duc <1218621+tamnd@users.noreply.github.com> Date: Sat, 22 Aug 2026 17:32:33 +0700 Subject: [PATCH] Install it on a machine that has never heard of it Every job in this repository so far runs against a checkout with the engine built beside it and an environment variable pointing at the library. That is right for a test suite and it is no answer at all to the question a new reader asks, which is whether a pom and a JDK are enough. This adds the job that answers it somewhere else. A container with a JDK and Maven in it and nothing else. The script asserts what is not there before it starts: no rustc, no cargo, no compiler, no libzu anywhere on the filesystem, no header, no pkg-config file, no ZU_LIBRARY. Then it lifts the quickstart off the README character for character, lifts the dependency elements off the same page, writes a pom nobody here has touched, and builds against the artifacts this commit would publish, staged into a repository standing in for Central. It runs the program in an empty directory somewhere else and compares what it printed against the block the page says it prints. Two rows, because the claim this client makes that no other client of this engine makes is that it installs on 17 and on 25, over two different bindings, from the same dependency. zudb-ffm on JDK 25 and zudb-jni on JDK 17, and the page has to print a dependency element for both, so it now prints one for the JNI pair as well. The last step is the one that matters most. It takes the library artifact off the classpath and asserts the run fails naming the line to add, because a job that has only ever passed is a job nobody has watched fail. Running it turned up something the page was wrong about. From JDK 24 native access is granted by whoever starts the JVM, and the manifest attribute these jars carry only speaks for the jar that java -jar names. A -cp run needs the flag on the command line, whatever any dependency's manifest says. The README claimed the attribute covered the class path case and it does not, so the paragraph now says who grants it and where. The error the provider raises was already right about this, which is how the script found out. --- .github/workflows/install.yml | 149 +++++++++ README.md | 17 +- scripts/install.sh | 287 ++++++++++++++++++ .../test/java/dev/zudb/ffm/ReadmeTest.java | 6 +- 4 files changed, 454 insertions(+), 5 deletions(-) create mode 100644 .github/workflows/install.yml create mode 100755 scripts/install.sh diff --git a/.github/workflows/install.yml b/.github/workflows/install.yml new file mode 100644 index 0000000..1254485 --- /dev/null +++ b/.github/workflows/install.yml @@ -0,0 +1,149 @@ +# Installing this client on a machine that has nothing on it. +# +# Every other job in this repository runs against a checkout: the engine +# is built beside it, an environment variable points at the library, and +# the tests are told where everything is so that they test the binding +# rather than the search. That is the right thing for a test suite and +# it is the wrong thing entirely for the question a new user asks, which +# is whether three lines in a pom and a JDK are enough. +# +# So this job answers that question somewhere else. A container with a +# JDK and Maven in it and nothing else, no toolchain, no engine, no +# variable, and the program off the front page built against the +# artifacts this commit would publish. It runs nightly rather than on +# every push, because what breaks it is rarely a commit here: it is a +# base image that changed, a JDK that tightened a default, a +# dependency's dependency that moved. +# +# Both providers get a row, because the claim this client makes that no +# other client of this engine makes is that it installs on a JDK from +# 17 and on a JDK from 25, over two different bindings, out of the same +# dependency. A row each is what keeps that from being a paragraph. +name: Install + +on: + schedule: + # Nightly, an hour that is nobody's working day and not on the hour, + # where a queue forms. + - cron: "41 4 * * *" + workflow_dispatch: + pull_request: + paths: + - .github/workflows/install.yml + - scripts/install.sh + - README.md + - "**/pom.xml" + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + MAVEN_ARGS: -B -ntp + +jobs: + clean-machine: + strategy: + fail-fast: false + matrix: + include: + - provider: zudb-ffm + image: maven:3-eclipse-temurin-25 + java: "25" + - provider: zudb-jni + image: maven:3-eclipse-temurin-17 + java: "17" + name: ${{ matrix.provider }} on JDK ${{ matrix.java }} + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v5 + + - uses: actions/checkout@v5 + with: + repository: tamnd/zu + path: engine + + # This JDK is the one that builds the artifacts. The one that + # installs them is inside the container and is a different JDK, + # which is the point of the row. + - uses: actions/setup-java@v5 + with: + distribution: temurin + java-version: "25" + cache: maven + + - uses: Swatinem/rust-cache@v2 + with: + workspaces: engine + + - name: Build libzu + working-directory: engine + run: cargo build --release -p zu-capi + + # One platform, this runner's, because a runner builds the one it + # is. The release is where all seven have to be there, and the + # enforcer that says so is stood down here for the same reason it + # is in the natives job. + - name: Stage the one platform this runner is + run: | + set -eu + mkdir -p zudb-native/lib/linux-amd64 + cp engine/target/release/libzu.so zudb-native/lib/linux-amd64/libzu.so + + # The JNI row needs a shim, and a shim needs a C compiler, which is + # exactly the tool the container is asserted not to have. That is + # not a contradiction: the compiler is ours to have at build time + # and the user's not to need at install time, and the shim ends up + # in the jar. + - name: Build the JNI shim + run: ./scripts/build-shim.sh + + # A file repository cannot serve a snapshot without the metadata a + # deploy writes, and a version with SNAPSHOT in it is not what a + # reader of the page would type anyway. So the artifacts are + # staged under the version this pom is on its way to. + - name: Which version this would be + run: | + set -eu + snapshot="$(mvn $MAVEN_ARGS help:evaluate -Dexpression=project.version -q -DforceStdout)" + echo "VERSION=${snapshot%-SNAPSHOT}" >> "$GITHUB_ENV" + + - name: Stage the artifacts + run: | + set -eu + mvn $MAVEN_ARGS versions:set -DnewVersion="$VERSION" -DgenerateBackupPoms=false + # A cached local repository can hold this version from a run + # before this one, and an install that quietly resolved that + # is an install nobody tested. + rm -rf "$HOME/.m2/repository/dev/zudb" + mvn $MAVEN_ARGS -Pnatives -Denforcer.skip=true -DskipTests \ + -pl zudb,zudb-ffm,zudb-jni,zudb-native -am clean install + mkdir -p "$RUNNER_TEMP/repo" + cp -r "$HOME/.m2/repository/dev" "$RUNNER_TEMP/repo/" + # Notes the local repository keeps about where it downloaded + # something from. A repository being served has no use for + # them and they are one more way for a resolve to go + # sideways. + find "$RUNNER_TEMP/repo" -name _remote.repositories -delete + # A repository serves a checksum beside every file, and a + # resolver that finds none says so at length. Central has + # them, so this has them, or else the install being tested is + # noisier than the one a user gets. + find "$RUNNER_TEMP/repo" -type f \ + -exec sh -c 'sha1sum "$1" | cut -d" " -f1 > "$1.sha1"' _ {} \; + find "$RUNNER_TEMP/repo" -name '*.jar' -o -name '*.pom' | sort + + - name: Install it the way a reader of the page would + run: | + set -eu + mkdir -p "$RUNNER_TEMP/app" + docker run --rm \ + -v "$PWD:/src:ro" \ + -v "$RUNNER_TEMP/repo:/repo:ro" \ + -v "$RUNNER_TEMP/app:/app" \ + -e SRC=/src -e REPO=/repo -e APP=/app \ + -e VERSION="$VERSION" \ + -e PROVIDER=${{ matrix.provider }} \ + -e ABSENT="rustc cargo cc gcc g++ make cmake ninja" \ + ${{ matrix.image }} \ + bash /src/scripts/install.sh diff --git a/README.md b/README.md index f6d72c4..da1a05c 100644 --- a/README.md +++ b/README.md @@ -277,6 +277,8 @@ The library inside the jar is a resource, and no loader on any platform can map A GraalVM native image needs no configuration for any of this. Both artifacts carry their own reachability metadata: `zudb-ffm` lists every signature it binds, because an image has no linker in it and each downcall stub is machine code the builder has to be told to write, and `zudb-native` registers the libraries so that one ends up inside the image rather than being looked for on a machine that does not have it. Use a classifier rather than the platform-complete jar, or the image carries seven libraries and uses one. CI builds an image on Linux and macOS every run and makes it answer a query, because a metadata file that is wrong produces an image that builds clean and dies on the first call. +None of that is taken on trust. A nightly job installs this client the way this page says to, in a container holding a JDK and Maven and nothing else, and runs the program at the top of the page against the artifacts the commit would publish. Before it starts it asserts the machine has no libzu on it, no header, no pkg-config file, no `ZU_LIBRARY`, and no compiler of any kind, because an install that works for a reason the reader does not have is the one failure a test on our own machine cannot see. Then it takes the library artifact away again and checks that what comes back names the line to add, since a job that has only ever passed is a job nobody has watched fail. + On the module path the artifact needs `--add-modules dev.zudb.natives`. Nothing `requires` it, since there is no code in it to require, and a jar nothing requires is a jar that is never resolved and whose resources are therefore invisible. The search says so itself when it comes up empty on a module path, so the failure names the flag rather than leaving a user to work out why the same classpath run worked. ## How it binds @@ -296,7 +298,18 @@ An SDK that requires a recent JDK in 2026 excludes a large part of the enterpris A `ServiceLoader` picks the provider at run time and application code never names one, and the same suite of cases runs against both every build, so a difference between the two is a red job rather than something you find. Add `zudb-jni` beside `zudb` on 17 through 21, add `zudb-ffm` on 25 and later, or add both and let the loader pick. The FFM artifact targets Java 25 rather than the Java 22 that finalised the API, because 22 has been out of support since September 2024 and shipping against an unsupported release only moves the problem. CI runs 17, 21, 25, and 26. -One thing to know before your first run: from JDK 24, native access must be granted explicitly. The jars carry `Enable-Native-Access: ALL-UNNAMED` for the class path case, the module path case wants `--enable-native-access=dev.zudb.ffm` or `--enable-native-access=dev.zudb.jni` for whichever provider is in play, and the FFM provider checks `Module::isNativeAccessEnabled` before the first downcall so that the failure is an exception naming the flag rather than a JVM warning on stderr three frames from any of our code. +On 17 through 21, the second dependency at the top of this page is the only line that changes: + +```xml + + dev.zudb + zudb-jni + ${zu.version} + runtime + +``` + +One thing to know before your first run: from JDK 24, native access is granted by whoever starts the JVM rather than by the library being called. On a class path that means `--enable-native-access=ALL-UNNAMED` on the command line, or the same thing as an `Enable-Native-Access` line in the manifest of the jar that `java -jar` names. On a module path it names the provider instead, `--enable-native-access=dev.zudb.ffm` or `--enable-native-access=dev.zudb.jni` for whichever one is in play. The FFM provider checks `Module::isNativeAccessEnabled` before the first downcall, so a run without the grant is an exception naming the flag rather than a JVM warning on stderr three frames from any of our code. The JNI provider is not restricted this way on a class path, which is one more thing the 17 artifact is quieter about. ## Errors @@ -385,7 +398,7 @@ Inside this repository: | JMH benchmarks | `zudb-bench` | | The staged libraries, built by the release rather than by a clone | `zudb-native` | | Every published name and the shape it is published in | `api/surface.txt` | -| Building the shim, staging the libraries | `scripts` | +| Building the shim, staging the libraries, installing on a clean machine | `scripts` | `api/surface.txt` is generated, one line per exported type and per member a caller outside the module can name, in the spirit of the `api/go1.N.txt` files Go holds itself to. `SurfaceTest` regenerates it and compares, so a change to the API is a change to that file in the same commit, where it is the first thing in the diff rather than something a user finds after the release. Write it down with `mvn -pl zudb test -Dzu.surface.write=true` and review it like any other file: a name that arrived is a minor release, a name that went or changed shape is a major one or a mistake, and the gate says which of the three a diff is while it is still a diff. It reads the compiled classes of the API module and links nothing, so it answers on a clone with no library staged and no Rust installed. diff --git a/scripts/install.sh b/scripts/install.sh new file mode 100755 index 0000000..3fe054a --- /dev/null +++ b/scripts/install.sh @@ -0,0 +1,287 @@ +#!/usr/bin/env bash +# Install this client the way a user does, on a machine that has never +# heard of it. +# +# This runs inside a container that holds a JDK, Maven, and nothing +# else: no Rust, no C compiler, no libzu, no header, no pkg-config file, +# and no checkout of the engine. What it does is what the front page +# tells a reader to do, which is add three lines to a pom and write the +# program on the page, and what it proves is that those two things are +# the whole of it. +# +# The point is not that the build passes. It is that the build passes +# here. A repository's own machine has the engine on it, has a toolchain +# on it, and has an environment variable pointing at a library, and +# every one of those is a way for an install to work for a reason the +# user does not have. So this asserts each of them is absent before it +# starts, and then it lets Maven do the rest. +# +# Read from the environment, all set by .github/workflows/install.yml: +# +# SRC the checkout, mounted read only, for the README +# REPO the repository the artifacts were staged into, read only +# APP a writable directory of this run's own +# VERSION the version that was staged +# PROVIDER zudb-ffm or zudb-jni, whichever this JDK is for +# ABSENT the tools that must not be on this machine +set -euo pipefail + +SRC="${SRC:-/src}" +REPO="${REPO:-/repo}" +APP="${APP:-/app}" +VERSION="${VERSION:?the version that was staged}" +PROVIDER="${PROVIDER:?zudb-ffm or zudb-jni}" +ABSENT="${ABSENT:-}" + +readme="$SRC/README.md" + +step() { printf '\n=== %s\n' "$1"; } + +step "what this machine is" +java -version 2>&1 | head -1 +mvn -v 2>&1 | head -1 +uname -sm + +# Every Maven run here resolves into a repository of this run's own, so +# that what came over the wire is a directory that can be measured and +# not a number nobody has. +mvn() { command mvn -B -ntp -Dmaven.repo.local="$APP/m2" "$@"; } + +step "and what it is not" +# A tool that is here anyway makes this whole run mean nothing, and it +# is the kind of thing that arrives in a base image without anybody +# choosing it, so it is asked about rather than assumed. +for tool in $ABSENT; do + if command -v "$tool" >/dev/null 2>&1; then + echo "$tool is on this machine, so it is not the clean one this test needs" >&2 + exit 1 + fi + echo "no $tool" +done + +# The engine, in each of the shapes a machine can already have it in. +# The last two are the ones that would make this pass for the wrong +# reason: a library on the loader's path, or a variable naming one. +if command -v pkg-config >/dev/null 2>&1 && pkg-config --exists zu; then + echo "pkg-config knows zu on this machine" >&2 + exit 1 +fi +for header in /usr/include/zu.h /usr/local/include/zu.h; do + if [ -e "$header" ]; then + echo "$header is on this machine" >&2 + exit 1 + fi +done +found="$(find / -xdev \( -name 'libzu.so*' -o -name 'libzu.a' -o -name 'libzu.dylib' \) \ + -not -path "$SRC/*" 2>/dev/null || true)" +if [ -n "$found" ]; then + echo "there is already a libzu on this machine:" >&2 + echo "$found" >&2 + exit 1 +fi +if [ -n "${ZU_LIBRARY:-}" ]; then + echo "ZU_LIBRARY is set, which is the other way this passes for the wrong reason" >&2 + exit 1 +fi +echo "no libzu, no header, no pkg-config file, no ZU_LIBRARY" + +step "the program off the front page" +# The same rule the README test uses: a fenced Java block that declares +# a public class is a whole program, and the block after it is what it +# prints. Taken off the page character for character, because a +# quickstart that was retyped here is a quickstart nobody checked. +program="$(awk ' + $0 == "```java" && !taken { inside = 1; count = 0; whole = 0; next } + inside && $0 == "```" { + inside = 0 + if (whole) { for (i = 1; i <= count; i++) print line[i]; taken = 1; exit } + next + } + inside { line[++count] = $0; if ($0 ~ /^public class /) whole = 1 } +' "$readme")" +expected="$(awk ' + $0 == "```java" && !after { inside = 1; whole = 0; next } + inside && $0 == "```" { inside = 0; if (whole) after = 1; next } + inside { if ($0 ~ /^public class /) whole = 1; next } + after && $0 == "```" { if (out) exit; out = 1; next } + out { print } +' "$readme")" +name="$(printf '%s\n' "$program" | sed -n 's/^public class \([A-Za-z0-9_]*\) .*/\1/p' | head -1)" +if [ -z "$name" ] || [ -z "$expected" ]; then + echo "no whole program with an output block on $readme" >&2 + exit 1 +fi +echo "$name, and the ${#expected} characters it says it prints" + +step "the dependencies off the front page" +# Every dependency element the page prints, filtered to the three this +# row installs. Lifting them rather than writing them here is what makes +# a groupId that changed on the page a red job: a reader who copies +# those lines gets exactly what this pom gets. +dependencies="$(awk -v want=" zudb $PROVIDER zudb-native " ' + $0 == "```xml" { inside = 1; next } + inside && $0 == "```" { inside = 0; next } + !inside { next } + // { count = 0; keep = 0 } + { block[++count] = $0 } + // { + id = $0 + sub(/.*/, "", id) + sub(/<\/artifactId>.*/, "", id) + if (index(want, " " id " ")) keep = 1 + } + /<\/dependency>/ { if (keep) for (i = 1; i <= count; i++) print block[i] } +' "$readme")" +for wanted in zudb "$PROVIDER" zudb-native; do + if ! printf '%s\n' "$dependencies" | grep -q "$wanted"; then + echo "$readme prints no dependency on $wanted, so a reader cannot install one" >&2 + exit 1 + fi +done +printf '%s\n' "$dependencies" | grep '' | sed 's/[[:space:]]*//' + +step "a project of two files" +work="$APP/quickstart" +rm -rf "$work" +mkdir -p "$work/src/main/java" +printf '%s\n' "$program" > "$work/src/main/java/$name.java" + +# The staged repository stands in for Maven Central, which is where +# these coordinates resolve from the day the release goes out. Nothing +# else about the pom is special, and that is the claim: a dependency, a +# compiler release, and no plugin of ours anywhere in it. +cat > "$work/pom.xml" < + + 4.0.0 + example + quickstart + 1 + + + UTF-8 + 17 + $VERSION + + + + + staged + file://$REPO + true + false + + + + +$dependencies + + +POM + +step "mvn package, on a machine with no engine on it" +(cd "$work" && mvn package) + +step "what a user's classpath came out as" +(cd "$work" && mvn dependency:copy-dependencies -DincludeScope=runtime -DoutputDirectory=target/lib) +ls -l "$work/target/lib" +jars="$(find "$work/target/lib" -name '*.jar' | wc -l)" +if [ "$jars" -ne 3 ]; then + echo "three jars is what the page asks for and $jars is what arrived" >&2 + exit 1 +fi + +# The library is inside one of them, which is the whole reason a user +# who installed nothing has an engine. The platform asked about is this +# one, because a jar full of libraries for other machines is a jar that +# fails here and passes a test that only counted them. +here="dev/zudb/native/linux-amd64/libzu.so" +if ! jar tf "$work"/target/lib/zudb-native-*.jar | grep -qx "$here"; then + echo "the zudb-native jar has no $here in it, so this machine has no engine" >&2 + jar tf "$work"/target/lib/zudb-native-*.jar | grep libzu >&2 || true + exit 1 +fi +echo "the engine came down inside the jar, at $here" + +step "the program, in a directory of its own" +# From JDK 24 a program that calls native code has to say so when it +# starts, and the flag that says it arrived in 22. Which of those this +# JDK is gets asked of the JDK rather than looked up in a table here, +# because the row that runs on 17 and the row that runs on 25 are meant +# to be the same run with a different image under it. +grant="" +if java --enable-native-access=ALL-UNNAMED -version >/dev/null 2>&1; then + grant="--enable-native-access=ALL-UNNAMED" + echo "this JVM wants the grant, so it gets it: $grant" +else + echo "this JVM has no such flag, which is every JDK before 22" +fi + +# Somewhere else on the machine, empty, because the quickstart writes a +# database beside itself and a directory with one already in it is a +# different program. +run="$APP/run" +rm -rf "$run" +mkdir -p "$run" +set +e +(cd "$run" && java ${grant:+$grant} -cp "$work/target/classes:$work/target/lib/*" "$name" \ + > "$APP/stdout.txt" 2> "$APP/stderr.txt") +code=$? +set -e +if [ -s "$APP/stderr.txt" ]; then + echo "it wrote to stderr, and a first run should not:" >&2 + cat "$APP/stderr.txt" >&2 + exit 1 +fi +if [ "$code" -ne 0 ]; then + echo "$name exited $code" >&2 + exit 1 +fi +cat "$APP/stdout.txt" +if [ "$(cat "$APP/stdout.txt")" != "$expected" ]; then + echo "the page says it prints" >&2 + printf '%s\n' "$expected" >&2 + echo "and it printed" >&2 + cat "$APP/stdout.txt" >&2 + exit 1 +fi +echo "which is what $readme says it prints" + +step "what it cost to get here" +du -sh "$APP/m2/dev/zudb" | cut -f1 | sed 's/$/ of artifacts, one platform of the seven/' + +step "the failure this job is meant to catch, caught" +# A job that has only ever passed is a job nobody has seen fail. The way +# an install goes wrong on a machine like this one is the library: three +# dependencies work and two of them compile, so a reader who left the +# runtime one out has a program that builds and dies. What has to happen +# then is a message naming the line to add, and this is where that gets +# checked. +without="$APP/without" +rm -rf "$without" +cp -r "$work" "$without" +rm -f "$without"/target/lib/zudb-native-*.jar +set +e +(cd "$run" && java ${grant:+$grant} -cp "$without/target/classes:$without/target/lib/*" "$name" \ + > "$APP/without-stdout.txt" 2> "$APP/without-stderr.txt") +code=$? +set -e +if [ "$code" -eq 0 ]; then + echo "it ran with no library artifact on the classpath, so it found one somewhere else" >&2 + exit 1 +fi +said="$(cat "$APP/without-stderr.txt")" +case "$said" in + *"a zudb-native artifact for "*) ;; + *) + echo "it failed without saying which artifact is missing:" >&2 + echo "$said" >&2 + exit 1 + ;; +esac +printf '%s\n' "$said" | grep -o 'no provider could bind libzu.*' | head -c 400 +printf '\n' + +step "installed, on a machine that had nothing" diff --git a/zudb-ffm/src/test/java/dev/zudb/ffm/ReadmeTest.java b/zudb-ffm/src/test/java/dev/zudb/ffm/ReadmeTest.java index c466d24..ed8d7bf 100644 --- a/zudb-ffm/src/test/java/dev/zudb/ffm/ReadmeTest.java +++ b/zudb-ffm/src/test/java/dev/zudb/ffm/ReadmeTest.java @@ -96,9 +96,9 @@ private static String run(Program program) throws Exception { List command = new ArrayList<>(); command.add(Paths.get(System.getProperty("java.home"), "bin", "java").toString()); - // The jars carry Enable-Native-Access in their manifest and a class path - // run off them needs no flag. This runs off a directory of class files, - // which has no manifest to carry it, so the grant is made here instead. + // From JDK 24 the grant belongs to whoever starts the JVM, which here is + // this process starting that one. A manifest carries it only for the jar + // that java -jar names, and a class path run is not that. command.add("--enable-native-access=ALL-UNNAMED"); command.add("-Dzu.library=" + System.getProperty("zu.library")); command.add("-classpath");