diff --git a/.github/actions/build-foundation/action.yml b/.github/actions/build-foundation/action.yml new file mode 100644 index 0000000..4b40290 --- /dev/null +++ b/.github/actions/build-foundation/action.yml @@ -0,0 +1,33 @@ +name: Build DataWeave foundation +description: >- + Shared foundation for the DataWeave CLI workflows: sets up Gradle and GraalVM, + then runs the single `gradlew build` that compiles both native images (the CLI + `dw` and the `dwlib` shared library). Callers must run actions/checkout BEFORE + this action, then invoke the per-artifact actions (cli/python/node/native-lib) + AFTER it in the same job so Gradle up-to-date checks reuse the compiled images. +inputs: + github-token: + description: Token for graalvm/setup-graalvm component downloads. + required: true + native-version: + description: >- + Value passed as -PnativeVersion to Gradle. Empty string omits the flag + (the build then uses the nativeVersion default). + required: false + default: '' +runs: + using: composite + steps: + - name: Setup Gradle + uses: gradle/actions/setup-gradle@v3 + + - name: Setup Graalvm + uses: graalvm/setup-graalvm@v1 + with: + java-version: '24' + distribution: 'graalvm-community' + github-token: ${{ inputs.github-token }} + + - name: Run Build + run: ./gradlew --stacktrace --no-problems-report -PskipNodeTests=true build ${{ inputs.native-version != '' && format('-PnativeVersion={0}', inputs.native-version) || '' }} + shell: bash diff --git a/.github/actions/cli/action.yml b/.github/actions/cli/action.yml new file mode 100644 index 0000000..39636c9 --- /dev/null +++ b/.github/actions/cli/action.yml @@ -0,0 +1,77 @@ +name: CLI artifact +description: >- + Produces the DataWeave `dw` CLI distro zip; optionally runs the master-only + native-cli regression/TCK suites; optionally publishes the distro as a CI + artifact (main) or a release asset (release). Requires build-foundation to + have run earlier in the same job. +inputs: + native-version: + description: -PnativeVersion value; empty omits the flag. + required: false + default: '' + run-tck: + description: When 'true', run the master-only native-cli regression suites. + required: false + default: 'false' + publish: + description: "'none' | 'artifact' | 'release'." + required: false + default: 'none' + repo-token: + description: Token for svenstaro release upload (publish=release). + required: false + default: '' + arch: + description: Runtime arch token for artifact/asset names. + required: false + default: '' + script-name: + description: OS naming token (linux/windows/macos) for artifact/asset names. + required: false + default: '' + distro-os: + description: Gradle distro classifier (linux/windows/osx) for the source zip name. + required: false + default: '' + tag: + description: Release tag (publish=release). + required: false + default: '' +runs: + using: composite + steps: + - name: Create Distro + run: ./gradlew --stacktrace --no-problems-report native-cli:distro ${{ inputs.native-version != '' && format('-PnativeVersion={0}', inputs.native-version) || '' }} + shell: bash + + - name: Run regression test 2.12.2-SNAPSHOT + if: inputs.run-tck == 'true' + run: ./gradlew --stacktrace -PweaveTestSuiteVersion=2.12.2-SNAPSHOT -DweaveSuiteVersion=2.12.2-SNAPSHOT native-cli-integration-tests:test + shell: bash + + - name: Run regression test 2.13.0-SNAPSHOT + if: inputs.run-tck == 'true' + run: ./gradlew --stacktrace -PweaveTestSuiteVersion=2.13.0-SNAPSHOT -DweaveSuiteVersion=2.13.0-SNAPSHOT native-cli-integration-tests:test + shell: bash + + - name: Stage renamed CLI distro + if: inputs.publish == 'artifact' + run: cp "native-cli/build/distributions/native-cli-${{ inputs.native-version }}-native-distro-${{ inputs.distro-os }}.zip" "native-cli/build/distributions/dw-cli-${{ inputs.native-version }}-${{ inputs.script-name }}-${{ inputs.arch }}.zip" + shell: bash + + - name: Upload CLI distro (artifact) + if: inputs.publish == 'artifact' + uses: actions/upload-artifact@v7.0.1 + with: + path: native-cli/build/distributions/dw-cli-${{ inputs.native-version }}-${{ inputs.script-name }}-${{ inputs.arch }}.zip + archive: false + + - name: Upload binaries to release + if: inputs.publish == 'release' + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ inputs.repo-token }} + file: native-cli/build/distributions/native-cli-${{ inputs.native-version }}-native-distro-${{ inputs.distro-os }}.zip + asset_name: dw-cli-${{ inputs.native-version }}-${{ inputs.script-name }}-${{ inputs.arch }}.zip + tag: ${{ inputs.tag }} + overwrite: true diff --git a/.github/actions/native-lib/action.yml b/.github/actions/native-lib/action.yml new file mode 100644 index 0000000..e644da2 --- /dev/null +++ b/.github/actions/native-lib/action.yml @@ -0,0 +1,88 @@ +name: Native shared library artifact +description: >- + Stages and publishes the raw dwlib shared library (dwlib.so/.dll/.dylib) and + its C header (dwlib.h) — the FFI surface both the Python and Node bindings + embed, published here as standalone downloads. Requires build-foundation + earlier in the same job. +inputs: + native-version: + description: -PnativeVersion value; empty omits the flag. + required: false + default: '' + publish: + description: "'none' | 'artifact' | 'release'." + required: false + default: 'none' + repo-token: + description: Token for svenstaro release upload (publish=release). + required: false + default: '' + arch: + description: Runtime arch token for artifact/asset names. + required: false + default: '' + script-name: + description: OS naming token (linux/windows/macos) for artifact/asset names. + required: false + default: '' + tag: + description: Release tag (publish=release). + required: false + default: '' +runs: + using: composite + steps: + - name: Stage native shared library + run: ./gradlew --stacktrace --no-problems-report native-lib:stagePythonNativeLib ${{ inputs.native-version != '' && format('-PnativeVersion={0}', inputs.native-version) || '' }} + shell: bash + + - name: Upload native shared library (artifact) + if: inputs.publish == 'artifact' + uses: actions/upload-artifact@v7.0.1 + with: + name: dwlib-${{ inputs.native-version }}-${{ inputs.script-name }}-${{ inputs.arch }} + path: | + native-lib/python/src/dataweave/native/dwlib.dylib + native-lib/python/src/dataweave/native/dwlib.so + native-lib/python/src/dataweave/native/dwlib.dll + native-lib/python/src/dataweave/native/dwlib.h + + - name: Upload native shared library to release (Linux) + if: inputs.publish == 'release' && runner.os == 'Linux' + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ inputs.repo-token }} + file: native-lib/python/src/dataweave/native/dwlib.so + asset_name: dwlib-${{ inputs.native-version }}-${{ inputs.script-name }}-${{ inputs.arch }}.so + tag: ${{ inputs.tag }} + overwrite: true + + - name: Upload native shared library to release (Windows) + if: inputs.publish == 'release' && runner.os == 'Windows' + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ inputs.repo-token }} + file: native-lib/python/src/dataweave/native/dwlib.dll + asset_name: dwlib-${{ inputs.native-version }}-${{ inputs.script-name }}-${{ inputs.arch }}.dll + tag: ${{ inputs.tag }} + overwrite: true + + - name: Upload native shared library to release (macOS) + if: inputs.publish == 'release' && runner.os == 'macOS' + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ inputs.repo-token }} + file: native-lib/python/src/dataweave/native/dwlib.dylib + asset_name: dwlib-${{ inputs.native-version }}-${{ inputs.script-name }}-${{ inputs.arch }}.dylib + tag: ${{ inputs.tag }} + overwrite: true + + - name: Upload native library header to release + if: inputs.publish == 'release' + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ inputs.repo-token }} + file: native-lib/python/src/dataweave/native/dwlib.h + asset_name: dwlib-${{ inputs.native-version }}.h + tag: ${{ inputs.tag }} + overwrite: true diff --git a/.github/actions/node/action.yml b/.github/actions/node/action.yml new file mode 100644 index 0000000..b27cea5 --- /dev/null +++ b/.github/actions/node/action.yml @@ -0,0 +1,87 @@ +name: Node artifact +description: >- + Sets up Node.js, builds the DataWeave Node package (which embeds dwlib), and + runs the Node unit/integration tests (always). Optionally runs the master-only + Node TCK conformance lane, and optionally publishes the .tgz as a CI artifact + (main) or a release asset (release). Requires build-foundation earlier in the + same job. +inputs: + native-version: + description: -PnativeVersion value; empty omits the flag. + required: false + default: '' + run-tck: + description: When 'true', run the master-only Node TCK conformance lane. + required: false + default: 'false' + publish: + description: "'none' | 'artifact' | 'release'." + required: false + default: 'none' + repo-token: + description: Token for svenstaro release upload (publish=release). + required: false + default: '' + arch: + description: Runtime arch token for artifact/asset names. + required: false + default: '' + script-name: + description: OS naming token (linux/windows/macos) for artifact/asset names. + required: false + default: '' + tag: + description: Release tag (publish=release). + required: false + default: '' +runs: + using: composite + steps: + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: '18' + + - name: Create Native Lib Node Package + run: ./gradlew --stacktrace --no-problems-report native-lib:buildNodePackage ${{ inputs.native-version != '' && format('-PnativeVersion={0}', inputs.native-version) || '' }} + shell: bash + + - name: Run Node.js Tests + run: ./gradlew --stacktrace --no-problems-report native-lib:nodeTest ${{ inputs.native-version != '' && format('-PnativeVersion={0}', inputs.native-version) || '' }} + shell: bash + + - name: Run Node.js TCK Conformance 2.12.2-SNAPSHOT + if: inputs.run-tck == 'true' + run: | + ./gradlew --stacktrace --no-problems-report -PweaveTestSuiteVersion=2.12.2-SNAPSHOT native-lib:stageTckSuites + cd native-lib/node && npm run test:tck + shell: bash + + - name: Run Node.js TCK Conformance 2.13.0-SNAPSHOT + if: inputs.run-tck == 'true' + run: | + ./gradlew --stacktrace --no-problems-report -PweaveTestSuiteVersion=2.13.0-SNAPSHOT native-lib:stageTckSuites + cd native-lib/node && npm run test:tck + shell: bash + + - name: Stage OS-qualified Node package + if: inputs.publish == 'artifact' + run: cp native-lib/node/dataweave-native-0.0.1.tgz "native-lib/node/dataweave-node-0.0.1-${{ inputs.script-name }}-${{ inputs.arch }}.tgz" + shell: bash + + - name: Upload Node package (artifact) + if: inputs.publish == 'artifact' + uses: actions/upload-artifact@v7.0.1 + with: + path: native-lib/node/dataweave-node-0.0.1-${{ inputs.script-name }}-${{ inputs.arch }}.tgz + archive: false + + - name: Upload Node package to release + if: inputs.publish == 'release' + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ inputs.repo-token }} + file: native-lib/node/dataweave-native-0.0.1.tgz + asset_name: dataweave-node-0.0.1-${{ inputs.script-name }}-${{ inputs.arch }}.tgz + tag: ${{ inputs.tag }} + overwrite: true diff --git a/.github/actions/python/action.yml b/.github/actions/python/action.yml new file mode 100644 index 0000000..760fc03 --- /dev/null +++ b/.github/actions/python/action.yml @@ -0,0 +1,57 @@ +name: Python artifact +description: >- + Installs Python build dependencies and builds the DataWeave Python wheel + (which embeds dwlib); optionally publishes the wheel as a CI artifact (main) + or a release asset (release). No TCK phase today. Requires build-foundation to + have run earlier in the same job. +inputs: + native-version: + description: -PnativeVersion value; empty omits the flag. + required: false + default: '' + break-system-packages: + description: >- + When 'true', add --break-system-packages to the pip install (required on + GitHub-hosted runners per PEP 668; not needed on self-hosted mulesoft + runners). + required: false + default: 'false' + publish: + description: "'none' | 'artifact' | 'release'." + required: false + default: 'none' + repo-token: + description: Token for svenstaro release upload (publish=release). + required: false + default: '' + tag: + description: Release tag (publish=release). + required: false + default: '' +runs: + using: composite + steps: + - name: Install Python build dependencies + run: python3 -m pip install ${{ inputs.break-system-packages == 'true' && '--break-system-packages' || '' }} --upgrade setuptools wheel + shell: bash + + - name: Create Native Lib Python Wheel + run: ./gradlew --stacktrace --no-problems-report native-lib:buildPythonWheel ${{ inputs.native-version != '' && format('-PnativeVersion={0}', inputs.native-version) || '' }} + shell: bash + + - name: Upload Python wheel (artifact) + if: inputs.publish == 'artifact' + uses: actions/upload-artifact@v7.0.1 + with: + path: native-lib/python/dist/dataweave_native-0.0.1-py3-*.whl + archive: false + + - name: Upload Python wheel to release + if: inputs.publish == 'release' + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ inputs.repo-token }} + file: native-lib/python/dist/dataweave_native-0.0.1-py3-none-*.whl + file_glob: true + tag: ${{ inputs.tag }} + overwrite: true diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bedae26..97cb2b3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -22,50 +22,19 @@ jobs: # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it - uses: actions/checkout@v4 - - name: Setup Gradle - uses: gradle/actions/setup-gradle@v3 - - # Setup Graalvm - - name: Setup Graalvm - uses: graalvm/setup-graalvm@v1 + - name: Build foundation + uses: ./.github/actions/build-foundation with: - java-version: '24' - distribution: 'graalvm-community' github-token: ${{ secrets.GITHUB_TOKEN }} - # Runs a single command using the runners shell - - name: Run Build (Latest) - run: | - ./gradlew --stacktrace --no-problems-report -PskipNodeTests=true build - shell: bash - - # Generate distro - - name: Create Distro - run: ./gradlew --stacktrace --no-problems-report native-cli:distro - shell: bash - - # Install Python build dependencies (setuptools/wheel may be missing on Windows runners) - - name: Install Python build dependencies - run: python3 -m pip install --upgrade setuptools wheel - shell: bash + - name: CLI + uses: ./.github/actions/cli - # Generate native-lib python wheel - - name: Create Native Lib Python Wheel - run: ./gradlew --stacktrace --no-problems-report native-lib:buildPythonWheel - shell: bash - - # Setup Node.js for native-lib Node package - - name: Setup Node.js - uses: actions/setup-node@v4 - with: - node-version: '18' + - name: Python + uses: ./.github/actions/python - # Stage the native lib and build Node package (npm install, node-gyp, tsc, npm pack) - - name: Create Native Lib Node Package - run: ./gradlew --stacktrace --no-problems-report native-lib:buildNodePackage - shell: bash + - name: Node + uses: ./.github/actions/node - # Run Node.js tests - - name: Run Node.js Tests - run: ./gradlew --stacktrace --no-problems-report native-lib:nodeTest - shell: bash + - name: Native library + uses: ./.github/actions/native-lib diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 1a1a255..7ca64c0 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -39,148 +39,45 @@ jobs: # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it - uses: actions/checkout@v4 - - name: Setup Gradle - uses: gradle/actions/setup-gradle@v3 - - # Setup Graalvm - - name: Setup Graalvm - uses: graalvm/setup-graalvm@v1 - with: - java-version: '24' - distribution: 'graalvm-community' - github-token: ${{ secrets.GITHUB_TOKEN }} - - # Runs a single command using the runners shell - - name: Run Build - run: | - ./gradlew --stacktrace --no-problems-report -PskipNodeTests=true build - shell: bash - # Run regression tests (only on master branch to save CI time on PRs) - # TCK artifact available in 2.12.2-SNAPSHOT and 2.13.0-SNAPSHOT - - name: Run regression test 2.12.2-SNAPSHOT - if: github.ref == 'refs/heads/master' - run: | - ./gradlew --stacktrace -PweaveTestSuiteVersion=2.12.2-SNAPSHOT -DweaveSuiteVersion=2.12.2-SNAPSHOT native-cli-integration-tests:test - shell: bash - - name: Run regression test 2.13.0-SNAPSHOT - if: github.ref == 'refs/heads/master' - run: | - ./gradlew --stacktrace -PweaveTestSuiteVersion=2.13.0-SNAPSHOT -DweaveSuiteVersion=2.13.0-SNAPSHOT native-cli-integration-tests:test - shell: bash - - # Generate distro - - name: Create Distro - run: ./gradlew --stacktrace --no-problems-report native-cli:distro - shell: bash - - # Install Python build dependencies (setuptools/wheel may be missing on Windows runners) - - name: Install Python build dependencies - run: python3 -m pip install --break-system-packages --upgrade setuptools wheel - shell: bash - - # Generate native-lib python wheel - - name: Create Native Lib Python Wheel - run: ./gradlew --stacktrace --no-problems-report native-lib:buildPythonWheel - shell: bash - - # Setup Node.js for native-lib Node package - - name: Setup Node.js - uses: actions/setup-node@v4 - with: - node-version: '18' - - # Stage the native lib and build Node package (npm install, node-gyp, tsc, npm pack) - - name: Create Native Lib Node Package - run: ./gradlew --stacktrace --no-problems-report native-lib:buildNodePackage - shell: bash - - # Run Node.js tests - - name: Run Node.js Tests - run: ./gradlew --stacktrace --no-problems-report native-lib:nodeTest - shell: bash - - # Run the Node.js TCK conformance lane (only on master to save CI time on - # PRs — mirrors the native-cli regression gating). Stages the DataWeave - # tck@zip corpus per supported version and runs the tck vitest project - # against the built package. The Node package was already built by "Create - # Native Lib Node Package" above; dwlib is the single 2.13.0 build and the - # 2.12.2 corpus replays against it (the binding exposes no language-level). - - name: Run Node.js TCK Conformance 2.12.2-SNAPSHOT - if: github.ref == 'refs/heads/master' - run: | - ./gradlew --stacktrace --no-problems-report -PweaveTestSuiteVersion=2.12.2-SNAPSHOT native-lib:stageTckSuites - cd native-lib/node && npm run test:tck - shell: bash - - name: Run Node.js TCK Conformance 2.13.0-SNAPSHOT - if: github.ref == 'refs/heads/master' - run: | - ./gradlew --stacktrace --no-problems-report -PweaveTestSuiteVersion=2.13.0-SNAPSHOT native-lib:stageTckSuites - cd native-lib/node && npm run test:tck - shell: bash - - # Derive lowercase OS + runtime arch tokens for artifact names. - # OS comes from matrix.script_name (linux/windows); arch from uname -m - # (x86_64 on linux/windows, arm64 on macos). - name: Derive platform tokens run: echo "ARCH=$(uname -m)" >> "$GITHUB_ENV" shell: bash - # archive:false makes the artifact name equal the uploaded file name. - # Copy the Gradle-named distro zip to the convention name so the CLI - # artifact reads dw-cli---.zip. - - name: Stage renamed CLI distro - run: cp "native-cli/build/distributions/native-cli-${{env.NATIVE_VERSION}}-native-distro-${{ matrix.distro_os }}.zip" "native-cli/build/distributions/dw-cli-${{env.NATIVE_VERSION}}-${{ matrix.script_name }}-${{ env.ARCH }}.zip" - shell: bash - - # Upload the artifact file - - name: Upload generated script - uses: actions/upload-artifact@v7.0.1 + - name: Build foundation + uses: ./.github/actions/build-foundation with: - # archive:false skips the redundant outer zip (the distro is already a - # .zip) and names the artifact after the file — which already carries - # ${matrix.script_name}, so the matrix legs don't collide. No `name:`: - # it would be ignored under archive:false. - path: native-cli/build/distributions/dw-cli-${{env.NATIVE_VERSION}}-${{ matrix.script_name }}-${{ env.ARCH }}.zip - archive: false + github-token: ${{ secrets.GITHUB_TOKEN }} - # Upload the Python wheel - - name: Upload Python wheel - uses: actions/upload-artifact@v7.0.1 + - name: CLI + uses: ./.github/actions/cli with: - # archive:false skips the redundant outer zip and names the artifact - # after the wheel — whose platform tag (manylinux/win_amd64/…) is - # already OS-unique, so the matrix legs don't collide. No `name:`: it - # would be ignored under archive:false. - path: native-lib/python/dist/dataweave_native-0.0.1-py3-*.whl - archive: false - - # npm pack emits the same filename (dataweave-native-0.0.1.tgz) on every - # OS. With archive:false the file NAME becomes the artifact name (the - # `name:` input is ignored), so copy to an OS-qualified name first to keep - # the matrix legs from colliding. - - name: Stage OS-qualified Node package - run: cp native-lib/node/dataweave-native-0.0.1.tgz "native-lib/node/dataweave-node-0.0.1-${{ matrix.script_name }}-${{ env.ARCH }}.tgz" - shell: bash - - # Upload the Node.js package - - name: Upload Node package - uses: actions/upload-artifact@v7.0.1 + native-version: ${{ env.NATIVE_VERSION }} + run-tck: ${{ github.ref == 'refs/heads/master' }} + publish: 'artifact' + arch: ${{ env.ARCH }} + script-name: ${{ matrix.script_name }} + distro-os: ${{ matrix.distro_os }} + + - name: Python + uses: ./.github/actions/python with: - path: native-lib/node/dataweave-node-0.0.1-${{ matrix.script_name }}-${{ env.ARCH }}.tgz - # Single .tgz (already gzip-compressed); skip the redundant outer zip - # (v7+ feature). archive:false ignores `name:` and uses the file name, - # which the copy above made OS-unique. - archive: false + native-version: ${{ env.NATIVE_VERSION }} + break-system-packages: 'true' + publish: 'artifact' - # Upload the native shared library + header together per OS. Multiple - # files, so this stays archived (archive:false allows only one file); the - # zip is wanted here and `name:` still applies. - - name: Upload native shared library - uses: actions/upload-artifact@v7.0.1 + - name: Node + uses: ./.github/actions/node + with: + native-version: ${{ env.NATIVE_VERSION }} + run-tck: ${{ github.ref == 'refs/heads/master' }} + publish: 'artifact' + arch: ${{ env.ARCH }} + script-name: ${{ matrix.script_name }} + + - name: Native library + uses: ./.github/actions/native-lib with: - name: dwlib-${{env.NATIVE_VERSION}}-${{ matrix.script_name }}-${{ env.ARCH }} - path: | - native-lib/python/src/dataweave/native/dwlib.dylib - native-lib/python/src/dataweave/native/dwlib.so - native-lib/python/src/dataweave/native/dwlib.dll - native-lib/python/src/dataweave/native/dwlib.h + native-version: ${{ env.NATIVE_VERSION }} + publish: 'artifact' + arch: ${{ env.ARCH }} + script-name: ${{ matrix.script_name }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 01e7cf7..4239bd5 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -34,127 +34,54 @@ jobs: # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it - uses: actions/checkout@v4 - - name: Setup Gradle - uses: gradle/actions/setup-gradle@v3 - - # Setup Graalvm - - name: Setup Graalvm - uses: graalvm/setup-graalvm@v1 - with: - java-version: '24' - distribution: 'graalvm-community' - github-token: ${{ secrets.GITHUB_TOKEN }} - - name: Guess Extension Version run: | echo "NATIVE_VERSION=$(echo '${{github.ref}}' | sed -e 's,.*/v\(.*\),\1,')" >> $GITHUB_ENV echo "ARCH=$(uname -m)" >> $GITHUB_ENV shell: bash - # Runs a single command using the runners shell - - name: Run Build - run: | - ./gradlew --stacktrace --no-problems-report -PskipNodeTests=true build -PnativeVersion=${{env.NATIVE_VERSION}} - shell: bash - - # Generate distro - - name: Create Distro - run: ./gradlew --stacktrace --no-problems-report native-cli:distro -PnativeVersion=${{env.NATIVE_VERSION}} - shell: bash - - # Install Python build dependencies (setuptools/wheel may be missing on Windows runners) - - name: Install Python build dependencies - run: python3 -m pip install --break-system-packages --upgrade setuptools wheel - shell: bash - - # Generate native-lib python wheel - - name: Create Native Lib Python Wheel - run: ./gradlew --stacktrace --no-problems-report native-lib:buildPythonWheel -PnativeVersion=${{env.NATIVE_VERSION}} - shell: bash - - # Setup Node.js for native-lib Node package - - name: Setup Node.js - uses: actions/setup-node@v4 + - name: Build foundation + uses: ./.github/actions/build-foundation with: - node-version: '18' - - # Stage the native lib and build Node package (npm install, node-gyp, tsc, npm pack) - - name: Create Native Lib Node Package - run: ./gradlew --stacktrace --no-problems-report native-lib:buildNodePackage -PnativeVersion=${{env.NATIVE_VERSION}} - shell: bash - - # Run Node.js tests - - name: Run Node.js Tests - run: ./gradlew --stacktrace --no-problems-report native-lib:nodeTest -PnativeVersion=${{env.NATIVE_VERSION}} - shell: bash - - # Upload the artifact file - - name: Upload binaries to release - uses: svenstaro/upload-release-action@v2 - with: - repo_token: ${{ secrets.GITHUB_TOKEN }} - file: native-cli/build/distributions/native-cli-${{env.NATIVE_VERSION}}-native-distro-${{ matrix.distro_os }}.zip - asset_name: dw-cli-${{env.NATIVE_VERSION}}-${{ matrix.script_name }}-${{ env.ARCH }}.zip - tag: ${{ github.ref }} - overwrite: true - - # Upload the Python wheel - - name: Upload Python wheel to release - uses: svenstaro/upload-release-action@v2 - with: - repo_token: ${{ secrets.GITHUB_TOKEN }} - file: native-lib/python/dist/dataweave_native-0.0.1-py3-none-*.whl - file_glob: true - tag: ${{ github.ref }} - overwrite: true - - # Upload the Node.js package - - name: Upload Node package to release - uses: svenstaro/upload-release-action@v2 - with: - repo_token: ${{ secrets.GITHUB_TOKEN }} - file: native-lib/node/dataweave-native-0.0.1.tgz - asset_name: dataweave-node-0.0.1-${{ matrix.script_name }}-${{ env.ARCH }}.tgz - tag: ${{ github.ref }} - overwrite: true + github-token: ${{ secrets.GITHUB_TOKEN }} + native-version: ${{ env.NATIVE_VERSION }} - # Upload the native shared library - - name: Upload native shared library to release (Linux) - if: runner.os == 'Linux' - uses: svenstaro/upload-release-action@v2 + - name: CLI + uses: ./.github/actions/cli with: - repo_token: ${{ secrets.GITHUB_TOKEN }} - file: native-lib/python/src/dataweave/native/dwlib.so - asset_name: dwlib-${{env.NATIVE_VERSION}}-${{ matrix.script_name }}-${{ env.ARCH }}.so + native-version: ${{ env.NATIVE_VERSION }} + publish: 'release' + repo-token: ${{ secrets.GITHUB_TOKEN }} tag: ${{ github.ref }} - overwrite: true + arch: ${{ env.ARCH }} + script-name: ${{ matrix.script_name }} + distro-os: ${{ matrix.distro_os }} - - name: Upload native shared library to release (Windows) - if: runner.os == 'Windows' - uses: svenstaro/upload-release-action@v2 + - name: Python + uses: ./.github/actions/python with: - repo_token: ${{ secrets.GITHUB_TOKEN }} - file: native-lib/python/src/dataweave/native/dwlib.dll - asset_name: dwlib-${{env.NATIVE_VERSION}}-${{ matrix.script_name }}-${{ env.ARCH }}.dll + native-version: ${{ env.NATIVE_VERSION }} + break-system-packages: 'true' + publish: 'release' + repo-token: ${{ secrets.GITHUB_TOKEN }} tag: ${{ github.ref }} - overwrite: true - - name: Upload native shared library to release (macOS) - if: runner.os == 'macOS' - uses: svenstaro/upload-release-action@v2 + - name: Node + uses: ./.github/actions/node with: - repo_token: ${{ secrets.GITHUB_TOKEN }} - file: native-lib/python/src/dataweave/native/dwlib.dylib - asset_name: dwlib-${{env.NATIVE_VERSION}}-${{ matrix.script_name }}-${{ env.ARCH }}.dylib + native-version: ${{ env.NATIVE_VERSION }} + publish: 'release' + repo-token: ${{ secrets.GITHUB_TOKEN }} tag: ${{ github.ref }} - overwrite: true + arch: ${{ env.ARCH }} + script-name: ${{ matrix.script_name }} - # Upload the native library header - - name: Upload native library header to release - uses: svenstaro/upload-release-action@v2 + - name: Native library + uses: ./.github/actions/native-lib with: - repo_token: ${{ secrets.GITHUB_TOKEN }} - file: native-lib/python/src/dataweave/native/dwlib.h - asset_name: dwlib-${{env.NATIVE_VERSION}}.h + native-version: ${{ env.NATIVE_VERSION }} + publish: 'release' + repo-token: ${{ secrets.GITHUB_TOKEN }} tag: ${{ github.ref }} - overwrite: true + arch: ${{ env.ARCH }} + script-name: ${{ matrix.script_name }} diff --git a/docs/superpowers/specs/2026-07-30-ci-composite-action-design.md b/docs/superpowers/specs/2026-07-30-ci-composite-action-design.md new file mode 100644 index 0000000..83893ab --- /dev/null +++ b/docs/superpowers/specs/2026-07-30-ci-composite-action-design.md @@ -0,0 +1,165 @@ +# CI `build-native` Composite Action + +**Date:** 2026-07-30 + +## Goal + +Remove the duplicated build prefix across the three GitHub Actions workflows +(`ci.yml`, `main.yml`, `release.yml`) by extracting it into a single composite +action — one source of truth for how the DataWeave native artifacts (CLI +distro, Python wheel, Node package, native lib) are built and tested in CI. + +## Context + +The three workflows each repeat the same ~9-step build prefix verbatim: +setup Gradle, setup GraalVM, run build, create distro, install Python build +deps, build the Python wheel, setup Node, build the Node package, run Node +tests. Every fix to that prefix (recent examples: `--break-system-packages` +for macOS PEP 668, the `windows-2022`/`distro_os` matrix work) currently has to +be applied in up to three places. + +The actual build *logic* already lives in Gradle tasks (`nativeCompile`, +`distro`, `buildPythonWheel`, `buildNodePackage`); the YAML only invokes them. +So the duplication being removed is the **CI orchestration** of those tasks, +not the build logic itself. + +This is a follow-up to the runner-migration work +(`2026-07-29-github-runners-migration-design.md`) and is delivered as a +**separate branch + PR** (`ci-composite-action`, stacked on +`runners-migration`). + +## Current state + +Shared prefix present in all three workflows (identical except `release.yml` +threads `-PnativeVersion=` and `ci.yml`'s pip step omits +`--break-system-packages`): + +1. Setup Gradle (`gradle/actions/setup-gradle@v3`) +2. Setup GraalVM (`graalvm/setup-graalvm@v1`, Java 24, `graalvm-community`) +3. Run Build (`./gradlew … -PskipNodeTests=true build`) +4. Create Distro (`native-cli:distro`) +5. Install Python build dependencies (`pip … setuptools wheel`) +6. Create Native Lib Python Wheel (`native-lib:buildPythonWheel`) +7. Setup Node.js (`actions/setup-node@v4`, node 18) +8. Create Native Lib Node Package (`native-lib:buildNodePackage`) +9. Run Node.js Tests (`native-lib:nodeTest`) + +Divergent tails (stay in each workflow, unchanged): + +- `ci.yml` — nothing after the prefix (build + test only). +- `main.yml` — master-only regression tests (currently **interleaved** between + Run Build and Create Distro) + master-only Node TCK conformance (after Run + Node.js Tests), then artifact staging + `upload-artifact` steps. +- `release.yml` — tag-version derivation (`NATIVE_VERSION`, `ARCH`) and + `svenstaro/upload-release-action` release-asset uploads (incl. the per-OS + `dwlib` steps). + +## Design + +### The composite action + +Location: `.github/actions/build-native/action.yml` +(`runs.using: "composite"`). + +It owns the 9-step prefix as one contiguous block, each step preserved as a +separate step (so per-target failure granularity stays visible in the CI UI). +`actions/checkout` is **not** in the action — it must run first in the calling +job so the action's own files are on disk. + +**Inputs:** + +```yaml +inputs: + github-token: + description: Token for graalvm/setup-graalvm component downloads. + required: true + native-version: + description: Passed as -PnativeVersion to Gradle. Empty string = omit the flag. + required: false + default: '' + break-system-packages: + description: >- + Add --break-system-packages to the pip install (needed on GitHub-hosted + macOS per PEP 668; not needed on the self-hosted mulesoft runners). + required: false + default: 'false' +``` + +Composite actions cannot read `secrets` or the caller's matrix directly, which +is why `github-token` is an input rather than a secret reference. + +**Conditional `-PnativeVersion`** — the four Gradle steps (build, distro, +wheel, node package) use: + +```yaml +run: ./gradlew … ${{ inputs.native-version != '' && format('-PnativeVersion={0}', inputs.native-version) || '' }} +shell: bash +``` + +Empty input → bare command (today's `ci`/`main` behavior); non-empty → +`-PnativeVersion=` (today's `release` behavior). + +**Conditional pip flag:** + +```yaml +run: python3 -m pip install ${{ inputs.break-system-packages == 'true' && '--break-system-packages' || '' }} --upgrade setuptools wheel +shell: bash +``` + +Every `run` step in a composite action **must** declare `shell: bash`. + +### Caller wiring + +| Input | `ci.yml` | `main.yml` | `release.yml` | +|---|---|---|---| +| `github-token` | `${{ secrets.GITHUB_TOKEN }}` | same | same | +| `native-version` | `''` (omit) | `''` (omit) | `${{ env.NATIVE_VERSION }}` | +| `break-system-packages` | `'false'` | `'true'` | `'true'` | + +Note: `main.yml` intentionally does **not** thread a native version — it relies +on the `NATIVE_VERSION: 100.100.100` workflow-env default today, so +`native-version: ''` preserves current behavior exactly. + +Each job becomes: `actions/checkout` → `uses: ./.github/actions/build-native` +(with the inputs above) → the workflow's own divergent tail. + +### Ordering change in `main.yml` + +`main.yml`'s master-only regression steps are currently interleaved *inside* +the prefix (between Run Build and Create Distro). To let the action own one +contiguous block, the master-only regression + TCK steps **move to after** the +action call. + +This is functionally identical: the regression tests only need the compiled +CLI that Run Build already produced, and Gradle's up-to-date checks mean +running them after the Node build does not recompile anything. Only the CI +step *list order* changes. + +## Risks / notes + +- **Local-path action requires checkout first.** `uses: ./.github/actions/…` + resolves against the checked-out workspace, so `actions/checkout` must be the + first step in every calling job (it already is). +- **`shell:` is mandatory** on composite-action `run` steps — omitting it is a + load-bearing gotcha (the action fails to parse). +- Expression fallbacks (`… && X || ''`) are how the conditional flag/version + args are injected without duplicating whole steps; verify the rendered + command matches the pre-refactor command per workflow. +- `ci.yml` runs on `mulesoft-*` runners; the action runs on whatever runner the + calling job selects, so mixed runner types are fine. + +## Testing + +No unit tests apply. Validation is: + +- YAML parse for `action.yml` and all three workflows. +- Confirm the rendered Gradle commands per workflow match the pre-refactor + commands: `ci`/`main` have no `-PnativeVersion`; `release` has + `-PnativeVersion=` on build, distro, wheel, and node-package steps. +- Confirm the pip step renders with `--break-system-packages` for + `main`/`release` and without it for `ci`. +- Confirm each workflow still has its divergent tail intact (ci: none; + main: regression/TCK + upload-artifact; release: svenstaro uploads). +- End-to-end: the `Build Native CLI` (`main.yml`) run is green on all three + legs after the refactor; `ci.yml` and `release.yml` parse and their step + lists are unchanged in behavior. diff --git a/docs/superpowers/specs/2026-07-30-ci-layered-actions-design.md b/docs/superpowers/specs/2026-07-30-ci-layered-actions-design.md new file mode 100644 index 0000000..710f719 --- /dev/null +++ b/docs/superpowers/specs/2026-07-30-ci-layered-actions-design.md @@ -0,0 +1,214 @@ +# CI Layered Actions: Foundation + Per-Artifact Actions + +**Date:** 2026-07-30 + +## Goal + +Restructure CI so **adding a new product artifact touches the workflows as +little as possible** and **everything about one artifact lives in one place**. + +Two layers: + +- **One shared foundation action** — Gradle + GraalVM setup and the single + `gradlew build` that compiles both native images. +- **One action per artifact** (`cli`, `python`, `node`, later `go`/`rust`) — + each owns its full lifecycle (produce → optional test/tck → optional + publish), with the phases selected by inputs. + +Target outcome: a workflow reads `checkout → build-foundation → cli → python → +node`, one call per artifact, and adding **Go** is *one new action directory + +one line per workflow that wants it*. + +## Context & motivation + +PR #150 (`build-native`) removed the duplicated build prefix but is a monolith +that bakes Node- and Python-specific setup into a "generic" block. The roadmap +adds Go/Rust bindings; we want new artifacts to be drop-in. This evolves PR #150 +in place (branch `ci-composite-action`, PR #150) — master never carries the +monolith. + +### GitHub Actions constraint (decisive for the shape) + +A composite action is **one directory / one `action.yml` / one callable unit** — +you cannot define several named sub-actions (`produce-cli`, `upload-cli`, …) in +one file. The other primitive, a reusable *workflow* (`workflow_call`), runs as +its **own job on its own runner**, so it would not see the native images the +foundation compiled without uploading/downloading them between jobs — which +breaks "build once, package many." + +Therefore each artifact is **one composite action** that performs all its +phases, gated by inputs (Shape A). Naming is **by artifact, not phase** (`cli`, +not `produce-cli`), because the action does more than produce. + +### Gradle dependency facts (verified in `native-lib/build.gradle`) + +The layering is safe because packaging/test tasks declare their own +dependencies back to compilation, and Gradle up-to-date checks prevent +recompilation: + +- `native-cli:distro` → `nativeCompile` (CLI image). +- `native-lib:buildPythonWheel` → `stagePythonNativeLib` → `stripNativeLibrary` + → `nativeCompile` (`dwlib`). +- `native-lib:buildNodePackage` / `nodeTest` → `stageNodeNativeLib` → + `stripNativeLibrary` → `nativeCompile`. + +`gradlew build` in the foundation compiles both native images once; +`setup-gradle`/`setup-graalvm` configure the **job** env (PATH, `JAVA_HOME`), +so later actions in the same job invoke `./gradlew` with no re-setup. The +foundation must run before any artifact action. + +### Per-artifact test/tck phases are asymmetric (verified) + +- **CLI:** regression/TCK = `native-cli-integration-tests:test` (a *separate* + Gradle module), master-only, two weave suite versions (2.12.2-SNAPSHOT, + 2.13.0-SNAPSHOT). +- **Node:** `nodeTest` (unit/integration, always) **plus** a master-only TCK + conformance lane (`stageTckSuites` + `npm run test:tck`, two suite versions). +- **Python:** a `pythonTest` Gradle task exists but is **not** wired into any + workflow today — no tck phase. + +A rigid produce→tck→upload triple would misfit this. Each artifact action +therefore encodes *its own* test/tck phase (or none), gated by `run-tck`. +This spec preserves today's exact test wiring — it does not add Python tests or +change which tests run. + +## Design + +### Layer 1 — foundation action + +`.github/actions/build-foundation/action.yml` (composite; renamed from +`build-native`). + +Steps: Setup Gradle → Setup GraalVM (Java 24, `graalvm-community`, +`github-token`) → Run Build (`./gradlew … -PskipNodeTests=true build +`). + +Inputs: `github-token` (required); `native-version` (default `''`, empty omits +`-PnativeVersion`). Version arg: `${{ inputs.native-version != '' && +format('-PnativeVersion={0}', inputs.native-version) || '' }}`. + +### Layer 2 — per-artifact actions + +Each is one composite action named for the artifact; every `run` step declares +`shell: bash`; each applies the version-arg conditional. Common inputs: + +- `native-version` (default `''`). +- `run-tck` (default `'false'`) — when `'true'`, run this artifact's + master-only test/tck phase in addition to its always-on tests. +- `publish` (default `'none'`) — `'none' | 'artifact' | 'release'`, selecting + the publish phase. `'artifact'` = `actions/upload-artifact@v7` (main.yml CI + retention, incl. any staging/rename this artifact needs); `'release'` = + `svenstaro/upload-release-action@v2` (release assets, per-OS names, tag). +- Publish-only inputs, consumed only when `publish != 'none'`: + `github-token` / `repo-token`, `native-version`, `arch`, `script-name`, + `distro-os`, `tag` as each artifact requires. (Exact per-artifact input set + is finalized in the plan.) + +**`.github/actions/cli/action.yml`** — produce: `native-cli:distro`; tck +(if `run-tck`): `native-cli-integration-tests:test` × two suite versions; +publish: distro zip (`artifact` stages `dw-cli-…` and uploads; `release` +uploads via svenstaro with `asset_name`). + +**`.github/actions/python/action.yml`** — produce: pip deps (owns +`--break-system-packages`, gated by a `break-system-packages` input, default +`'false'`) + `native-lib:buildPythonWheel`; tck: none (preserves today); +publish: the wheel. + +**`.github/actions/node/action.yml`** — produce: `setup-node` + +`native-lib:buildNodePackage` + `nodeTest` (always); tck (if `run-tck`): +`stageTckSuites` + `npm run test:tck` × two suite versions; publish: the `.tgz` +(`artifact` stages OS-qualified name; `release` via svenstaro). + +**dwlib (raw `.so`/`.dll`/`.dylib` + `.h`)** is a 4th uploaded thing today, +separate from the wheel/tgz that embed it. Whether it becomes its own +`native-lib` artifact action or rides along with one binding is **deferred to +the implementation plan.** + +**Future** `go`, `rust`: same shape, added without touching foundation or other +artifact actions. + +### Workflow composition (target) + +Checkout stays first (local `uses:` needs the workspace). Foundation second. + +**`ci.yml`** (mulesoft matrix; produce only, no tck, no publish): +``` +checkout +build-foundation (github-token) +cli (defaults: run-tck false, publish none) +python (break-system-packages omitted → false) +node +``` + +**`main.yml`** (ubuntu-latest / windows-2022 / macos-latest): +``` +checkout +build-foundation (github-token) +cli (run-tck: master?, publish: 'artifact', + arch/script-name/distro-os) +python (break-system-packages: 'true', publish: 'artifact') +node (run-tck: master?, publish: 'artifact', + arch/script-name) +``` +`run-tck` is passed `${{ github.ref == 'refs/heads/master' }}` so the tck phase +stays master-only, matching today. + +**`release.yml`** (tag builds): +``` +checkout +Guess Extension Version (sets NATIVE_VERSION, ARCH — must precede foundation) +build-foundation (github-token, native-version: ${{ env.NATIVE_VERSION }}) +cli (native-version, publish: 'release', repo-token, tag, arch, script-name, distro-os) +python (native-version, break-system-packages: 'true', publish: 'release', repo-token, tag) +node (native-version, publish: 'release', repo-token, tag, arch, script-name) +``` +Plus whatever the dwlib decision (planning) yields, and the shared header +upload. + +### Behavioral equivalence + +Pure restructuring — the effective commands, the artifacts produced, their +names, and which tests run per trigger must be **identical** to PR #150's +current green state: + +- version arg: empty for ci/main, `${{ env.NATIVE_VERSION }}` for release. +- `--break-system-packages`: only on main/release python. +- master-only tck: CLI regression + Node TCK gated exactly as today; Python + still has none. +- publish: ci none; main `upload-artifact` (same staged names); release + svenstaro (same `asset_name`s, per-OS dwlib, header). + +## Naming decisions + +- Foundation: **`build-foundation`**. +- Artifact actions named **by artifact**: `cli`, `python`, `node` (referenced + `uses: ./.github/actions/cli`). Not `produce-cli` — the action owns + produce + tck + publish, selected by inputs. + +## Risks / notes + +- **Input surface grows** on each artifact action (publish/tck knobs). This is + the deliberate tradeoff for one-call-per-artifact + one-file-per-artifact; + accepted per the goal. Keep inputs documented in each `action.yml`. +- **Produce/publish now co-located** inside each artifact action (reverses the + earlier separation proposal) — a conscious choice favoring fewer workflow + touch-points over strict separation, reasonable for a small, stable set of + publish mechanisms. +- **Local composite actions require checkout first** (unchanged). **`shell: + bash` mandatory** on every composite `run` step. +- **`release.yml` ordering:** `Guess Extension Version` before + `build-foundation` (populates `NATIVE_VERSION`/`ARCH`). +- **Job-scoped setup ordering:** foundation before all artifact actions. +- More step groups per job in the CI UI; negligible vs. native-image compile + time. + +## Testing / validation + +No unit tests apply. Validation: +- YAML parse for every action file and all three workflows. +- Rendered commands per workflow equal PR #150's green state (version arg, + break-system-packages, tck gating, publish mechanism + artifact names). +- No `build-native` reference remains; foundation is `build-foundation`. +- Each workflow composes `build-foundation → cli → python → node` (foundation + first), one call per artifact. +- End-to-end: the `Build Native CLI` (`main.yml`) PR run green on all three + legs — the live proof, same bar PR #150 cleared. `ci.yml`/`release.yml` + validated by parse + command inspection (not triggered on PRs).