ci: smoke-test the computing-unit cluster (master boot + worker join)

.github/scripts/smoke-cluster.sh

@@ -0,0 +1,179 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# smoke-cluster.sh -- boot a computing-unit-master and a computing-unit-worker
+# from their unpacked dist and assert the worker actually JOINS the master's
+# amber Pekko cluster, not merely that its process survives.
+#
+# Why this can't reuse smoke-boot.sh: ComputingUnitWorker.main only calls
+# AmberRuntime.startActorWorker, which binds pekko.remote.artery.canonical.port=0
+# (a random ephemeral port) and starts no HTTP server -- so there is no fixed
+# port to probe. A worker booted standalone just retries its seed forever and
+# stays up, so "did not crash" proves nothing. The real health signal is cluster
+# membership, which ClusterListener prints on every membership change:
+#     ---------Now we have N nodes in the cluster---------  (N = cluster.state.members.size)
+# master-alone => 1, master+worker joined => 2. We wait for the "2" line -- a
+# deliberate readiness signal the app emits, not a crash-scan (contrast #6332).
+# See issue #6523 (raised from the #6377 review) and #6220 for the rationale.
+#
+# Single host, no external network: the master runs in non-cluster mode, which
+# self-seeds pekko://Amber@localhost:2552; the worker is launched with NO
+# --serverAddr, so it defaults its seed to localhost:2552 and skips the
+# getNodeIpAddress -> http://checkip.amazonaws.com lookup (that only fires when
+# --serverAddr is supplied).
+#
+# Usage:
+#   smoke-cluster.sh <master-glob> <worker-glob> <master-http-port> [join_timeout]
+#
+# Requirements: run from the checkout root -- the master resolves its config via
+# Utils.amberHomePath -> ./amber/src/main/resources/computing-unit-master-config.yml
+# -- with postgres already provisioned (the master calls SqlServer.initConnection).
+
+set -euo pipefail
+
+master_glob="${1:?usage: smoke-cluster.sh <master-glob> <worker-glob> <master-http-port> [join_timeout]}"
+worker_glob="${2:?worker launcher glob required}"
+http_port="${3:?master http port required}"
+join_timeout="${4:-40}"
+master_ready_timeout="${SMOKE_MASTER_READY_TIMEOUT:-60}"
+
+PHRASE_JOINED='Now we have 2 nodes in the cluster'
+
+# Resolve a (globbed) launcher to exactly one executable -- same contract as
+# smoke-boot.sh: erroring on >1 match avoids booting the wrong binary.
+RESOLVED=""
+resolve_launcher() {
+  local glob="$1" role="$2" matches count
+  # shellcheck disable=SC2086  # intentional: $glob must glob-expand here
+  matches="$(ls -d $glob 2>/dev/null || true)"
+  count="$(printf '%s' "$matches" | grep -c . || true)"
+  if [[ "$count" -eq 0 ]]; then
+    echo "::error::smoke-cluster: $role launcher not found: $glob"; return 1
+  fi
+  if [[ "$count" -gt 1 ]]; then
+    echo "::error::smoke-cluster: $role launcher glob matched $count files, expected exactly 1: $glob"; return 1
+  fi
+  if [[ ! -x "$matches" ]]; then
+    echo "::error::smoke-cluster: $role launcher not executable: $matches"; return 1
+  fi
+  RESOLVED="$matches"
+}
+resolve_launcher "$master_glob" master || exit 1
+master_bin="$RESOLVED"
+resolve_launcher "$worker_glob" worker || exit 1
+worker_bin="$RESOLVED"
+
+port_open() {
+  # Probe 127.0.0.1 explicitly (not "localhost", which can resolve to ::1 first
+  # while the JVM binds IPv4, giving a false "not listening").
+  if command -v nc >/dev/null 2>&1; then
+    nc -z 127.0.0.1 "$http_port" >/dev/null 2>&1
+  else
+    (exec 3<>"/dev/tcp/127.0.0.1/$http_port") 2>/dev/null
+  fi
+}
+if port_open; then
+  echo "::error::smoke-cluster: port $http_port is already in use before launching the master"
+  exit 1
+fi
+
+master_log="$(mktemp)"
+worker_log="$(mktemp)"
+master_pid=""
+worker_pid=""
+
+alive() { kill -0 "$1" 2>/dev/null; }
+
+dump_logs() {
+  echo "----- master (last 120 lines) -----"; tail -n 120 "$master_log" 2>/dev/null || true
+  echo "----- worker (last 120 lines) -----"; tail -n 120 "$worker_log" 2>/dev/null || true
+}
+
+# Reap both JVMs (SIGTERM, grace, SIGKILL) and remove temp logs on any exit --
+# including a set -e abort -- so a failing run can't orphan a JVM or leak files.
+# shellcheck disable=SC2329  # invoked indirectly via `trap cleanup EXIT` below
+cleanup() {
+  for p in "$worker_pid" "$master_pid"; do [[ -n "$p" ]] && kill "$p" 2>/dev/null || true; done
+  for _ in $(seq 1 10); do
+    { [[ -z "$master_pid" ]] || ! alive "$master_pid"; } &&
+      { [[ -z "$worker_pid" ]] || ! alive "$worker_pid"; } && break
+    sleep 1
+  done
+  for p in "$worker_pid" "$master_pid"; do [[ -n "$p" ]] && kill -9 "$p" 2>/dev/null || true; done
+  wait 2>/dev/null || true
+  rm -f "$master_log" "$worker_log"
+}
+trap cleanup EXIT
+
+# 1) Master: non-cluster mode self-seeds localhost:2552 and serves Dropwizard on
+#    :http_port. Launch from the current dir (checkout root) so its config resolves.
+echo "smoke-cluster: launching master '$master_bin'"
+"$master_bin" >"$master_log" 2>&1 &
+master_pid=$!
+
+# Wait for the master to be ready: its HTTP port coming up implies the actor
+# system bound :2552 and DB init finished.
+ready=0
+for ((i = 0; i < master_ready_timeout; i++)); do
+  if port_open; then ready=1; break; fi
+  if ! alive "$master_pid"; then
+    echo "::error::smoke-cluster: master exited before listening on :$http_port"; dump_logs; exit 1
+  fi
+  sleep 1
+done
+if [[ "$ready" != 1 ]]; then
+  echo "::error::smoke-cluster: master did not listen on :$http_port within ${master_ready_timeout}s"; dump_logs; exit 1
+fi
+
+# NB: we deliberately do NOT wait here for the master's own "1 nodes" self-join
+# line. The master opens :8085 as soon as its actor system + DB are up, but its
+# own MemberUp is gated by cluster.conf's leader-actions-interval (~10s) and can
+# land *after* the port is already open -- keying on it here caused a false
+# failure. The worker-join assertion below is the real signal; a readiness-log
+# drift is reported there instead.
+echo "smoke-cluster: master up (listening on :$http_port); launching worker '$worker_bin'"
+
+# 2) Worker: no --serverAddr => seed localhost:2552, random remote port, no egress.
+"$worker_bin" >"$worker_log" 2>&1 &
+worker_pid=$!
+
+# 3) Assert the worker joins: the master observes a 2-member cluster. Poll a
+#    deliberate readiness line (not a crash-scan); fail fast if either JVM dies.
+joined=0
+for ((i = 0; i < join_timeout; i++)); do
+  if grep -Fq "$PHRASE_JOINED" "$master_log"; then joined=1; break; fi
+  if ! alive "$master_pid"; then
+    echo "::error::smoke-cluster: master exited before the worker joined"; dump_logs; exit 1
+  fi
+  if ! alive "$worker_pid"; then
+    echo "::error::smoke-cluster: worker exited before joining the cluster"; dump_logs; exit 1
+  fi
+  sleep 1
+done
+
+if [[ "$joined" == 1 ]]; then
+  echo "smoke-cluster: OK -- worker joined; master observes 2 nodes in the cluster"
+  grep -F "$PHRASE_JOINED" "$master_log" | tail -n1
+  exit 0
+fi
+echo "::error::smoke-cluster: worker did not join the cluster within ${join_timeout}s"
+if ! grep -Fq "nodes in the cluster" "$master_log"; then
+  echo "::error::smoke-cluster: the master logged no 'N nodes in the cluster' line at all -- ClusterListener's readiness log may have changed; this test keys on it"
+fi
+dump_logs
+exit 1

.github/scripts/test_smoke_cluster.sh

@@ -0,0 +1,159 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# End-to-end regression tests for smoke-cluster.sh, driven by fake master/worker
+# launchers (no real amber dist). These guard the harness's own decision logic:
+#   - worker joins (master logs the 2-node readiness line)  -> PASS
+#   - worker exits before joining                           -> FAIL ("worker exited...")
+#   - worker never joins within the timeout                 -> FAIL ("did not join...")
+#   - master dies during the join wait                      -> FAIL ("master exited...joined")
+#   - master logs no "N nodes" line at all (readiness drift) -> FAIL (timeout + hint)
+#   - master crashes before it listens                      -> FAIL ("...before listening")
+# The failure cases assert the SPECIFIC error message, so a future edit that
+# deletes a fast-fail branch (and lets the case pass only via the slow timeout)
+# turns the test red instead of silently passing. The happy path ties the join
+# signal to the worker actually running, so it can't pass if the worker is never
+# launched. The real cluster join is exercised against the packaged dist in
+# amber-integration. See https://github.com/apache/texera/issues/6523.
+
+set -uo pipefail
+
+command -v python3 >/dev/null || { echo "python3 is required to run these tests" >&2; exit 1; }
+
+script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+smoke="$script_dir/smoke-cluster.sh"
+work="$(mktemp -d 2>/dev/null || mktemp -d -t smoke-cluster)"
+trap 'rm -rf "$work"' EXIT
+rc=0
+
+PHRASE_SELF='Now we have 1 nodes in the cluster'
+PHRASE_JOINED='Now we have 2 nodes in the cluster'
+
+free_port() {
+  python3 -c 'import socket; s=socket.socket(); s.bind(("127.0.0.1",0)); print(s.getsockname()[1]); s.close()'
+}
+pass()   { echo "ok:   $1"; }
+failed() { echo "FAIL: $1"; rc=1; }
+
+py_listen() {  # emit a python one-liner that binds $1 and holds it for $2 seconds
+  echo "exec python3 -c \"import socket,time; s=socket.socket(); s.setsockopt(socket.SOL_SOCKET,socket.SO_REUSEADDR,1); s.bind(('127.0.0.1',$1)); s.listen(); time.sleep($2)\""
+}
+
+# Fake master: binds $port so smoke-cluster's port-wait succeeds. $self=y prints
+# the self-join line. $trigger is "n" (never emit the joined line), a number N
+# (emit it N seconds after start), or "wait:<file>" (emit it only once <file>
+# exists -- used to couple the join to the worker actually running). $hold is how
+# long to hold the port (short => master dies mid-run).
+#   $1=path $2=port $3=self(y/n) $4=trigger $5=hold_secs
+make_master() {
+  local path="$1" port="$2" self="$3" trigger="$4" hold="$5"
+  {
+    echo '#!/usr/bin/env bash'
+    [[ "$self" == y ]] && echo "echo '$PHRASE_SELF'"
+    case "$trigger" in
+      n) : ;;
+      wait:*) echo "( while [ ! -f '${trigger#wait:}' ]; do sleep 0.2; done; echo '$PHRASE_JOINED' ) &" ;;
+      *) echo "( sleep $trigger; echo '$PHRASE_JOINED' ) &" ;;
+    esac
+    py_listen "$port" "$hold"
+  } >"$path"
+  chmod +x "$path"
+}
+
+# Fake worker: sleeps (healthy), crashes on boot, or touches a signal file (to
+# trigger a coupled master) then sleeps.  $1=path $2=sleep|crash|touch:<file>
+make_worker() {
+  case "$2" in
+    crash)   printf '#!/usr/bin/env bash\necho "worker boom" >&2\nexit 1\n' >"$1" ;;
+    touch:*) printf '#!/usr/bin/env bash\ntouch "%s"\nexec sleep 120\n' "${2#touch:}" >"$1" ;;
+    *)       printf '#!/usr/bin/env bash\nexec sleep 120\n' >"$1" ;;
+  esac
+  chmod +x "$1"
+}
+
+# Assert smoke-cluster FAILS and its output contains $needle (locks in the branch
+# that produces that message). $1=desc $2=needle, remaining args = smoke-cluster argv.
+assert_fail_msg() {
+  local desc="$1" needle="$2"; shift 2
+  local out
+  if out="$("$smoke" "$@" 2>&1)"; then
+    failed "$desc: expected FAIL but it passed"
+  elif printf '%s\n' "$out" | grep -Fq "$needle"; then
+    pass "$desc"
+  else
+    failed "$desc: failed but message missing '$needle'"; printf '%s\n' "$out" | tail -n3
+  fi
+}
+
+# --- happy path: the worker triggers the join; master observes 2 nodes -> PASS.
+# The joined line fires only after the worker touches the signal, so this cannot
+# pass if smoke-cluster never launches the worker. ---
+port="$(free_port)"; sig="$work/joined.signal"
+make_master "$work/m_ok" "$port" y "wait:$sig" 120
+make_worker "$work/w_ok" "touch:$sig"
+if "$smoke" "$work/m_ok" "$work/w_ok" "$port" 30 >/dev/null 2>&1; then
+  pass "worker joins cluster (master sees 2 nodes) -> OK"
+else
+  failed "worker join should be OK"
+fi
+
+# --- worker exits before joining -> FAIL fast (assert the fast-fail message) ---
+port="$(free_port)"
+make_master "$work/m_nojoin" "$port" y n 120
+make_worker "$work/w_crash" crash
+assert_fail_msg \
+  "worker exits before join -> FAIL fast" "worker exited before joining" \
+  "$work/m_nojoin" "$work/w_crash" "$port" 15
+
+# --- worker never joins within the timeout -> FAIL (assert the timeout message) ---
+port="$(free_port)"
+make_master "$work/m_nojoin2" "$port" y n 120
+make_worker "$work/w_sleep" sleep
+assert_fail_msg \
+  "worker never joins -> FAIL (timeout)" "did not join the cluster within" \
+  "$work/m_nojoin2" "$work/w_sleep" "$port" 3
+
+# --- master dies during the join wait -> FAIL (assert the master-death message) ---
+port="$(free_port)"
+make_master "$work/m_die" "$port" y n 4
+make_worker "$work/w_sleep_b" sleep
+assert_fail_msg \
+  "master dies during join wait -> FAIL" "master exited before the worker joined" \
+  "$work/m_die" "$work/w_sleep_b" "$port" 20
+
+# --- master logs no "N nodes" line at all (readiness log drift) -> FAIL with a hint ---
+port="$(free_port)"
+make_master "$work/m_drift" "$port" n n 120
+make_worker "$work/w_sleep2" sleep
+assert_fail_msg \
+  "master readiness log drift -> FAIL (timeout + drift hint)" "readiness log may have changed" \
+  "$work/m_drift" "$work/w_sleep2" "$port" 3
+
+# --- master crashes before it listens -> FAIL (assert the pre-listen message) ---
+port="$(free_port)"
+printf '#!/usr/bin/env bash\necho "master boom" >&2\nexit 1\n' >"$work/m_crash"; chmod +x "$work/m_crash"
+make_worker "$work/w_sleep3" sleep
+SMOKE_MASTER_READY_TIMEOUT=15 assert_fail_msg \
+  "master crashes before listening -> FAIL" "master exited before listening" \
+  "$work/m_crash" "$work/w_sleep3" "$port" 20
+
+if [[ "$rc" -ne 0 ]]; then
+  echo "smoke-cluster regression tests FAILED"
+  exit 1
+fi
+echo "smoke-cluster regression tests passed"

.github/workflows/build.yml

@@ -639,19 +639,22 @@ jobs:
         # amber/src/main/resources/web-config.yml (port 8080).
         if: matrix.os == 'ubuntu-latest'
         run: .github/scripts/smoke-boot.sh "/tmp/dists/amber-*/bin/texera-web-application" 8080
-      - name: Smoke-test computing-unit-master boots
-        # computing-unit-master boots on postgres only, like texera-web: main
-        # starts a LOCAL Pekko actor master (clusterMode defaults to false with
-        # no --cluster arg, so no external seed/discovery and the cluster-only
-        # checkip.amazonaws.com lookup is never reached), then run() calls
-        # SqlServer.initConnection against the postgres provisioned above. The
-        # boot-time result-store cleanup is gated on
-        # ApplicationConfig.cleanupAllExecutionResults, which defaults to false,
-        # so no iceberg / S3 access. Config resolves via Utils.amberHomePath to
-        # amber/src/main/resources/computing-unit-master-config.yml (port 8085).
-        # Reuses the amber dist built + unzipped above for the texera-web boot.
+      - name: Smoke-test computing-unit-master boots and the worker joins its cluster
+        # One check for the whole computing-unit cluster: boot a
+        # computing-unit-master and a no-arg computing-unit-worker from the
+        # packaged dist and assert the worker JOINS -- the master's ClusterListener
+        # logging "2 nodes in the cluster". This supersedes a standalone master
+        # boot check: smoke-cluster.sh already boots the master, waits for it to
+        # listen on :8085, and verifies it self-joins its 1-node cluster before the
+        # worker is launched, so a #6204-class linkage crash on master startup is
+        # still caught (reported as "master exited before listening"). The master
+        # runs non-cluster mode (self-seeds localhost:2552) and the worker is
+        # launched with no --serverAddr (joins :2552, skipping the
+        # checkip.amazonaws.com lookup), so no external network; it fails fast if
+        # either JVM dies. Reuses the amber dist built + unzipped above; postgres
+        # already provisioned.
         if: matrix.os == 'ubuntu-latest'
-        run: .github/scripts/smoke-boot.sh "/tmp/dists/amber-*/bin/computing-unit-master" 8085
+        run: .github/scripts/smoke-cluster.sh "/tmp/dists/amber-*/bin/computing-unit-master" "/tmp/dists/amber-*/bin/computing-unit-worker" 8085
       - name: Run Python integration tests
         # --junit-xml feeds the Test Analytics upload below.
         run: |