From 30bf9754d0fc53876293b785873c56864acb4be5 Mon Sep 17 00:00:00 2001 From: Himanshu Verma Date: Sun, 26 Jul 2026 15:21:11 +0530 Subject: [PATCH 01/10] fix(dist): replace lsof-based port check with ss/netstat//dev/tcp fallbacks and harden startup scripts Replace lsof with layered probe (ss, netstat, /dev/tcp) in server check_port to eliminate startup hang in Kubernetes pods where ulimit -n can be ~1 billion causing lsof to scan the entire FD table. Server util.sh: - Token-based listener matching with normalize_addr() for IPv6 canonicalization - IPv6 addresses canonicalized via getent ahosts (Linux) or python3 (macOS/BSD) - Authority extraction strips ? and # in addition to / and path - Same-family wildcard matching (IPv4/IPv6 isolated) - run_with_deadline() helper: deadline-bounded /dev/tcp without timeout - Atomic download(): temp file + mv on success, rm on failure - curl -fL with correct -o before -- argument ordering - wget progress flags in indexed array for set -u / Bash 3.2 safety - Helper functions scoped with _check_port_ prefix and unset -f cleanup - netstat -an fallback uses Bash built-in case-insensitive match - normalize_addr() wraps getent in timeout 2, tr fallback for case - process_num() returns boolean 0/1, process_id() echoes result - command_available() uses command -v instead of [[ -x ]] - get_ip() falls back to loopback on empty parse - free_memory() Darwin branch guards empty values - wait_for_startup() separates local/assignment, quotes http_code - read_property() localizes file_name/property_name - port_checked=1 set when ss/netstat succeeds and comparison is reliable PD and Store util.sh: - Remove dead check_port, fix command_available - Atomic download() matching server pattern - wget progress flags in indexed array - read_property() localizes file_name/property_name CI and test scripts: - test-check-port.sh runs as fast gate after Compile - fuser preflight on Linux, lsof removed - OS-aware port cleanup (fuser on Linux, safe lsof loop on macOS) - PD/Store/Server tests include fuser in non-Darwin prereqs - PD/Store tests call STOP_SCRIPT before manual cleanup - test-start-hugegraph.sh no longer masks java_home exit status Tests: test-check-port.sh - 40 mock-driven tests, 0 failures - timeout mock makes ss/netstat free cases deterministic - IPv6 canonicalization (compressed vs expanded) - download tests verify temp file + atomic rename + cleanup - wget success-path tests for PD and Store - Bash 3.2-safe array expansion throughout --- .github/workflows/server-ci.yml | 20 +- .../src/assembly/static/bin/util.sh | 149 +-- .../src/assembly/static/bin/util.sh | 475 ++++++++- .../src/assembly/travis/test-check-port.sh | 984 ++++++++++++++++++ .../travis/test-start-hugegraph-pd.sh | 29 +- .../travis/test-start-hugegraph-store.sh | 28 +- .../assembly/travis/test-start-hugegraph.sh | 34 +- .../static/bin/start-hugegraph-store.sh | 13 +- .../src/assembly/static/bin/util.sh | 176 ++-- 9 files changed, 1725 insertions(+), 183 deletions(-) create mode 100755 hugegraph-server/hugegraph-dist/src/assembly/travis/test-check-port.sh diff --git a/.github/workflows/server-ci.yml b/.github/workflows/server-ci.yml index 72f28b6692..c9d895be7e 100644 --- a/.github/workflows/server-ci.yml +++ b/.github/workflows/server-ci.yml @@ -79,17 +79,30 @@ jobs: run: | mvn clean compile -U -Dmaven.javadoc.skip=true -ntp + - name: Run check_port unit tests + if: ${{ env.BACKEND == 'rocksdb' }} + run: | + # Validates ss/netstat//dev/tcp replacement for lsof + $TRAVIS_DIR/test-check-port.sh hugegraph-server/hugegraph-dist/src/assembly/static + - name: Check startup test prerequisites id: server-preflight if: ${{ env.BACKEND == 'rocksdb' }} run: | - for tool in lsof crontab curl java; do + # Note: lsof removed from prerequisites (replaced with ss/netstat//dev/tcp in check_port). + # This job always runs on Linux (ubuntu-22.04), which uses fuser for port cleanup. + for tool in crontab curl java; do if ! command -v "$tool" >/dev/null 2>&1; then echo "can_run=false" >> "$GITHUB_OUTPUT" echo "skip_reason=missing tool: $tool" >> "$GITHUB_OUTPUT" exit 0 fi done + if ! command -v fuser >/dev/null 2>&1; then + echo "can_run=false" >> "$GITHUB_OUTPUT" + echo "skip_reason=missing tool: fuser (required for Linux port cleanup)" >> "$GITHUB_OUTPUT" + exit 0 + fi echo "can_run=true" >> "$GITHUB_OUTPUT" - name: Run start-hugegraph.sh foreground mode tests @@ -187,6 +200,11 @@ jobs: run: | mvn clean compile -pl hugegraph-server/hugegraph-test -am -U -Dmaven.javadoc.skip=true -ntp + - name: Run check_port unit tests + # Validates ss/netstat//dev/tcp replacement for lsof + run: | + $TRAVIS_DIR/test-check-port.sh hugegraph-server/hugegraph-dist/src/assembly/static + - name: Run RocksDB core test run: | $TRAVIS_DIR/run-core-test.sh $BACKEND diff --git a/hugegraph-pd/hg-pd-dist/src/assembly/static/bin/util.sh b/hugegraph-pd/hg-pd-dist/src/assembly/static/bin/util.sh index 0b7ad0f0f0..e80255672e 100644 --- a/hugegraph-pd/hg-pd-dist/src/assembly/static/bin/util.sh +++ b/hugegraph-pd/hg-pd-dist/src/assembly/static/bin/util.sh @@ -20,20 +20,21 @@ # TODO: consider reuse it with server-dist module (almost same as it) function command_available() { local cmd=$1 - if [ $(command -v $cmd >/dev/null 2>&1) ]; then - return 1 - else + if command -v "$cmd" >/dev/null 2>&1; then return 0 fi + return 1 } # read a property from .properties file function read_property() { # file path + local file_name + local property_name file_name=$1 # replace "." to "\." - property_name=$(echo $2 | sed 's/\./\\\./g') - cat $file_name | sed -n -e "s/^[ ]*//g;/^#/d;s/^$property_name=//p" | tail -1 + property_name=$(echo "$2" | sed 's/\./\\\./g') + cat "$file_name" | sed -n -e "s/^[ ]*//g;/^#/d;s/^$property_name=//p" | tail -1 } function write_property() { @@ -56,7 +57,7 @@ function parse_yaml() { local version=$2 local module=$3 - cat $file | tr -d '\n {}'| awk -F',+|:' '''{ + cat "$file" | tr -d '\n {}'| awk -F',+|:' '''{ pre=""; for(i=1; i<=NF; ) { if(match($i, /version/)) { @@ -72,27 +73,19 @@ function parse_yaml() { } function process_num() { - num=`ps -ef | grep $1 | grep -v grep | wc -l` - return $num + local num + num=$(ps -ef | grep "$1" | grep -v grep | wc -l) + if (( num > 0 )); then + return 1 + fi + return 0 } function process_id() { - pid=`ps -ef | grep $1 | grep -v grep | awk '{print $2}'` - return $pid -} - -# check the port of rest server is occupied -function check_port() { - local port=$(echo "$1" | sed 's|.*:||' | sed 's|/.*||') - if ! command_available "lsof"; then - echo "Required lsof but it is unavailable" - exit 1 - fi - lsof -i :$port >/dev/null - if [ $? -eq 0 ]; then - echo "The port $port has already been used" - exit 1 - fi + local pid + pid=$(ps -ef | grep "$1" | grep -v grep | awk '{print $2}') + echo "$pid" + return 0 } function crontab_append() { @@ -130,13 +123,14 @@ function wait_for_startup() { local server_url="$3" local timeout_s="$4" - local now_s=`date '+%s'` - local stop_s=$(( $now_s + $timeout_s )) + local now_s + now_s=$(date '+%s') + local stop_s=$(( now_s + timeout_s )) local status echo -n "Connecting to $server_name ($server_url)" - while [ $now_s -le $stop_s ]; do + while [ "$now_s" -le "$stop_s" ]; do echo -n . process_status "$server_name" "$pid" >/dev/null if [ $? -eq 1 ]; then @@ -144,14 +138,14 @@ function wait_for_startup() { return 1 fi - status=`curl -o /dev/null -s -k -w %{http_code} $server_url` - if [[ $status -eq 200 || $status -eq 401 ]]; then + status=$(curl -o /dev/null -s -k -w '%{http_code}' "$server_url") + if [[ "$status" -eq 200 || "$status" -eq 401 ]]; then echo "OK" echo "Started [pid $pid]" return 0 fi sleep 2 - now_s=`date '+%s'` + now_s=$(date '+%s') done echo "The operation timed out when attempting to connect to $server_url" >&2 @@ -160,22 +154,28 @@ function wait_for_startup() { function free_memory() { local free="" - local os=`uname` + local os=$(uname) if [ "$os" == "Linux" ]; then - local mem_free=`cat /proc/meminfo | grep -w "MemFree" | awk '{print $2}'` - local mem_buffer=`cat /proc/meminfo | grep -w "Buffers" | awk '{print $2}'` - local mem_cached=`cat /proc/meminfo | grep -w "Cached" | awk '{print $2}'` + local mem_free=$(cat /proc/meminfo | grep -w "MemFree" | awk '{print $2}') + local mem_buffer=$(cat /proc/meminfo | grep -w "Buffers" | awk '{print $2}') + local mem_cached=$(cat /proc/meminfo | grep -w "Cached" | awk '{print $2}') if [[ "$mem_free" == "" || "$mem_buffer" == "" || "$mem_cached" == "" ]]; then echo "Failed to get free memory" exit 1 fi - free=`expr $mem_free + $mem_buffer + $mem_cached` - free=`expr $free / 1024` + free=$(expr "$mem_free" + "$mem_buffer" + "$mem_cached") + free=$(expr "$free" / 1024) elif [ "$os" == "Darwin" ]; then - local pages_free=`vm_stat | awk '/Pages free/{print $0}' | awk -F'[:.]+' '{print $2}' | tr -d " "` - local pages_inactive=`vm_stat | awk '/Pages inactive/{print $0}' | awk -F'[:.]+' '{print $2}' | tr -d " "` - local pages_available=`expr $pages_free + $pages_inactive` - free=`expr $pages_available \* 4096 / 1024 / 1024` + local pages_free pages_inactive + pages_free=$(vm_stat | awk '/Pages free/{print $0}' | awk -F'[:.]+' '{print $2}' | tr -d " ") + pages_inactive=$(vm_stat | awk '/Pages inactive/{print $0}' | awk -F'[:.]+' '{print $2}' | tr -d " ") + if [[ -z "$pages_free" || -z "$pages_inactive" ]]; then + echo "Failed to get free memory" + exit 1 + fi + local pages_available + pages_available=$(expr "$pages_free" + "$pages_inactive") + free=$(expr "$pages_available" \* 4096 / 1024 / 1024) else echo "Unsupported operating system $os" exit 1 @@ -187,8 +187,8 @@ function calc_xmx() { local min_mem=$1 local max_mem=$2 # Get machine available memory - local free=`free_memory` - local half_free=$[free/2] + local free=$(free_memory) + local half_free=$((free/2)) local xmx=$min_mem if [[ "$free" -lt "$min_mem" ]]; then @@ -236,47 +236,74 @@ function ensure_path_writable() { } function get_ip() { - local os=`uname` + local os=$(uname) local loopback="127.0.0.1" local ip="" case $os in Linux) if command_available "ifconfig"; then - ip=`ifconfig | grep 'inet addr:' | grep -v "$loopback" | cut -d: -f2 | awk '{ print $1}'` + ip=$(ifconfig | grep 'inet addr:' | grep -v "$loopback" | cut -d: -f2 | awk '{ print $1}') elif command_available "ip"; then - ip=`ip addr | grep 'state UP' -A2 | tail -n1 | awk '{print $2}' | awk -F"/" '{print $1}'` + ip=$(ip addr | grep 'state UP' -A2 | tail -n1 | awk '{print $2}' | awk -F"/" '{print $1}') else ip=$loopback fi ;; FreeBSD|OpenBSD|Darwin) if command_available "ifconfig"; then - ip=`ifconfig | grep -E 'inet.[0-9]' | grep -v "$loopback" | awk '{ print $2}'` + ip=$(ifconfig | grep -E 'inet.[0-9]' | grep -v "$loopback" | awk '{ print $2}') else ip=$loopback fi ;; SunOS) if command_available "ifconfig"; then - ip=`ifconfig -a | grep inet | grep -v "$loopback" | awk '{ print $2} '` + ip=$(ifconfig -a | grep inet | grep -v "$loopback" | awk '{ print $2} ') else ip=$loopback fi ;; *) ip=$loopback;; esac - echo $ip + [[ -z "$ip" ]] && ip=$loopback + echo "$ip" } function download() { local path=$1 local link_url=$2 - if command_available "wget"; then - wget --help | grep -q '\--show-progress' && progress_opt="-q --show-progress" || progress_opt="" - wget ${link_url} -P ${path} $progress_opt - elif command_available "curl"; then - curl ${link_url} -o ${path}/${link_url} + if [ ! -d "$path" ]; then + mkdir -p "$path" || { + echo "Failed to create directory: $path" + exit 1 + } + fi + + local filename + filename=$(basename "${link_url%%[?#]*}") + local tmp="${path}/.${filename}.tmp.$$" + local dest="${path}/${filename}" + + if command_available "curl"; then + # -o must appear before -- so it is parsed as an option, not an extra URL. + if curl -fL -o "$tmp" -- "${link_url}"; then + mv -f -- "$tmp" "$dest" + else + rm -f -- "$tmp" + return 1 + fi + elif command_available "wget"; then + local -a progress_opt=() + if wget --help 2>&1 | grep -q -- '--show-progress'; then + progress_opt=(-q --show-progress) + fi + if wget ${progress_opt[@]+"${progress_opt[@]}"} -O "$tmp" -- "${link_url}"; then + mv -f -- "$tmp" "$dest" + else + rm -f -- "$tmp" + return 1 + fi else echo "Required wget or curl but they are unavailable" exit 1 @@ -289,19 +316,17 @@ function ensure_package_exist() { local tar=$3 local link=$4 - if [ ! -d ${path}/${dir} ]; then - if [ ! -f ${path}/${tar} ]; then + if [ ! -d "${path}/${dir}" ]; then + if [ ! -f "${path}/${tar}" ]; then echo "Downloading the compressed package '${tar}'" - download ${path} ${link} - if [ $? -ne 0 ]; then + if ! download "${path}" "${link}"; then echo "Failed to download, please ensure the network is available and link is valid" exit 1 fi echo "[OK] Finished download" fi echo "Unzip the compressed package '$tar'" - tar -zxvf ${path}/${tar} -C ${path} >/dev/null 2>&1 - if [ $? -ne 0 ]; then + if ! tar -zxvf "${path}/${tar}" -C "${path}" >/dev/null 2>&1; then echo "Failed to unzip, please check the compressed package" exit 1 fi @@ -316,7 +341,7 @@ function wait_for_shutdown() { local pid="$2" local timeout_s="$3" - local now_s=`date '+%s'` + local now_s=$(date '+%s') local stop_s=$(( $now_s + $timeout_s )) echo -n "Killing $process_name(pid $pid)" >&2 @@ -328,7 +353,7 @@ function wait_for_shutdown() { return 0 fi sleep 2 - now_s=`date '+%s'` + now_s=$(date '+%s') done echo "$process_name shutdown timeout(exceeded $timeout_s seconds)" >&2 return 1 @@ -357,7 +382,7 @@ function kill_process() { return 0 fi - case "`uname`" in + case "$(uname)" in CYGWIN*) taskkill /F /PID "$pid" ;; *) kill "$pid" ;; esac diff --git a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/util.sh b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/util.sh index 91c0c3efbf..5040daa652 100755 --- a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/util.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/util.sh @@ -18,11 +18,10 @@ function command_available() { local cmd=$1 - if [[ -x "$(command -v "$cmd")" ]]; then + if command -v "$cmd" >/dev/null 2>&1; then return 0 - else - return 1 fi + return 1 } function configure_riscv64_libatomic() { @@ -109,27 +108,417 @@ function parse_yaml() { } function process_num() { + local num num=$(ps -ef | grep "$1" | grep -v grep | wc -l) - return "$num" + # Return 0 when no process, 1 when one or more. Using $num directly as an + # exit code would truncate values > 255, so treat this as a boolean result. + if (( num > 0 )); then + return 1 + fi + return 0 } function process_id() { + local pid pid=$(ps -ef | grep "$1" | grep -v grep | awk '{print $2}') - return "$pid" + echo "$pid" + return 0 +} + +# Run a command with a hard deadline via background watchdog. +# Returns the command's exit code if it finishes in time. +# If the deadline expires, the command is killed (exit code reflects signal). +# Works without the timeout command — uses sleep + kill -9 pattern. +function run_with_deadline() { + local cmd="$1" + local deadline="$2" + shift 2 + + bash -c "$cmd" bash "$@" & + local child_pid=$! + ( + local sleep_pid="" + cleanup_watchdog() { + [[ -n "$sleep_pid" ]] && kill -9 "$sleep_pid" 2>/dev/null + } + # Kill any direct children of the target PID (e.g. a spawned sleep) + # in case killing the wrapper left them orphaned. + kill_children() { + local child + for child in $(pgrep -P "$1" 2>/dev/null); do + kill -9 "$child" 2>/dev/null || true + done + } + trap 'cleanup_watchdog' EXIT TERM + sleep "$deadline" & sleep_pid=$! + wait "$sleep_pid" 2>/dev/null + if kill -0 "$child_pid" 2>/dev/null; then + kill -9 "$child_pid" 2>/dev/null + kill_children "$child_pid" + fi + ) 2>/dev/null & + local watchdog_pid=$! + + wait "$child_pid" 2>/dev/null + local rc=$? + # Kill the watchdog with SIGTERM so its EXIT trap reaps the sleep child. + kill -TERM "$watchdog_pid" 2>/dev/null || true + wait "$watchdog_pid" 2>/dev/null || true + return $rc +} + + +# Normalize an IP address to a canonical string for comparison. +# - IPv4 addresses are returned unchanged. +# - IPv6 addresses are stripped of brackets/zone scope, lowercased and +# compressed to the canonical textual form (via getent ahosts). +# - IPv4-mapped IPv6 (::ffff:1.2.3.4) is collapsed to the IPv4 form. +# - Hostnames are returned unchanged; callers should resolve them first. +function normalize_addr() { + local addr="$1" + + # Strip brackets + if [[ "$addr" =~ ^\[.*\]$ ]]; then + addr="${addr#\[}" + addr="${addr%\]}" + fi + + # Drop IPv6 zone scope (e.g. 127.0.0.53%lo, fe80::1%eth0) + addr="${addr%%\%*}" + + # IPv4-mapped IPv6 -> IPv4 so a bound IPv4-mapped socket is compared + # against an IPv4-configured address. + if [[ "$addr" =~ ^::ffff:([0-9]+\.[0-9]+\.[0-9]+\.[0-9]+)$ ]]; then + echo "${BASH_REMATCH[1]}" + return + fi + + # Plain IPv4 + if [[ "$addr" =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + echo "$addr" + return + fi + + # IPv6: ask glibc for the canonical compressed form + if [[ "$addr" =~ ^[0-9a-fA-F:]*:[0-9a-fA-F:]*$ ]]; then + local norm + if command_available "timeout"; then + norm=$(timeout 2 getent ahosts "$addr" 2>/dev/null | awk '{print $1; exit}') + else + norm=$(getent ahosts "$addr" 2>/dev/null | awk '{print $1; exit}') + fi + if [[ -n "$norm" ]]; then + echo "$norm" + return + fi + # macOS/BSD fallback: python3's socket module can canonicalise + # numeric IPv6 even when getent is unavailable. + if command_available "python3"; then + norm=$(python3 -c 'import socket, sys; print(socket.inet_ntop(socket.AF_INET6, socket.inet_pton(socket.AF_INET6, sys.argv[1])))' "$addr" 2>/dev/null) + if [[ -n "$norm" ]]; then + echo "$norm" + return + fi + fi + # No canonicaliser available: at least normalise hex case so ss/netstat + # lowercase output still matches an uppercase configured address. + echo "$addr" | tr '[:upper:]' '[:lower:]' + return + fi + + # Fall back to the cleaned original + echo "$addr" } -# check the port of rest server is occupied +# check whether the REST server port is occupied function check_port() { - local port=$(echo "$1" | sed 's|.*:||' | sed 's|/.*||') - if ! command_available "lsof"; then - echo "Required lsof but it is unavailable" - exit 1 + local url="$1" + local host + local port + + # Strip leading/trailing whitespace from URL (handles whitespace from ServerOptions) + url="${url#"${url%%[![:space:]]*}"}" + url="${url%"${url##*[![:space:]]}"}" + + # Extract authority: strip scheme and stop at the first /, ? or #. + local authority + authority="${url#*://}" + authority="${authority%%[/?#]*}" + + # Extract host and port from authority. + if [[ "$authority" =~ ^\[([^\]]*)\]:([0-9]+)$ ]]; then + # IPv6 with port: [::1]:8080 + host="${BASH_REMATCH[1]}" + port="${BASH_REMATCH[2]}" + elif [[ "$authority" =~ ^\[([^\]]*)\]$ ]]; then + # IPv6 without port: [::1] + host="${BASH_REMATCH[1]}" + port="" + elif [[ "$authority" =~ :([0-9]+)$ ]]; then + # IPv4 or hostname with port: 127.0.0.1:8080, localhost:8080 + port="${BASH_REMATCH[1]}" + host="${authority%:*}" + else + # No explicit port in authority + host="$authority" + port="" fi - lsof -i :"$port" >/dev/null - if [ $? -eq 0 ]; then + + # Handle default ports from scheme when no explicit port is given + if [[ -z "$port" ]]; then + if [[ "$url" == https://* ]]; then + port="443" + elif [[ "$url" == http://* ]]; then + port="80" + else + return 0 + fi + fi + + # Validate port as a decimal number + if ! [[ "$port" =~ ^[0-9]+$ ]]; then + return 0 + fi + port=$((10#$port)) + if (( port < 1 || port > 65535 )); then + return 0 + fi + + # Strip any leading/trailing whitespace from host + host="${host#"${host%%[![:space:]]*}"}" + host="${host%"${host##*[![:space:]]}"}" + + local norm_host + norm_host=$(normalize_addr "$host") + + # Determine the address family of the configured host so we only treat + # same-family wildcard listeners as conflicts (IPv4 vs IPv6 sockets are + # separate unless explicitly dual-stacked). + local host_family="" + if [[ "$norm_host" =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + host_family="ipv4" + elif [[ "$norm_host" =~ ^[0-9a-fA-F:]*:[0-9a-fA-F:]*$ ]]; then + host_family="ipv6" + else + host_family="hostname" + fi + + local in_use=0 + local port_checked=0 + + # Wildcard binds are detected by looking for any listener on the port. + # Specific hosts are matched against the local address; if the listener + # is a same-family wildcard (0.0.0.0 / :: / *), that is a conflict too. + local is_wildcard=0 + if [[ -z "$host" || "$host" == "0.0.0.0" || "$host" == "::" || "$host" == "*" ]]; then + is_wildcard=1 + fi + + # Build the list of acceptable normalized addresses for a specific host. + # Hostnames must be resolved to numeric IPs first because ss/netstat + # output is always numeric. + local candidate_addrs="" + if [[ $is_wildcard -eq 0 ]]; then + if [[ "$host_family" == "ipv4" || "$host_family" == "ipv6" ]]; then + # Already numeric + candidate_addrs="$norm_host" + else + # Hostname: resolve with deadline + if command_available "getent" && command_available "timeout"; then + candidate_addrs=$(timeout 2 getent ahosts "$host" 2>/dev/null | awk '{print $1}') + elif command_available "dscacheutil" && command_available "timeout"; then + candidate_addrs=$(timeout 2 dscacheutil -q host -a name "$host" 2>/dev/null \ + | awk '/ip_address:/{print $2}') + fi + fi + fi + + # Helper: scan a line of listener-table output and return 0 if it matches + # the configured host/port. Sets 'matched_token' to 1 when it evaluates a + # token that we can trust, so callers know whether "no match" is reliable. + local out line listener_addr norm_listener token matched_token + + # Returns true if the listener address is a wildcard on the same family + # as the configured host (or any family, if the host is a wildcard). + _check_port_wildcard_conflicts() { + local wl_addr="$1" + if [[ $is_wildcard -eq 1 ]]; then + return 0 + fi + # BSD netstat prints *. for an any-family wildcard. + if [[ "$wl_addr" == "*" ]]; then + return 0 + fi + if [[ "$wl_addr" == "0.0.0.0" ]]; then + [[ "$host_family" == "ipv4" ]] && return 0 + # A hostname that resolved to IPv4 can also bind an IPv4 wildcard + if [[ "$host_family" == "hostname" ]]; then + local addr + while IFS= read -r addr; do + [[ -z "$addr" ]] && continue + if [[ "$addr" =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + return 0 + fi + done <<< "$candidate_addrs" + fi + fi + if [[ "$wl_addr" == "::" ]]; then + [[ "$host_family" == "ipv6" ]] && return 0 + if [[ "$host_family" == "hostname" ]]; then + local addr + while IFS= read -r addr; do + [[ -z "$addr" ]] && continue + if [[ "$addr" =~ ^[0-9a-fA-F:]*:[0-9a-fA-F:]*$ ]]; then + return 0 + fi + done <<< "$candidate_addrs" + fi + fi + return 1 + } + + _check_port_match_listener_line() { + local out_line="$1" + # Skip empty or whitespace-only lines before read -a to avoid an empty + # array expansion under set -u on Bash 3.2. + if [[ -z "${out_line//[[:space:]]/}" ]]; then + return 1 + fi + local -a tokens + read -r -a tokens <<< "$out_line" + for token in ${tokens[@]+"${tokens[@]}"}; do + # We only care about the "local address:port" token. For the + # common tools this is the first token that ends with the + # target port after ':' or '.' (peer addresses use :* on Linux + # and *.* on BSD, so they never match a numeric port). + if [[ "$token" =~ ^(.*):(${port})$ ]]; then + listener_addr="${BASH_REMATCH[1]}" + elif [[ "$token" =~ ^(.*)\.(${port})$ ]]; then + listener_addr="${BASH_REMATCH[1]}" + else + continue + fi + + # Wildcard host: any listener on this port is a conflict. + if [[ $is_wildcard -eq 1 ]]; then + matched_token=1 + return 0 + fi + + # No resolved candidates (hostname resolution failed/unsupported): + # we cannot reliably compare against the numeric listener table. + if [[ -z "$candidate_addrs" ]]; then + continue + fi + + matched_token=1 + norm_listener=$(normalize_addr "$listener_addr") + + # A same-family wildcard listener on this port conflicts with any + # specific host of that family. + if _check_port_wildcard_conflicts "$norm_listener"; then + return 0 + fi + + local addr + while IFS= read -r addr; do + [[ -z "$addr" ]] && continue + if [[ "$norm_listener" == "$(normalize_addr "$addr")" ]]; then + return 0 + fi + done <<< "$candidate_addrs" + done + return 1 + } + + if command_available "ss"; then + matched_token=0 + if out=$(ss -ltn 2>/dev/null); then + while IFS= read -r line; do + if _check_port_match_listener_line "$line"; then + in_use=1 + break + fi + done <<< "$out" + # ss -ltn succeeded. We can trust a non-match when we have a + # numeric host or resolved addresses (or a wildcard); otherwise + # fall through to /dev/tcp for unresolved hostnames. + if [[ $in_use -eq 0 && ( $is_wildcard -eq 1 || -n "$candidate_addrs" || $matched_token -eq 1 ) ]]; then + port_checked=1 + fi + fi + fi + + if [[ $in_use -eq 0 && $port_checked -eq 0 ]] && command_available "netstat"; then + matched_token=0 + if out=$(netstat -ltn 2>/dev/null) && echo "$out" | grep -qi "listen"; then + while IFS= read -r line; do + if _check_port_match_listener_line "$line"; then + in_use=1 + break + fi + done <<< "$out" + # netstat -ltn output is the complete Linux listener table. + if [[ $in_use -eq 0 && ( $is_wildcard -eq 1 || -n "$candidate_addrs" || $matched_token -eq 1 ) ]]; then + port_checked=1 + fi + elif out=$(netstat -an 2>/dev/null) && [[ -n "$out" ]]; then + local old_nocasematch + old_nocasematch=$(shopt -p nocasematch 2>/dev/null || true) + shopt -s nocasematch 2>/dev/null || true + while IFS= read -r line; do + if [[ "$line" == *listen* ]] && _check_port_match_listener_line "$line"; then + in_use=1 + break + fi + done <<< "$out" + eval "$old_nocasematch" 2>/dev/null || true + # netstat -an output is the complete BSD listener table. + if [[ $in_use -eq 0 && ( $is_wildcard -eq 1 || -n "$candidate_addrs" || $matched_token -eq 1 ) ]]; then + port_checked=1 + fi + fi + fi + + if [[ $in_use -eq 0 && $port_checked -eq 0 ]]; then + # Probe the actual configured endpoint(s) with a short deadline. + local probe_addrs="$candidate_addrs" + if [[ -z "$probe_addrs" ]]; then + # Could not resolve (or wildcard with only loopback probe needed) + if [[ $is_wildcard -eq 1 ]]; then + probe_addrs="127.0.0.1 ::1" + else + probe_addrs="$host" + fi + fi + + local addr + for addr in $probe_addrs; do + # /dev/tcp needs unbracketed, normalized addresses + addr=$(normalize_addr "$addr") + [[ -z "$addr" ]] && continue + if command_available "timeout"; then + if timeout 1 bash -c ': >/dev/tcp/"$1"/"$2"' _ "$addr" "$port" 2>/dev/null; then + in_use=1 + break + fi + else + if run_with_deadline ': >/dev/tcp/"$1"/"$2" 2>/dev/null' 2 "$addr" "$port"; then + in_use=1 + break + fi + fi + done + fi + + local _rc=0 + if [[ "$in_use" -eq 1 ]]; then echo "The port $port has already been used" - exit 1 + _rc=1 fi + unset -f _check_port_wildcard_conflicts _check_port_match_listener_line || true + [[ $_rc -eq 1 ]] && exit 1 + return 0 } function crontab_append() { @@ -167,14 +556,15 @@ function wait_for_startup() { local server_url="$3" local timeout_s="$4" - local now_s=$(date '+%s') + local now_s + now_s=$(date '+%s') local stop_s=$((now_s + timeout_s)) local status local error_file_name="startup_error.txt" echo -n "Connecting to $server_name ($server_url)" - while [ "$now_s" -le $stop_s ]; do + while [ "$now_s" -le "$stop_s" ]; do echo -n . process_status "$server_name" "$pid" >/dev/null if [ $? -eq 1 ]; then @@ -186,7 +576,7 @@ function wait_for_startup() { fi status=$(curl -I -sS -k -w "%{http_code}" -o /dev/null "$server_url" 2> "$error_file_name") - if [[ $status -eq 200 || $status -eq 401 ]]; then + if [[ "$status" -eq 200 || "$status" -eq 401 ]]; then echo "OK" echo "Started [pid $pid]" if [ -e "$error_file_name" ]; then @@ -219,9 +609,15 @@ function free_memory() { free=$(expr "$mem_free" + "$mem_buffer" + "$mem_cached") free=$(expr "$free" / 1024) elif [ "$os" == "Darwin" ]; then - local pages_free=$(vm_stat | awk '/Pages free/{print $0}' | awk -F'[:.]+' '{print $2}' | tr -d " ") - local pages_inactive=$(vm_stat | awk '/Pages inactive/{print $0}' | awk -F'[:.]+' '{print $2}' | tr -d " ") - local pages_available=$(expr "$pages_free" + "$pages_inactive") + local pages_free pages_inactive + pages_free=$(vm_stat | awk '/Pages free/{print $0}' | awk -F'[:.]+' '{print $2}' | tr -d " ") + pages_inactive=$(vm_stat | awk '/Pages inactive/{print $0}' | awk -F'[:.]+' '{print $2}' | tr -d " ") + if [[ -z "$pages_free" || -z "$pages_inactive" ]]; then + echo "Failed to get free memory" + exit 1 + fi + local pages_available + pages_available=$(expr "$pages_free" + "$pages_inactive") free=$(expr "$pages_available" \* 4096 / 1024 / 1024) else echo "Unsupported operating system $os" @@ -312,23 +708,46 @@ function get_ip() { ;; *) ip=$loopback;; esac - echo $ip + [[ -z "$ip" ]] && ip=$loopback + echo "$ip" } function download() { local path=$1 local download_url=$2 + + if [ ! -d "$path" ]; then + mkdir -p "$path" || { + echo "Failed to create directory: $path" + exit 1 + } + fi + + # Strip query/fragment so the on-disk name matches the server-side artifact. + local filename + filename=$(basename "${download_url%%[?#]*}") + local tmp="${path}/.${filename}.tmp.$$" + local dest="${path}/${filename}" + if command_available "curl"; then - if [ ! -d "$path" ]; then - mkdir -p "$path" || { - echo "Failed to create directory: $path" - exit 1 - } + # -o must appear before -- so it is parsed as an option, not an extra URL. + if curl -fL -o "$tmp" -- "${download_url}"; then + mv -f -- "$tmp" "$dest" + else + rm -f -- "$tmp" + return 1 fi - curl -L "${download_url}" -o "${path}/$(basename "${download_url}")" elif command_available "wget"; then - wget --help | grep -q '\--show-progress' && progress_opt="-q --show-progress" || progress_opt="" - wget "${download_url}" -P "${path}" $progress_opt + local -a progress_opt=() + if wget --help 2>&1 | grep -q -- '--show-progress'; then + progress_opt=(-q --show-progress) + fi + if wget ${progress_opt[@]+"${progress_opt[@]}"} -O "$tmp" -- "${download_url}"; then + mv -f -- "$tmp" "$dest" + else + rm -f -- "$tmp" + return 1 + fi else echo "Required curl or wget but they are unavailable" exit 1 diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/test-check-port.sh b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-check-port.sh new file mode 100755 index 0000000000..c886f6f67e --- /dev/null +++ b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-check-port.sh @@ -0,0 +1,984 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# test-check-port.sh — Unit tests for check_port() in util.sh +# +# Strategy: each test runs check_port in a subshell that overrides +# command_available() to control which probe branch is taken, and +# overrides the tool functions (ss, netstat, timeout) to control +# what they return — no real network connections needed. +# +# check_port calls `exit 1` when the port is in use, so the subshell +# exits 1; it returns normally (exit 0) when the port is free. +# +# Usage: ./test-check-port.sh [path-to-hugegraph-static-dir] +# path-to-hugegraph-static-dir: directory containing bin/util.sh +# Defaults to current directory. +# In CI: $TRAVIS_DIR/test-check-port.sh hugegraph-server/hugegraph-dist/src/assembly/static + +set -uo pipefail +# -u: fail on undefined variables (catches typos in test assertions) +# -o pipefail: pipeline exit status is the last non-zero component +# shellcheck disable=SC1090,SC1091 # UTIL_SH / PD_UTIL_SH sourced dynamically at runtime + +STATIC_DIR="${1:-$(pwd)}" +UTIL_SH="$STATIC_DIR/bin/util.sh" + +REPO_ROOT="$(cd "$(dirname "$0")/../../../../.." && pwd)" +PD_UTIL_SH="$REPO_ROOT/hugegraph-pd/hg-pd-dist/src/assembly/static/bin/util.sh" +STORE_UTIL_SH="$REPO_ROOT/hugegraph-store/hg-store-dist/src/assembly/static/bin/util.sh" + +PASS=0 +FAIL=0 +ERRORS=() + +GREEN='\033[0;32m' +RED='\033[0;31m' +NC='\033[0m' + +pass() { echo -e "${GREEN} PASS${NC} $1"; PASS=$((PASS + 1)); } +fail() { echo -e "${RED} FAIL${NC} $1"; ERRORS+=("$1"); FAIL=$((FAIL + 1)); } +section() { echo ""; echo "── $1 ──"; } + +# timeout mock used by ss/netstat test cases. It passes getent through so +# normalize_addr can canonicalise numeric IPv6, and pretends every other +# probe (the /dev/tcp fallback) failed, so free-port assertions are not +# influenced by real network state. +timeout() { + if [[ "${2:-}" == "getent" ]]; then + shift + "$@" 2>/dev/null + else + return 1 + fi +} + +echo "" +echo "check_port() unit test suite" +echo "util.sh: $UTIL_SH" +echo "" + +if [[ ! -f "$UTIL_SH" ]]; then + echo -e "${RED}ERROR:${NC} $UTIL_SH not found." + echo " Pass the HugeGraph static assembly dir as \$1" + exit 1 +fi + +# ── ss branch ───────────────────────────────────────────────────────────────── + +section "ss branch — IPv4" + +( + # shellcheck source=/dev/null + source "$UTIL_SH" + command_available() { [[ "$1" == "ss" || "$1" == "timeout" ]]; } + ss() { echo "tcp LISTEN 0 128 0.0.0.0:8080 0.0.0.0:*"; } + check_port "http://127.0.0.1:8080" +) +[[ $? -eq 1 ]] \ + && pass "ss: IPv4 port occupied → exit 1" \ + || fail "ss: IPv4 port occupied → expected exit 1, got 0" + +( + source "$UTIL_SH" + command_available() { [[ "$1" == "ss" || "$1" == "timeout" ]]; } + ss() { echo "tcp LISTEN 0 128 0.0.0.0:9090 0.0.0.0:*"; } + check_port "http://127.0.0.1:8080" +) +[[ $? -eq 0 ]] \ + && pass "ss: IPv4 port free → exit 0" \ + || fail "ss: IPv4 port free → expected exit 0, got 1" + +section "ss branch — IPv6 URL with scheme (http://[::1]:8080)" + +( + source "$UTIL_SH" + command_available() { [[ "$1" == "ss" || "$1" == "timeout" ]]; } + ss() { echo "tcp LISTEN 0 128 [::]:8080 [::]:*"; } + check_port "http://[::1]:8080" +) +[[ $? -eq 1 ]] \ + && pass "ss: http://[::1]:8080 occupied → exit 1" \ + || fail "ss: http://[::1]:8080 occupied → expected exit 1, got 0" + +( + source "$UTIL_SH" + command_available() { [[ "$1" == "ss" || "$1" == "timeout" ]]; } + ss() { echo "tcp LISTEN 0 128 [::]:9090 [::]:*"; } + check_port "http://[::1]:8080" +) +[[ $? -eq 0 ]] \ + && pass "ss: http://[::1]:8080 free → exit 0" \ + || fail "ss: http://[::1]:8080 free → expected exit 0, got 1" + +section "ss branch — IPv6 URL without scheme ([::1]:8080)" + +( + source "$UTIL_SH" + command_available() { [[ "$1" == "ss" || "$1" == "timeout" ]]; } + ss() { echo "tcp LISTEN 0 128 [::]:8080 [::]:*"; } + check_port "[::1]:8080" +) +[[ $? -eq 1 ]] \ + && pass "ss: [::1]:8080 (no scheme) occupied → exit 1" \ + || fail "ss: [::1]:8080 (no scheme) occupied → expected exit 1, got 0" + +( + source "$UTIL_SH" + command_available() { [[ "$1" == "ss" || "$1" == "timeout" ]]; } + ss() { echo "tcp LISTEN 0 128 [::]:9090 [::]:*"; } + check_port "[::1]:8080" +) +[[ $? -eq 0 ]] \ + && pass "ss: [::1]:8080 (no scheme) free → exit 0" \ + || fail "ss: [::1]:8080 (no scheme) free → expected exit 0, got 1" + +section "ss branch — wildcard 0.0.0.0" + +( + source "$UTIL_SH" + command_available() { [[ "$1" == "ss" || "$1" == "timeout" ]]; } + ss() { echo "tcp LISTEN 0 128 0.0.0.0:8080 0.0.0.0:*"; } + check_port "http://0.0.0.0:8080" +) +[[ $? -eq 1 ]] \ + && pass "ss: 0.0.0.0:8080 occupied → exit 1" \ + || fail "ss: 0.0.0.0:8080 occupied → expected exit 1, got 0" + +section "ss branch — wildcard ::" + +( + source "$UTIL_SH" + command_available() { [[ "$1" == "ss" || "$1" == "timeout" ]]; } + ss() { echo "tcp LISTEN 0 128 [::]:8080 [::]:*"; } + check_port "http://[::]:8080" +) +[[ $? -eq 1 ]] \ + && pass "ss: [::]:8080 occupied → exit 1" \ + || fail "ss: [::]:8080 occupied → expected exit 1, got 0" + +( + source "$UTIL_SH" + command_available() { [[ "$1" == "ss" || "$1" == "timeout" ]]; } + ss() { echo "tcp LISTEN 0 128 [::]:9090 [::]:*"; } + check_port "http://[::]:8080" +) +[[ $? -eq 0 ]] \ + && pass "ss: [::]:8080 free → exit 0" \ + || fail "ss: [::]:8080 free → expected exit 0, got 1" + +# ── netstat branch ──────────────────────────────────────────────────────────── + +section "netstat branch — Linux format (-ltn), occupied" + +( + source "$UTIL_SH" + command_available() { [[ "$1" == "netstat" || "$1" == "timeout" ]]; } + netstat() { echo "tcp 0 0 0.0.0.0:8080 0.0.0.0:* LISTEN"; } + check_port "http://127.0.0.1:8080" +) +[[ $? -eq 1 ]] \ + && pass "netstat -ltn: port 8080 occupied → exit 1" \ + || fail "netstat -ltn: port 8080 occupied → expected exit 1, got 0" + +section "netstat branch — Linux format (-ltn), free" + +( + source "$UTIL_SH" + command_available() { [[ "$1" == "netstat" || "$1" == "timeout" ]]; } + netstat() { echo "tcp 0 0 0.0.0.0:9090 0.0.0.0:* LISTEN"; } + check_port "http://127.0.0.1:8080" +) +[[ $? -eq 0 ]] \ + && pass "netstat -ltn: port 8080 free → exit 0" \ + || fail "netstat -ltn: port 8080 free → expected exit 0, got 1" + +section "netstat branch — BSD/macOS fallback (-an), occupied" + +# Simulate netstat that produces no output for -ltn (Linux flag unsupported) +# but outputs BSD-format lines for -an +( + source "$UTIL_SH" + command_available() { [[ "$1" == "netstat" || "$1" == "timeout" ]]; } + netstat() { + if [[ "$1" == "-ltn" ]]; then + return 1 # flag not supported on BSD + fi + echo "tcp4 0 0 *.8080 *.* LISTEN" + } + check_port "http://127.0.0.1:8080" +) +[[ $? -eq 1 ]] \ + && pass "netstat -an BSD: port 8080 occupied → exit 1" \ + || fail "netstat -an BSD: port 8080 occupied → expected exit 1, got 0" + +section "netstat branch — IP octet false-positive guard" + +# Port 80 check; netstat output contains 192.168.80.1:443 +# The .80 in the IP address must NOT match port 80 +( + source "$UTIL_SH" + command_available() { [[ "$1" == "netstat" || "$1" == "timeout" ]]; } + netstat() { echo "tcp 0 0 192.168.80.1:443 0.0.0.0:* LISTEN"; } + check_port "http://127.0.0.1:80" +) +[[ $? -eq 0 ]] \ + && pass "netstat: IP octet .80 does not false-positive for port 80 → exit 0" \ + || fail "netstat: IP octet .80 false-positived for port 80 → expected exit 0, got 1" + +section "ss branch — host dot-escaping guard" + +# Host 127.0.0.1 must be matched literally: a listener whose address merely +# matches the pattern with '.' as a regex wildcard (127a0b0c1) must NOT count +( + source "$UTIL_SH" + command_available() { [[ "$1" == "ss" || "$1" == "timeout" ]]; } + ss() { echo "tcp LISTEN 0 128 127a0b0c1:8080 0.0.0.0:*"; } + check_port "http://127.0.0.1:8080" +) +[[ $? -eq 0 ]] \ + && pass "ss: unescaped-dot lookalike 127a0b0c1 does not false-positive → exit 0" \ + || fail "ss: unescaped-dot lookalike 127a0b0c1 false-positived → expected exit 0, got 1" + +# ── /dev/tcp fallback branch ────────────────────────────────────────────────── + +section "/dev/tcp fallback — timeout available, port occupied" + +# timeout exits 0 → connection succeeded → port in use +( + source "$UTIL_SH" + command_available() { [[ "$1" == "timeout" ]]; } + timeout() { + # Assert correct invocation: timeout 1 bash -c SCRIPT _ HOST PORT + [[ "$1" == "1" && "$2" == "bash" && "$3" == "-c" && "$5" == "_" ]] \ + || { echo "timeout mock: unexpected argv: $*"; return 2; } + return 0 + } + check_port "http://127.0.0.1:8080" +) +[[ $? -eq 1 ]] \ + && pass "/dev/tcp+timeout: connection succeeded (exit 0) → port occupied → exit 1" \ + || fail "/dev/tcp+timeout: connection succeeded → expected exit 1, got 0" + +section "/dev/tcp fallback — timeout available, port free" + +# timeout exits 1 → connection refused → port free +( + source "$UTIL_SH" + command_available() { [[ "$1" == "timeout" ]]; } + timeout() { + [[ "$1" == "1" && "$2" == "bash" && "$3" == "-c" && "$5" == "_" ]] \ + || { echo "timeout mock: unexpected argv: $*"; return 2; } + return 1 + } + check_port "http://127.0.0.1:8080" +) +[[ $? -eq 0 ]] \ + && pass "/dev/tcp+timeout: connection refused (exit 1) → port free → exit 0" \ + || fail "/dev/tcp+timeout: connection refused → expected exit 0, got 1" + +section "/dev/tcp fallback — real loopback (ephemeral port)" + +# Start Python server on ephemeral port 0, capture actual port from child stdout +( + source "$UTIL_SH" + command_available() { [[ "$1" == "timeout" ]]; } + # Mock timeout: on hosts without timeout (macOS), run the probe directly. + # The probe args are: timeout 1 bash -c SCRIPT _ HOST PORT + timeout() { + [[ "$1" == "1" && "$2" == "bash" && "$3" == "-c" && "$5" == "_" ]] \ + || { echo "timeout mock: unexpected argv: $*" >&2; return 2; } + # Run the probe with a 2-second hard deadline via background + watchdog + bash -c "$4" "$5" "$6" "$7" 2>/dev/null & + local probe_pid=$! + (sleep 2; kill -9 "$probe_pid" 2>/dev/null) & + local watchdog_pid=$! + wait "$probe_pid" 2>/dev/null + local rc=$? + kill -9 "$watchdog_pid" 2>/dev/null + wait "$watchdog_pid" 2>/dev/null + return $rc + } + # Use a temp file to capture the bound port from child + port_file=$(mktemp) + trap 'rm -f "$port_file"; [[ -n "${PY_PID:-}" ]] && { kill -9 "$PY_PID" 2>/dev/null; wait "$PY_PID" 2>/dev/null; }' EXIT + + # Start Python server that prints the bound port to stdout + python3 -c " +import socket +s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) +s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) +s.bind(('127.0.0.1', 0)) +s.listen(1) # actually listen so /dev/tcp can connect +port = s.getsockname()[1] +print(port, flush=True) +import time +time.sleep(30) # keep server alive +" > "$port_file" 2>/dev/null & + PY_PID=$! + # Poll for port file up to ~4s (Python cold-start can exceed 0.5s on busy CI) + bound_port="" + for _ in $(seq 1 40); do + bound_port=$(head -1 "$port_file" 2>/dev/null) + [[ -n "$bound_port" ]] && break + kill -0 $PY_PID 2>/dev/null || break + sleep 0.1 + done + if [[ -z "$bound_port" || ! "$bound_port" =~ ^[0-9]+$ ]]; then + echo "SKIP: failed to get bound port" + kill -9 $PY_PID 2>/dev/null || true + exit 77 + fi + # Verify child is alive + if ! kill -0 $PY_PID 2>/dev/null; then + echo "SKIP: Python child died" + exit 77 + fi + check_port "http://127.0.0.1:$bound_port" +) +_rc=$? +if [[ $_rc -eq 77 ]]; then + echo " SKIP /dev/tcp real ephemeral port (setup failed)" +elif [[ $_rc -eq 1 ]]; then + pass "/dev/tcp real ephemeral port → detects occupation → exit 1" +else + fail "/dev/tcp real ephemeral port → expected exit 1, got $_rc" +fi + +section "/dev/tcp fallback — no-timeout watchdog kills stuck probe" + +# run_with_deadline "sleep 5" 2 should return in ~2s, not ~5s. +# Proves the watchdog actually kills a stuck child, not just that the code path is taken. +( + source "$UTIL_SH" + start=$(date +%s) + run_with_deadline 'sleep 5' 2 + rc=$? + elapsed=$(($(date +%s) - start)) + # Must complete well before the 5s sleep (2s deadline + overhead) and return non-zero (killed) + if [[ $rc -ne 0 && $elapsed -le 4 ]]; then + exit 0 + else + echo "run_with_deadline: rc=$rc elapsed=${elapsed}s (expected rc≠0, ≤4s)" >&2 + exit 1 + fi +) +[[ $? -eq 0 ]] \ + && pass "/dev/tcp no-timeout watchdog: sleep 5 killed in ≤4s" \ + || fail "/dev/tcp no-timeout watchdog: watchdog did not kill in time" + +# ── host/port conflict semantics ────────────────────────────────────────────── + +section "Host/port conflict semantics" + +# Configured host 192.168.1.50, but only 127.0.0.1 is listening -> free +( + source "$UTIL_SH" + command_available() { [[ "$1" == "ss" || "$1" == "timeout" ]]; } + ss() { echo "LISTEN 0 128 127.0.0.1:8080 0.0.0.0:*"; return 0; } + check_port "http://192.168.1.50:8080" +) +[[ $? -eq 0 ]] \ + && pass "specific IP configured vs loopback listener → free → exit 0" \ + || fail "specific IP configured vs loopback listener → expected free, got occupied" + +# Configured host 192.168.1.50, 0.0.0.0 is listening -> occupied +( + source "$UTIL_SH" + command_available() { [[ "$1" == "ss" || "$1" == "timeout" ]]; } + ss() { echo "LISTEN 0 128 0.0.0.0:8080 0.0.0.0:*"; return 0; } + check_port "http://192.168.1.50:8080" +) +[[ $? -eq 1 ]] \ + && pass "specific IP configured vs wildcard listener → occupied → exit 1" \ + || fail "specific IP configured vs wildcard listener → expected occupied, got free" + +# Specific IPv4 host should not conflict with IPv6 wildcard listener +( + source "$UTIL_SH" + command_available() { [[ "$1" == "ss" || "$1" == "timeout" ]]; } + ss() { echo "LISTEN 0 128 [::]:8080 [::]:*"; return 0; } + check_port "http://127.0.0.1:8080" +) +[[ $? -eq 0 ]] \ + && pass "specific IPv4 host vs IPv6 wildcard listener → free → exit 0" \ + || fail "specific IPv4 host vs IPv6 wildcard listener → expected free, got occupied" + +# Specific IPv6 host should not conflict with IPv4 wildcard listener +( + source "$UTIL_SH" + command_available() { [[ "$1" == "ss" || "$1" == "timeout" ]]; } + ss() { echo "LISTEN 0 128 0.0.0.0:8080 0.0.0.0:*"; return 0; } + check_port "http://[::1]:8080" +) +[[ $? -eq 0 ]] \ + && pass "specific IPv6 host vs IPv4 wildcard listener → free → exit 0" \ + || fail "specific IPv6 host vs IPv4 wildcard listener → expected free, got occupied" + +# ── tool failure fallthrough ────────────────────────────────────────────────── + +section "Tool failure fallthrough" + +# ss fails, netstat succeeds and finds occupied port +( + source "$UTIL_SH" + command_available() { [[ "$1" == "ss" || "$1" == "netstat" || "$1" == "timeout" ]]; } + ss() { return 1; } # ss execution fails + netstat() { + if [[ "$1" == "-ltn" ]]; then echo "tcp 0 0 0.0.0.0:8080 0.0.0.0:* LISTEN"; return 0; fi + return 1 + } + check_port "http://127.0.0.1:8080" +) +[[ $? -eq 1 ]] \ + && pass "ss exits 1 → fallthrough to netstat → occupied → exit 1" \ + || fail "ss exits 1 → fallthrough to netstat → expected occupied, got free" + +# both ss and netstat fail, /dev/tcp timeout path must still detect occupation +( + source "$UTIL_SH" + command_available() { [[ "$1" == "ss" || "$1" == "netstat" || "$1" == "timeout" ]]; } + ss() { return 1; } + netstat() { return 1; } + timeout() { return 0; } # /dev/tcp connect succeeds + check_port "http://127.0.0.1:8080" +) +[[ $? -eq 1 ]] \ + && pass "ss/netstat both fail → /dev/tcp timeout path occupied → exit 1" \ + || fail "ss/netstat both fail → expected /dev/tcp fallback occupation, got free" + +# ── hostname and URL normalization ──────────────────────────────────────────── + +section "Hostname collision detection" + +# localhost should resolve to numeric addresses before ss matching +( + source "$UTIL_SH" + command_available() { [[ "$1" == "ss" || "$1" == "getent" || "$1" == "timeout" ]]; } + timeout() { shift; "$@" 2>/dev/null; } # pass-through for mocked getent + getent() { + if [[ "$1" == "hosts" || "$1" == "ahosts" ]] && [[ "$2" == "localhost" ]]; then + echo "127.0.0.1" + return 0 + fi + return 1 + } + ss() { echo "tcp LISTEN 0 128 127.0.0.1:8080 0.0.0.0:*"; } + check_port "http://localhost:8080" +) +[[ $? -eq 1 ]] \ + && pass "hostname localhost resolves to 127.0.0.1 and detects occupation → exit 1" \ + || fail "hostname localhost occupation was not detected via resolved address" + +# unresolved hostnames should skip ss/netstat text matching and use /dev/tcp fallback +( + source "$UTIL_SH" + command_available() { [[ "$1" == "ss" || "$1" == "getent" || "$1" == "timeout" ]]; } + getent() { return 2; } # resolution fails + ss() { echo "tcp LISTEN 0 128 127.0.0.1:8080 0.0.0.0:*"; return 0; } + timeout() { return 0; } # fallback connect succeeds + check_port "http://unresolved.localdomain:8080" +) +[[ $? -eq 1 ]] \ + && pass "unresolved hostname falls through to /dev/tcp fallback and detects occupation" \ + || fail "unresolved hostname should use /dev/tcp fallback" + +section "URL normalization and IPv6 hextet guard" + +# check_port should trim URL whitespace before parsing host/port +( + source "$UTIL_SH" + command_available() { [[ "$1" == "ss" || "$1" == "timeout" ]]; } + ss() { echo "tcp LISTEN 0 128 127.0.0.1:8080 0.0.0.0:*"; } + check_port " http://127.0.0.1:8080 " +) +[[ $? -eq 1 ]] \ + && pass "leading/trailing URL whitespace is normalized → occupied detected" \ + || fail "URL whitespace normalization failed to detect occupied port" + +# Ensure ":8080" inside an IPv6 hextet does not match listener port 8080 +( + source "$UTIL_SH" + command_available() { [[ "$1" == "ss" || "$1" == "timeout" ]]; } + timeout() { + if [[ "$2" == "getent" ]]; then + shift + "$@" 2>/dev/null + else + return 1 + fi + } + ss() { echo "tcp LISTEN 0 128 [2001:db8:8080::1]:9090 [::]:*"; } + check_port "http://[::1]:8080" +) +[[ $? -eq 0 ]] \ + && pass "IPv6 hextet containing 8080 does not false-positive port 8080" \ + || fail "IPv6 hextet false-positive: expected free, got occupied" + +# ── IPv6 address canonicalisation ───────────────────────────────────────────── + +section "IPv6 address canonicalisation" + +# getent mock that normalises the two spellings of loopback to ::1 +getent_canonical_loopback() { + if [[ "$1" == "ahosts" ]] && [[ "$2" == "::1" || "$2" == "0:0:0:0:0:0:0:1" ]]; then + echo "::1" + return 0 + fi + return 2 +} + +# Host ::1, listener expanded 0:0:0:0:0:0:0:1 +( + source "$UTIL_SH" + command_available() { [[ "$1" == "ss" || "$1" == "timeout" ]]; } + getent() { getent_canonical_loopback "$@"; } + ss() { echo "tcp LISTEN 0 128 [0:0:0:0:0:0:0:1]:8080 [::]:*"; } + check_port "http://[::1]:8080" +) +[[ $? -eq 1 ]] \ + && pass "IPv6 host [::1] matches expanded listener [0:0:0:0:0:0:0:1]:8080" \ + || fail "IPv6 compressed vs expanded spelling did not match" + +# Host expanded 0:0:0:0:0:0:0:1, listener ::1 (mixed case) +( + source "$UTIL_SH" + command_available() { [[ "$1" == "ss" || "$1" == "timeout" ]]; } + getent() { getent_canonical_loopback "$@"; } + ss() { echo "tcp LISTEN 0 128 [::1]:8080 [::]:*"; } + check_port "http://[0:0:0:0:0:0:0:1]:8080" +) +[[ $? -eq 1 ]] \ + && pass "IPv6 expanded host matches compressed listener" \ + || fail "IPv6 expanded vs compressed spelling did not match" + +# Host ::1, listener expanded on a different port → free +( + source "$UTIL_SH" + command_available() { [[ "$1" == "ss" || "$1" == "timeout" ]]; } + timeout() { + if [[ "$2" == "getent" ]]; then + shift + "$@" 2>/dev/null + else + return 1 + fi + } + getent() { getent_canonical_loopback "$@"; } + ss() { echo "tcp LISTEN 0 128 [0:0:0:0:0:0:0:1]:9090 [::]:*"; } + check_port "http://[::1]:8080" +) +[[ $? -eq 0 ]] \ + && pass "IPv6 canonicalisation does not false-positive on different port" \ + || fail "IPv6 canonicalisation matched the wrong port" + +# ── edge cases ──────────────────────────────────────────────────────────────── + +section "Edge cases" + +# Empty URL → port string is empty → return 0 immediately +( + source "$UTIL_SH" + command_available() { false; } + check_port "" +) +[[ $? -eq 0 ]] \ + && pass "empty URL → exit 0 (graceful)" \ + || fail "empty URL → expected exit 0, got 1" + +# Port out of range (> 65535) → return 0 immediately +( + source "$UTIL_SH" + command_available() { false; } + check_port "http://127.0.0.1:99999" +) +[[ $? -eq 0 ]] \ + && pass "port 99999 (out of range) → exit 0 (graceful)" \ + || fail "port 99999 (out of range) → expected exit 0, got 1" + +# Port 0 (invalid) → return 0 immediately +( + source "$UTIL_SH" + command_available() { false; } + check_port "http://127.0.0.1:0" +) +[[ $? -eq 0 ]] \ + && pass "port 0 (invalid) → exit 0 (graceful)" \ + || fail "port 0 (invalid) → expected exit 0, got 1" + +# Non-numeric port → return 0 immediately +( + source "$UTIL_SH" + command_available() { false; } + check_port "127.0.0.1:abc" +) +[[ $? -eq 0 ]] \ + && pass "non-numeric port → exit 0 (graceful)" \ + || fail "non-numeric port → expected exit 0, got 1" + +# ── download() fallback ─────────────────────────────────────────────────────── + +section "download() atomic curl path (pd)" + +( + if [[ ! -f "$PD_UTIL_SH" ]]; then + exit 77 + fi + + source "$PD_UTIL_SH" + + OUTDIR="$(mktemp -d)/outdir" + trap 'rm -rf "$(dirname "$OUTDIR")"' EXIT + + command_available() { + if [[ "$1" == "wget" ]]; then return 1; fi + if [[ "$1" == "curl" ]]; then return 0; fi + command -v "$1" >/dev/null 2>&1 + } + + MKDIR_CALLED=0 + mkdir() { + if [[ "$1" == "-p" && "$2" == "$OUTDIR" ]]; then + MKDIR_CALLED=1 + command mkdir -p "$OUTDIR" + return 0 + fi + echo "mkdir mock called with unexpected args: $*" + return 1 + } + + CURL_CALLED=0 + curl() { + # Expected: curl -fL -o -- + if [[ "$1" == "-fL" && "$2" == "-o" && \ + "$4" == "--" && \ + "$5" == "https://example.com/some/path/file.tar.gz" ]]; then + CURL_CALLED=1 + # The temp name must be hidden and contain the PID. + if [[ "$3" != "$OUTDIR/.file.tar.gz.tmp."* ]]; then + echo "curl mock: unexpected output path: $3" + return 1 + fi + touch "$3" + return 0 + fi + echo "curl mock called with unexpected args: $*" + return 1 + } + + download "$OUTDIR" "https://example.com/some/path/file.tar.gz" >/dev/null 2>&1 + RC=$? + + if [[ $RC -eq 0 && $CURL_CALLED -eq 1 && $MKDIR_CALLED -eq 1 && \ + -f "$OUTDIR/file.tar.gz" ]]; then + exit 0 + else + echo "RC=$RC CURL_CALLED=$CURL_CALLED MKDIR_CALLED=$MKDIR_CALLED" + ls -la "$OUTDIR" 2>&1 || true + exit 1 + fi +) +_rc=$? +if [[ $_rc -eq 77 ]]; then + echo " SKIP download() atomic curl path: pd util.sh not found" +elif [[ $_rc -eq 0 ]]; then + pass "download() (PD) writes to temp and renames on success" +else + fail "download() (PD) atomic curl path failed" +fi + +section "download() atomic curl path (store)" + +( + if [[ ! -f "$STORE_UTIL_SH" ]]; then + exit 77 + fi + + source "$STORE_UTIL_SH" + + OUTDIR="$(mktemp -d)/outdir" + trap 'rm -rf "$(dirname "$OUTDIR")"' EXIT + + command_available() { + if [[ "$1" == "wget" ]]; then return 1; fi + if [[ "$1" == "curl" ]]; then return 0; fi + command -v "$1" >/dev/null 2>&1 + } + + MKDIR_CALLED=0 + mkdir() { + if [[ "$1" == "-p" && "$2" == "$OUTDIR" ]]; then + MKDIR_CALLED=1 + command mkdir -p "$OUTDIR" + return 0 + fi + echo "mkdir mock called with unexpected args: $*" + return 1 + } + + CURL_CALLED=0 + curl() { + if [[ "$1" == "-fL" && "$2" == "-o" && \ + "$4" == "--" && \ + "$5" == "https://example.com/some/path/file.tar.gz" ]]; then + CURL_CALLED=1 + if [[ "$3" != "$OUTDIR/.file.tar.gz.tmp."* ]]; then + echo "curl mock: unexpected output path: $3" + return 1 + fi + touch "$3" + return 0 + fi + echo "curl mock called with unexpected args: $*" + return 1 + } + + download "$OUTDIR" "https://example.com/some/path/file.tar.gz" >/dev/null 2>&1 + RC=$? + + if [[ $RC -eq 0 && $CURL_CALLED -eq 1 && $MKDIR_CALLED -eq 1 && \ + -f "$OUTDIR/file.tar.gz" ]]; then + exit 0 + else + echo "RC=$RC CURL_CALLED=$CURL_CALLED MKDIR_CALLED=$MKDIR_CALLED" + ls -la "$OUTDIR" 2>&1 || true + exit 1 + fi +) +_rc=$? +if [[ $_rc -eq 77 ]]; then + echo " SKIP download() atomic curl path: store util.sh not found" +elif [[ $_rc -eq 0 ]]; then + pass "download() (Store) writes to temp and renames on success" +else + fail "download() (Store) atomic curl path failed" +fi + +section "download() atomic wget path (pd)" + +( + if [[ ! -f "$PD_UTIL_SH" ]]; then + exit 77 + fi + + source "$PD_UTIL_SH" + + OUTDIR="$(mktemp -d)/outdir" + trap 'rm -rf "$(dirname "$OUTDIR")"' EXIT + + command_available() { + if [[ "$1" == "curl" ]]; then return 1; fi + if [[ "$1" == "wget" ]]; then return 0; fi + command -v "$1" >/dev/null 2>&1 + } + + MKDIR_CALLED=0 + mkdir() { + if [[ "$1" == "-p" && "$2" == "$OUTDIR" ]]; then + MKDIR_CALLED=1 + command mkdir -p "$OUTDIR" + return 0 + fi + echo "mkdir mock called with unexpected args: $*" + return 1 + } + + WGET_CALLED=0 + wget() { + if [[ "$1" == "--help" ]]; then + # wget --help runs in a pipeline subshell, so side-effect counters + # cannot be observed from the parent. Only the stdout matters here. + echo "--show-progress" + return 0 + fi + # Expected: wget -q --show-progress -O -- + if [[ "$1" == "-q" && "$2" == "--show-progress" && "$3" == "-O" && \ + "$5" == "--" && \ + "$6" == "https://example.com/some/path/file.tar.gz" ]]; then + WGET_CALLED=1 + if [[ "$4" != "$OUTDIR/.file.tar.gz.tmp."* ]]; then + echo "wget mock: unexpected output path: $4" + return 1 + fi + touch "$4" + return 0 + fi + echo "wget mock called with unexpected args: $*" + return 1 + } + + download "$OUTDIR" "https://example.com/some/path/file.tar.gz" >/dev/null 2>&1 + RC=$? + + if [[ $RC -eq 0 && $WGET_CALLED -eq 1 && $MKDIR_CALLED -eq 1 && \ + -f "$OUTDIR/file.tar.gz" ]]; then + exit 0 + else + echo "RC=$RC WGET_CALLED=$WGET_CALLED MKDIR_CALLED=$MKDIR_CALLED" + ls -la "$OUTDIR" 2>&1 || true + exit 1 + fi +) +_rc=$? +if [[ $_rc -eq 77 ]]; then + echo " SKIP download() atomic wget path: pd util.sh not found" +elif [[ $_rc -eq 0 ]]; then + pass "download() (PD) wget writes to temp and renames on success" +else + fail "download() (PD) atomic wget path failed" +fi + +section "download() atomic wget path (store)" + +( + if [[ ! -f "$STORE_UTIL_SH" ]]; then + exit 77 + fi + + source "$STORE_UTIL_SH" + + OUTDIR="$(mktemp -d)/outdir" + trap 'rm -rf "$(dirname "$OUTDIR")"' EXIT + + command_available() { + if [[ "$1" == "curl" ]]; then return 1; fi + if [[ "$1" == "wget" ]]; then return 0; fi + command -v "$1" >/dev/null 2>&1 + } + + MKDIR_CALLED=0 + mkdir() { + if [[ "$1" == "-p" && "$2" == "$OUTDIR" ]]; then + MKDIR_CALLED=1 + command mkdir -p "$OUTDIR" + return 0 + fi + echo "mkdir mock called with unexpected args: $*" + return 1 + } + + WGET_CALLED=0 + wget() { + if [[ "$1" == "--help" ]]; then + # wget --help runs in a pipeline subshell, so side-effect counters + # cannot be observed from the parent. Only the stdout matters here. + echo "--show-progress" + return 0 + fi + # Expected: wget -q --show-progress -O -- + if [[ "$1" == "-q" && "$2" == "--show-progress" && "$3" == "-O" && \ + "$5" == "--" && \ + "$6" == "https://example.com/some/path/file.tar.gz" ]]; then + WGET_CALLED=1 + if [[ "$4" != "$OUTDIR/.file.tar.gz.tmp."* ]]; then + echo "wget mock: unexpected output path: $4" + return 1 + fi + touch "$4" + return 0 + fi + echo "wget mock called with unexpected args: $*" + return 1 + } + + download "$OUTDIR" "https://example.com/some/path/file.tar.gz" >/dev/null 2>&1 + RC=$? + + if [[ $RC -eq 0 && $WGET_CALLED -eq 1 && $MKDIR_CALLED -eq 1 && \ + -f "$OUTDIR/file.tar.gz" ]]; then + exit 0 + else + echo "RC=$RC WGET_CALLED=$WGET_CALLED MKDIR_CALLED=$MKDIR_CALLED" + ls -la "$OUTDIR" 2>&1 || true + exit 1 + fi +) +_rc=$? +if [[ $_rc -eq 77 ]]; then + echo " SKIP download() atomic wget path: store util.sh not found" +elif [[ $_rc -eq 0 ]]; then + pass "download() (Store) wget writes to temp and renames on success" +else + fail "download() (Store) atomic wget path failed" +fi + +section "download() curl failure cleanup (pd)" + +# curl failure must clean up the partial file and never leave a poisoned artifact +( + if [[ ! -f "$PD_UTIL_SH" ]]; then exit 77; fi + source "$PD_UTIL_SH" + + OUTDIR=$(mktemp -d) + trap 'rm -rf "$OUTDIR"' EXIT + + command_available() { + if [[ "$1" == "wget" ]]; then return 1; fi + if [[ "$1" == "curl" ]]; then return 0; fi + command -v "$1" >/dev/null 2>&1 + } + mkdir() { command mkdir -p "$2"; return 0; } + PARTIAL="" + curl() { + if [[ "$1" == "-fL" && "$2" == "-o" && \ + "$4" == "--" && \ + "$5" == "https://example.com/some/path/file.tar.gz" ]]; then + PARTIAL="$3" + # Simulate a partial/corrupted transfer: write something then fail. + echo "partial" > "$PARTIAL" + return 1 + fi + return 1 + } + download "$OUTDIR" "https://example.com/some/path/file.tar.gz" >/dev/null 2>&1 + rc=$? + + if [[ $rc -ne 0 && ! -f "$OUTDIR/file.tar.gz" && \ + ( -z "$PARTIAL" || ! -f "$PARTIAL" ) ]]; then + exit 0 + else + echo "rc=$rc PARTIAL=$PARTIAL DEST=$([[ -f $OUTDIR/file.tar.gz ]] && echo exists || echo missing)" + ls -la "$OUTDIR" 2>&1 || true + exit 1 + fi +) +_rc=$? +if [[ $_rc -eq 77 ]]; then + echo " SKIP download() curl failure cleanup: pd util.sh not found" +elif [[ $_rc -eq 0 ]]; then + pass "download() curl failure cleans temp and does not poison destination" +else + fail "download() curl failure did not clean partial file" +fi + +# ── summary ─────────────────────────────────────────────────────────────────── + +echo "" +echo "════════════════════════════════" +echo -e " Results: ${GREEN}$PASS passed${NC} ${RED}$FAIL failed${NC}" +echo "════════════════════════════════" + +if [[ $FAIL -gt 0 ]]; then + echo "" + echo "Failed tests:" + for err in ${ERRORS[@]+"${ERRORS[@]}"}; do + echo -e " ${RED}✗${NC} $err" + done +fi + +echo "" +[[ $FAIL -eq 0 ]] && exit 0 || exit 1 diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/test-start-hugegraph-pd.sh b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-start-hugegraph-pd.sh index 260e7f5711..c6926c6dfd 100755 --- a/hugegraph-server/hugegraph-dist/src/assembly/travis/test-start-hugegraph-pd.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-start-hugegraph-pd.sh @@ -30,6 +30,7 @@ set -uo pipefail PD_ROOT="${1:-$(pwd)}" BIN="$PD_ROOT/bin" START_SCRIPT="$BIN/start-hugegraph-pd.sh" +STOP_SCRIPT="$BIN/stop-hugegraph-pd.sh" PID_FILE="$BIN/pid" PD_URL="http://localhost:8620" STARTUP_WAIT=60 # seconds to wait for PD HTTP to respond @@ -67,14 +68,29 @@ section() { cleanup() { info "Cleaning up..." + "$STOP_SCRIPT" >/dev/null 2>&1 || true if [[ -s "$PID_FILE" ]]; then kill "$(cat "$PID_FILE")" 2>/dev/null || true fi rm -f "$PID_FILE" rm -rf "$PD_ROOT/logs/" - # kill anything still holding the PD port - lsof -ti :8620 | xargs kill -9 2>/dev/null || true - lsof -ti :8686 | xargs kill -9 2>/dev/null || true + # kill anything still holding the PD port (fuser avoids lsof dependency) + if [[ "$(uname)" == "Darwin" ]]; then + local port pids pid + for port in 8620 8686; do + pids=$(lsof -ti ":$port" 2>/dev/null) + if [[ -n "$pids" ]]; then + for pid in $pids; do + kill -9 "$pid" 2>/dev/null || true + done + fi + done + elif command -v fuser >/dev/null 2>&1; then + fuser -k 8620/tcp 2>/dev/null || true + fuser -k 8686/tcp 2>/dev/null || true + else + echo "Warning: Neither lsof nor fuser available for PD port cleanup" + fi sleep 3 } @@ -129,7 +145,12 @@ if [[ ! -f "$START_SCRIPT" ]]; then exit 1 fi -for tool in lsof curl java; do +if [[ "$(uname)" == "Darwin" ]]; then + _prereq_tools="lsof curl java" +else + _prereq_tools="fuser curl java" +fi +for tool in $_prereq_tools; do if ! command -v "$tool" >/dev/null 2>&1; then echo "SKIP: required tool '$tool' not found — skipping test suite" exit 77 diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/test-start-hugegraph-store.sh b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-start-hugegraph-store.sh index 7e11da28dd..9ff324ae29 100755 --- a/hugegraph-server/hugegraph-dist/src/assembly/travis/test-start-hugegraph-store.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-start-hugegraph-store.sh @@ -51,15 +51,30 @@ section() { echo ""; echo "── $1 ──"; } cleanup() { info "Cleaning up..." + "$STOP_SCRIPT" >/dev/null 2>&1 || true if [[ -s "$PID_FILE" ]]; then kill "$(cat "$PID_FILE")" 2>/dev/null || true fi rm -f "$PID_FILE" rm -rf "$STORE_ROOT/logs/" # kill anything holding Store ports (8520 REST, 8510 raft, 8500 gRPC) - lsof -ti :8520 | xargs kill -9 2>/dev/null || true - lsof -ti :8510 | xargs kill -9 2>/dev/null || true - lsof -ti :8500 | xargs kill -9 2>/dev/null || true + if [[ "$(uname)" == "Darwin" ]]; then + local port pids pid + for port in 8520 8510 8500; do + pids=$(lsof -ti ":$port" 2>/dev/null) + if [[ -n "$pids" ]]; then + for pid in $pids; do + kill -9 "$pid" 2>/dev/null || true + done + fi + done + elif command -v fuser >/dev/null 2>&1; then + fuser -k 8520/tcp 2>/dev/null || true + fuser -k 8510/tcp 2>/dev/null || true + fuser -k 8500/tcp 2>/dev/null || true + else + echo "Warning: Neither lsof nor fuser available for Store port cleanup" + fi sleep 3 } @@ -110,7 +125,12 @@ if [[ ! -f "$START_SCRIPT" ]]; then exit 1 fi -for tool in lsof curl java; do +if [[ "$(uname)" == "Darwin" ]]; then + _prereq_tools="lsof curl java" +else + _prereq_tools="fuser curl java" +fi +for tool in $_prereq_tools; do if ! command -v "$tool" >/dev/null 2>&1; then echo "SKIP: required tool '$tool' not found — skipping test suite" exit 77 diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/test-start-hugegraph.sh b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-start-hugegraph.sh index 353cab6518..c8e5bfced5 100755 --- a/hugegraph-server/hugegraph-dist/src/assembly/travis/test-start-hugegraph.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-start-hugegraph.sh @@ -76,10 +76,26 @@ cleanup() { kill "$(cat "$PID_FILE")" 2>/dev/null || true fi rm -f "$PID_FILE" - # kill anything still holding server ports so check_port doesn't fail next test - lsof -ti :8080 | xargs kill -9 2>/dev/null || true - lsof -ti :8182 | xargs kill -9 2>/dev/null || true - lsof -ti :8088 | xargs kill -9 2>/dev/null || true + # kill anything still holding server ports (fuser avoids lsof dependency) + if [[ "$(uname)" == "Darwin" ]]; then + local port pids pid + for port in 8080 8182 8088; do + pids=$(lsof -ti ":$port" 2>/dev/null) + if [[ -n "$pids" ]]; then + for pid in $pids; do + kill -9 "$pid" 2>/dev/null || true + done + fi + done + elif command -v fuser >/dev/null 2>&1; then + # fuser exists and should work on Linux; suppress failures but log if debug needed + fuser -k 8080/tcp 2>/dev/null || true + fuser -k 8182/tcp 2>/dev/null || true + fuser -k 8088/tcp 2>/dev/null || true + else + # Neither lsof nor fuser available; ports may remain occupied + echo "Warning: Neither lsof nor fuser available for port cleanup" + fi sleep 3 crontab -l 2>/dev/null | grep -v monitor-hugegraph | crontab - 2>/dev/null || true } @@ -145,7 +161,12 @@ if [[ ! -f "$STOP_SCRIPT" ]]; then exit 1 fi -for tool in lsof crontab curl java; do +if [[ "$(uname)" == "Darwin" ]]; then + _prereq_tools="lsof crontab curl java" +else + _prereq_tools="fuser crontab curl java" +fi +for tool in $_prereq_tools; do if ! command -v "$tool" >/dev/null 2>&1; then echo "SKIP: required tool '$tool' not found — skipping test suite" exit 77 @@ -155,7 +176,8 @@ done # start-monitor.sh requires JAVA_HOME if [[ -z "${JAVA_HOME:-}" ]]; then if command -v /usr/libexec/java_home >/dev/null 2>&1; then - export JAVA_HOME="$(/usr/libexec/java_home 2>/dev/null)" + JAVA_HOME="$(/usr/libexec/java_home 2>/dev/null)" + export JAVA_HOME fi fi diff --git a/hugegraph-store/hg-store-dist/src/assembly/static/bin/start-hugegraph-store.sh b/hugegraph-store/hg-store-dist/src/assembly/static/bin/start-hugegraph-store.sh index 5a11eeab39..88165e47b6 100755 --- a/hugegraph-store/hg-store-dist/src/assembly/static/bin/start-hugegraph-store.sh +++ b/hugegraph-store/hg-store-dist/src/assembly/static/bin/start-hugegraph-store.sh @@ -46,8 +46,8 @@ if [[ $arch == "aarch64" || $arch == "arm64" ]]; then lib_file="$TOP/bin/libjemalloc_aarch64.so" download_url="${GITHUB}/apache/hugegraph-doc/raw/binary-1.5/dist/server/libjemalloc_aarch64.so" expected_md5="2a631d2f81837f9d5864586761c5e380" - if download_and_verify $download_url $lib_file $expected_md5; then - export LD_PRELOAD=$lib_file + if download_and_verify "$download_url" "$lib_file" "$expected_md5"; then + export LD_PRELOAD="$lib_file" else echo "Failed to verify or download $lib_file, skip it" fi @@ -55,8 +55,8 @@ elif [[ $arch == "x86_64" ]]; then lib_file="$TOP/bin/libjemalloc.so" download_url="${GITHUB}/apache/hugegraph-doc/raw/binary-1.5/dist/server/libjemalloc.so" expected_md5="fd61765eec3bfea961b646c269f298df" - if download_and_verify $download_url $lib_file $expected_md5; then - export LD_PRELOAD=$lib_file + if download_and_verify "$download_url" "$lib_file" "$expected_md5"; then + export LD_PRELOAD="$lib_file" else echo "Failed to verify or download $lib_file, skip it" fi @@ -73,7 +73,8 @@ export FILE_LIMITN=1024 #export FILE_LIMITN=1024000 function check_evn_limit() { - local limit_check=$(ulimit -n) + local limit_check + limit_check=$(ulimit -n) if [[ ${limit_check} != "unlimited" && ${limit_check} -lt ${FILE_LIMITN} ]]; then echo -e "${BASH_SOURCE[0]##*/}:${LINENO}:\E[1;32m ulimit -n can open too few maximum file descriptors, need (${FILE_LIMITN})!! \E[0m" return 1 @@ -218,7 +219,7 @@ fi # JAVA_OPTIONS="${JAVA_OPTIONS} -javaagent:${LIB}/jmx_prometheus_javaagent-0.16.1.jar=${JMX_EXPORT_PORT}:${CONF}/jmx_exporter.yml" #fi -if [ $(ps -ef|grep -v grep| grep java|grep -cE ${CONF}) -ne 0 ]; then +if [ "$(ps -ef | grep -v grep | grep java | grep -cE "${CONF}")" -ne 0 ]; then echo "HugeGraphStoreServer is already running..." exit 0 fi diff --git a/hugegraph-store/hg-store-dist/src/assembly/static/bin/util.sh b/hugegraph-store/hg-store-dist/src/assembly/static/bin/util.sh index 3b3d660102..5e68d4cbb3 100644 --- a/hugegraph-store/hg-store-dist/src/assembly/static/bin/util.sh +++ b/hugegraph-store/hg-store-dist/src/assembly/static/bin/util.sh @@ -19,20 +19,21 @@ function command_available() { local cmd=$1 - if [ `command -v $cmd >/dev/null 2>&1` ]; then - return 1 - else + if command -v "$cmd" >/dev/null 2>&1; then return 0 fi + return 1 } # read a property from .properties file function read_property() { # file path + local file_name + local property_name file_name=$1 # replace "." to "\." - property_name=`echo $2 | sed 's/\./\\\./g'` - cat $file_name | sed -n -e "s/^[ ]*//g;/^#/d;s/^$property_name=//p" | tail -1 + property_name=$(echo "$2" | sed 's/\./\\\./g') + cat "$file_name" | sed -n -e "s/^[ ]*//g;/^#/d;s/^$property_name=//p" | tail -1 } function write_property() { @@ -40,7 +41,7 @@ function write_property() { local key=$2 local value=$3 - local os=`uname` + local os=$(uname) case $os in # Note: in mac os should use sed -i '' "xxx" to replace string, # otherwise prompt 'command c expects \ followed by text'. @@ -55,7 +56,7 @@ function parse_yaml() { local version=$2 local module=$3 - cat $file | tr -d '\n {}'| awk -F',+|:' '''{ + cat "$file" | tr -d '\n {}'| awk -F',+|:' '''{ pre=""; for(i=1; i<=NF; ) { if(match($i, /version/)) { @@ -71,27 +72,19 @@ function parse_yaml() { } function process_num() { - num=`ps -ef | grep $1 | grep -v grep | wc -l` - return $num + local num + num=$(ps -ef | grep "$1" | grep -v grep | wc -l) + if (( num > 0 )); then + return 1 + fi + return 0 } function process_id() { - pid=`ps -ef | grep $1 | grep -v grep | awk '{print $2}'` - return $pid -} - -# check the port of rest server is occupied -function check_port() { - local port=$(echo "$1" | sed 's|.*:||' | sed 's|/.*||') - if ! command_available "lsof"; then - echo "Required lsof but it is unavailable" - exit 1 - fi - lsof -i :$port >/dev/null - if [ $? -eq 0 ]; then - echo "The port $port has already been used" - exit 1 - fi + local pid + pid=$(ps -ef | grep "$1" | grep -v grep | awk '{print $2}') + echo "$pid" + return 0 } function crontab_append() { @@ -129,13 +122,14 @@ function wait_for_startup() { local server_url="$3" local timeout_s="$4" - local now_s=`date '+%s'` - local stop_s=$(( $now_s + $timeout_s )) + local now_s + now_s=$(date '+%s') + local stop_s=$(( now_s + timeout_s )) local status echo -n "Connecting to $server_name ($server_url)" - while [ $now_s -le $stop_s ]; do + while [ "$now_s" -le "$stop_s" ]; do echo -n . process_status "$server_name" "$pid" >/dev/null if [ $? -eq 1 ]; then @@ -143,14 +137,14 @@ function wait_for_startup() { return 1 fi - status=`curl -o /dev/null -s -k -w %{http_code} $server_url` - if [[ $status -eq 200 || $status -eq 401 ]]; then + status=$(curl -o /dev/null -s -k -w '%{http_code}' "$server_url") + if [[ "$status" -eq 200 || "$status" -eq 401 ]]; then echo "OK" echo "Started [pid $pid]" return 0 fi sleep 2 - now_s=`date '+%s'` + now_s=$(date '+%s') done echo "The operation timed out when attempting to connect to $server_url" >&2 @@ -159,22 +153,28 @@ function wait_for_startup() { function free_memory() { local free="" - local os=`uname` + local os=$(uname) if [ "$os" == "Linux" ]; then - local mem_free=`cat /proc/meminfo | grep -w "MemFree" | awk '{print $2}'` - local mem_buffer=`cat /proc/meminfo | grep -w "Buffers" | awk '{print $2}'` - local mem_cached=`cat /proc/meminfo | grep -w "Cached" | awk '{print $2}'` + local mem_free=$(cat /proc/meminfo | grep -w "MemFree" | awk '{print $2}') + local mem_buffer=$(cat /proc/meminfo | grep -w "Buffers" | awk '{print $2}') + local mem_cached=$(cat /proc/meminfo | grep -w "Cached" | awk '{print $2}') if [[ "$mem_free" == "" || "$mem_buffer" == "" || "$mem_cached" == "" ]]; then echo "Failed to get free memory" exit 1 fi - free=`expr $mem_free + $mem_buffer + $mem_cached` - free=`expr $free / 1024` + free=$(expr "$mem_free" + "$mem_buffer" + "$mem_cached") + free=$(expr "$free" / 1024) elif [ "$os" == "Darwin" ]; then - local pages_free=`vm_stat | awk '/Pages free/{print $0}' | awk -F'[:.]+' '{print $2}' | tr -d " "` - local pages_inactive=`vm_stat | awk '/Pages inactive/{print $0}' | awk -F'[:.]+' '{print $2}' | tr -d " "` - local pages_available=`expr $pages_free + $pages_inactive` - free=`expr $pages_available \* 4096 / 1024 / 1024` + local pages_free pages_inactive + pages_free=$(vm_stat | awk '/Pages free/{print $0}' | awk -F'[:.]+' '{print $2}' | tr -d " ") + pages_inactive=$(vm_stat | awk '/Pages inactive/{print $0}' | awk -F'[:.]+' '{print $2}' | tr -d " ") + if [[ -z "$pages_free" || -z "$pages_inactive" ]]; then + echo "Failed to get free memory" + exit 1 + fi + local pages_available + pages_available=$(expr "$pages_free" + "$pages_inactive") + free=$(expr "$pages_available" \* 4096 / 1024 / 1024) else echo "Unsupported operating system $os" exit 1 @@ -186,8 +186,8 @@ function calc_xmx() { local min_mem=$1 local max_mem=$2 # Get machine available memory - local free=`free_memory` - local half_free=$[free/2] + local free=$(free_memory) + local half_free=$((free/2)) local xmx=$min_mem if [[ "$free" -lt "$min_mem" ]]; then @@ -235,47 +235,74 @@ function ensure_path_writable() { } function get_ip() { - local os=`uname` + local os=$(uname) local loopback="127.0.0.1" local ip="" case $os in Linux) if command_available "ifconfig"; then - ip=`ifconfig | grep 'inet addr:' | grep -v "$loopback" | cut -d: -f2 | awk '{ print $1}'` + ip=$(ifconfig | grep 'inet addr:' | grep -v "$loopback" | cut -d: -f2 | awk '{ print $1}') elif command_available "ip"; then - ip=`ip addr | grep 'state UP' -A2 | tail -n1 | awk '{print $2}' | awk -F"/" '{print $1}'` + ip=$(ip addr | grep 'state UP' -A2 | tail -n1 | awk '{print $2}' | awk -F"/" '{print $1}') else ip=$loopback fi ;; FreeBSD|OpenBSD|Darwin) if command_available "ifconfig"; then - ip=`ifconfig | grep -E 'inet.[0-9]' | grep -v "$loopback" | awk '{ print $2}'` + ip=$(ifconfig | grep -E 'inet.[0-9]' | grep -v "$loopback" | awk '{ print $2}') else ip=$loopback fi ;; SunOS) if command_available "ifconfig"; then - ip=`ifconfig -a | grep inet | grep -v "$loopback" | awk '{ print $2} '` + ip=$(ifconfig -a | grep inet | grep -v "$loopback" | awk '{ print $2} ') else ip=$loopback fi ;; *) ip=$loopback;; esac - echo $ip + [[ -z "$ip" ]] && ip=$loopback + echo "$ip" } function download() { local path=$1 local link_url=$2 - if command_available "wget"; then - wget --help | grep -q '\--show-progress' && progress_opt="-q --show-progress" || progress_opt="" - wget ${link_url} -P ${path} $progress_opt - elif command_available "curl"; then - curl ${link_url} -o ${path}/${link_url} + if [ ! -d "$path" ]; then + mkdir -p "$path" || { + echo "Failed to create directory: $path" + exit 1 + } + fi + + local filename + filename=$(basename "${link_url%%[?#]*}") + local tmp="${path}/.${filename}.tmp.$$" + local dest="${path}/${filename}" + + if command_available "curl"; then + # -o must appear before -- so it is parsed as an option, not an extra URL. + if curl -fL -o "$tmp" -- "${link_url}"; then + mv -f -- "$tmp" "$dest" + else + rm -f -- "$tmp" + return 1 + fi + elif command_available "wget"; then + local -a progress_opt=() + if wget --help 2>&1 | grep -q -- '--show-progress'; then + progress_opt=(-q --show-progress) + fi + if wget ${progress_opt[@]+"${progress_opt[@]}"} -O "$tmp" -- "${link_url}"; then + mv -f -- "$tmp" "$dest" + else + rm -f -- "$tmp" + return 1 + fi else echo "Required wget or curl but they are unavailable" exit 1 @@ -286,14 +313,15 @@ download_and_verify() { local url=$1 local filepath=$2 local expected_md5=$3 + local actual_md5 - if [[ -f $filepath ]]; then + if [[ -f "$filepath" ]]; then echo "File $filepath exists. Verifying MD5 checksum..." - actual_md5=$(md5sum $filepath | awk '{ print $1 }') - if [[ $actual_md5 != $expected_md5 ]]; then + actual_md5=$(md5sum -- "$filepath" | awk '{ print $1 }') + if [[ "$actual_md5" != "$expected_md5" ]]; then echo "MD5 checksum verification failed for $filepath. Expected: $expected_md5, but got: $actual_md5" echo "Deleting $filepath..." - rm -f $filepath + rm -f -- "$filepath" else echo "MD5 checksum verification succeeded for $filepath." return 0 @@ -301,11 +329,17 @@ download_and_verify() { fi echo "Downloading $filepath..." - curl -L -o $filepath $url - - actual_md5=$(md5sum $filepath | awk '{ print $1 }') - if [[ $actual_md5 != $expected_md5 ]]; then - echo "MD5 checksum verification failed for $filepath after download. Expected: $expected_md5, but got: $actual_md5" + local tmp="${filepath}.tmp.$$" + if curl -fL -o "$tmp" -- "$url"; then + actual_md5=$(md5sum -- "$tmp" | awk '{ print $1 }') + if [[ "$actual_md5" != "$expected_md5" ]]; then + echo "MD5 checksum verification failed for $filepath after download. Expected: $expected_md5, but got: $actual_md5" + rm -f -- "$tmp" + return 1 + fi + mv -f -- "$tmp" "$filepath" + else + rm -f -- "$tmp" return 1 fi @@ -318,19 +352,17 @@ function ensure_package_exist() { local tar=$3 local link=$4 - if [ ! -d ${path}/${dir} ]; then - if [ ! -f ${path}/${tar} ]; then + if [ ! -d "${path}/${dir}" ]; then + if [ ! -f "${path}/${tar}" ]; then echo "Downloading the compressed package '${tar}'" - download ${path} ${link} - if [ $? -ne 0 ]; then + if ! download "${path}" "${link}"; then echo "Failed to download, please ensure the network is available and link is valid" exit 1 fi echo "[OK] Finished download" fi echo "Unzip the compressed package '$tar'" - tar -zxvf ${path}/${tar} -C ${path} >/dev/null 2>&1 - if [ $? -ne 0 ]; then + if ! tar -zxvf "${path}/${tar}" -C "${path}" >/dev/null 2>&1; then echo "Failed to unzip, please check the compressed package" exit 1 fi @@ -345,7 +377,7 @@ function wait_for_shutdown() { local pid="$2" local timeout_s="$3" - local now_s=`date '+%s'` + local now_s=$(date '+%s') local stop_s=$(( $now_s + $timeout_s )) echo -n "Killing $process_name(pid $pid)" >&2 @@ -357,7 +389,7 @@ function wait_for_shutdown() { return 0 fi sleep 2 - now_s=`date '+%s'` + now_s=$(date '+%s') done echo "$process_name shutdown timeout(exceeded $timeout_s seconds)" >&2 return 1 @@ -386,7 +418,7 @@ function kill_process() { return 0 fi - case "`uname`" in + case "$(uname)" in CYGWIN*) taskkill /F /PID "$pid" ;; *) kill "$pid" ;; esac From bf0dd0331fd27d6189fde071781e3f8d2d433b4b Mon Sep 17 00:00:00 2001 From: Himanshu Verma Date: Sun, 26 Jul 2026 15:45:22 +0530 Subject: [PATCH 02/10] fix(bin): polish server util helpers after lsof port-check review - Localize read_property variables (match PD/Store) - Use if ! download / if ! tar in ensure_package_exist so atomic download failures are detected without a fragile $? check Also merges upstream master so the branch is no longer diverged (RISC-V libatomic + CI from #3102 kept intact with check_port). --- .../hugegraph-dist/src/assembly/static/bin/util.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/util.sh b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/util.sh index 5040daa652..687e5d2001 100755 --- a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/util.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/util.sh @@ -66,6 +66,8 @@ function configure_riscv64_libatomic() { # read a property from .properties file function read_property() { # file path + local file_name + local property_name file_name=$1 # replace "." to "\." property_name=$(echo "$2" | sed 's/\./\\\./g') @@ -760,19 +762,17 @@ function ensure_package_exist() { local tar=$3 local link=$4 - if [ ! -d "${path}"/"${dir}" ]; then - if [ ! -f "${path}"/"${tar}" ]; then + if [ ! -d "${path}/${dir}" ]; then + if [ ! -f "${path}/${tar}" ]; then echo "Downloading the compressed package '${tar}'" - download "${path}" "${link}" - if [ $? -ne 0 ]; then + if ! download "${path}" "${link}"; then echo "Failed to download, please ensure the network is available and link is valid" exit 1 fi echo "[OK] Finished download" fi echo "Unzip the compressed package '$tar'" - tar zxvf "${path}"/"${tar}" -C "${path}" >/dev/null 2>&1 - if [ $? -ne 0 ]; then + if ! tar -zxvf "${path}/${tar}" -C "${path}" >/dev/null 2>&1; then echo "Failed to unzip, please check the compressed package" exit 1 fi From 263d8182298d110186fe24770b9dbce25ecb0fe9 Mon Sep 17 00:00:00 2001 From: Himanshu Verma Date: Mon, 27 Jul 2026 20:13:26 +0530 Subject: [PATCH 03/10] fix(bin): converge check_port on a conservative port-only preflight Rework the startup port check to the minimal design agreed in review. The preflight is best effort - the server's own bind stays authoritative - so it now answers one question only: is anything already listening on this port? check_port: - Parse just enough of the configured URL to obtain a validated port: case-insensitive scheme, explicit or default port, scheme-less value, bracketed IPv6. Ambiguous unbracketed IPv6 is rejected with a warning instead of guessed. - Detect listeners per OS, inspecting only LISTEN rows and only the local-address column: ss -H -ltn on Linux, netstat -an -p tcp on BSD. Splitting on the last separator keeps IPv6 hextets from being read as the port. - Return busy, free or unknown. A probe that is missing, fails, or yields no recognisable listener row is unknown: warn and let the bind decide. - Drop DNS resolution, address canonicalisation, cross-family endpoint matching, /dev/tcp and its watchdog. Port-level detection covers the dual-stack and wildcard cases conservatively without reproducing kernel socket semantics in shell, and removes the last unbounded call from the startup path. This fixes the macOS false positive where netstat -ltn returned non-listening rows and an unrelated ESTABLISHED peer port aborted startup. Also in this pass: - store: a failed atomic rename no longer reports success, so LD_PRELOAD is never exported for a library that did not install. Same fix applied to download() in all three copies. - download(): mktemp instead of a PID-derived name, which is shared by concurrent background subshells. - wait_for_startup(): bound each probe with --connect-timeout / --max-time by the time left in the deadline, so one blackholed request cannot outlive the configured timeout. - pd-store-ci: preflight checked lsof while cleanup used fuser; align it. - Revert unrelated process_num / process_id / free_memory / get_ip changes. netstat is used as the Linux fallback rather than fuser: fuser cannot distinguish "not found" from "insufficient privileges", which would break the busy/free/unknown contract. Tests: test-check-port.sh replaced with a compact contract suite - URL to port, ss busy/free/failure, BSD LISTEN vs ESTABLISHED vs TIME_WAIT, unknown fallback, a real ephemeral listener, rename failure, and a bounded startup probe. 41 passed, 0 failed. Verified non-vacuous by mutation: dropping the LISTEN filter, downgrading unknown to free, and reading the foreign-address column each fail the suite. --- .github/workflows/pd-store-ci.yml | 8 +- .../src/assembly/static/bin/util.sh | 12 +- .../src/assembly/static/bin/util.sh | 525 ++------ .../src/assembly/travis/test-check-port.sh | 1120 ++++------------- .../src/assembly/static/bin/util.sh | 20 +- 5 files changed, 387 insertions(+), 1298 deletions(-) diff --git a/.github/workflows/pd-store-ci.yml b/.github/workflows/pd-store-ci.yml index 62006794c7..6f670e1cb9 100644 --- a/.github/workflows/pd-store-ci.yml +++ b/.github/workflows/pd-store-ci.yml @@ -110,7 +110,9 @@ jobs: - name: Check startup test prerequisites (PD) id: pd-preflight run: | - for tool in lsof curl java; do + # These jobs run on Linux, where the suites clean up ports with fuser. + # lsof is no longer required: check_port now uses ss/netstat. + for tool in fuser curl java; do if ! command -v "$tool" >/dev/null 2>&1; then echo "can_run=false" >> "$GITHUB_OUTPUT" echo "skip_reason=missing tool: $tool" >> "$GITHUB_OUTPUT" @@ -190,7 +192,9 @@ jobs: - name: Check startup test prerequisites (Store) id: store-preflight run: | - for tool in lsof curl java; do + # These jobs run on Linux, where the suites clean up ports with fuser. + # lsof is no longer required: check_port now uses ss/netstat. + for tool in fuser curl java; do if ! command -v "$tool" >/dev/null 2>&1; then echo "can_run=false" >> "$GITHUB_OUTPUT" echo "skip_reason=missing tool: $tool" >> "$GITHUB_OUTPUT" diff --git a/hugegraph-pd/hg-pd-dist/src/assembly/static/bin/util.sh b/hugegraph-pd/hg-pd-dist/src/assembly/static/bin/util.sh index e80255672e..4f4a90b45f 100644 --- a/hugegraph-pd/hg-pd-dist/src/assembly/static/bin/util.sh +++ b/hugegraph-pd/hg-pd-dist/src/assembly/static/bin/util.sh @@ -280,15 +280,19 @@ function download() { } fi - local filename + local filename tmp filename=$(basename "${link_url%%[?#]*}") - local tmp="${path}/.${filename}.tmp.$$" local dest="${path}/${filename}" + # mktemp, not $$: a PID is shared by concurrent background subshells. + tmp=$(mktemp -- "${path}/.${filename}.XXXXXX") || { + echo "Failed to create a temporary file in $path" + return 1 + } if command_available "curl"; then # -o must appear before -- so it is parsed as an option, not an extra URL. if curl -fL -o "$tmp" -- "${link_url}"; then - mv -f -- "$tmp" "$dest" + mv -f -- "$tmp" "$dest" || { rm -f -- "$tmp"; return 1; } else rm -f -- "$tmp" return 1 @@ -299,7 +303,7 @@ function download() { progress_opt=(-q --show-progress) fi if wget ${progress_opt[@]+"${progress_opt[@]}"} -O "$tmp" -- "${link_url}"; then - mv -f -- "$tmp" "$dest" + mv -f -- "$tmp" "$dest" || { rm -f -- "$tmp"; return 1; } else rm -f -- "$tmp" return 1 diff --git a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/util.sh b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/util.sh index 687e5d2001..c794bd6d2e 100755 --- a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/util.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/util.sh @@ -110,416 +110,150 @@ function parse_yaml() { } function process_num() { - local num num=$(ps -ef | grep "$1" | grep -v grep | wc -l) - # Return 0 when no process, 1 when one or more. Using $num directly as an - # exit code would truncate values > 255, so treat this as a boolean result. - if (( num > 0 )); then - return 1 - fi - return 0 + return "$num" } function process_id() { - local pid pid=$(ps -ef | grep "$1" | grep -v grep | awk '{print $2}') - echo "$pid" - return 0 -} - -# Run a command with a hard deadline via background watchdog. -# Returns the command's exit code if it finishes in time. -# If the deadline expires, the command is killed (exit code reflects signal). -# Works without the timeout command — uses sleep + kill -9 pattern. -function run_with_deadline() { - local cmd="$1" - local deadline="$2" - shift 2 - - bash -c "$cmd" bash "$@" & - local child_pid=$! - ( - local sleep_pid="" - cleanup_watchdog() { - [[ -n "$sleep_pid" ]] && kill -9 "$sleep_pid" 2>/dev/null - } - # Kill any direct children of the target PID (e.g. a spawned sleep) - # in case killing the wrapper left them orphaned. - kill_children() { - local child - for child in $(pgrep -P "$1" 2>/dev/null); do - kill -9 "$child" 2>/dev/null || true - done - } - trap 'cleanup_watchdog' EXIT TERM - sleep "$deadline" & sleep_pid=$! - wait "$sleep_pid" 2>/dev/null - if kill -0 "$child_pid" 2>/dev/null; then - kill -9 "$child_pid" 2>/dev/null - kill_children "$child_pid" - fi - ) 2>/dev/null & - local watchdog_pid=$! - - wait "$child_pid" 2>/dev/null - local rc=$? - # Kill the watchdog with SIGTERM so its EXIT trap reaps the sleep child. - kill -TERM "$watchdog_pid" 2>/dev/null || true - wait "$watchdog_pid" 2>/dev/null || true - return $rc -} - - -# Normalize an IP address to a canonical string for comparison. -# - IPv4 addresses are returned unchanged. -# - IPv6 addresses are stripped of brackets/zone scope, lowercased and -# compressed to the canonical textual form (via getent ahosts). -# - IPv4-mapped IPv6 (::ffff:1.2.3.4) is collapsed to the IPv4 form. -# - Hostnames are returned unchanged; callers should resolve them first. -function normalize_addr() { - local addr="$1" - - # Strip brackets - if [[ "$addr" =~ ^\[.*\]$ ]]; then - addr="${addr#\[}" - addr="${addr%\]}" - fi - - # Drop IPv6 zone scope (e.g. 127.0.0.53%lo, fe80::1%eth0) - addr="${addr%%\%*}" - - # IPv4-mapped IPv6 -> IPv4 so a bound IPv4-mapped socket is compared - # against an IPv4-configured address. - if [[ "$addr" =~ ^::ffff:([0-9]+\.[0-9]+\.[0-9]+\.[0-9]+)$ ]]; then - echo "${BASH_REMATCH[1]}" - return - fi - - # Plain IPv4 - if [[ "$addr" =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then - echo "$addr" - return - fi - - # IPv6: ask glibc for the canonical compressed form - if [[ "$addr" =~ ^[0-9a-fA-F:]*:[0-9a-fA-F:]*$ ]]; then - local norm - if command_available "timeout"; then - norm=$(timeout 2 getent ahosts "$addr" 2>/dev/null | awk '{print $1; exit}') - else - norm=$(getent ahosts "$addr" 2>/dev/null | awk '{print $1; exit}') - fi - if [[ -n "$norm" ]]; then - echo "$norm" - return - fi - # macOS/BSD fallback: python3's socket module can canonicalise - # numeric IPv6 even when getent is unavailable. - if command_available "python3"; then - norm=$(python3 -c 'import socket, sys; print(socket.inet_ntop(socket.AF_INET6, socket.inet_pton(socket.AF_INET6, sys.argv[1])))' "$addr" 2>/dev/null) - if [[ -n "$norm" ]]; then - echo "$norm" - return - fi - fi - # No canonicaliser available: at least normalise hex case so ss/netstat - # lowercase output still matches an uppercase configured address. - echo "$addr" | tr '[:upper:]' '[:lower:]' - return - fi - - # Fall back to the cleaned original - echo "$addr" + return "$pid" } -# check whether the REST server port is occupied -function check_port() { +# Extract a validated TCP port from a configured server URL. +# Echoes the port on success. Returns 1 when the value carries no usable port +# or is ambiguous, in which case the caller skips the preflight. +function parse_port_from_url() { local url="$1" - local host - local port - # Strip leading/trailing whitespace from URL (handles whitespace from ServerOptions) + # ServerOptions tolerates surrounding whitespace, so strip it first. url="${url#"${url%%[![:space:]]*}"}" url="${url%"${url##*[![:space:]]}"}" + [[ -z "$url" ]] && return 1 + + # The scheme is optional and case-insensitive. + local scheme="" rest="$url" + if [[ "$url" == *"://"* ]]; then + scheme=$(echo "${url%%://*}" | tr '[:upper:]' '[:lower:]') + rest="${url#*://}" + fi - # Extract authority: strip scheme and stop at the first /, ? or #. - local authority - authority="${url#*://}" - authority="${authority%%[/?#]*}" + # The authority ends at the first '/', '?' or '#'. + local authority="${rest%%[/?#]*}" + [[ -z "$authority" ]] && return 1 - # Extract host and port from authority. - if [[ "$authority" =~ ^\[([^\]]*)\]:([0-9]+)$ ]]; then - # IPv6 with port: [::1]:8080 - host="${BASH_REMATCH[1]}" + local port="" + if [[ "$authority" =~ ^\[[^]]*\](:([0-9]+))?$ ]]; then + # Bracketed IPv6, with or without a port: [::1] or [::1]:8080 port="${BASH_REMATCH[2]}" - elif [[ "$authority" =~ ^\[([^\]]*)\]$ ]]; then - # IPv6 without port: [::1] - host="${BASH_REMATCH[1]}" - port="" - elif [[ "$authority" =~ :([0-9]+)$ ]]; then - # IPv4 or hostname with port: 127.0.0.1:8080, localhost:8080 - port="${BASH_REMATCH[1]}" - host="${authority%:*}" - else - # No explicit port in authority - host="$authority" - port="" + elif [[ "$authority" == *:*:* ]]; then + # Unbracketed IPv6 is ambiguous: in "::1:8080" the trailing group may + # be a port or another hextet. Refuse to guess. + echo "WARN: ambiguous IPv6 authority '$authority' in server URL;" \ + "use bracket notation such as [::1]:8080." >&2 + return 1 + elif [[ "$authority" == *:* ]]; then + port="${authority##*:}" fi - # Handle default ports from scheme when no explicit port is given + # Fall back to the scheme's default port. if [[ -z "$port" ]]; then - if [[ "$url" == https://* ]]; then - port="443" - elif [[ "$url" == http://* ]]; then - port="80" - else - return 0 - fi + case "$scheme" in + http) port="80" ;; + https) port="443" ;; + *) return 1 ;; + esac fi - # Validate port as a decimal number - if ! [[ "$port" =~ ^[0-9]+$ ]]; then - return 0 - fi + [[ "$port" =~ ^[0-9]+$ ]] || return 1 + # Normalise leading-zero forms; Java reads 08080 as decimal 8080. port=$((10#$port)) - if (( port < 1 || port > 65535 )); then - return 0 - fi + (( port >= 1 && port <= 65535 )) || return 1 - # Strip any leading/trailing whitespace from host - host="${host#"${host%%[![:space:]]*}"}" - host="${host%"${host##*[![:space:]]}"}" - - local norm_host - norm_host=$(normalize_addr "$host") - - # Determine the address family of the configured host so we only treat - # same-family wildcard listeners as conflicts (IPv4 vs IPv6 sockets are - # separate unless explicitly dual-stacked). - local host_family="" - if [[ "$norm_host" =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then - host_family="ipv4" - elif [[ "$norm_host" =~ ^[0-9a-fA-F:]*:[0-9a-fA-F:]*$ ]]; then - host_family="ipv6" - else - host_family="hostname" - fi - - local in_use=0 - local port_checked=0 - - # Wildcard binds are detected by looking for any listener on the port. - # Specific hosts are matched against the local address; if the listener - # is a same-family wildcard (0.0.0.0 / :: / *), that is a conflict too. - local is_wildcard=0 - if [[ -z "$host" || "$host" == "0.0.0.0" || "$host" == "::" || "$host" == "*" ]]; then - is_wildcard=1 - fi - - # Build the list of acceptable normalized addresses for a specific host. - # Hostnames must be resolved to numeric IPs first because ss/netstat - # output is always numeric. - local candidate_addrs="" - if [[ $is_wildcard -eq 0 ]]; then - if [[ "$host_family" == "ipv4" || "$host_family" == "ipv6" ]]; then - # Already numeric - candidate_addrs="$norm_host" - else - # Hostname: resolve with deadline - if command_available "getent" && command_available "timeout"; then - candidate_addrs=$(timeout 2 getent ahosts "$host" 2>/dev/null | awk '{print $1}') - elif command_available "dscacheutil" && command_available "timeout"; then - candidate_addrs=$(timeout 2 dscacheutil -q host -a name "$host" 2>/dev/null \ - | awk '/ip_address:/{print $2}') - fi - fi - fi - - # Helper: scan a line of listener-table output and return 0 if it matches - # the configured host/port. Sets 'matched_token' to 1 when it evaluates a - # token that we can trust, so callers know whether "no match" is reliable. - local out line listener_addr norm_listener token matched_token + echo "$port" +} - # Returns true if the listener address is a wildcard on the same family - # as the configured host (or any family, if the host is a wildcard). - _check_port_wildcard_conflicts() { - local wl_addr="$1" - if [[ $is_wildcard -eq 1 ]]; then +# Echo "busy", "free" or "unknown" for the given TCP port. +# +# Detection is deliberately port-only, matching the conservative behaviour of +# the `lsof -i :PORT` call this replaces. Reproducing kernel socket semantics +# in Bash - dual-stack IPV6_V6ONLY, wildcard versus specific binds, address +# canonicalisation - produced more wrong answers than it prevented, so we only +# ask "is anything already listening on this port?". +# +# Only LISTEN rows and only the local-address column are inspected, so an +# unrelated outbound connection to the same port number is never mistaken for +# a local listener. A tool that is missing, fails, or yields no recognisable +# listener row reports "unknown" rather than "free". +function port_listen_state() { + local port="$1" + local out + local os + os=$(uname) + + # $4 is the local address for both `ss -ltn` and BSD `netstat -an`. + # Splitting on the last separator keeps IPv6 hextets (for example + # [2001:db8::80]:443) from being misread as the port. + local parser=' + NF >= 4 && (!want_listen || $NF == "LISTEN") { + addr = $4 + cut = 0 + for (k = length(addr); k > 0; k--) { + if (substr(addr, k, 1) == sep) { cut = k; break } + } + if (cut == 0) next + rows++ + if (substr(addr, cut + 1) == port) { found = 1; exit } + } + END { + if (found) print "busy" + else if (rows > 0) print "free" + else print "unknown" + }' + + if [[ "$os" == "Darwin" || "$os" == *BSD* ]]; then + if command_available "netstat" && out=$(netstat -an -p tcp 2>/dev/null) \ + && [[ -n "$out" ]]; then + echo "$out" | awk -v port="$port" -v sep="." -v want_listen=1 "$parser" return 0 fi - # BSD netstat prints *. for an any-family wildcard. - if [[ "$wl_addr" == "*" ]]; then + else + # `ss -H -ltn` already restricts output to listening sockets. + if command_available "ss" && out=$(ss -H -ltn 2>/dev/null) && [[ -n "$out" ]]; then + echo "$out" | awk -v port="$port" -v sep=":" -v want_listen=0 "$parser" return 0 fi - if [[ "$wl_addr" == "0.0.0.0" ]]; then - [[ "$host_family" == "ipv4" ]] && return 0 - # A hostname that resolved to IPv4 can also bind an IPv4 wildcard - if [[ "$host_family" == "hostname" ]]; then - local addr - while IFS= read -r addr; do - [[ -z "$addr" ]] && continue - if [[ "$addr" =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then - return 0 - fi - done <<< "$candidate_addrs" - fi - fi - if [[ "$wl_addr" == "::" ]]; then - [[ "$host_family" == "ipv6" ]] && return 0 - if [[ "$host_family" == "hostname" ]]; then - local addr - while IFS= read -r addr; do - [[ -z "$addr" ]] && continue - if [[ "$addr" =~ ^[0-9a-fA-F:]*:[0-9a-fA-F:]*$ ]]; then - return 0 - fi - done <<< "$candidate_addrs" - fi - fi - return 1 - } - - _check_port_match_listener_line() { - local out_line="$1" - # Skip empty or whitespace-only lines before read -a to avoid an empty - # array expansion under set -u on Bash 3.2. - if [[ -z "${out_line//[[:space:]]/}" ]]; then - return 1 - fi - local -a tokens - read -r -a tokens <<< "$out_line" - for token in ${tokens[@]+"${tokens[@]}"}; do - # We only care about the "local address:port" token. For the - # common tools this is the first token that ends with the - # target port after ':' or '.' (peer addresses use :* on Linux - # and *.* on BSD, so they never match a numeric port). - if [[ "$token" =~ ^(.*):(${port})$ ]]; then - listener_addr="${BASH_REMATCH[1]}" - elif [[ "$token" =~ ^(.*)\.(${port})$ ]]; then - listener_addr="${BASH_REMATCH[1]}" - else - continue - fi - - # Wildcard host: any listener on this port is a conflict. - if [[ $is_wildcard -eq 1 ]]; then - matched_token=1 - return 0 - fi - - # No resolved candidates (hostname resolution failed/unsupported): - # we cannot reliably compare against the numeric listener table. - if [[ -z "$candidate_addrs" ]]; then - continue - fi - - matched_token=1 - norm_listener=$(normalize_addr "$listener_addr") - - # A same-family wildcard listener on this port conflicts with any - # specific host of that family. - if _check_port_wildcard_conflicts "$norm_listener"; then - return 0 - fi - - local addr - while IFS= read -r addr; do - [[ -z "$addr" ]] && continue - if [[ "$norm_listener" == "$(normalize_addr "$addr")" ]]; then - return 0 - fi - done <<< "$candidate_addrs" - done - return 1 - } - - if command_available "ss"; then - matched_token=0 - if out=$(ss -ltn 2>/dev/null); then - while IFS= read -r line; do - if _check_port_match_listener_line "$line"; then - in_use=1 - break - fi - done <<< "$out" - # ss -ltn succeeded. We can trust a non-match when we have a - # numeric host or resolved addresses (or a wildcard); otherwise - # fall through to /dev/tcp for unresolved hostnames. - if [[ $in_use -eq 0 && ( $is_wildcard -eq 1 || -n "$candidate_addrs" || $matched_token -eq 1 ) ]]; then - port_checked=1 - fi + if command_available "netstat" && out=$(netstat -ltn 2>/dev/null) \ + && [[ -n "$out" ]]; then + echo "$out" | awk -v port="$port" -v sep=":" -v want_listen=1 "$parser" + return 0 fi fi - if [[ $in_use -eq 0 && $port_checked -eq 0 ]] && command_available "netstat"; then - matched_token=0 - if out=$(netstat -ltn 2>/dev/null) && echo "$out" | grep -qi "listen"; then - while IFS= read -r line; do - if _check_port_match_listener_line "$line"; then - in_use=1 - break - fi - done <<< "$out" - # netstat -ltn output is the complete Linux listener table. - if [[ $in_use -eq 0 && ( $is_wildcard -eq 1 || -n "$candidate_addrs" || $matched_token -eq 1 ) ]]; then - port_checked=1 - fi - elif out=$(netstat -an 2>/dev/null) && [[ -n "$out" ]]; then - local old_nocasematch - old_nocasematch=$(shopt -p nocasematch 2>/dev/null || true) - shopt -s nocasematch 2>/dev/null || true - while IFS= read -r line; do - if [[ "$line" == *listen* ]] && _check_port_match_listener_line "$line"; then - in_use=1 - break - fi - done <<< "$out" - eval "$old_nocasematch" 2>/dev/null || true - # netstat -an output is the complete BSD listener table. - if [[ $in_use -eq 0 && ( $is_wildcard -eq 1 || -n "$candidate_addrs" || $matched_token -eq 1 ) ]]; then - port_checked=1 - fi - fi - fi + echo "unknown" +} - if [[ $in_use -eq 0 && $port_checked -eq 0 ]]; then - # Probe the actual configured endpoint(s) with a short deadline. - local probe_addrs="$candidate_addrs" - if [[ -z "$probe_addrs" ]]; then - # Could not resolve (or wildcard with only loopback probe needed) - if [[ $is_wildcard -eq 1 ]]; then - probe_addrs="127.0.0.1 ::1" - else - probe_addrs="$host" - fi - fi +# Best-effort startup preflight. Exits 1 when the configured port is already +# in use. The server's own bind stays authoritative, so an inconclusive +# result only warns and lets startup proceed. +function check_port() { + local url="$1" + local port + local state - local addr - for addr in $probe_addrs; do - # /dev/tcp needs unbracketed, normalized addresses - addr=$(normalize_addr "$addr") - [[ -z "$addr" ]] && continue - if command_available "timeout"; then - if timeout 1 bash -c ': >/dev/tcp/"$1"/"$2"' _ "$addr" "$port" 2>/dev/null; then - in_use=1 - break - fi - else - if run_with_deadline ': >/dev/tcp/"$1"/"$2" 2>/dev/null' 2 "$addr" "$port"; then - in_use=1 - break - fi - fi - done - fi + port=$(parse_port_from_url "$url") || return 0 + + state=$(port_listen_state "$port") + case "$state" in + busy) + echo "The port $port has already been used" + exit 1 + ;; + unknown) + echo "WARN: could not determine whether port $port is free;" \ + "continuing and letting the server bind decide." >&2 + ;; + esac - local _rc=0 - if [[ "$in_use" -eq 1 ]]; then - echo "The port $port has already been used" - _rc=1 - fi - unset -f _check_port_wildcard_conflicts _check_port_match_listener_line || true - [[ $_rc -eq 1 ]] && exit 1 return 0 } @@ -577,7 +311,13 @@ function wait_for_startup() { return 1 fi - status=$(curl -I -sS -k -w "%{http_code}" -o /dev/null "$server_url" 2> "$error_file_name") + # Bound each probe by the time left in the overall deadline: without + # --max-time a single blackholed request blocks past ${timeout_s}s. + local remain_s=$((stop_s - now_s)) + [ "$remain_s" -lt 1 ] && remain_s=1 + local connect_s=$((remain_s < 5 ? remain_s : 5)) + status=$(curl -I -sS -k --connect-timeout "$connect_s" --max-time "$remain_s" \ + -w "%{http_code}" -o /dev/null "$server_url" 2> "$error_file_name") if [[ "$status" -eq 200 || "$status" -eq 401 ]]; then echo "OK" echo "Started [pid $pid]" @@ -611,15 +351,9 @@ function free_memory() { free=$(expr "$mem_free" + "$mem_buffer" + "$mem_cached") free=$(expr "$free" / 1024) elif [ "$os" == "Darwin" ]; then - local pages_free pages_inactive - pages_free=$(vm_stat | awk '/Pages free/{print $0}' | awk -F'[:.]+' '{print $2}' | tr -d " ") - pages_inactive=$(vm_stat | awk '/Pages inactive/{print $0}' | awk -F'[:.]+' '{print $2}' | tr -d " ") - if [[ -z "$pages_free" || -z "$pages_inactive" ]]; then - echo "Failed to get free memory" - exit 1 - fi - local pages_available - pages_available=$(expr "$pages_free" + "$pages_inactive") + local pages_free=$(vm_stat | awk '/Pages free/{print $0}' | awk -F'[:.]+' '{print $2}' | tr -d " ") + local pages_inactive=$(vm_stat | awk '/Pages inactive/{print $0}' | awk -F'[:.]+' '{print $2}' | tr -d " ") + local pages_available=$(expr "$pages_free" + "$pages_inactive") free=$(expr "$pages_available" \* 4096 / 1024 / 1024) else echo "Unsupported operating system $os" @@ -710,8 +444,7 @@ function get_ip() { ;; *) ip=$loopback;; esac - [[ -z "$ip" ]] && ip=$loopback - echo "$ip" + echo $ip } function download() { @@ -726,15 +459,19 @@ function download() { fi # Strip query/fragment so the on-disk name matches the server-side artifact. - local filename + local filename tmp filename=$(basename "${download_url%%[?#]*}") - local tmp="${path}/.${filename}.tmp.$$" local dest="${path}/${filename}" + # mktemp, not $$: a PID is shared by concurrent background subshells. + tmp=$(mktemp -- "${path}/.${filename}.XXXXXX") || { + echo "Failed to create a temporary file in $path" + return 1 + } if command_available "curl"; then # -o must appear before -- so it is parsed as an option, not an extra URL. if curl -fL -o "$tmp" -- "${download_url}"; then - mv -f -- "$tmp" "$dest" + mv -f -- "$tmp" "$dest" || { rm -f -- "$tmp"; return 1; } else rm -f -- "$tmp" return 1 @@ -745,7 +482,7 @@ function download() { progress_opt=(-q --show-progress) fi if wget ${progress_opt[@]+"${progress_opt[@]}"} -O "$tmp" -- "${download_url}"; then - mv -f -- "$tmp" "$dest" + mv -f -- "$tmp" "$dest" || { rm -f -- "$tmp"; return 1; } else rm -f -- "$tmp" return 1 diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/test-check-port.sh b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-check-port.sh index c886f6f67e..45102ece24 100755 --- a/hugegraph-server/hugegraph-dist/src/assembly/travis/test-check-port.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-check-port.sh @@ -15,970 +15,306 @@ # See the License for the specific language governing permissions and # limitations under the License. # -# test-check-port.sh — Unit tests for check_port() in util.sh -# -# Strategy: each test runs check_port in a subshell that overrides -# command_available() to control which probe branch is taken, and -# overrides the tool functions (ss, netstat, timeout) to control -# what they return — no real network connections needed. -# -# check_port calls `exit 1` when the port is in use, so the subshell -# exits 1; it returns normally (exit 0) when the port is free. + +# Contract tests for the startup port preflight in bin/util.sh. # -# Usage: ./test-check-port.sh [path-to-hugegraph-static-dir] -# path-to-hugegraph-static-dir: directory containing bin/util.sh -# Defaults to current directory. -# In CI: $TRAVIS_DIR/test-check-port.sh hugegraph-server/hugegraph-dist/src/assembly/static +# The preflight is best effort: the server's own bind is authoritative. These +# tests pin the three-state contract (busy / free / unknown) rather than the +# internals of any one probe. -set -uo pipefail -# -u: fail on undefined variables (catches typos in test assertions) -# -o pipefail: pipeline exit status is the last non-zero component -# shellcheck disable=SC1090,SC1091 # UTIL_SH / PD_UTIL_SH sourced dynamically at runtime +set -u -STATIC_DIR="${1:-$(pwd)}" +STATIC_DIR="${1:-hugegraph-server/hugegraph-dist/src/assembly/static}" UTIL_SH="$STATIC_DIR/bin/util.sh" -REPO_ROOT="$(cd "$(dirname "$0")/../../../../.." && pwd)" -PD_UTIL_SH="$REPO_ROOT/hugegraph-pd/hg-pd-dist/src/assembly/static/bin/util.sh" -STORE_UTIL_SH="$REPO_ROOT/hugegraph-store/hg-store-dist/src/assembly/static/bin/util.sh" - -PASS=0 -FAIL=0 -ERRORS=() - -GREEN='\033[0;32m' -RED='\033[0;31m' -NC='\033[0m' - -pass() { echo -e "${GREEN} PASS${NC} $1"; PASS=$((PASS + 1)); } -fail() { echo -e "${RED} FAIL${NC} $1"; ERRORS+=("$1"); FAIL=$((FAIL + 1)); } -section() { echo ""; echo "── $1 ──"; } - -# timeout mock used by ss/netstat test cases. It passes getent through so -# normalize_addr can canonicalise numeric IPv6, and pretends every other -# probe (the /dev/tcp fallback) failed, so free-port assertions are not -# influenced by real network state. -timeout() { - if [[ "${2:-}" == "getent" ]]; then - shift - "$@" 2>/dev/null - else - return 1 - fi -} - -echo "" -echo "check_port() unit test suite" -echo "util.sh: $UTIL_SH" -echo "" - if [[ ! -f "$UTIL_SH" ]]; then - echo -e "${RED}ERROR:${NC} $UTIL_SH not found." - echo " Pass the HugeGraph static assembly dir as \$1" - exit 1 + echo "SKIP: util.sh not found at $UTIL_SH" + exit 0 fi -# ── ss branch ───────────────────────────────────────────────────────────────── - -section "ss branch — IPv4" - -( - # shellcheck source=/dev/null - source "$UTIL_SH" - command_available() { [[ "$1" == "ss" || "$1" == "timeout" ]]; } - ss() { echo "tcp LISTEN 0 128 0.0.0.0:8080 0.0.0.0:*"; } - check_port "http://127.0.0.1:8080" -) -[[ $? -eq 1 ]] \ - && pass "ss: IPv4 port occupied → exit 1" \ - || fail "ss: IPv4 port occupied → expected exit 1, got 0" - -( - source "$UTIL_SH" - command_available() { [[ "$1" == "ss" || "$1" == "timeout" ]]; } - ss() { echo "tcp LISTEN 0 128 0.0.0.0:9090 0.0.0.0:*"; } - check_port "http://127.0.0.1:8080" -) -[[ $? -eq 0 ]] \ - && pass "ss: IPv4 port free → exit 0" \ - || fail "ss: IPv4 port free → expected exit 0, got 1" - -section "ss branch — IPv6 URL with scheme (http://[::1]:8080)" - -( - source "$UTIL_SH" - command_available() { [[ "$1" == "ss" || "$1" == "timeout" ]]; } - ss() { echo "tcp LISTEN 0 128 [::]:8080 [::]:*"; } - check_port "http://[::1]:8080" -) -[[ $? -eq 1 ]] \ - && pass "ss: http://[::1]:8080 occupied → exit 1" \ - || fail "ss: http://[::1]:8080 occupied → expected exit 1, got 0" +# shellcheck source=/dev/null +source "$UTIL_SH" -( - source "$UTIL_SH" - command_available() { [[ "$1" == "ss" || "$1" == "timeout" ]]; } - ss() { echo "tcp LISTEN 0 128 [::]:9090 [::]:*"; } - check_port "http://[::1]:8080" -) -[[ $? -eq 0 ]] \ - && pass "ss: http://[::1]:8080 free → exit 0" \ - || fail "ss: http://[::1]:8080 free → expected exit 0, got 1" +# Sections run in subshells so their command overrides stay isolated, which +# means results have to be tallied through a file rather than a variable. +RESULTS=$(mktemp) -section "ss branch — IPv6 URL without scheme ([::1]:8080)" +pass() { + echo " PASS $1" + echo "P" >> "$RESULTS" +} -( - source "$UTIL_SH" - command_available() { [[ "$1" == "ss" || "$1" == "timeout" ]]; } - ss() { echo "tcp LISTEN 0 128 [::]:8080 [::]:*"; } - check_port "[::1]:8080" -) -[[ $? -eq 1 ]] \ - && pass "ss: [::1]:8080 (no scheme) occupied → exit 1" \ - || fail "ss: [::1]:8080 (no scheme) occupied → expected exit 1, got 0" +fail() { + echo " FAIL $1" + echo " $2" + echo "F" >> "$RESULTS" +} -( - source "$UTIL_SH" - command_available() { [[ "$1" == "ss" || "$1" == "timeout" ]]; } - ss() { echo "tcp LISTEN 0 128 [::]:9090 [::]:*"; } - check_port "[::1]:8080" -) -[[ $? -eq 0 ]] \ - && pass "ss: [::1]:8080 (no scheme) free → exit 0" \ - || fail "ss: [::1]:8080 (no scheme) free → expected exit 0, got 1" +expect() { + # expect + if [[ "$2" == "$3" ]]; then + pass "$1" + else + fail "$1" "expected '$2', got '$3'" + fi +} -section "ss branch — wildcard 0.0.0.0" +echo "" +echo "check_port contract tests ($UTIL_SH)" +# -------------------------------------------------------------------------- +echo "" +echo "1. URL to port" +# "|", where SKIP means "no usable port, preflight is skipped". +url_cases=( + 'http://127.0.0.1:8080|8080' + 'HTTP://127.0.0.1:8080|8080' + 'http://127.0.0.1|80' + 'https://127.0.0.1|443' + '127.0.0.1:8080|8080' + 'http://127.0.0.1:8080/path:9090|8080' + 'http://127.0.0.1:8080?probe=x|8080' + 'http://127.0.0.1:8080#frag|8080' + 'http://[::1]:8080|8080' + '[::1]:8080|8080' + 'https://[::1]|443' + 'http://127.0.0.1:08080|8080' + '::1:8080|SKIP' + 'http://127.0.0.1:abc|SKIP' + 'http://127.0.0.1:0|SKIP' + 'http://127.0.0.1:70000|SKIP' + '127.0.0.1|SKIP' +) +for case in "${url_cases[@]}"; do + url="${case%|*}" + want="${case##*|}" + got=$(parse_port_from_url " $url " 2>/dev/null) || got="SKIP" + expect "$url" "$want" "$got" +done + +# -------------------------------------------------------------------------- +echo "" +echo "2. Linux: ss busy / free / failure" ( - source "$UTIL_SH" - command_available() { [[ "$1" == "ss" || "$1" == "timeout" ]]; } - ss() { echo "tcp LISTEN 0 128 0.0.0.0:8080 0.0.0.0:*"; } - check_port "http://0.0.0.0:8080" -) -[[ $? -eq 1 ]] \ - && pass "ss: 0.0.0.0:8080 occupied → exit 1" \ - || fail "ss: 0.0.0.0:8080 occupied → expected exit 1, got 0" + uname() { echo "Linux"; } + command_available() { [[ "$1" == "ss" ]]; } + SS_OUT="" + SS_RC=0 + ss() { printf '%s' "$SS_OUT"; return "$SS_RC"; } -section "ss branch — wildcard ::" + SS_OUT='LISTEN 0 4096 0.0.0.0:8080 0.0.0.0:*' + expect "listener on 8080 is busy" "busy" "$(port_listen_state 8080)" -( - source "$UTIL_SH" - command_available() { [[ "$1" == "ss" || "$1" == "timeout" ]]; } - ss() { echo "tcp LISTEN 0 128 [::]:8080 [::]:*"; } - check_port "http://[::]:8080" -) -[[ $? -eq 1 ]] \ - && pass "ss: [::]:8080 occupied → exit 1" \ - || fail "ss: [::]:8080 occupied → expected exit 1, got 0" + SS_OUT='LISTEN 0 4096 0.0.0.0:22 0.0.0.0:*' + expect "no listener on 8080 is free" "free" "$(port_listen_state 8080)" -( - source "$UTIL_SH" - command_available() { [[ "$1" == "ss" || "$1" == "timeout" ]]; } - ss() { echo "tcp LISTEN 0 128 [::]:9090 [::]:*"; } - check_port "http://[::]:8080" -) -[[ $? -eq 0 ]] \ - && pass "ss: [::]:8080 free → exit 0" \ - || fail "ss: [::]:8080 free → expected exit 0, got 1" + # The port must come from the local-address field, not an IPv6 hextet. + SS_OUT='LISTEN 0 128 [2001:db8:8080::1]:9090 [::]:*' + expect "IPv6 hextet 8080 is not the port" "free" "$(port_listen_state 8080)" -# ── netstat branch ──────────────────────────────────────────────────────────── + SS_OUT='LISTEN 0 128 [::]:8080 [::]:*' + expect "IPv6 wildcard listener is busy" "busy" "$(port_listen_state 8080)" -section "netstat branch — Linux format (-ltn), occupied" + # A tool that runs but fails proves nothing. + SS_OUT='' + SS_RC=1 + expect "ss failure is unknown" "unknown" "$(port_listen_state 8080)" -( - source "$UTIL_SH" - command_available() { [[ "$1" == "netstat" || "$1" == "timeout" ]]; } - netstat() { echo "tcp 0 0 0.0.0.0:8080 0.0.0.0:* LISTEN"; } - check_port "http://127.0.0.1:8080" -) -[[ $? -eq 1 ]] \ - && pass "netstat -ltn: port 8080 occupied → exit 1" \ - || fail "netstat -ltn: port 8080 occupied → expected exit 1, got 0" + # Success with an unusable table proves nothing either. + SS_RC=0 + SS_OUT='' + expect "ss empty output is unknown" "unknown" "$(port_listen_state 8080)" -section "netstat branch — Linux format (-ltn), free" + SS_OUT='some diagnostic banner' + expect "ss unparseable output is unknown" "unknown" "$(port_listen_state 8080)" -( - source "$UTIL_SH" - command_available() { [[ "$1" == "netstat" || "$1" == "timeout" ]]; } - netstat() { echo "tcp 0 0 0.0.0.0:9090 0.0.0.0:* LISTEN"; } - check_port "http://127.0.0.1:8080" ) -[[ $? -eq 0 ]] \ - && pass "netstat -ltn: port 8080 free → exit 0" \ - || fail "netstat -ltn: port 8080 free → expected exit 0, got 1" -section "netstat branch — BSD/macOS fallback (-an), occupied" - -# Simulate netstat that produces no output for -ltn (Linux flag unsupported) -# but outputs BSD-format lines for -an +# -------------------------------------------------------------------------- +echo "" +echo "3. macOS/BSD: LISTEN versus ESTABLISHED" ( - source "$UTIL_SH" - command_available() { [[ "$1" == "netstat" || "$1" == "timeout" ]]; } - netstat() { - if [[ "$1" == "-ltn" ]]; then - return 1 # flag not supported on BSD - fi - echo "tcp4 0 0 *.8080 *.* LISTEN" - } - check_port "http://127.0.0.1:8080" -) -[[ $? -eq 1 ]] \ - && pass "netstat -an BSD: port 8080 occupied → exit 1" \ - || fail "netstat -an BSD: port 8080 occupied → expected exit 1, got 0" - -section "netstat branch — IP octet false-positive guard" + uname() { echo "Darwin"; } + command_available() { [[ "$1" == "netstat" ]]; } + NS_OUT="" + netstat() { printf '%s' "$NS_OUT"; } -# Port 80 check; netstat output contains 192.168.80.1:443 -# The .80 in the IP address must NOT match port 80 -( - source "$UTIL_SH" - command_available() { [[ "$1" == "netstat" || "$1" == "timeout" ]]; } - netstat() { echo "tcp 0 0 192.168.80.1:443 0.0.0.0:* LISTEN"; } - check_port "http://127.0.0.1:80" -) -[[ $? -eq 0 ]] \ - && pass "netstat: IP octet .80 does not false-positive for port 80 → exit 0" \ - || fail "netstat: IP octet .80 false-positived for port 80 → expected exit 0, got 1" + NS_OUT='Active Internet connections (including servers) +Proto Recv-Q Send-Q Local Address Foreign Address (state) +tcp4 0 0 *.8080 *.* LISTEN' + expect "BSD LISTEN on 8080 is busy" "busy" "$(port_listen_state 8080)" -section "ss branch — host dot-escaping guard" + # An outbound connection to :443 is not a local listener on 443. This is + # the false positive that blocked startup on macOS. + NS_OUT='Active Internet connections (including servers) +Proto Recv-Q Send-Q Local Address Foreign Address (state) +tcp4 0 0 192.168.1.3.60320 44.195.16.138.443 ESTABLISHED +tcp4 0 0 *.22 *.* LISTEN' + expect "BSD ESTABLISHED peer :443 is free" "free" "$(port_listen_state 443)" -# Host 127.0.0.1 must be matched literally: a listener whose address merely -# matches the pattern with '.' as a regex wildcard (127a0b0c1) must NOT count -( - source "$UTIL_SH" - command_available() { [[ "$1" == "ss" || "$1" == "timeout" ]]; } - ss() { echo "tcp LISTEN 0 128 127a0b0c1:8080 0.0.0.0:*"; } - check_port "http://127.0.0.1:8080" -) -[[ $? -eq 0 ]] \ - && pass "ss: unescaped-dot lookalike 127a0b0c1 does not false-positive → exit 0" \ - || fail "ss: unescaped-dot lookalike 127a0b0c1 false-positived → expected exit 0, got 1" + # A socket lingering in TIME_WAIT holds the port in its *local* address but + # is not a listener, so only the connection state can tell them apart. + NS_OUT='Proto Recv-Q Send-Q Local Address Foreign Address (state) +tcp4 0 0 127.0.0.1.8080 127.0.0.1.51000 TIME_WAIT +tcp4 0 0 *.22 *.* LISTEN' + expect "BSD TIME_WAIT on local 8080 is free" "free" "$(port_listen_state 8080)" -# ── /dev/tcp fallback branch ────────────────────────────────────────────────── + NS_OUT='Proto Recv-Q Send-Q Local Address Foreign Address (state) +tcp6 0 0 ::1.8080 *.* LISTEN' + expect "BSD IPv6 LISTEN is busy" "busy" "$(port_listen_state 8080)" -section "/dev/tcp fallback — timeout available, port occupied" + NS_OUT='Proto Recv-Q Send-Q Local Address Foreign Address (state)' + expect "BSD header only is unknown" "unknown" "$(port_listen_state 8080)" -# timeout exits 0 → connection succeeded → port in use -( - source "$UTIL_SH" - command_available() { [[ "$1" == "timeout" ]]; } - timeout() { - # Assert correct invocation: timeout 1 bash -c SCRIPT _ HOST PORT - [[ "$1" == "1" && "$2" == "bash" && "$3" == "-c" && "$5" == "_" ]] \ - || { echo "timeout mock: unexpected argv: $*"; return 2; } - return 0 - } - check_port "http://127.0.0.1:8080" ) -[[ $? -eq 1 ]] \ - && pass "/dev/tcp+timeout: connection succeeded (exit 0) → port occupied → exit 1" \ - || fail "/dev/tcp+timeout: connection succeeded → expected exit 1, got 0" -section "/dev/tcp fallback — timeout available, port free" - -# timeout exits 1 → connection refused → port free -( - source "$UTIL_SH" - command_available() { [[ "$1" == "timeout" ]]; } - timeout() { - [[ "$1" == "1" && "$2" == "bash" && "$3" == "-c" && "$5" == "_" ]] \ - || { echo "timeout mock: unexpected argv: $*"; return 2; } - return 1 - } - check_port "http://127.0.0.1:8080" -) -[[ $? -eq 0 ]] \ - && pass "/dev/tcp+timeout: connection refused (exit 1) → port free → exit 0" \ - || fail "/dev/tcp+timeout: connection refused → expected exit 0, got 1" - -section "/dev/tcp fallback — real loopback (ephemeral port)" - -# Start Python server on ephemeral port 0, capture actual port from child stdout +# -------------------------------------------------------------------------- +echo "" +echo "4. Unknown never blocks startup" ( - source "$UTIL_SH" - command_available() { [[ "$1" == "timeout" ]]; } - # Mock timeout: on hosts without timeout (macOS), run the probe directly. - # The probe args are: timeout 1 bash -c SCRIPT _ HOST PORT - timeout() { - [[ "$1" == "1" && "$2" == "bash" && "$3" == "-c" && "$5" == "_" ]] \ - || { echo "timeout mock: unexpected argv: $*" >&2; return 2; } - # Run the probe with a 2-second hard deadline via background + watchdog - bash -c "$4" "$5" "$6" "$7" 2>/dev/null & - local probe_pid=$! - (sleep 2; kill -9 "$probe_pid" 2>/dev/null) & - local watchdog_pid=$! - wait "$probe_pid" 2>/dev/null - local rc=$? - kill -9 "$watchdog_pid" 2>/dev/null - wait "$watchdog_pid" 2>/dev/null - return $rc - } - # Use a temp file to capture the bound port from child - port_file=$(mktemp) - trap 'rm -f "$port_file"; [[ -n "${PY_PID:-}" ]] && { kill -9 "$PY_PID" 2>/dev/null; wait "$PY_PID" 2>/dev/null; }' EXIT - - # Start Python server that prints the bound port to stdout - python3 -c " -import socket -s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) -s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) -s.bind(('127.0.0.1', 0)) -s.listen(1) # actually listen so /dev/tcp can connect -port = s.getsockname()[1] -print(port, flush=True) -import time -time.sleep(30) # keep server alive -" > "$port_file" 2>/dev/null & - PY_PID=$! - # Poll for port file up to ~4s (Python cold-start can exceed 0.5s on busy CI) - bound_port="" - for _ in $(seq 1 40); do - bound_port=$(head -1 "$port_file" 2>/dev/null) - [[ -n "$bound_port" ]] && break - kill -0 $PY_PID 2>/dev/null || break - sleep 0.1 - done - if [[ -z "$bound_port" || ! "$bound_port" =~ ^[0-9]+$ ]]; then - echo "SKIP: failed to get bound port" - kill -9 $PY_PID 2>/dev/null || true - exit 77 - fi - # Verify child is alive - if ! kill -0 $PY_PID 2>/dev/null; then - echo "SKIP: Python child died" - exit 77 - fi - check_port "http://127.0.0.1:$bound_port" -) -_rc=$? -if [[ $_rc -eq 77 ]]; then - echo " SKIP /dev/tcp real ephemeral port (setup failed)" -elif [[ $_rc -eq 1 ]]; then - pass "/dev/tcp real ephemeral port → detects occupation → exit 1" -else - fail "/dev/tcp real ephemeral port → expected exit 1, got $_rc" -fi + uname() { echo "Linux"; } + command_available() { return 1; } -section "/dev/tcp fallback — no-timeout watchdog kills stuck probe" + expect "no probe tool is unknown" "unknown" "$(port_listen_state 8080)" -# run_with_deadline "sleep 5" 2 should return in ~2s, not ~5s. -# Proves the watchdog actually kills a stuck child, not just that the code path is taken. -( - source "$UTIL_SH" - start=$(date +%s) - run_with_deadline 'sleep 5' 2 + # check_port must warn and let the server perform the authoritative bind. + err=$( (check_port "http://127.0.0.1:8080") 2>&1 >/dev/null ) rc=$? - elapsed=$(($(date +%s) - start)) - # Must complete well before the 5s sleep (2s deadline + overhead) and return non-zero (killed) - if [[ $rc -ne 0 && $elapsed -le 4 ]]; then - exit 0 + expect "unknown exits 0" "0" "$rc" + if [[ "$err" == *"could not determine"* ]]; then + pass "unknown warns on stderr" else - echo "run_with_deadline: rc=$rc elapsed=${elapsed}s (expected rc≠0, ≤4s)" >&2 - exit 1 - fi -) -[[ $? -eq 0 ]] \ - && pass "/dev/tcp no-timeout watchdog: sleep 5 killed in ≤4s" \ - || fail "/dev/tcp no-timeout watchdog: watchdog did not kill in time" - -# ── host/port conflict semantics ────────────────────────────────────────────── - -section "Host/port conflict semantics" - -# Configured host 192.168.1.50, but only 127.0.0.1 is listening -> free -( - source "$UTIL_SH" - command_available() { [[ "$1" == "ss" || "$1" == "timeout" ]]; } - ss() { echo "LISTEN 0 128 127.0.0.1:8080 0.0.0.0:*"; return 0; } - check_port "http://192.168.1.50:8080" -) -[[ $? -eq 0 ]] \ - && pass "specific IP configured vs loopback listener → free → exit 0" \ - || fail "specific IP configured vs loopback listener → expected free, got occupied" - -# Configured host 192.168.1.50, 0.0.0.0 is listening -> occupied -( - source "$UTIL_SH" - command_available() { [[ "$1" == "ss" || "$1" == "timeout" ]]; } - ss() { echo "LISTEN 0 128 0.0.0.0:8080 0.0.0.0:*"; return 0; } - check_port "http://192.168.1.50:8080" -) -[[ $? -eq 1 ]] \ - && pass "specific IP configured vs wildcard listener → occupied → exit 1" \ - || fail "specific IP configured vs wildcard listener → expected occupied, got free" - -# Specific IPv4 host should not conflict with IPv6 wildcard listener -( - source "$UTIL_SH" - command_available() { [[ "$1" == "ss" || "$1" == "timeout" ]]; } - ss() { echo "LISTEN 0 128 [::]:8080 [::]:*"; return 0; } - check_port "http://127.0.0.1:8080" -) -[[ $? -eq 0 ]] \ - && pass "specific IPv4 host vs IPv6 wildcard listener → free → exit 0" \ - || fail "specific IPv4 host vs IPv6 wildcard listener → expected free, got occupied" - -# Specific IPv6 host should not conflict with IPv4 wildcard listener -( - source "$UTIL_SH" - command_available() { [[ "$1" == "ss" || "$1" == "timeout" ]]; } - ss() { echo "LISTEN 0 128 0.0.0.0:8080 0.0.0.0:*"; return 0; } - check_port "http://[::1]:8080" -) -[[ $? -eq 0 ]] \ - && pass "specific IPv6 host vs IPv4 wildcard listener → free → exit 0" \ - || fail "specific IPv6 host vs IPv4 wildcard listener → expected free, got occupied" - -# ── tool failure fallthrough ────────────────────────────────────────────────── - -section "Tool failure fallthrough" - -# ss fails, netstat succeeds and finds occupied port -( - source "$UTIL_SH" - command_available() { [[ "$1" == "ss" || "$1" == "netstat" || "$1" == "timeout" ]]; } - ss() { return 1; } # ss execution fails - netstat() { - if [[ "$1" == "-ltn" ]]; then echo "tcp 0 0 0.0.0.0:8080 0.0.0.0:* LISTEN"; return 0; fi - return 1 - } - check_port "http://127.0.0.1:8080" -) -[[ $? -eq 1 ]] \ - && pass "ss exits 1 → fallthrough to netstat → occupied → exit 1" \ - || fail "ss exits 1 → fallthrough to netstat → expected occupied, got free" - -# both ss and netstat fail, /dev/tcp timeout path must still detect occupation -( - source "$UTIL_SH" - command_available() { [[ "$1" == "ss" || "$1" == "netstat" || "$1" == "timeout" ]]; } - ss() { return 1; } - netstat() { return 1; } - timeout() { return 0; } # /dev/tcp connect succeeds - check_port "http://127.0.0.1:8080" -) -[[ $? -eq 1 ]] \ - && pass "ss/netstat both fail → /dev/tcp timeout path occupied → exit 1" \ - || fail "ss/netstat both fail → expected /dev/tcp fallback occupation, got free" - -# ── hostname and URL normalization ──────────────────────────────────────────── - -section "Hostname collision detection" - -# localhost should resolve to numeric addresses before ss matching -( - source "$UTIL_SH" - command_available() { [[ "$1" == "ss" || "$1" == "getent" || "$1" == "timeout" ]]; } - timeout() { shift; "$@" 2>/dev/null; } # pass-through for mocked getent - getent() { - if [[ "$1" == "hosts" || "$1" == "ahosts" ]] && [[ "$2" == "localhost" ]]; then - echo "127.0.0.1" - return 0 - fi - return 1 - } - ss() { echo "tcp LISTEN 0 128 127.0.0.1:8080 0.0.0.0:*"; } - check_port "http://localhost:8080" -) -[[ $? -eq 1 ]] \ - && pass "hostname localhost resolves to 127.0.0.1 and detects occupation → exit 1" \ - || fail "hostname localhost occupation was not detected via resolved address" - -# unresolved hostnames should skip ss/netstat text matching and use /dev/tcp fallback -( - source "$UTIL_SH" - command_available() { [[ "$1" == "ss" || "$1" == "getent" || "$1" == "timeout" ]]; } - getent() { return 2; } # resolution fails - ss() { echo "tcp LISTEN 0 128 127.0.0.1:8080 0.0.0.0:*"; return 0; } - timeout() { return 0; } # fallback connect succeeds - check_port "http://unresolved.localdomain:8080" -) -[[ $? -eq 1 ]] \ - && pass "unresolved hostname falls through to /dev/tcp fallback and detects occupation" \ - || fail "unresolved hostname should use /dev/tcp fallback" - -section "URL normalization and IPv6 hextet guard" - -# check_port should trim URL whitespace before parsing host/port -( - source "$UTIL_SH" - command_available() { [[ "$1" == "ss" || "$1" == "timeout" ]]; } - ss() { echo "tcp LISTEN 0 128 127.0.0.1:8080 0.0.0.0:*"; } - check_port " http://127.0.0.1:8080 " -) -[[ $? -eq 1 ]] \ - && pass "leading/trailing URL whitespace is normalized → occupied detected" \ - || fail "URL whitespace normalization failed to detect occupied port" - -# Ensure ":8080" inside an IPv6 hextet does not match listener port 8080 -( - source "$UTIL_SH" - command_available() { [[ "$1" == "ss" || "$1" == "timeout" ]]; } - timeout() { - if [[ "$2" == "getent" ]]; then - shift - "$@" 2>/dev/null - else - return 1 - fi - } - ss() { echo "tcp LISTEN 0 128 [2001:db8:8080::1]:9090 [::]:*"; } - check_port "http://[::1]:8080" -) -[[ $? -eq 0 ]] \ - && pass "IPv6 hextet containing 8080 does not false-positive port 8080" \ - || fail "IPv6 hextet false-positive: expected free, got occupied" - -# ── IPv6 address canonicalisation ───────────────────────────────────────────── - -section "IPv6 address canonicalisation" - -# getent mock that normalises the two spellings of loopback to ::1 -getent_canonical_loopback() { - if [[ "$1" == "ahosts" ]] && [[ "$2" == "::1" || "$2" == "0:0:0:0:0:0:0:1" ]]; then - echo "::1" - return 0 + fail "unknown warns on stderr" "stderr was: $err" fi - return 2 -} - -# Host ::1, listener expanded 0:0:0:0:0:0:0:1 -( - source "$UTIL_SH" - command_available() { [[ "$1" == "ss" || "$1" == "timeout" ]]; } - getent() { getent_canonical_loopback "$@"; } - ss() { echo "tcp LISTEN 0 128 [0:0:0:0:0:0:0:1]:8080 [::]:*"; } - check_port "http://[::1]:8080" -) -[[ $? -eq 1 ]] \ - && pass "IPv6 host [::1] matches expanded listener [0:0:0:0:0:0:0:1]:8080" \ - || fail "IPv6 compressed vs expanded spelling did not match" - -# Host expanded 0:0:0:0:0:0:0:1, listener ::1 (mixed case) -( - source "$UTIL_SH" - command_available() { [[ "$1" == "ss" || "$1" == "timeout" ]]; } - getent() { getent_canonical_loopback "$@"; } - ss() { echo "tcp LISTEN 0 128 [::1]:8080 [::]:*"; } - check_port "http://[0:0:0:0:0:0:0:1]:8080" -) -[[ $? -eq 1 ]] \ - && pass "IPv6 expanded host matches compressed listener" \ - || fail "IPv6 expanded vs compressed spelling did not match" - -# Host ::1, listener expanded on a different port → free -( - source "$UTIL_SH" - command_available() { [[ "$1" == "ss" || "$1" == "timeout" ]]; } - timeout() { - if [[ "$2" == "getent" ]]; then - shift - "$@" 2>/dev/null - else - return 1 - fi - } - getent() { getent_canonical_loopback "$@"; } - ss() { echo "tcp LISTEN 0 128 [0:0:0:0:0:0:0:1]:9090 [::]:*"; } - check_port "http://[::1]:8080" -) -[[ $? -eq 0 ]] \ - && pass "IPv6 canonicalisation does not false-positive on different port" \ - || fail "IPv6 canonicalisation matched the wrong port" - -# ── edge cases ──────────────────────────────────────────────────────────────── - -section "Edge cases" - -# Empty URL → port string is empty → return 0 immediately -( - source "$UTIL_SH" - command_available() { false; } - check_port "" -) -[[ $? -eq 0 ]] \ - && pass "empty URL → exit 0 (graceful)" \ - || fail "empty URL → expected exit 0, got 1" - -# Port out of range (> 65535) → return 0 immediately -( - source "$UTIL_SH" - command_available() { false; } - check_port "http://127.0.0.1:99999" -) -[[ $? -eq 0 ]] \ - && pass "port 99999 (out of range) → exit 0 (graceful)" \ - || fail "port 99999 (out of range) → expected exit 0, got 1" -# Port 0 (invalid) → return 0 immediately -( - source "$UTIL_SH" - command_available() { false; } - check_port "http://127.0.0.1:0" -) -[[ $? -eq 0 ]] \ - && pass "port 0 (invalid) → exit 0 (graceful)" \ - || fail "port 0 (invalid) → expected exit 0, got 1" - -# Non-numeric port → return 0 immediately -( - source "$UTIL_SH" - command_available() { false; } - check_port "127.0.0.1:abc" ) -[[ $? -eq 0 ]] \ - && pass "non-numeric port → exit 0 (graceful)" \ - || fail "non-numeric port → expected exit 0, got 1" - -# ── download() fallback ─────────────────────────────────────────────────────── - -section "download() atomic curl path (pd)" - -( - if [[ ! -f "$PD_UTIL_SH" ]]; then - exit 77 - fi - - source "$PD_UTIL_SH" - - OUTDIR="$(mktemp -d)/outdir" - trap 'rm -rf "$(dirname "$OUTDIR")"' EXIT - - command_available() { - if [[ "$1" == "wget" ]]; then return 1; fi - if [[ "$1" == "curl" ]]; then return 0; fi - command -v "$1" >/dev/null 2>&1 - } - - MKDIR_CALLED=0 - mkdir() { - if [[ "$1" == "-p" && "$2" == "$OUTDIR" ]]; then - MKDIR_CALLED=1 - command mkdir -p "$OUTDIR" - return 0 - fi - echo "mkdir mock called with unexpected args: $*" - return 1 - } - CURL_CALLED=0 - curl() { - # Expected: curl -fL -o -- - if [[ "$1" == "-fL" && "$2" == "-o" && \ - "$4" == "--" && \ - "$5" == "https://example.com/some/path/file.tar.gz" ]]; then - CURL_CALLED=1 - # The temp name must be hidden and contain the PID. - if [[ "$3" != "$OUTDIR/.file.tar.gz.tmp."* ]]; then - echo "curl mock: unexpected output path: $3" - return 1 - fi - touch "$3" - return 0 - fi - echo "curl mock called with unexpected args: $*" - return 1 - } +# -------------------------------------------------------------------------- +echo "" +echo "5. Real listener on an ephemeral port" +if command -v python3 >/dev/null 2>&1; then + PORT_FILE=$(mktemp) + python3 -c ' +import socket, sys, time +s = socket.socket() +s.bind(("127.0.0.1", 0)) +s.listen(1) +print(s.getsockname()[1]) +sys.stdout.flush() +time.sleep(30) +' > "$PORT_FILE" & + PY_PID=$! + trap 'kill "$PY_PID" 2>/dev/null; rm -f "$PORT_FILE"' EXIT - download "$OUTDIR" "https://example.com/some/path/file.tar.gz" >/dev/null 2>&1 - RC=$? + BOUND="" + for _ in 1 2 3 4 5 6 7 8 9 10; do + BOUND=$(head -1 "$PORT_FILE" 2>/dev/null) + [[ -n "$BOUND" ]] && break + sleep 0.5 + done - if [[ $RC -eq 0 && $CURL_CALLED -eq 1 && $MKDIR_CALLED -eq 1 && \ - -f "$OUTDIR/file.tar.gz" ]]; then - exit 0 + if [[ -z "$BOUND" ]] || ! kill -0 "$PY_PID" 2>/dev/null; then + fail "listener bound an ephemeral port" "child did not report a port" else - echo "RC=$RC CURL_CALLED=$CURL_CALLED MKDIR_CALLED=$MKDIR_CALLED" - ls -la "$OUTDIR" 2>&1 || true - exit 1 + pass "listener bound an ephemeral port ($BOUND)" + expect "bound port is busy" "busy" "$(port_listen_state "$BOUND")" + (check_port "http://127.0.0.1:$BOUND" >/dev/null 2>&1) + expect "check_port exits 1 on the bound port" "1" "$?" + + # A port that was bound and released must not read as busy. + FREE=$(python3 -c 'import socket +s = socket.socket() +s.bind(("127.0.0.1", 0)) +p = s.getsockname()[1] +s.close() +print(p)') + expect "released port is not busy" "0" "$([[ "$(port_listen_state "$FREE")" == "busy" ]] && echo 1 || echo 0)" fi -) -_rc=$? -if [[ $_rc -eq 77 ]]; then - echo " SKIP download() atomic curl path: pd util.sh not found" -elif [[ $_rc -eq 0 ]]; then - pass "download() (PD) writes to temp and renames on success" + + kill "$PY_PID" 2>/dev/null + wait "$PY_PID" 2>/dev/null + rm -f "$PORT_FILE" + trap - EXIT else - fail "download() (PD) atomic curl path failed" + echo " SKIP real listener test: python3 not available" fi -section "download() atomic curl path (store)" - +# -------------------------------------------------------------------------- +echo "" +echo "6. download() rename failure is reported" ( - if [[ ! -f "$STORE_UTIL_SH" ]]; then - exit 77 - fi + WORK=$(mktemp -d) + trap 'rm -rf "$WORK"' EXIT - source "$STORE_UTIL_SH" - - OUTDIR="$(mktemp -d)/outdir" - trap 'rm -rf "$(dirname "$OUTDIR")"' EXIT - - command_available() { - if [[ "$1" == "wget" ]]; then return 1; fi - if [[ "$1" == "curl" ]]; then return 0; fi - command -v "$1" >/dev/null 2>&1 - } - - MKDIR_CALLED=0 - mkdir() { - if [[ "$1" == "-p" && "$2" == "$OUTDIR" ]]; then - MKDIR_CALLED=1 - command mkdir -p "$OUTDIR" - return 0 - fi - echo "mkdir mock called with unexpected args: $*" - return 1 - } - - CURL_CALLED=0 + command_available() { [[ "$1" == "curl" ]]; } curl() { - if [[ "$1" == "-fL" && "$2" == "-o" && \ - "$4" == "--" && \ - "$5" == "https://example.com/some/path/file.tar.gz" ]]; then - CURL_CALLED=1 - if [[ "$3" != "$OUTDIR/.file.tar.gz.tmp."* ]]; then - echo "curl mock: unexpected output path: $3" - return 1 - fi - touch "$3" - return 0 - fi - echo "curl mock called with unexpected args: $*" - return 1 + # Write to the -o destination so a temp file exists to rename. + local out="" + while [[ $# -gt 0 ]]; do + [[ "$1" == "-o" ]] && { out="$2"; shift; } + shift + done + echo "payload" > "$out" + return 0 } + # Simulate a rename that fails (read-only target, cross-device, ...). + mv() { return 1; } - download "$OUTDIR" "https://example.com/some/path/file.tar.gz" >/dev/null 2>&1 - RC=$? - - if [[ $RC -eq 0 && $CURL_CALLED -eq 1 && $MKDIR_CALLED -eq 1 && \ - -f "$OUTDIR/file.tar.gz" ]]; then - exit 0 + if download "$WORK" "https://example.com/pkg.tar.gz"; then + fail "download reports rename failure" "download returned 0 after mv failed" else - echo "RC=$RC CURL_CALLED=$CURL_CALLED MKDIR_CALLED=$MKDIR_CALLED" - ls -la "$OUTDIR" 2>&1 || true - exit 1 + pass "download reports rename failure" fi -) -_rc=$? -if [[ $_rc -eq 77 ]]; then - echo " SKIP download() atomic curl path: store util.sh not found" -elif [[ $_rc -eq 0 ]]; then - pass "download() (Store) writes to temp and renames on success" -else - fail "download() (Store) atomic curl path failed" -fi -section "download() atomic wget path (pd)" - -( - if [[ ! -f "$PD_UTIL_SH" ]]; then - exit 77 + if [[ -e "$WORK/pkg.tar.gz" ]]; then + fail "failed rename leaves no destination" "$WORK/pkg.tar.gz exists" + else + pass "failed rename leaves no destination" fi - source "$PD_UTIL_SH" - - OUTDIR="$(mktemp -d)/outdir" - trap 'rm -rf "$(dirname "$OUTDIR")"' EXIT - - command_available() { - if [[ "$1" == "curl" ]]; then return 1; fi - if [[ "$1" == "wget" ]]; then return 0; fi - command -v "$1" >/dev/null 2>&1 - } - - MKDIR_CALLED=0 - mkdir() { - if [[ "$1" == "-p" && "$2" == "$OUTDIR" ]]; then - MKDIR_CALLED=1 - command mkdir -p "$OUTDIR" - return 0 - fi - echo "mkdir mock called with unexpected args: $*" - return 1 - } + leftover=$(find "$WORK" -name '.pkg.tar.gz.*' 2>/dev/null | wc -l | tr -d ' ') + expect "failed rename cleans its temp file" "0" "$leftover" - WGET_CALLED=0 - wget() { - if [[ "$1" == "--help" ]]; then - # wget --help runs in a pipeline subshell, so side-effect counters - # cannot be observed from the parent. Only the stdout matters here. - echo "--show-progress" - return 0 - fi - # Expected: wget -q --show-progress -O -- - if [[ "$1" == "-q" && "$2" == "--show-progress" && "$3" == "-O" && \ - "$5" == "--" && \ - "$6" == "https://example.com/some/path/file.tar.gz" ]]; then - WGET_CALLED=1 - if [[ "$4" != "$OUTDIR/.file.tar.gz.tmp."* ]]; then - echo "wget mock: unexpected output path: $4" - return 1 - fi - touch "$4" - return 0 - fi - echo "wget mock called with unexpected args: $*" - return 1 - } - - download "$OUTDIR" "https://example.com/some/path/file.tar.gz" >/dev/null 2>&1 - RC=$? - - if [[ $RC -eq 0 && $WGET_CALLED -eq 1 && $MKDIR_CALLED -eq 1 && \ - -f "$OUTDIR/file.tar.gz" ]]; then - exit 0 - else - echo "RC=$RC WGET_CALLED=$WGET_CALLED MKDIR_CALLED=$MKDIR_CALLED" - ls -la "$OUTDIR" 2>&1 || true - exit 1 - fi ) -_rc=$? -if [[ $_rc -eq 77 ]]; then - echo " SKIP download() atomic wget path: pd util.sh not found" -elif [[ $_rc -eq 0 ]]; then - pass "download() (PD) wget writes to temp and renames on success" -else - fail "download() (PD) atomic wget path failed" -fi - -section "download() atomic wget path (store)" +# -------------------------------------------------------------------------- +echo "" +echo "7. Startup probe is bounded" ( - if [[ ! -f "$STORE_UTIL_SH" ]]; then - exit 77 - fi + WORK=$(mktemp -d) + cd "$WORK" || exit 1 + trap 'cd /; rm -rf "$WORK"' EXIT - source "$STORE_UTIL_SH" + ARGS_FILE="$WORK/curl-args" + curl() { echo "$*" >> "$ARGS_FILE"; echo "000"; return 28; } + process_status() { return 0; } + ps() { return 0; } - OUTDIR="$(mktemp -d)/outdir" - trap 'rm -rf "$(dirname "$OUTDIR")"' EXIT + wait_for_startup "$$" "test-server" "http://127.0.0.1:1" 1 >/dev/null 2>&1 - command_available() { - if [[ "$1" == "curl" ]]; then return 1; fi - if [[ "$1" == "wget" ]]; then return 0; fi - command -v "$1" >/dev/null 2>&1 - } - - MKDIR_CALLED=0 - mkdir() { - if [[ "$1" == "-p" && "$2" == "$OUTDIR" ]]; then - MKDIR_CALLED=1 - command mkdir -p "$OUTDIR" - return 0 - fi - echo "mkdir mock called with unexpected args: $*" - return 1 - } - - WGET_CALLED=0 - wget() { - if [[ "$1" == "--help" ]]; then - # wget --help runs in a pipeline subshell, so side-effect counters - # cannot be observed from the parent. Only the stdout matters here. - echo "--show-progress" - return 0 - fi - # Expected: wget -q --show-progress -O -- - if [[ "$1" == "-q" && "$2" == "--show-progress" && "$3" == "-O" && \ - "$5" == "--" && \ - "$6" == "https://example.com/some/path/file.tar.gz" ]]; then - WGET_CALLED=1 - if [[ "$4" != "$OUTDIR/.file.tar.gz.tmp."* ]]; then - echo "wget mock: unexpected output path: $4" - return 1 - fi - touch "$4" - return 0 - fi - echo "wget mock called with unexpected args: $*" - return 1 - } - - download "$OUTDIR" "https://example.com/some/path/file.tar.gz" >/dev/null 2>&1 - RC=$? - - if [[ $RC -eq 0 && $WGET_CALLED -eq 1 && $MKDIR_CALLED -eq 1 && \ - -f "$OUTDIR/file.tar.gz" ]]; then - exit 0 + if grep -q -- "--max-time" "$ARGS_FILE" 2>/dev/null; then + pass "startup probe passes --max-time" else - echo "RC=$RC WGET_CALLED=$WGET_CALLED MKDIR_CALLED=$MKDIR_CALLED" - ls -la "$OUTDIR" 2>&1 || true - exit 1 + fail "startup probe passes --max-time" "args were: $(cat "$ARGS_FILE" 2>/dev/null)" fi -) -_rc=$? -if [[ $_rc -eq 77 ]]; then - echo " SKIP download() atomic wget path: store util.sh not found" -elif [[ $_rc -eq 0 ]]; then - pass "download() (Store) wget writes to temp and renames on success" -else - fail "download() (Store) atomic wget path failed" -fi - -section "download() curl failure cleanup (pd)" - -# curl failure must clean up the partial file and never leave a poisoned artifact -( - if [[ ! -f "$PD_UTIL_SH" ]]; then exit 77; fi - source "$PD_UTIL_SH" - - OUTDIR=$(mktemp -d) - trap 'rm -rf "$OUTDIR"' EXIT - - command_available() { - if [[ "$1" == "wget" ]]; then return 1; fi - if [[ "$1" == "curl" ]]; then return 0; fi - command -v "$1" >/dev/null 2>&1 - } - mkdir() { command mkdir -p "$2"; return 0; } - PARTIAL="" - curl() { - if [[ "$1" == "-fL" && "$2" == "-o" && \ - "$4" == "--" && \ - "$5" == "https://example.com/some/path/file.tar.gz" ]]; then - PARTIAL="$3" - # Simulate a partial/corrupted transfer: write something then fail. - echo "partial" > "$PARTIAL" - return 1 - fi - return 1 - } - download "$OUTDIR" "https://example.com/some/path/file.tar.gz" >/dev/null 2>&1 - rc=$? - - if [[ $rc -ne 0 && ! -f "$OUTDIR/file.tar.gz" && \ - ( -z "$PARTIAL" || ! -f "$PARTIAL" ) ]]; then - exit 0 + if grep -q -- "--connect-timeout" "$ARGS_FILE" 2>/dev/null; then + pass "startup probe passes --connect-timeout" else - echo "rc=$rc PARTIAL=$PARTIAL DEST=$([[ -f $OUTDIR/file.tar.gz ]] && echo exists || echo missing)" - ls -la "$OUTDIR" 2>&1 || true - exit 1 + fail "startup probe passes --connect-timeout" "args were: $(cat "$ARGS_FILE" 2>/dev/null)" fi -) -_rc=$? -if [[ $_rc -eq 77 ]]; then - echo " SKIP download() curl failure cleanup: pd util.sh not found" -elif [[ $_rc -eq 0 ]]; then - pass "download() curl failure cleans temp and does not poison destination" -else - fail "download() curl failure did not clean partial file" -fi -# ── summary ─────────────────────────────────────────────────────────────────── +) +# -------------------------------------------------------------------------- echo "" -echo "════════════════════════════════" -echo -e " Results: ${GREEN}$PASS passed${NC} ${RED}$FAIL failed${NC}" -echo "════════════════════════════════" - -if [[ $FAIL -gt 0 ]]; then - echo "" - echo "Failed tests:" - for err in ${ERRORS[@]+"${ERRORS[@]}"}; do - echo -e " ${RED}✗${NC} $err" - done +echo "----------------------------------------" +# grep -c prints 0 and exits 1 when there are no matches; the count is what +# matters here, so the exit status is deliberately ignored. +PASSED=$(grep -c '^P' "$RESULTS" 2>/dev/null) +FAILED=$(grep -c '^F' "$RESULTS" 2>/dev/null) +rm -f "$RESULTS" +echo "passed: $PASSED failed: $FAILED" +if [[ "$FAILED" -gt 0 ]]; then + exit 1 fi - -echo "" -[[ $FAIL -eq 0 ]] && exit 0 || exit 1 +exit 0 diff --git a/hugegraph-store/hg-store-dist/src/assembly/static/bin/util.sh b/hugegraph-store/hg-store-dist/src/assembly/static/bin/util.sh index 5e68d4cbb3..b4011e2e47 100644 --- a/hugegraph-store/hg-store-dist/src/assembly/static/bin/util.sh +++ b/hugegraph-store/hg-store-dist/src/assembly/static/bin/util.sh @@ -279,15 +279,19 @@ function download() { } fi - local filename + local filename tmp filename=$(basename "${link_url%%[?#]*}") - local tmp="${path}/.${filename}.tmp.$$" local dest="${path}/${filename}" + # mktemp, not $$: a PID is shared by concurrent background subshells. + tmp=$(mktemp -- "${path}/.${filename}.XXXXXX") || { + echo "Failed to create a temporary file in $path" + return 1 + } if command_available "curl"; then # -o must appear before -- so it is parsed as an option, not an extra URL. if curl -fL -o "$tmp" -- "${link_url}"; then - mv -f -- "$tmp" "$dest" + mv -f -- "$tmp" "$dest" || { rm -f -- "$tmp"; return 1; } else rm -f -- "$tmp" return 1 @@ -298,7 +302,7 @@ function download() { progress_opt=(-q --show-progress) fi if wget ${progress_opt[@]+"${progress_opt[@]}"} -O "$tmp" -- "${link_url}"; then - mv -f -- "$tmp" "$dest" + mv -f -- "$tmp" "$dest" || { rm -f -- "$tmp"; return 1; } else rm -f -- "$tmp" return 1 @@ -329,7 +333,9 @@ download_and_verify() { fi echo "Downloading $filepath..." - local tmp="${filepath}.tmp.$$" + local tmp + # mktemp, not $$: a PID is shared by concurrent background subshells. + tmp=$(mktemp -- "${filepath}.XXXXXX") || return 1 if curl -fL -o "$tmp" -- "$url"; then actual_md5=$(md5sum -- "$tmp" | awk '{ print $1 }') if [[ "$actual_md5" != "$expected_md5" ]]; then @@ -337,7 +343,9 @@ download_and_verify() { rm -f -- "$tmp" return 1 fi - mv -f -- "$tmp" "$filepath" + # A failed rename must not report success: callers export LD_PRELOAD + # from $filepath and would otherwise use a library that never installed. + mv -f -- "$tmp" "$filepath" || { rm -f -- "$tmp"; return 1; } else rm -f -- "$tmp" return 1 From d58a98308040f91f549577a6f39a70a8c4810362 Mon Sep 17 00:00:00 2001 From: Himanshu Verma Date: Mon, 27 Jul 2026 20:22:03 +0530 Subject: [PATCH 04/10] docs(bin): record the port-preflight tradeoffs, and fix userinfo parsing Converging on port-only detection removed capability on purpose. Mark each gap in place with a namespaced TODO so it can be found and picked up later rather than rediscovered: - port-only matching ignores the listener's address, so a listener on one local address reports the port busy even when the server would bind a different one. This is the old `lsof -i :PORT` behaviour and fails safe, but it can refuse a bind that would have succeeded. - a host with genuinely zero LISTEN sockets is indistinguishable from a restricted or unparseable table; both report unknown and warn. - with neither ss nor netstat present there is no probe left, so the check is permanently unknown. A dependency-free fallback needs a bounded connect, which is exactly what was removed here. - unbracketed IPv6 and a scheme-less value with no port are skipped rather than guessed. - wait_for_startup overshoot is now bounded but not zero: the loop still sleeps between probes before re-reading the clock. - the Linux and BSD detection branches are mock-driven, so only the host's own branch runs against a real kernel on any one runner. Also fix a real parsing defect found while auditing those gaps: a URL carrying userinfo (http://user:pass@host:port) was matched by the unbracketed-IPv6 check, so it skipped the preflight and printed a misleading warning about bracket notation. Strip userinfo from the authority before that test, and cover the userinfo forms in the suite. Suite: 44 passed, 0 failed. --- .../src/assembly/static/bin/util.sh | 31 +++++++++++++++++++ .../src/assembly/travis/test-check-port.sh | 9 ++++++ 2 files changed, 40 insertions(+) diff --git a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/util.sh b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/util.sh index c794bd6d2e..6f9e62d0f7 100755 --- a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/util.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/util.sh @@ -141,6 +141,11 @@ function parse_port_from_url() { local authority="${rest%%[/?#]*}" [[ -z "$authority" ]] && return 1 + # Drop any userinfo prefix; its colon would otherwise look like an + # unbracketed IPv6 separator. + authority="${authority##*@}" + [[ -z "$authority" ]] && return 1 + local port="" if [[ "$authority" =~ ^\[[^]]*\](:([0-9]+))?$ ]]; then # Bracketed IPv6, with or without a port: [::1] or [::1]:8080 @@ -148,6 +153,9 @@ function parse_port_from_url() { elif [[ "$authority" == *:*:* ]]; then # Unbracketed IPv6 is ambiguous: in "::1:8080" the trailing group may # be a port or another hextet. Refuse to guess. + # TODO(check_port): no preflight runs at all for this form. If + # ServerOptions ever guarantees a normalized bracketed value here, this + # branch can resolve the port instead of skipping the check. echo "WARN: ambiguous IPv6 authority '$authority' in server URL;" \ "use bracket notation such as [::1]:8080." >&2 return 1 @@ -156,6 +164,9 @@ function parse_port_from_url() { fi # Fall back to the scheme's default port. + # TODO(check_port): a scheme-less value with no explicit port (e.g. plain + # "127.0.0.1") has no derivable port, so it is skipped rather than guessed. + # Reading the configured default from ServerOptions would close this gap. if [[ -z "$port" ]]; then case "$scheme" in http) port="80" ;; @@ -184,6 +195,18 @@ function parse_port_from_url() { # unrelated outbound connection to the same port number is never mistaken for # a local listener. A tool that is missing, fails, or yields no recognisable # listener row reports "unknown" rather than "free". +# +# TODO(check_port): port-only matching ignores the listener's address, so a +# listener bound to one local address (127.0.0.1:8080) reports the port busy +# even when the server would bind a different one (192.168.1.5:8080). This is +# deliberate - it is what `lsof -i :PORT` did, and it fails safe - but it can +# refuse a bind that would have succeeded. If that is reported in practice, +# revisit by comparing the local-address column instead of only its port. +# +# TODO(check_port): a host with genuinely zero LISTEN sockets is indistinguish- +# able from a restricted or unparseable table, so both report "unknown" and +# warn on every start. Distinguishing them needs a positive signal that the +# table was readable (e.g. an exit status ss/netstat do not currently give). function port_listen_state() { local port="$1" local out @@ -229,6 +252,11 @@ function port_listen_state() { fi fi + # TODO(check_port): with neither ss nor netstat present (some minimal + # container images ship neither) there is no probe left, so the preflight + # is permanently "unknown" and never detects a busy port. A dependency- + # free fallback would need a bounded connect, which was deliberately + # removed here; adding one back means re-solving the hang this PR fixes. echo "unknown" } @@ -313,6 +341,9 @@ function wait_for_startup() { # Bound each probe by the time left in the overall deadline: without # --max-time a single blackholed request blocks past ${timeout_s}s. + # TODO(wait_for_startup): overshoot is now bounded but not zero - the + # loop still sleeps 2s after a probe and only then re-reads the clock, + # so the total can exceed ${timeout_s}s by roughly one sleep interval. local remain_s=$((stop_s - now_s)) [ "$remain_s" -lt 1 ] && remain_s=1 local connect_s=$((remain_s < 5 ? remain_s : 5)) diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/test-check-port.sh b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-check-port.sh index 45102ece24..21bfbeb37c 100755 --- a/hugegraph-server/hugegraph-dist/src/assembly/travis/test-check-port.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-check-port.sh @@ -21,6 +21,12 @@ # The preflight is best effort: the server's own bind is authoritative. These # tests pin the three-state contract (busy / free / unknown) rather than the # internals of any one probe. +# +# TODO(test-check-port): the Linux and BSD detection branches are both driven +# by mocked tool output, so on any one runner only the host's own branch is +# ever exercised against a real kernel. The single real-listener case covers +# whichever OS the job runs on. Closing this needs the suite to run on both +# a Linux and a macOS runner, which CI already does for the server job. set -u @@ -75,7 +81,10 @@ url_cases=( 'http://127.0.0.1:8080/path:9090|8080' 'http://127.0.0.1:8080?probe=x|8080' 'http://127.0.0.1:8080#frag|8080' + 'http://user:pass@127.0.0.1:8080|8080' + 'http://user@127.0.0.1:8080|8080' 'http://[::1]:8080|8080' + 'http://user:pass@[::1]:8080|8080' '[::1]:8080|8080' 'https://[::1]|443' 'http://127.0.0.1:08080|8080' From cd9f8c261a1bdb03d2039983e7ffa765c4f97424 Mon Sep 17 00:00:00 2001 From: Himanshu Verma Date: Tue, 28 Jul 2026 06:45:00 +0530 Subject: [PATCH 05/10] fix(bin): reject oversized ports before arithmetic conversion parse_port_from_url() converted the digit string with $((10#$port)) before bounding it. Bash evaluates in 64 bits and wraps silently, so an out-of-range value could re-enter 1..65535 and be accepted as a real port: 18446744073709551617 parsed as 1, and 18446744073709559616 as 8000. The preflight would then abort startup naming a port that was never configured. Normalise the leading-zero form textually instead, reject anything longer than five digits, and only then convert and range-check. Java reads 08080 as 8080, so that form is still accepted. Covers both wrapping values, an all-zero port, and a long leading-zero form in the URL-parsing table. --- .../hugegraph-dist/src/assembly/static/bin/util.sh | 9 +++++++-- .../src/assembly/travis/test-check-port.sh | 6 ++++++ 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/util.sh b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/util.sh index 6f9e62d0f7..4a3c7f37b5 100755 --- a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/util.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/util.sh @@ -176,8 +176,13 @@ function parse_port_from_url() { fi [[ "$port" =~ ^[0-9]+$ ]] || return 1 - # Normalise leading-zero forms; Java reads 08080 as decimal 8080. - port=$((10#$port)) + # Normalise leading-zero forms textually; Java reads 08080 as decimal 8080. + # Arithmetic conversion must not happen before the value is bounded: Bash + # evaluates in 64-bit and wraps silently, so 18446744073709559616 would + # otherwise pass the range check as port 8000. + port="${port#"${port%%[!0]*}"}" + [[ -z "$port" ]] && return 1 + (( ${#port} <= 5 )) || return 1 (( port >= 1 && port <= 65535 )) || return 1 echo "$port" diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/test-check-port.sh b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-check-port.sh index 21bfbeb37c..8a57485525 100755 --- a/hugegraph-server/hugegraph-dist/src/assembly/travis/test-check-port.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-check-port.sh @@ -93,6 +93,12 @@ url_cases=( 'http://127.0.0.1:0|SKIP' 'http://127.0.0.1:70000|SKIP' '127.0.0.1|SKIP' + # Oversized values must be rejected on their digits, not after a 64-bit + # arithmetic conversion that would wrap them back into range. + 'http://127.0.0.1:18446744073709551617|SKIP' + 'http://127.0.0.1:18446744073709559616|SKIP' + 'http://127.0.0.1:0000000000000008080|8080' + 'http://127.0.0.1:00000|SKIP' ) for case in "${url_cases[@]}"; do url="${case%|*}" From e993c4b7bcf22d8d069bbbd18ffb189c86af383c Mon Sep 17 00:00:00 2001 From: Himanshu Verma Date: Tue, 28 Jul 2026 14:20:33 +0530 Subject: [PATCH 06/10] fix(bin): give the shipped images a working port preflight The server images install lsof, which check_port no longer calls, and the eclipse-temurin:11-jre-jammy base ships neither ss nor netstat. The preflight was therefore permanently inconclusive in every official image: a duplicate start was no longer refused, so it could overwrite bin/pid before the second JVM failed to bind, leaving the first process outside the stop script's reach. base image as shipped: port_listen_state 8080 -> unknown base image + iproute2: port_listen_state 8080 -> busy Replace lsof with iproute2 in both server images; no shipped script calls lsof any more. Assert the result in the image build, which already runs on any PR touching a Dockerfile. port_listen_state also ended its search on the first probe, so an ss response that ran but could not be parsed returned "unknown" even with a working netstat behind it. Capture the parser result, answer only on busy or free, and fall through otherwise. `ss -H` exiting zero with no output is the one case where empty is an answer rather than a failure - -H means there is no header to print - so read it as "no listeners" instead of warning on every clean start. wait_for_startup bounded each curl but not the loop: it slept a flat 2s and only then re-read the clock, so a 1s timeout ran for 2s. Recompute the remainder before pausing, cap the pause to it, and stop once it is spent. Drop the /dev/tcp wording from the server CI comments; no such branch exists. Suite is 53 passed, 0 failed on macOS bash 3.2 and on Linux bash 5.2 against a real ss and a real listener. Each fix is mutation-checked: restoring the early return fails only the two fallback cases, restoring unknown-on-empty fails one, restoring the flat sleep fails one. --- .github/workflows/docker-build-ci.yml | 19 +++++ .github/workflows/server-ci.yml | 7 +- hugegraph-server/Dockerfile | 6 +- hugegraph-server/Dockerfile-hstore | 6 +- .../src/assembly/static/bin/util.sh | 72 +++++++++++++------ .../src/assembly/travis/test-check-port.sh | 55 ++++++++++++-- 6 files changed, 134 insertions(+), 31 deletions(-) diff --git a/.github/workflows/docker-build-ci.yml b/.github/workflows/docker-build-ci.yml index 8d31ff266f..2247471a69 100644 --- a/.github/workflows/docker-build-ci.yml +++ b/.github/workflows/docker-build-ci.yml @@ -47,6 +47,25 @@ jobs: run: | IMAGE_ID=$(docker build -q -f ${{ matrix.dockerfile }} .) echo "Built: $IMAGE_ID" + echo "IMAGE_ID=$IMAGE_ID" >> "$GITHUB_ENV" HC=$(docker inspect --format='{{json .Config.Healthcheck}}' "$IMAGE_ID") echo "Healthcheck: $HC" [[ "$HC" != "null" ]] || { echo "ERROR: HEALTHCHECK missing in ${{ matrix.dockerfile }}"; exit 1; } + + # The startup preflight needs a socket-table tool, and the base image + # ships none of its own. Without one every start reports "unknown" and + # a duplicate start is no longer refused, so assert the image can + # actually answer. Only the server images run check_port. + # TODO(docker-ci): this pins the probe dependency, not the behaviour it + # protects. A full duplicate-start/stop check needs a booted server with + # a backend, which belongs with the e2e job rather than the image build. + - name: Port preflight can answer inside ${{ matrix.dockerfile }} + if: ${{ startsWith(matrix.dockerfile, 'hugegraph-server/') }} + run: | + STATE=$(docker run --rm "$IMAGE_ID" bash -c \ + 'source /hugegraph-server/bin/util.sh && port_listen_state 8080') + echo "port_listen_state 8080 -> $STATE" + [[ "$STATE" != "unknown" ]] || { + echo "ERROR: no usable socket-table tool (ss/netstat) in ${{ matrix.dockerfile }}" + exit 1 + } diff --git a/.github/workflows/server-ci.yml b/.github/workflows/server-ci.yml index c9d895be7e..08ddfdf44f 100644 --- a/.github/workflows/server-ci.yml +++ b/.github/workflows/server-ci.yml @@ -82,14 +82,15 @@ jobs: - name: Run check_port unit tests if: ${{ env.BACKEND == 'rocksdb' }} run: | - # Validates ss/netstat//dev/tcp replacement for lsof + # Validates the ss/netstat port preflight that replaced lsof $TRAVIS_DIR/test-check-port.sh hugegraph-server/hugegraph-dist/src/assembly/static - name: Check startup test prerequisites id: server-preflight if: ${{ env.BACKEND == 'rocksdb' }} run: | - # Note: lsof removed from prerequisites (replaced with ss/netstat//dev/tcp in check_port). + # Note: lsof removed from prerequisites; check_port now uses ss, falling back + # to netstat, and warns without blocking when neither can answer. # This job always runs on Linux (ubuntu-22.04), which uses fuser for port cleanup. for tool in crontab curl java; do if ! command -v "$tool" >/dev/null 2>&1; then @@ -201,7 +202,7 @@ jobs: mvn clean compile -pl hugegraph-server/hugegraph-test -am -U -Dmaven.javadoc.skip=true -ntp - name: Run check_port unit tests - # Validates ss/netstat//dev/tcp replacement for lsof + # Validates the ss/netstat port preflight that replaced lsof run: | $TRAVIS_DIR/test-check-port.sh hugegraph-server/hugegraph-dist/src/assembly/static diff --git a/hugegraph-server/Dockerfile b/hugegraph-server/Dockerfile index 2ca946e6ba..5caadd23cb 100644 --- a/hugegraph-server/Dockerfile +++ b/hugegraph-server/Dockerfile @@ -46,12 +46,16 @@ ENV JAVA_OPTS="-XX:+UnlockExperimentalVMOptions -XX:+UseContainerSupport -XX:Max WORKDIR /hugegraph-server/ # 1. Install runtime dependencies and configure server +# Note: iproute2 provides `ss`, which the bin/util.sh port preflight needs. The +# jammy base image ships neither ss nor netstat, so without it the preflight is +# permanently inconclusive and a duplicate start is no longer caught. It +# replaces lsof, which no shipped script calls any more. RUN apt-get -q update \ && apt-get -q install -y --no-install-recommends --no-install-suggests \ dumb-init \ procps \ curl \ - lsof \ + iproute2 \ vim \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* \ diff --git a/hugegraph-server/Dockerfile-hstore b/hugegraph-server/Dockerfile-hstore index 06d2d4b88c..7cd64e8f3b 100644 --- a/hugegraph-server/Dockerfile-hstore +++ b/hugegraph-server/Dockerfile-hstore @@ -48,12 +48,16 @@ ENV JAVA_OPTS="-XX:+UnlockExperimentalVMOptions -XX:+UseContainerSupport -XX:Max WORKDIR /hugegraph-server/ # 1. Install runtime dependencies and configure server +# Note: iproute2 provides `ss`, which the bin/util.sh port preflight needs. The +# jammy base image ships neither ss nor netstat, so without it the preflight is +# permanently inconclusive and a duplicate start is no longer caught. It +# replaces lsof, which no shipped script calls any more. RUN apt-get -q update \ && apt-get -q install -y --no-install-recommends --no-install-suggests \ dumb-init \ procps \ curl \ - lsof \ + iproute2 \ vim \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* \ diff --git a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/util.sh b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/util.sh index 4a3c7f37b5..5b08369d4d 100755 --- a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/util.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/util.sh @@ -199,7 +199,9 @@ function parse_port_from_url() { # Only LISTEN rows and only the local-address column are inspected, so an # unrelated outbound connection to the same port number is never mistaken for # a local listener. A tool that is missing, fails, or yields no recognisable -# listener row reports "unknown" rather than "free". +# listener row reports "unknown" rather than "free", and the next probe is +# tried; the sole exception is `ss -H` succeeding with no output at all, which +# says positively that the host has no TCP listeners. # # TODO(check_port): port-only matching ignores the listener's address, so a # listener bound to one local address (127.0.0.1:8080) reports the port busy @@ -208,13 +210,14 @@ function parse_port_from_url() { # refuse a bind that would have succeeded. If that is reported in practice, # revisit by comparing the local-address column instead of only its port. # -# TODO(check_port): a host with genuinely zero LISTEN sockets is indistinguish- -# able from a restricted or unparseable table, so both report "unknown" and -# warn on every start. Distinguishing them needs a positive signal that the -# table was readable (e.g. an exit status ss/netstat do not currently give). +# TODO(check_port): only the `ss` branch can tell "no listeners at all" from +# "table unreadable", because -H removes the header and leaves a zero exit with +# empty output as a positive signal. Both netstat branches print headers that +# the parser drops, so an empty result there stays "unknown" and warns. function port_listen_state() { local port="$1" local out + local state local os os=$(uname) @@ -238,30 +241,45 @@ function port_listen_state() { else print "unknown" }' + # Each probe answers only when it produced a usable listener table; an + # inconclusive one falls through to the next rather than ending the search. if [[ "$os" == "Darwin" || "$os" == *BSD* ]]; then if command_available "netstat" && out=$(netstat -an -p tcp 2>/dev/null) \ && [[ -n "$out" ]]; then - echo "$out" | awk -v port="$port" -v sep="." -v want_listen=1 "$parser" - return 0 + state=$(echo "$out" | awk -v port="$port" -v sep="." -v want_listen=1 "$parser") + case "$state" in + busy|free) echo "$state"; return 0 ;; + esac fi else - # `ss -H -ltn` already restricts output to listening sockets. - if command_available "ss" && out=$(ss -H -ltn 2>/dev/null) && [[ -n "$out" ]]; then - echo "$out" | awk -v port="$port" -v sep=":" -v want_listen=0 "$parser" - return 0 + # `ss -H -ltn` already restricts output to listening sockets, and -H + # drops the header, so a zero exit with no output means "nothing is + # listening" - the one case where empty is an answer, not a failure. + if command_available "ss" && out=$(ss -H -ltn 2>/dev/null); then + if [[ -z "$out" ]]; then + echo "free" + return 0 + fi + state=$(echo "$out" | awk -v port="$port" -v sep=":" -v want_listen=0 "$parser") + case "$state" in + busy|free) echo "$state"; return 0 ;; + esac fi if command_available "netstat" && out=$(netstat -ltn 2>/dev/null) \ && [[ -n "$out" ]]; then - echo "$out" | awk -v port="$port" -v sep=":" -v want_listen=1 "$parser" - return 0 + state=$(echo "$out" | awk -v port="$port" -v sep=":" -v want_listen=1 "$parser") + case "$state" in + busy|free) echo "$state"; return 0 ;; + esac fi fi - # TODO(check_port): with neither ss nor netstat present (some minimal - # container images ship neither) there is no probe left, so the preflight - # is permanently "unknown" and never detects a busy port. A dependency- - # free fallback would need a bounded connect, which was deliberately - # removed here; adding one back means re-solving the hang this PR fixes. + # TODO(check_port): with neither ss nor netstat present there is no probe + # left, so the preflight is permanently "unknown" and never detects a busy + # port. The published server images carry iproute2 for this reason, but a + # hand-built minimal image can still ship neither. A dependency-free + # fallback would need a bounded connect, which was deliberately removed + # here; adding one back means re-solving the hang this PR fixes. echo "unknown" } @@ -346,9 +364,6 @@ function wait_for_startup() { # Bound each probe by the time left in the overall deadline: without # --max-time a single blackholed request blocks past ${timeout_s}s. - # TODO(wait_for_startup): overshoot is now bounded but not zero - the - # loop still sleeps 2s after a probe and only then re-reads the clock, - # so the total can exceed ${timeout_s}s by roughly one sleep interval. local remain_s=$((stop_s - now_s)) [ "$remain_s" -lt 1 ] && remain_s=1 local connect_s=$((remain_s < 5 ? remain_s : 5)) @@ -362,7 +377,16 @@ function wait_for_startup() { fi return 0 fi - sleep 2 + + # Pause for the retry interval, but never past the deadline: sleeping a + # fixed 2s and only then re-reading the clock made a 1s timeout take 2s. + # Nothing left to wait for means the deadline is spent, so stop here + # rather than spin until the clock ticks past it. + now_s=$(date '+%s') + local sleep_s=$((stop_s - now_s)) + [ "$sleep_s" -le 0 ] && break + [ "$sleep_s" -gt 2 ] && sleep_s=2 + sleep "$sleep_s" now_s=$(date '+%s') done @@ -555,6 +579,10 @@ function ensure_package_exist() { ########################################################################### +# TODO(wait_for_shutdown): this loop has the same fixed-2s-then-check-the-clock +# shape wait_for_startup had, so it can also overrun its timeout by roughly one +# sleep interval. Left alone here to keep this PR to the port preflight; the +# fix is the same remaining-deadline cap used above. function wait_for_shutdown() { local process_name="$1" local pid="$2" diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/test-check-port.sh b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-check-port.sh index 8a57485525..07aa071d84 100755 --- a/hugegraph-server/hugegraph-dist/src/assembly/travis/test-check-port.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-check-port.sh @@ -109,7 +109,7 @@ done # -------------------------------------------------------------------------- echo "" -echo "2. Linux: ss busy / free / failure" +echo "2. Linux: ss busy / free / failure, and the netstat fallback" ( uname() { echo "Linux"; } command_available() { [[ "$1" == "ss" ]]; } @@ -135,14 +135,34 @@ echo "2. Linux: ss busy / free / failure" SS_RC=1 expect "ss failure is unknown" "unknown" "$(port_listen_state 8080)" - # Success with an unusable table proves nothing either. + # `-H` prints no header, so a zero exit with no output is a positive "this + # host has no TCP listeners" - the state a fresh container starts in. SS_RC=0 SS_OUT='' - expect "ss empty output is unknown" "unknown" "$(port_listen_state 8080)" + expect "ss empty output is free" "free" "$(port_listen_state 8080)" + # Success with an unusable table proves nothing. SS_OUT='some diagnostic banner' expect "ss unparseable output is unknown" "unknown" "$(port_listen_state 8080)" + # ...and it must not end the search either: netstat may still have a + # readable table. Returning on the first probe left the port "unknown" + # even when the fallback could have answered. + command_available() { [[ "$1" == "ss" || "$1" == "netstat" ]]; } + netstat() { + echo 'Active Internet connections (only servers)' + echo 'Proto Recv-Q Send-Q Local Address Foreign Address State' + echo 'tcp 0 0 0.0.0.0:8080 0.0.0.0:* LISTEN' + } + + SS_OUT='some diagnostic banner' + expect "unparseable ss falls back to netstat" "busy" "$(port_listen_state 8080)" + expect "netstat fallback can also report free" "free" "$(port_listen_state 9999)" + + SS_RC=1 + SS_OUT='' + expect "failed ss falls back to netstat" "busy" "$(port_listen_state 8080)" + ) # -------------------------------------------------------------------------- @@ -301,11 +321,38 @@ echo "7. Startup probe is bounded" trap 'cd /; rm -rf "$WORK"' EXIT ARGS_FILE="$WORK/curl-args" + SLEEP_FILE="$WORK/sleep-args" + : > "$SLEEP_FILE" curl() { echo "$*" >> "$ARGS_FILE"; echo "000"; return 28; } + # Record what the retry pause asks for, but still take it, so the + # end-to-end duration below stays a real measurement. + sleep() { echo "$1" >> "$SLEEP_FILE"; command sleep "$1"; } process_status() { return 0; } ps() { return 0; } - wait_for_startup "$$" "test-server" "http://127.0.0.1:1" 1 >/dev/null 2>&1 + TIMEOUT_S=1 + START_S=$(date '+%s') + wait_for_startup "$$" "test-server" "http://127.0.0.1:1" "$TIMEOUT_S" >/dev/null 2>&1 + ELAPSED_S=$(( $(date '+%s') - START_S )) + + # The deterministic half: bounding each curl does not bound the loop, which + # used to sleep a flat 2s and only then re-read the clock. A 1s timeout + # must never ask for more than 1s of sleeping. + TOTAL_SLEEP_S=$(awk '{ total += $1 } END { print total + 0 }' "$SLEEP_FILE") + if [[ "$TOTAL_SLEEP_S" -le "$TIMEOUT_S" ]]; then + pass "retry sleeps stay inside the ${TIMEOUT_S}s deadline" + else + fail "retry sleeps stay inside the ${TIMEOUT_S}s deadline" \ + "slept ${TOTAL_SLEEP_S}s in total: $(tr '\n' ' ' < "$SLEEP_FILE")" + fi + + # The end-to-end half, with a second of slack for whole-second clock reads. + if [[ "$ELAPSED_S" -le $((TIMEOUT_S + 1)) ]]; then + pass "wait_for_startup returns within the deadline (${ELAPSED_S}s)" + else + fail "wait_for_startup returns within the deadline" \ + "took ${ELAPSED_S}s for a ${TIMEOUT_S}s timeout" + fi if grep -q -- "--max-time" "$ARGS_FILE" 2>/dev/null; then pass "startup probe passes --max-time" From 5264ecfe18a29bc6acbaa98215628cae685ed63b Mon Sep 17 00:00:00 2001 From: Himanshu Verma Date: Tue, 28 Jul 2026 15:07:34 +0530 Subject: [PATCH 07/10] fix(bin): revert PD/Store helper contracts and drop their dead lsof dep Self-review pass over the port-preflight change. `process_num` and `process_id` in the PD and Store `util.sh` still carried a changed contract - a boolean return and an echoed pid - while the server copy had already been reverted. Neither has a caller in PD or Store, but the review asked for unrelated helper refactors to be reverted or split, so all three copies now match master's contract again. The PD and Store images installed `lsof` solely for the dead `check_port` this change removes; nothing in `hg-pd-dist` or `hg-store-dist` calls it any more. Unlike the server images they run no preflight, so nothing replaces it. Also in this pass: - `test-check-port.sh`: the tally file is now covered by a trap for the whole run, section 5 registers its trap before forking the listener rather than after, and it restores that handler instead of clearing it globally. Verified against the previous head: a SIGTERM mid-run leaked one temp file, now none. - `test-check-port.sh`: `sleep 0.5` -> `sleep 1`. Fractional sleep is not POSIX, and the suite runs `set -u` without `-e`, so a busybox sleep would fail silently and burn all ten readiness iterations at once. - `docker-build-ci.yml`: assert `free` or `busy` rather than "not unknown", so an empty result cannot satisfy the check. - `port_listen_state`: record that the port must already be normalised, since it is matched as text. - Correct the port-cleanup comments in the server and PD startup tests, which credited fuser on the branch that uses lsof for macOS. Suite is 53 passed, 0 failed on macOS bash 3.2 and on Linux bash 5.2 against a real ss; the five implementation mutations all still fail it. --- .github/workflows/docker-build-ci.yml | 5 ++++- hugegraph-pd/Dockerfile | 4 +++- .../src/assembly/static/bin/util.sh | 10 ++-------- .../src/assembly/static/bin/util.sh | 4 ++++ .../src/assembly/travis/test-check-port.sh | 19 ++++++++++++++++--- .../travis/test-start-hugegraph-pd.sh | 3 ++- .../assembly/travis/test-start-hugegraph.sh | 3 ++- hugegraph-store/Dockerfile | 4 +++- .../src/assembly/static/bin/util.sh | 10 ++-------- 9 files changed, 38 insertions(+), 24 deletions(-) diff --git a/.github/workflows/docker-build-ci.yml b/.github/workflows/docker-build-ci.yml index 2247471a69..19667147db 100644 --- a/.github/workflows/docker-build-ci.yml +++ b/.github/workflows/docker-build-ci.yml @@ -65,7 +65,10 @@ jobs: STATE=$(docker run --rm "$IMAGE_ID" bash -c \ 'source /hugegraph-server/bin/util.sh && port_listen_state 8080') echo "port_listen_state 8080 -> $STATE" - [[ "$STATE" != "unknown" ]] || { + # Assert a positive answer rather than "not unknown": an empty $STATE + # would otherwise satisfy the check. The step's default `bash -e` + # already aborts on a failed `docker run`, so this only states intent. + [[ "$STATE" == "free" || "$STATE" == "busy" ]] || { echo "ERROR: no usable socket-table tool (ss/netstat) in ${{ matrix.dockerfile }}" exit 1 } diff --git a/hugegraph-pd/Dockerfile b/hugegraph-pd/Dockerfile index d55ac180ae..68e6e2555b 100644 --- a/hugegraph-pd/Dockerfile +++ b/hugegraph-pd/Dockerfile @@ -45,12 +45,14 @@ ENV JAVA_OPTS="-XX:+UnlockExperimentalVMOptions -XX:+UseContainerSupport -XX:Max WORKDIR /hugegraph-pd/ # 1. Install runtime dependencies +# Note: lsof is gone because its only consumer was the dead check_port removed +# from hg-pd-dist bin/util.sh. PD runs no port preflight, so unlike the server +# images this needs no socket-table tool in its place. RUN apt-get -q update \ && apt-get -q install -y --no-install-recommends --no-install-suggests \ dumb-init \ procps \ curl \ - lsof \ vim \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* diff --git a/hugegraph-pd/hg-pd-dist/src/assembly/static/bin/util.sh b/hugegraph-pd/hg-pd-dist/src/assembly/static/bin/util.sh index 4f4a90b45f..b476d7935b 100644 --- a/hugegraph-pd/hg-pd-dist/src/assembly/static/bin/util.sh +++ b/hugegraph-pd/hg-pd-dist/src/assembly/static/bin/util.sh @@ -73,19 +73,13 @@ function parse_yaml() { } function process_num() { - local num num=$(ps -ef | grep "$1" | grep -v grep | wc -l) - if (( num > 0 )); then - return 1 - fi - return 0 + return "$num" } function process_id() { - local pid pid=$(ps -ef | grep "$1" | grep -v grep | awk '{print $2}') - echo "$pid" - return 0 + return "$pid" } function crontab_append() { diff --git a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/util.sh b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/util.sh index 5b08369d4d..ea841b0d14 100755 --- a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/util.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/util.sh @@ -190,6 +190,10 @@ function parse_port_from_url() { # Echo "busy", "free" or "unknown" for the given TCP port. # +# The port is compared as text against the listener table, so the argument must +# already be normalised - parse_port_from_url() strips leading zeroes for this +# reason, and "08080" passed directly here would not match a listener on 8080. +# # Detection is deliberately port-only, matching the conservative behaviour of # the `lsof -i :PORT` call this replaces. Reproducing kernel socket semantics # in Bash - dual-stack IPV6_V6ONLY, wildcard versus specific binds, address diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/test-check-port.sh b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-check-port.sh index 07aa071d84..bc6783fad0 100755 --- a/hugegraph-server/hugegraph-dist/src/assembly/travis/test-check-port.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-check-port.sh @@ -44,6 +44,10 @@ source "$UTIL_SH" # Sections run in subshells so their command overrides stay isolated, which # means results have to be tallied through a file rather than a variable. RESULTS=$(mktemp) +# Clean the tally on any exit, not only the normal one. Section 5 runs outside +# a subshell and has to extend this trap, so it restores this handler rather +# than clearing it. +trap 'rm -f "$RESULTS"' EXIT pass() { echo " PASS $1" @@ -229,6 +233,10 @@ echo "" echo "5. Real listener on an ephemeral port" if command -v python3 >/dev/null 2>&1; then PORT_FILE=$(mktemp) + # Cover the temp file from the moment it exists, then widen the trap to the + # child as soon as there is a pid; registering only after the fork leaves a + # window where an interrupt orphans the listener for its full 30s sleep. + trap 'rm -f "$PORT_FILE" "$RESULTS"' EXIT python3 -c ' import socket, sys, time s = socket.socket() @@ -239,13 +247,16 @@ sys.stdout.flush() time.sleep(30) ' > "$PORT_FILE" & PY_PID=$! - trap 'kill "$PY_PID" 2>/dev/null; rm -f "$PORT_FILE"' EXIT + trap 'kill "$PY_PID" 2>/dev/null; rm -f "$PORT_FILE" "$RESULTS"' EXIT BOUND="" for _ in 1 2 3 4 5 6 7 8 9 10; do BOUND=$(head -1 "$PORT_FILE" 2>/dev/null) [[ -n "$BOUND" ]] && break - sleep 0.5 + # Whole seconds only: fractional sleep is not POSIX, and this suite runs + # with `set -u` but no `-e`, so a busybox sleep would fail silently and + # spin all ten iterations instantly. + sleep 1 done if [[ -z "$BOUND" ]] || ! kill -0 "$PY_PID" 2>/dev/null; then @@ -269,7 +280,9 @@ print(p)') kill "$PY_PID" 2>/dev/null wait "$PY_PID" 2>/dev/null rm -f "$PORT_FILE" - trap - EXIT + # Restore the script-level handler rather than clearing it, so the tally + # file stays covered for the remaining sections. + trap 'rm -f "$RESULTS"' EXIT else echo " SKIP real listener test: python3 not available" fi diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/test-start-hugegraph-pd.sh b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-start-hugegraph-pd.sh index c6926c6dfd..ab73255b8c 100755 --- a/hugegraph-server/hugegraph-dist/src/assembly/travis/test-start-hugegraph-pd.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-start-hugegraph-pd.sh @@ -74,7 +74,8 @@ cleanup() { fi rm -f "$PID_FILE" rm -rf "$PD_ROOT/logs/" - # kill anything still holding the PD port (fuser avoids lsof dependency) + # kill anything still holding the PD port: fuser on Linux, lsof on macOS, + # which ships no fuser. Only the Linux path had to lose its lsof dependency. if [[ "$(uname)" == "Darwin" ]]; then local port pids pid for port in 8620 8686; do diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/test-start-hugegraph.sh b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-start-hugegraph.sh index c8e5bfced5..9f0bcfaa63 100755 --- a/hugegraph-server/hugegraph-dist/src/assembly/travis/test-start-hugegraph.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-start-hugegraph.sh @@ -76,7 +76,8 @@ cleanup() { kill "$(cat "$PID_FILE")" 2>/dev/null || true fi rm -f "$PID_FILE" - # kill anything still holding server ports (fuser avoids lsof dependency) + # kill anything still holding server ports: fuser on Linux, lsof on macOS, + # which ships no fuser. Only the Linux path had to lose its lsof dependency. if [[ "$(uname)" == "Darwin" ]]; then local port pids pid for port in 8080 8182 8088; do diff --git a/hugegraph-store/Dockerfile b/hugegraph-store/Dockerfile index 49d41f6460..f1b9db5fd4 100644 --- a/hugegraph-store/Dockerfile +++ b/hugegraph-store/Dockerfile @@ -45,12 +45,14 @@ ENV JAVA_OPTS="-XX:+UnlockExperimentalVMOptions -XX:+UseContainerSupport -XX:Max WORKDIR /hugegraph-store/ # 1. Install runtime dependencies +# Note: lsof is gone because its only consumer was the dead check_port removed +# from hg-store-dist bin/util.sh. Store runs no port preflight, so unlike the +# server images this needs no socket-table tool in its place. RUN apt-get -q update \ && apt-get -q install -y --no-install-recommends --no-install-suggests \ dumb-init \ procps \ curl \ - lsof \ vim \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* diff --git a/hugegraph-store/hg-store-dist/src/assembly/static/bin/util.sh b/hugegraph-store/hg-store-dist/src/assembly/static/bin/util.sh index b4011e2e47..735b75a31b 100644 --- a/hugegraph-store/hg-store-dist/src/assembly/static/bin/util.sh +++ b/hugegraph-store/hg-store-dist/src/assembly/static/bin/util.sh @@ -72,19 +72,13 @@ function parse_yaml() { } function process_num() { - local num num=$(ps -ef | grep "$1" | grep -v grep | wc -l) - if (( num > 0 )); then - return 1 - fi - return 0 + return "$num" } function process_id() { - local pid pid=$(ps -ef | grep "$1" | grep -v grep | awk '{print $2}') - echo "$pid" - return 0 + return "$pid" } function crontab_append() { From 1709d907727be430160332dd5abc68cfb467957b Mon Sep 17 00:00:00 2001 From: Himanshu Verma Date: Wed, 29 Jul 2026 19:48:31 +0530 Subject: [PATCH 08/10] fix(bin): close remaining port preflight review gaps Honor the startup deadline at the probe boundary, preserve verified Store downloads across concurrent replacement failures, and make the focused CI wiring fail closed. Add deterministic regressions for both race boundaries and trigger the image probe when server util.sh changes. --- .github/workflows/docker-build-ci.yml | 1 + .github/workflows/server-ci.yml | 8 +- .../src/assembly/static/bin/util.sh | 14 +- .../src/assembly/travis/test-check-port.sh | 173 +++++++++++++++++- .../src/assembly/static/bin/util.sh | 4 +- 5 files changed, 191 insertions(+), 9 deletions(-) diff --git a/.github/workflows/docker-build-ci.yml b/.github/workflows/docker-build-ci.yml index 19667147db..eb1f0d9853 100644 --- a/.github/workflows/docker-build-ci.yml +++ b/.github/workflows/docker-build-ci.yml @@ -26,6 +26,7 @@ on: paths: - '**/Dockerfile*' - '.dockerignore' + - 'hugegraph-server/hugegraph-dist/src/assembly/static/bin/util.sh' jobs: docker-build: diff --git a/.github/workflows/server-ci.yml b/.github/workflows/server-ci.yml index 08ddfdf44f..7b9802174a 100644 --- a/.github/workflows/server-ci.yml +++ b/.github/workflows/server-ci.yml @@ -83,7 +83,9 @@ jobs: if: ${{ env.BACKEND == 'rocksdb' }} run: | # Validates the ss/netstat port preflight that replaced lsof - $TRAVIS_DIR/test-check-port.sh hugegraph-server/hugegraph-dist/src/assembly/static + $TRAVIS_DIR/test-check-port.sh \ + hugegraph-server/hugegraph-dist/src/assembly/static \ + hugegraph-store/hg-store-dist/src/assembly/static/bin/util.sh - name: Check startup test prerequisites id: server-preflight @@ -204,7 +206,9 @@ jobs: - name: Run check_port unit tests # Validates the ss/netstat port preflight that replaced lsof run: | - $TRAVIS_DIR/test-check-port.sh hugegraph-server/hugegraph-dist/src/assembly/static + $TRAVIS_DIR/test-check-port.sh \ + hugegraph-server/hugegraph-dist/src/assembly/static \ + hugegraph-store/hg-store-dist/src/assembly/static/bin/util.sh - name: Run RocksDB core test run: | diff --git a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/util.sh b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/util.sh index ea841b0d14..8ae46e88a4 100755 --- a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/util.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/util.sh @@ -355,7 +355,7 @@ function wait_for_startup() { local error_file_name="startup_error.txt" echo -n "Connecting to $server_name ($server_url)" - while [ "$now_s" -le "$stop_s" ]; do + while [ "$now_s" -lt "$stop_s" ]; do echo -n . process_status "$server_name" "$pid" >/dev/null if [ $? -eq 1 ]; then @@ -368,8 +368,12 @@ function wait_for_startup() { # Bound each probe by the time left in the overall deadline: without # --max-time a single blackholed request blocks past ${timeout_s}s. + # Refresh the clock after the process check and immediately before the + # probe. A stale loop-entry time must not grant curl a new one-second + # budget when the overall deadline has already been reached. + now_s=$(date '+%s') local remain_s=$((stop_s - now_s)) - [ "$remain_s" -lt 1 ] && remain_s=1 + [ "$remain_s" -le 0 ] && break local connect_s=$((remain_s < 5 ? remain_s : 5)) status=$(curl -I -sS -k --connect-timeout "$connect_s" --max-time "$remain_s" \ -w "%{http_code}" -o /dev/null "$server_url" 2> "$error_file_name") @@ -395,8 +399,10 @@ function wait_for_startup() { done echo "" - cat "$error_file_name" - rm "$error_file_name" + if [ -e "$error_file_name" ]; then + cat "$error_file_name" + rm "$error_file_name" + fi echo "The operation timed out(${timeout_s}s) when attempting to connect to $server_url" >&2 return 1 } diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/test-check-port.sh b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-check-port.sh index bc6783fad0..84521d07bf 100755 --- a/hugegraph-server/hugegraph-dist/src/assembly/travis/test-check-port.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-check-port.sh @@ -32,12 +32,25 @@ set -u STATIC_DIR="${1:-hugegraph-server/hugegraph-dist/src/assembly/static}" UTIL_SH="$STATIC_DIR/bin/util.sh" +STORE_UTIL_SH="${2:-}" if [[ ! -f "$UTIL_SH" ]]; then - echo "SKIP: util.sh not found at $UTIL_SH" + if [[ $# -gt 0 ]]; then + echo "ERROR: required util.sh not found at $UTIL_SH" >&2 + exit 1 + fi + # With no argument this script may be discovered outside the repository by + # an optional test runner. CI always supplies STATIC_DIR explicitly, so a + # bad workflow path takes the required failure branch above. + echo "SKIP: default util.sh not found at $UTIL_SH" exit 0 fi +if [[ -n "$STORE_UTIL_SH" && ! -f "$STORE_UTIL_SH" ]]; then + echo "ERROR: required Store util.sh not found at $STORE_UTIL_SH" >&2 + exit 1 +fi + # shellcheck source=/dev/null source "$UTIL_SH" @@ -380,6 +393,164 @@ echo "7. Startup probe is bounded" ) +# -------------------------------------------------------------------------- +echo "" +echo "8. Deadline boundary starts no probe" +( + WORK=$(mktemp -d) + cd "$WORK" || exit 1 + trap 'cd /; rm -rf "$WORK"' EXIT + + CLOCK_FILE="$WORK/clock-calls" + CURL_FILE="$WORK/curl-calls" + echo 0 > "$CLOCK_FILE" + : > "$CURL_FILE" + + # The first read establishes [100, 101) as the allowed interval. The + # process check consumes that interval, so the mandatory pre-curl refresh + # observes the deadline exactly. File-backed state works through the + # command substitutions used by wait_for_startup, including Bash 3.2. + date() { + local calls + calls=$(cat "$CLOCK_FILE") + calls=$((calls + 1)) + echo "$calls" > "$CLOCK_FILE" + if [[ "$calls" -eq 1 ]]; then + echo 100 + else + echo 101 + fi + } + process_status() { return 0; } + curl() { echo called >> "$CURL_FILE"; echo "000"; return 28; } + sleep() { echo called >> "$WORK/sleep-calls"; } + + wait_for_startup "$$" "boundary-server" "http://127.0.0.1:1" 1 \ + >/dev/null 2>&1 + + CLOCK_CALLS=$(cat "$CLOCK_FILE") + if [[ "$CLOCK_CALLS" -ge 2 ]]; then + pass "startup refreshes the clock at the probe boundary" + else + fail "startup refreshes the clock at the probe boundary" \ + "clock was read only ${CLOCK_CALLS} time(s)" + fi + + PROBE_CALLS=$(wc -l < "$CURL_FILE" | tr -d ' ') + expect "no startup probe begins at the deadline" "0" "$PROBE_CALLS" + +) + +if [[ -n "$STORE_UTIL_SH" ]]; then + # ---------------------------------------------------------------------- + echo "" + echo "9. Store verified replacement survives a stale concurrent caller" + ( + # Keep the Store helper overrides isolated from the server helpers used + # by every other section in this suite. + # shellcheck source=/dev/null + source "$STORE_UTIL_SH" + + WORK=$(mktemp -d) + DEST="$WORK/lib.so" + HASHED="$WORK/stale-hashed" + RELEASE="$WORK/release-stale" + STALE_DOWNLOAD="$WORK/stale-download" + STALE_PID="" + trap ' + touch "$RELEASE" + if [[ -n "$STALE_PID" ]]; then + kill "$STALE_PID" 2>/dev/null + wait "$STALE_PID" 2>/dev/null + fi + rm -rf "$WORK" + ' EXIT + + echo "invalid" > "$DEST" + + wait_for_file() { + local path="$1" + local _ + for _ in 1 2 3 4 5 6 7 8 9 10; do + [[ -e "$path" ]] && return 0 + command sleep 1 + done + return 1 + } + + # Both callers first observe the invalid destination. The stale caller + # then holds that checksum until the installer has atomically published + # valid bytes. Reintroducing the old pre-download rm makes the stale + # caller delete that valid replacement before its own download fails. + md5sum() { + local target="${@: -1}" + if [[ "$target" == "$DEST" ]]; then + if [[ "$CALLER" == "stale" ]]; then + touch "$HASHED" + wait_for_file "$RELEASE" || return 1 + fi + echo "bad $target" + else + echo "good $target" + fi + } + + curl() { + [[ "$1" == "-fL" && "$2" == "-o" && "$4" == "--" ]] || return 64 + if [[ "$CALLER" == "installer" ]]; then + echo "valid" > "$3" + return 0 + fi + touch "$STALE_DOWNLOAD" + return 22 + } + + ( + CALLER="stale" + download_and_verify "https://example.com/lib.so" "$DEST" "good" \ + >/dev/null 2>&1 + ) & + STALE_PID=$! + + if ! wait_for_file "$HASHED"; then + fail "stale caller captured the original checksum" \ + "caller did not reach the checksum barrier" + exit 0 + fi + pass "stale caller captured the original checksum" + + CALLER="installer" + if download_and_verify "https://example.com/lib.so" "$DEST" "good" \ + >/dev/null 2>&1; then + pass "concurrent installer succeeds" + else + fail "concurrent installer succeeds" "installer returned nonzero" + fi + + touch "$RELEASE" + wait "$STALE_PID" 2>/dev/null + STALE_WAIT_RC=$? + STALE_PID="" + if [[ "$STALE_WAIT_RC" -ne 0 && -e "$STALE_DOWNLOAD" ]]; then + pass "stale caller reports its failed replacement download" + else + CURL_REACHED=$([[ -e "$STALE_DOWNLOAD" ]] && echo yes || echo no) + fail "stale caller reports its failed replacement download" \ + "rc=$STALE_WAIT_RC, curl reached=$CURL_REACHED" + fi + + if [[ -f "$DEST" && "$(cat "$DEST")" == "valid" ]]; then + pass "failed stale caller preserves the valid replacement" + else + fail "failed stale caller preserves the valid replacement" \ + "destination is missing or no longer contains valid bytes" + fi + + LEFTOVERS=$(find "$WORK" -name 'lib.so.*' -type f | wc -l | tr -d ' ') + expect "concurrent replacement cleans private temp files" "0" "$LEFTOVERS" + ) +fi + # -------------------------------------------------------------------------- echo "" echo "----------------------------------------" diff --git a/hugegraph-store/hg-store-dist/src/assembly/static/bin/util.sh b/hugegraph-store/hg-store-dist/src/assembly/static/bin/util.sh index 735b75a31b..e45249a04b 100644 --- a/hugegraph-store/hg-store-dist/src/assembly/static/bin/util.sh +++ b/hugegraph-store/hg-store-dist/src/assembly/static/bin/util.sh @@ -318,8 +318,8 @@ download_and_verify() { actual_md5=$(md5sum -- "$filepath" | awk '{ print $1 }') if [[ "$actual_md5" != "$expected_md5" ]]; then echo "MD5 checksum verification failed for $filepath. Expected: $expected_md5, but got: $actual_md5" - echo "Deleting $filepath..." - rm -f -- "$filepath" + # Keep the shared destination until a verified temporary file is + # ready; a stale reader must not delete another caller's replacement. else echo "MD5 checksum verification succeeded for $filepath." return 0 From 8a64a1fb791eb436df0aef2919021a579f1129c1 Mon Sep 17 00:00:00 2001 From: Himanshu Verma Date: Thu, 30 Jul 2026 00:34:17 +0530 Subject: [PATCH 09/10] test(bin): enforce port preflight contracts --- .../src/assembly/travis/test-check-port.sh | 84 ++++++++++++++++--- 1 file changed, 74 insertions(+), 10 deletions(-) diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/test-check-port.sh b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-check-port.sh index 84521d07bf..9ba8e2dd54 100755 --- a/hugegraph-server/hugegraph-dist/src/assembly/travis/test-check-port.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-check-port.sh @@ -130,12 +130,35 @@ echo "2. Linux: ss busy / free / failure, and the netstat fallback" ( uname() { echo "Linux"; } command_available() { [[ "$1" == "ss" ]]; } + WORK=$(mktemp -d) + trap 'rm -rf "$WORK"' EXIT + SS_ARGS_FILE="$WORK/ss-args" SS_OUT="" SS_RC=0 - ss() { printf '%s' "$SS_OUT"; return "$SS_RC"; } + SS_FILTER_LISTEN=0 + ss() { + echo "$*" > "$SS_ARGS_FILE" + [[ $# -eq 2 && "$1" == "-H" && "$2" == "-ltn" ]] || return 64 + if [[ "$SS_FILTER_LISTEN" -eq 1 ]]; then + echo "$SS_OUT" | awk '$1 == "LISTEN"' + else + printf '%s' "$SS_OUT" + fi + return "$SS_RC" + } SS_OUT='LISTEN 0 4096 0.0.0.0:8080 0.0.0.0:*' expect "listener on 8080 is busy" "busy" "$(port_listen_state 8080)" + expect "ss receives the exact listener-table arguments" \ + "-H -ltn" "$(cat "$SS_ARGS_FILE")" + + # Model the filtering done by `ss -l`: an established connection may use + # the target as its local ephemeral port, but it is not a listener. The + # mock refuses changed argv, so dropping -l makes this unknown, not free. + SS_FILTER_LISTEN=1 + SS_OUT='ESTAB 0 0 127.0.0.1:8080 127.0.0.1:443' + expect "Linux non-LISTEN local port is free" "free" "$(port_listen_state 8080)" + SS_FILTER_LISTEN=0 SS_OUT='LISTEN 0 4096 0.0.0.0:22 0.0.0.0:*' expect "no listener on 8080 is free" "free" "$(port_listen_state 8080)" @@ -358,9 +381,16 @@ echo "7. Startup probe is bounded" TIMEOUT_S=1 START_S=$(date '+%s') - wait_for_startup "$$" "test-server" "http://127.0.0.1:1" "$TIMEOUT_S" >/dev/null 2>&1 + if wait_for_startup "$$" "test-server" "http://127.0.0.1:1" "$TIMEOUT_S" \ + >/dev/null 2>&1; then + WAIT_STATUS=0 + else + WAIT_STATUS=$? + fi ELAPSED_S=$(( $(date '+%s') - START_S )) + expect "failed startup probe returns failure" "1" "$WAIT_STATUS" + # The deterministic half: bounding each curl does not bound the loop, which # used to sleep a flat 2s and only then re-read the clock. A 1s timeout # must never ask for more than 1s of sleeping. @@ -380,15 +410,49 @@ echo "7. Startup probe is bounded" "took ${ELAPSED_S}s for a ${TIMEOUT_S}s timeout" fi - if grep -q -- "--max-time" "$ARGS_FILE" 2>/dev/null; then - pass "startup probe passes --max-time" - else - fail "startup probe passes --max-time" "args were: $(cat "$ARGS_FILE" 2>/dev/null)" - fi - if grep -q -- "--connect-timeout" "$ARGS_FILE" 2>/dev/null; then - pass "startup probe passes --connect-timeout" + # With a one-second overall deadline, every positive remaining budget is + # exactly one second. Parse each curl call so option presence, values, and + # the deadline bound are all part of the contract rather than string-grep + # smoke checks. + CURL_CALLS=0 + TIMEOUT_VALUES_VALID=1 + TIMEOUT_ERROR="" + while IFS= read -r curl_args; do + [[ -z "$curl_args" ]] && continue + CURL_CALLS=$((CURL_CALLS + 1)) + CONNECT_TIMEOUT="" + MAX_TIME="" + set -- $curl_args + while [[ $# -gt 0 ]]; do + case "$1" in + --connect-timeout) + shift + [[ $# -gt 0 ]] && CONNECT_TIMEOUT="$1" + ;; + --max-time) + shift + [[ $# -gt 0 ]] && MAX_TIME="$1" + ;; + esac + [[ $# -gt 0 ]] && shift + done + + if [[ ! "$CONNECT_TIMEOUT" =~ ^[0-9]+$ || + ! "$MAX_TIME" =~ ^[0-9]+$ || + "$CONNECT_TIMEOUT" -le 0 || "$CONNECT_TIMEOUT" -gt "$TIMEOUT_S" || + "$MAX_TIME" -le 0 || "$MAX_TIME" -gt "$TIMEOUT_S" ]]; then + TIMEOUT_VALUES_VALID=0 + TIMEOUT_ERROR="invalid timeouts connect='$CONNECT_TIMEOUT' max='$MAX_TIME' in: $curl_args" + break + fi + done < "$ARGS_FILE" + + if [[ "$CURL_CALLS" -gt 0 && "$TIMEOUT_VALUES_VALID" -eq 1 ]]; then + pass "startup probe timeouts are positive and within the remaining deadline" else - fail "startup probe passes --connect-timeout" "args were: $(cat "$ARGS_FILE" 2>/dev/null)" + [[ -n "$TIMEOUT_ERROR" ]] || TIMEOUT_ERROR="no curl calls recorded" + fail "startup probe timeouts are positive and within the remaining deadline" \ + "$TIMEOUT_ERROR" fi ) From 09dbc744776968f9928eea8f1f4a6833c7579b89 Mon Sep 17 00:00:00 2001 From: imbajin Date: Fri, 31 Jul 2026 21:05:39 +0800 Subject: [PATCH 10/10] fix(server): stabilize port deadline tests - control time for probe timeout assertions - cover probes that exhaust the remaining budget - retain a bounded real-clock startup smoke test --- .../src/assembly/travis/test-check-port.sh | 73 +++++++++++++++---- 1 file changed, 60 insertions(+), 13 deletions(-) diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/test-check-port.sh b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-check-port.sh index 9ba8e2dd54..e8950355c7 100755 --- a/hugegraph-server/hugegraph-dist/src/assembly/travis/test-check-port.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-check-port.sh @@ -369,25 +369,31 @@ echo "7. Startup probe is bounded" cd "$WORK" || exit 1 trap 'cd /; rm -rf "$WORK"' EXIT + CLOCK_FILE="$WORK/clock" ARGS_FILE="$WORK/curl-args" SLEEP_FILE="$WORK/sleep-args" + echo 100 > "$CLOCK_FILE" + : > "$ARGS_FILE" : > "$SLEEP_FILE" curl() { echo "$*" >> "$ARGS_FILE"; echo "000"; return 28; } - # Record what the retry pause asks for, but still take it, so the - # end-to-end duration below stays a real measurement. - sleep() { echo "$1" >> "$SLEEP_FILE"; command sleep "$1"; } + # Keep the contract checks deterministic across whole-second boundaries. + # The retry pause consumes the remaining simulated second without making + # this half of the test depend on scheduler timing. + date() { cat "$CLOCK_FILE"; } + sleep() { + echo "$1" >> "$SLEEP_FILE" + echo 101 > "$CLOCK_FILE" + } process_status() { return 0; } ps() { return 0; } TIMEOUT_S=1 - START_S=$(date '+%s') if wait_for_startup "$$" "test-server" "http://127.0.0.1:1" "$TIMEOUT_S" \ >/dev/null 2>&1; then WAIT_STATUS=0 else WAIT_STATUS=$? fi - ELAPSED_S=$(( $(date '+%s') - START_S )) expect "failed startup probe returns failure" "1" "$WAIT_STATUS" @@ -402,14 +408,6 @@ echo "7. Startup probe is bounded" "slept ${TOTAL_SLEEP_S}s in total: $(tr '\n' ' ' < "$SLEEP_FILE")" fi - # The end-to-end half, with a second of slack for whole-second clock reads. - if [[ "$ELAPSED_S" -le $((TIMEOUT_S + 1)) ]]; then - pass "wait_for_startup returns within the deadline (${ELAPSED_S}s)" - else - fail "wait_for_startup returns within the deadline" \ - "took ${ELAPSED_S}s for a ${TIMEOUT_S}s timeout" - fi - # With a one-second overall deadline, every positive remaining budget is # exactly one second. Parse each curl call so option presence, values, and # the deadline bound are all part of the contract rather than string-grep @@ -455,6 +453,55 @@ echo "7. Startup probe is bounded" "$TIMEOUT_ERROR" fi + # A probe may consume all of its assigned budget. Refreshing the clock + # after curl must then suppress the retry pause rather than sleep beyond + # the overall deadline. + EXHAUST_SLEEP_FILE="$WORK/exhausted-sleep-calls" + echo 100 > "$CLOCK_FILE" + : > "$EXHAUST_SLEEP_FILE" + curl() { + echo 101 > "$CLOCK_FILE" + echo "000" + return 28 + } + sleep() { echo "$1" >> "$EXHAUST_SLEEP_FILE"; } + if wait_for_startup "$$" "budget-server" "http://127.0.0.1:1" \ + "$TIMEOUT_S" >/dev/null 2>&1; then + EXHAUST_STATUS=0 + else + EXHAUST_STATUS=$? + fi + + expect "budget-exhausting probe returns failure" "1" "$EXHAUST_STATUS" + EXHAUST_SLEEP_CALLS=$(wc -l < "$EXHAUST_SLEEP_FILE" | tr -d ' ') + expect "budget-exhausting probe starts no retry sleep" \ + "0" "$EXHAUST_SLEEP_CALLS" + + # Keep a small real-clock smoke test for the end-to-end deadline. Crossing + # an integer-second boundary between the initial and pre-probe clock reads + # may legitimately produce zero curl calls, so this half checks only the + # return status and wall-clock bound and does not require an args file. + date() { command date "$@"; } + curl() { echo "000"; return 28; } + sleep() { command sleep "$1"; } + START_S=$(date '+%s') + if wait_for_startup "$$" "wall-clock-server" "http://127.0.0.1:1" \ + "$TIMEOUT_S" >/dev/null 2>&1; then + WALL_STATUS=0 + else + WALL_STATUS=$? + fi + ELAPSED_S=$(( $(date '+%s') - START_S )) + + expect "real-clock failed startup probe returns failure" "1" "$WALL_STATUS" + # One second of slack accounts for the granularity of date '+%s'. + if [[ "$ELAPSED_S" -le $((TIMEOUT_S + 1)) ]]; then + pass "wait_for_startup returns within the deadline (${ELAPSED_S}s)" + else + fail "wait_for_startup returns within the deadline" \ + "took ${ELAPSED_S}s for a ${TIMEOUT_S}s timeout" + fi + ) # --------------------------------------------------------------------------