diff --git a/.kokoro/system.sh b/.kokoro/system.sh index e0c7e71c1ad7..1d4414b8a92a 100755 --- a/.kokoro/system.sh +++ b/.kokoro/system.sh @@ -35,10 +35,24 @@ RETVAL=0 pwd +echo "=== KOKORO VM SCOUTING ===" +echo "CPU Count:" +nproc +echo "--------------------------" +echo "Detailed CPU Info:" +lscpu | grep -E "^(Model name|CPU\(s\)|Thread\(s\) per core|Core\(s\) per socket)" +echo "--------------------------" +echo "Memory Status:" +free -h +echo "--------------------------" +echo "Disk Usage:" +df -h / +echo "==========================" + run_package_test() { local package_name=$1 local package_path="packages/${package_name}" - + # Declare local overrides to prevent bleeding into the next loop iteration local PROJECT_ID local GOOGLE_APPLICATION_CREDENTIALS @@ -46,6 +60,14 @@ run_package_test() { # Inherit NOX_SESSION from environment to allow configs (like prerelease.cfg) to pass it in local NOX_SESSION="${NOX_SESSION}" + # ISOLATION: Create a unique gcloud config dir for this run + local gcloud_config_dir=$(mktemp -d -t "gcloud-config-${package_name}-XXXXXX") + local CLOUDSDK_CONFIG="${gcloud_config_dir}" + + # 🪤 TRAP: Ensure cleanup of THIS specific temp dir on exit of this subshell + trap 'rm -rf "$gcloud_config_dir"' EXIT + + echo "------------------------------------------------------------" echo "Configuring environment for: ${package_name}" echo "------------------------------------------------------------" @@ -80,11 +102,13 @@ run_package_test() { esac # Export variables for the duration of this function's sub-processes - export PROJECT_ID GOOGLE_APPLICATION_CREDENTIALS NOX_FILE NOX_SESSION + export PROJECT_ID GOOGLE_APPLICATION_CREDENTIALS NOX_FILE NOX_SESSION CLOUDSDK_CONFIG export GOOGLE_CLOUD_PROJECT="${PROJECT_ID}" - gcloud auth activate-service-account --key-file="$GOOGLE_APPLICATION_CREDENTIALS" - gcloud config set project "$PROJECT_ID" + # 🛡️ Explicit check: Fail early if auth fails + gcloud auth activate-service-account --key-file="$GOOGLE_APPLICATION_CREDENTIALS" || return 1 + export CLOUDSDK_CORE_PROJECT="${PROJECT_ID}" + # Run the actual test pushd "${package_path}" > /dev/null @@ -93,13 +117,81 @@ run_package_test() { local res=$? set -e popd > /dev/null - + return $res } +# Analyzes results of parallel test runs, prints summary, and dumps failed logs +reap_parallel_results() { + local retval=0 + local failed_count=0 + local passed_count=0 + + if [ -z "$LOG_DIR" ]; then + echo "Error: LOG_DIR is not set." + return 1 + fi + + # Count failed packages by checking for .failed marker files + for failed in "$LOG_DIR"/*.failed; do + if [ -f "$failed" ]; then + failed_count=$((failed_count + 1)) + fi + done + + local total_tested=${#PACKAGES_TO_TEST[@]} + passed_count=$((total_tested - failed_count)) + + echo "" + echo "==================================================" + echo " TEST RUN SUMMARY " + echo "==================================================" + echo "Total tested: $total_tested" + echo "Passed: $passed_count" + echo "Failed: $failed_count" + echo "==================================================" + + local passed_packages=() + for pkg in "${PACKAGES_TO_TEST[@]}"; do + if [ ! -f "$LOG_DIR/$pkg.failed" ]; then + passed_packages+=("$pkg") + fi + done + + if [ ${#passed_packages[@]} -gt 0 ]; then + echo "" + echo "PASSED PACKAGES:" + printf "%s\n" "${passed_packages[@]}" | sort | sed 's/^/- /' + fi + + if [ "$failed_count" -gt 0 ]; then + echo "" + echo "!!! DETAILED LOGS FOR FAILED PACKAGES !!!" + for failed in "$LOG_DIR"/*.failed; do + if [ -f "$failed" ]; then + local pkg=$(basename "$failed" .failed) + echo "--------------------------------------------------" + echo "LOGS FOR: $pkg" + echo "--------------------------------------------------" + if [ -n "$KOKORO_ARTIFACTS_DIR" ] && [ -f "$KOKORO_ARTIFACTS_DIR/$pkg/sponge_log.log" ]; then + cat "$KOKORO_ARTIFACTS_DIR/$pkg/sponge_log.log" + else + cat "$LOG_DIR/$pkg.log" + fi + echo "" + fi + done + retval=1 + fi + return $retval +} + + # A file for running system tests system_test_script="${PROJECT_ROOT}/.kokoro/system-single.sh" +PACKAGES_TO_TEST=() + # Run system tests for each package with directory packages/*/tests/system for path in `find 'packages' \ \( -type d -wholename 'packages/*/tests/system' \) -o \ @@ -113,6 +205,7 @@ for path in `find 'packages' \ package_name=${package_name%%/*} package_path="packages/${package_name}" + # Determine if we should skip based on git diff # We always check for changes in these specific versioning/config files files_to_check=( @@ -147,10 +240,54 @@ for path in `find 'packages' \ set -e if [[ "${package_modified}" -gt 0 || "$KOKORO_BUILD_ARTIFACTS_SUBDIR" == *"continuous"* ]]; then - # Call the function - its internal exports won't affect the next loop - run_package_test "$package_name" || RETVAL=$? + PACKAGES_TO_TEST+=("$package_name") else echo "No changes in ${package_name} and not a continuous build, skipping." fi done + +# Parallel Execution Logic +MAX_JOBS=${MAX_JOBS:-4} + +# Temporary directory for clean log segregation +LOG_DIR=$(mktemp -d -t test-logs-XXXXXX) +# Clean up logs on exit +trap 'rm -rf "$LOG_DIR"' EXIT + +if [ ${#PACKAGES_TO_TEST[@]} -eq 0 ]; then + echo "No packages to test." + exit 0 +fi + +echo "==================================================" +echo "Starting parallel test execution for ${#PACKAGES_TO_TEST[@]} packages" +echo "Concurrency limit: ${MAX_JOBS}" +echo "==================================================" + +export LOG_DIR +export -f run_package_test +export system_test_script PROJECT_ROOT KOKORO_GFILE_DIR + +# Stream package names to xargs for parallel execution +# -P "$MAX_JOBS" controls concurrency +# -I {} replaces {} with the package name +printf '%s\n' "${PACKAGES_TO_TEST[@]}" \ + | xargs -P "$MAX_JOBS" -I {} \ + bash -c ' + pkg="$1" + # Determine log location: prefer Sponge artifacts directory if available + if [ -n "$KOKORO_ARTIFACTS_DIR" ]; then + pkg_log_dir="$KOKORO_ARTIFACTS_DIR/$pkg" + mkdir -p "$pkg_log_dir" || { touch "$LOG_DIR/$pkg.failed"; exit 1; } + log_file="$pkg_log_dir/sponge_log.log" + else + log_file="$LOG_DIR/$pkg.log" + fi + + # Run test; if it fails, create a .failed file to signal failure to the reaper + run_package_test "$pkg" > "$log_file" 2>&1 || touch "$LOG_DIR/$pkg.failed" + ' _ "{}" + +reap_parallel_results || RETVAL=1 + exit ${RETVAL} diff --git a/packages/django-google-spanner/setup.py b/packages/django-google-spanner/setup.py index 973dc622047e..079139850bac 100644 --- a/packages/django-google-spanner/setup.py +++ b/packages/django-google-spanner/setup.py @@ -1,4 +1,6 @@ # Copyright 2020 Google LLC +# Trigger parallel system test run + # # Use of this source code is governed by a BSD-style # license that can be found in the LICENSE file or at diff --git a/packages/google-api-core/setup.py b/packages/google-api-core/setup.py index aacce68213d1..38ec2b3865b3 100644 --- a/packages/google-api-core/setup.py +++ b/packages/google-api-core/setup.py @@ -1,4 +1,6 @@ # Copyright 2018 Google LLC +# Trigger parallel system test run + # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/google-auth/setup.py b/packages/google-auth/setup.py index 1dfb9ee734ff..3de9b367c111 100644 --- a/packages/google-auth/setup.py +++ b/packages/google-auth/setup.py @@ -1,4 +1,5 @@ # Copyright 2014 Google Inc. +# Trigger parallel system test run # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -15,8 +16,7 @@ import io import os -from setuptools import find_namespace_packages -from setuptools import setup +from setuptools import find_namespace_packages, setup cryptography_base_require = [ "cryptography >= 38.0.3; python_version < '3.14'", diff --git a/packages/google-cloud-bigquery/setup.py b/packages/google-cloud-bigquery/setup.py index 13aa8b6ca346..3035f8492d4d 100644 --- a/packages/google-cloud-bigquery/setup.py +++ b/packages/google-cloud-bigquery/setup.py @@ -1,4 +1,5 @@ # Copyright 2018 Google LLC +# Trigger parallel system test run # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/google-cloud-bigtable/setup.py b/packages/google-cloud-bigtable/setup.py index cc161af33de1..7e9cf84c0350 100644 --- a/packages/google-cloud-bigtable/setup.py +++ b/packages/google-cloud-bigtable/setup.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- # Copyright 2026 Google LLC +# Trigger parallel system test run # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/google-cloud-core/setup.py b/packages/google-cloud-core/setup.py index 360fdf4cccb5..3bf0191f2a55 100644 --- a/packages/google-cloud-core/setup.py +++ b/packages/google-cloud-core/setup.py @@ -1,4 +1,5 @@ # Copyright 2018 Google LLC +# Trigger parallel system test run # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -17,7 +18,6 @@ import setuptools - # Package metadata. name = "google-cloud-core" diff --git a/packages/google-cloud-datastore/setup.py b/packages/google-cloud-datastore/setup.py index 6a169c1fd630..649997f74386 100644 --- a/packages/google-cloud-datastore/setup.py +++ b/packages/google-cloud-datastore/setup.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- # Copyright 2026 Google LLC +# Trigger parallel system test run # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/google-cloud-error-reporting/setup.py b/packages/google-cloud-error-reporting/setup.py index 2d7a6a1b6746..b49a012f973a 100644 --- a/packages/google-cloud-error-reporting/setup.py +++ b/packages/google-cloud-error-reporting/setup.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- # Copyright 2026 Google LLC +# Trigger parallel system test run # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/google-cloud-firestore/setup.py b/packages/google-cloud-firestore/setup.py index 5b5adc3ce374..6af93b8945ce 100644 --- a/packages/google-cloud-firestore/setup.py +++ b/packages/google-cloud-firestore/setup.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- # Copyright 2026 Google LLC +# Trigger parallel system test run # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/google-cloud-logging/setup.py b/packages/google-cloud-logging/setup.py index f1243f3e63b6..e3db60667faa 100644 --- a/packages/google-cloud-logging/setup.py +++ b/packages/google-cloud-logging/setup.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- # Copyright 2026 Google LLC +# Trigger parallel system test run # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/google-cloud-ndb/setup.py b/packages/google-cloud-ndb/setup.py index 8c22f5349b16..98573a0580ee 100644 --- a/packages/google-cloud-ndb/setup.py +++ b/packages/google-cloud-ndb/setup.py @@ -1,4 +1,6 @@ # Copyright 2018 Google LLC +# Trigger parallel system test run + # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/google-cloud-pubsub/setup.py b/packages/google-cloud-pubsub/setup.py index cfb787d161fc..08b13aec55f7 100644 --- a/packages/google-cloud-pubsub/setup.py +++ b/packages/google-cloud-pubsub/setup.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- # Copyright 2026 Google LLC +# Trigger parallel system test run # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/google-cloud-storage/setup.py b/packages/google-cloud-storage/setup.py index b872eff7dbd8..41a00895d1bd 100644 --- a/packages/google-cloud-storage/setup.py +++ b/packages/google-cloud-storage/setup.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- # Copyright 2026 Google LLC +# Trigger parallel system test run # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License.