From 6817ebab6ab326f00680f02a55221f3a08968b1d Mon Sep 17 00:00:00 2001 From: Manuel Mendez Date: Tue, 18 Aug 2026 11:45:40 -0400 Subject: [PATCH 1/8] ansible: Setup swap in stage2 not stage1 Having swap setup in stage1 uses up 10% of the disk while we need as much space as possible (for fetching, building, etc). We are extremely sensitive to cache clean up ordering so that we don't go over, but thats a big pain that can be allievated by moving swap to the end of the run. We run stage2 on beefy machines that don't need swap so lets use the space better. --- ansible/playbook.yml | 5 ++ ansible/tasks/setup-tuned.yml | 37 -------------- ansible/tasks/stage2/setup-swap.yml | 49 +++++++++++++++++++ .../scripts/surrogate-bootstrap-nix.sh | 14 +----- 4 files changed, 55 insertions(+), 50 deletions(-) create mode 100644 ansible/tasks/stage2/setup-swap.yml diff --git a/ansible/playbook.yml b/ansible/playbook.yml index 613c4a99d..5202fe987 100644 --- a/ansible/playbook.yml +++ b/ansible/playbook.yml @@ -212,3 +212,8 @@ become: yes shell: | sudo -u ubuntu bash -c ". /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && nix-collect-garbage -d" + + - name: Setup Swap + when: stage2 and not qemu + become: yes + import_tasks: tasks/stage2/setup-swap.yml diff --git a/ansible/tasks/setup-tuned.yml b/ansible/tasks/setup-tuned.yml index 30f2f611d..1a02becc9 100644 --- a/ansible/tasks/setup-tuned.yml +++ b/ansible/tasks/setup-tuned.yml @@ -185,43 +185,6 @@ value: '10' become: true - - name: 'tuned - Enable zswap if swap is present' # noqa: name[casing] - when: ansible_facts['swaptotal_mb'] > 0 - block: - - name: 'tuned - Decrease the kernel swappiness' # noqa: name[casing] - community.general.ini_file: - create: true - group: 'root' - mode: '0644' - no_extra_spaces: true - option: 'vm.swappiness' - path: '/etc/tuned/profiles/postgresql/tuned.conf' - section: 'sysctl' - state: 'present' - value: '10' - become: true - - - name: 'tuned - Load zstd compressor module' # noqa: name[casing] - community.general.modprobe: - name: 'zstd' - persistent: 'present' - state: 'present' - become: true - - - name: 'tuned - Configure and enable zswap' # noqa: name[casing] - ansible.builtin.shell: - cmd: "echo {{ zswap_item['value'] }} > /sys/module/zswap/parameters/{{ zswap_item['param'] }}" - changed_when: true - loop: - - param: 'compressor' - value: 'zstd' - - param: 'max_pool_percent' - value: '10' - - param: 'enabled' - value: 'Y' - loop_control: - loop_var: 'zswap_item' - - name: 'tuned - Activate the tuned service' # noqa: name[casing] ansible.builtin.systemd_service: daemon_reload: true diff --git a/ansible/tasks/stage2/setup-swap.yml b/ansible/tasks/stage2/setup-swap.yml new file mode 100644 index 000000000..a1823e058 --- /dev/null +++ b/ansible/tasks/stage2/setup-swap.yml @@ -0,0 +1,49 @@ +- name: Allocate and format swapfile + ansible.builtin.shell: | + fallocate -l 1G /swapfile + chmod 600 /swapfile + mkswap /swapfile + args: + creates: /swapfile + +- name: Add swapfile to fstab + ansible.posix.mount: + src: /swapfile + path: none + fstype: swap + opts: sw + dump: "0" + passno: "0" + state: present + +- name: 'tuned - Decrease the kernel swappiness' # noqa: name[casing] + community.general.ini_file: + create: true + group: 'root' + mode: '0644' + no_extra_spaces: true + option: 'vm.swappiness' + path: '/etc/tuned/profiles/postgresql/tuned.conf' + section: 'sysctl' + state: 'present' + value: '10' + +- name: 'tuned - Load zstd compressor module' # noqa: name[casing] + community.general.modprobe: + name: 'zstd' + persistent: 'present' + state: 'present' + +- name: 'tuned - Configure and enable zswap' # noqa: name[casing] + ansible.builtin.shell: + cmd: "echo {{ zswap_item['value'] }} > /sys/module/zswap/parameters/{{ zswap_item['param'] }}" + changed_when: true + loop: + - param: 'compressor' + value: 'zstd' + - param: 'max_pool_percent' + value: '10' + - param: 'enabled' + value: 'Y' + loop_control: + loop_var: 'zswap_item' diff --git a/ebssurrogate/scripts/surrogate-bootstrap-nix.sh b/ebssurrogate/scripts/surrogate-bootstrap-nix.sh index de44160b0..a24072b35 100755 --- a/ebssurrogate/scripts/surrogate-bootstrap-nix.sh +++ b/ebssurrogate/scripts/surrogate-bootstrap-nix.sh @@ -202,24 +202,15 @@ function format_and_mount_rootfs { mount -o defaults,discard /dev/xvdh /mnt/data } -function create_swapfile { - fallocate -l 1G /mnt/swapfile - chmod 600 /mnt/swapfile - mkswap /mnt/swapfile -} - function format_build_partition { mkfs.ext4 -O ^has_journal /dev/xvdc } # Create fstab function create_fstab { - local FMT="%-42s %-11s %-5s %-17s %-5s %s" ROOT_LINE DATA_LINE SWAP_LINE + local FMT="%-42s %-11s %-5s %-17s %-5s %s" ROOT_LINE DATA_LINE ROOT_LINE=$(findmnt -no SOURCE /mnt | xargs blkid -o export | awk -v FMT="$FMT" '/^UUID=/ { printf(FMT, $0, "/", "ext4", "defaults,discard", "0", "1" ) }') DATA_LINE=$(findmnt -no SOURCE /mnt/data | xargs blkid -o export | awk -v FMT="$FMT" '/^UUID=/ { printf(FMT, $0, "/data", "ext4", "defaults,discard", "0", "2" ) }') - # shellcheck disable=SC2059 - SWAP_LINE=$(printf "$FMT" "/swapfile" "none" "swap" "sw" "0" "0") - local EFI_LINE="" if [[ $ARCH == arm64 ]]; then EFI_LINE=$(findmnt -no SOURCE /mnt/boot/efi | xargs blkid -o export | awk -v FMT="$FMT" '/^UUID=/ { printf(FMT, $0, "/boot/efi", "vfat", "umask=0077", "0", "1" ) }') @@ -231,7 +222,6 @@ function create_fstab { echo "$ROOT_LINE" [ -n "$EFI_LINE" ] && echo "$EFI_LINE" echo "$DATA_LINE" - echo "$SWAP_LINE" } >/mnt/etc/fstab } @@ -387,7 +377,6 @@ function clean_system { rm -rf /mnt/root/.vpython* rm -rf /mnt/root/go rm -rf /mnt/usr/share/doc - } # Unmount bind mounts @@ -419,7 +408,6 @@ waitfor_boot_finished install_packages device_partition_mappings format_and_mount_rootfs -create_swapfile format_build_partition setup_chroot_environment execute_playbook From c0eb0c0684a1bbe2ce5570fbae196a97327f7b8d Mon Sep 17 00:00:00 2001 From: Manuel Mendez Date: Fri, 14 Aug 2026 13:50:51 -0400 Subject: [PATCH 2/8] packer: Simplify apt mirror fallback It turns out that most of the code deleted here was actually not doing anything useful. For example, switch_mirror modifies /etc/apt/sources.list in place but the file doesn't have any mirrors configured there so it's really a no-op! Here's the contents from an instance I just fired up[^1]: ubuntu@ip-172-31-26-227:~$ tail -n+1 /etc/apt/sources.list /etc/apt/sources.list.d/* ==> /etc/apt/sources.list <== # Ubuntu sources have moved to the /etc/apt/sources.list.d/ubuntu.sources # file, which uses the deb822 format. Use deb822-formatted .sources files # to manage package sources in the /etc/apt/sources.list.d/ directory. # See the sources.list(5) manual page for details. ==> /etc/apt/sources.list.d/ubuntu.sources <== ## Note, this file is written by cloud-init on first boot of an instance ## modifications made here will not survive a re-bundle. --- 8< --- Types: deb URIs: http://us-east-2.ec2.archive.ubuntu.com/ubuntu/ Suites: noble noble-updates noble-backports Components: main universe restricted multiverse Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg Types: deb URIs: http://us-east-2.ec2.archive.ubuntu.com/ubuntu/ Suites: noble noble-updates noble-backports Components: main universe restricted multiverse Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg ## Ubuntu security updates. Aside from URIs and Suites, ## this should mirror your choices in the previous section. Types: deb URIs: http://security.ubuntu.com/ubuntu Suites: noble-security Components: main universe restricted multiverse Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg We get fallback handling by apt itself by adding multiple mirrors in URIs, apt tries first (the regional) and falls back to global ubuntu repos if there's an issue. We also setup a temporary apt config that forces the APT_OPTIONS for all apt calls so we don't need to do it in every call site. I ended up dropping the in-repo sources file since they are strictly worse than what we get from AWS in the build. There's no change in suites or components between old and new, just that we get ubuntu upstream as a fallback. Well technically there's a slight difference since we are basing off of cloud-init generated files and they can theoretically change under us but I'll gamble that it'll be fine or better off. Besides, one day we'll be on NixOS as the ultimate "make sure we know everything in the instance" ;). I also got rid of the `add-apt-repository --yes universe` because universe is already enabled. [^1]: AMI=ubuntu/images/hvm-ssd-gp3/ubuntu-noble-24.04-amd64-server-20260604 --- amazon-amd64-nix.pkr.hcl | 5 - amazon-arm64-nix.pkr.hcl | 5 - ebssurrogate/files/sources-arm64.cfg | 10 -- ebssurrogate/files/sources.cfg | 10 -- ebssurrogate/scripts/chroot-bootstrap-nix.sh | 164 +----------------- .../scripts/surrogate-bootstrap-nix.sh | 128 ++++---------- 6 files changed, 44 insertions(+), 278 deletions(-) delete mode 100644 ebssurrogate/files/sources-arm64.cfg delete mode 100644 ebssurrogate/files/sources.cfg diff --git a/amazon-amd64-nix.pkr.hcl b/amazon-amd64-nix.pkr.hcl index 445462186..57e36e894 100644 --- a/amazon-amd64-nix.pkr.hcl +++ b/amazon-amd64-nix.pkr.hcl @@ -170,11 +170,6 @@ source "amazon-ebssurrogate" "source" { build { sources = ["source.amazon-ebssurrogate.source"] - provisioner "file" { - source = "ebssurrogate/files/sources.cfg" - destination = "/tmp/sources.list" - } - provisioner "file" { source = "ebssurrogate/files/ebsnvme-id" destination = "/tmp/ebsnvme-id" diff --git a/amazon-arm64-nix.pkr.hcl b/amazon-arm64-nix.pkr.hcl index 7bca54bbc..422886c0c 100644 --- a/amazon-arm64-nix.pkr.hcl +++ b/amazon-arm64-nix.pkr.hcl @@ -170,11 +170,6 @@ source "amazon-ebssurrogate" "source" { build { sources = ["source.amazon-ebssurrogate.source"] - provisioner "file" { - source = "ebssurrogate/files/sources-arm64.cfg" - destination = "/tmp/sources.list" - } - provisioner "file" { source = "ebssurrogate/files/ebsnvme-id" destination = "/tmp/ebsnvme-id" diff --git a/ebssurrogate/files/sources-arm64.cfg b/ebssurrogate/files/sources-arm64.cfg deleted file mode 100644 index eed6c0f16..000000000 --- a/ebssurrogate/files/sources-arm64.cfg +++ /dev/null @@ -1,10 +0,0 @@ -deb http://REGION.clouds.ports.ubuntu.com/ubuntu-ports/ noble main restricted -deb http://REGION.clouds.ports.ubuntu.com/ubuntu-ports/ noble-updates main restricted -deb http://REGION.clouds.ports.ubuntu.com/ubuntu-ports/ noble universe -deb http://REGION.clouds.ports.ubuntu.com/ubuntu-ports/ noble-updates universe -deb http://REGION.clouds.ports.ubuntu.com/ubuntu-ports/ noble multiverse -deb http://REGION.clouds.ports.ubuntu.com/ubuntu-ports/ noble-updates multiverse -deb http://REGION.clouds.ports.ubuntu.com/ubuntu-ports/ noble-backports main restricted universe multiverse -deb http://ports.ubuntu.com/ubuntu-ports noble-security main restricted -deb http://ports.ubuntu.com/ubuntu-ports noble-security universe -deb http://ports.ubuntu.com/ubuntu-ports noble-security multiverse diff --git a/ebssurrogate/files/sources.cfg b/ebssurrogate/files/sources.cfg deleted file mode 100644 index a27be052f..000000000 --- a/ebssurrogate/files/sources.cfg +++ /dev/null @@ -1,10 +0,0 @@ -deb http://REGION.ec2.archive.ubuntu.com/ubuntu/ noble main restricted -deb http://REGION.ec2.archive.ubuntu.com/ubuntu/ noble-updates main restricted -deb http://REGION.ec2.archive.ubuntu.com/ubuntu/ noble universe -deb http://REGION.ec2.archive.ubuntu.com/ubuntu/ noble-updates universe -deb http://REGION.ec2.archive.ubuntu.com/ubuntu/ noble multiverse -deb http://REGION.ec2.archive.ubuntu.com/ubuntu/ noble-updates multiverse -deb http://REGION.ec2.archive.ubuntu.com/ubuntu/ noble-backports main restricted universe multiverse -deb http://security.ubuntu.com/ubuntu noble-security main restricted -deb http://security.ubuntu.com/ubuntu noble-security universe -deb http://security.ubuntu.com/ubuntu noble-security multiverse diff --git a/ebssurrogate/scripts/chroot-bootstrap-nix.sh b/ebssurrogate/scripts/chroot-bootstrap-nix.sh index 140f6802d..862f8f190 100755 --- a/ebssurrogate/scripts/chroot-bootstrap-nix.sh +++ b/ebssurrogate/scripts/chroot-bootstrap-nix.sh @@ -8,166 +8,20 @@ set -o errexit set -o pipefail set -o xtrace -# Switch to a different mirror -function switch_mirror { - local new_mirror=$1 - local sources_file=/etc/apt/sources.list - - echo "Switching to mirror: $new_mirror" - if [[ $ARCH == amd64 ]]; then - sed -i "s|http://[^/]*/ubuntu/|http://$new_mirror/ubuntu/|g" "$sources_file" - else - sed -i "s|http://[^/]*/ubuntu-ports/|http://$new_mirror/ubuntu-ports/|g" "$sources_file" - fi - - # Show what we're using - echo "Current sources.list configuration:" - grep -E '^deb ' "$sources_file" | head -3 -} - -# Get list of mirrors to try -function get_mirror_list { - local sources_file=/etc/apt/sources.list - local -a mirrors=() - - # Priority order: - # 1. Country-specific mirror (most reliable) - # 2. Regional CDN (can be inconsistent) - # 3. Global fallback - - local current_region - if [[ $ARCH == amd64 ]]; then - current_region=$(grep -oP '(?<=http://)[^.]+(?=\.ec2\.archive\.ubuntu\.com)' "$sources_file" | head -1 || echo "") - - if [[ -n $current_region ]]; then - mirrors+=("$current_region.ec2.archive.ubuntu.com") - fi - - mirrors+=("archive.ubuntu.com") - else - current_region=$(grep -oP '(?<=http://)[^.]+(?=\.clouds\.ports\.ubuntu\.com)' "$sources_file" | head -1 || echo "") - - # Singapore country mirror for ap-southeast-1 - if [[ $current_region == "ap-southeast-1" ]]; then - mirrors+=("sg.ports.ubuntu.com") - fi - - if [[ -n $current_region ]]; then - mirrors+=("$current_region.clouds.ports.ubuntu.com") - fi - mirrors+=("ports.ubuntu.com") - fi - - echo "${mirrors[@]}" -} - -# Mirror fallback function for resilient apt-get update +# The following 2 functions don'treally do much since we are now using deb822 formatted sources with fallbacks in the URIs. +# This means apt-get handles fallback on its own, much better and cleaner than we are doing. +# Leaving the functions as is for now to make the diff smaller, soon will go away. function apt_update_with_fallback { - local sources_file=/etc/apt/sources.list - local -a mirror_list - readarray mirror_list < <(get_mirror_list) - local attempt=1 - local max_attempts=${#mirror_list[@]} - - for mirror in "${mirror_list[@]}"; do - echo "=========================================" - echo "Attempting apt-get update with mirror: $mirror" - echo "Attempt $attempt of $max_attempts" - echo "=========================================" - - switch_mirror "$mirror" - - # Attempt update with timeout (5 minutes) - if timeout 300 apt-get "${APT_OPTIONS[@]}" update 2>&1; then - echo "=========================================" - echo "✓ Successfully updated apt cache using mirror: $mirror" - echo "=========================================" - return 0 - else - local ret=$? - echo "=========================================" - echo "✗ Failed to update using mirror: $mirror" - echo "Exit code: $ret" - echo "=========================================" - - # Clean partial downloads - apt-get clean - rm -rf /var/lib/apt/lists/* - - # Exponential backoff before next attempt - if [[ $attempt -lt $max_attempts ]]; then - local sleep_time=$((attempt * 5)) - echo "Waiting $sleep_time seconds before trying next mirror..." - sleep $sleep_time - fi - fi - - attempt=$((attempt + 1)) - done - - echo "=========================================" - echo "ERROR: All mirror tiers failed after $max_attempts attempts" - echo "=========================================" - return 1 + timeout 300 apt-get "${APT_OPTIONS[@]}" update 2>&1 } -# Wrapper for apt-get install with mirror fallback on 404 errors function apt_install_with_fallback { - local -a mirror_list - readarray mirror_list < <(get_mirror_list) - local attempt=1 - local max_attempts=${#mirror_list[@]} - - for mirror in "${mirror_list[@]}"; do - echo "=========================================" - echo "Attempting apt-get install with mirror: $mirror" - echo "Attempt $attempt of $max_attempts" - echo "=========================================" - - switch_mirror "$mirror" - - # Re-run apt-get update to get package lists from new mirror - if ! timeout 300 apt-get "${APT_OPTIONS[@]}" update 2>&1; then - echo "Warning: apt-get update failed for mirror $mirror, trying next..." - attempt=$((attempt + 1)) - continue - fi - - # Run apt-get install directly (no output capture to avoid buffering/timeout issues) - if apt-get "$@"; then - echo "=========================================" - echo "✓ Successfully installed packages using mirror: ${mirror}" - echo "=========================================" - return 0 - else - local ret=$? - # On failure, check if it's a mirror issue worth retrying - echo "=========================================" - echo "✗ apt-get failed with exit code: $ret" - echo "=========================================" - fi - - # Clean apt cache before potential retry - apt-get clean - - if ((attempt < max_attempts)); then - local sleep_time=$((attempt * 5)) - echo "Waiting $sleep_time seconds before trying next mirror..." - sleep $sleep_time - fi - - attempt=$((attempt + 1)) - done - - echo "=========================================" - echo "ERROR: All mirror tiers failed for apt-get install after $max_attempts attempts" - echo "=========================================" - return 1 + apt-get "$@" } function update_install_packages { # Update APT with new sources (using fallback mechanism) - cat /etc/apt/sources.list + tail -n+1 /etc/apt/sources.list /etc/apt/sources.list.d/* if ! apt_update_with_fallback; then echo "FATAL: Failed to update package lists with any mirror tier" exit 1 @@ -208,12 +62,6 @@ function update_install_packages { # apt upgrade apt-get upgrade -y - # Install OpenSSH and other packages - add-apt-repository --yes universe - if ! apt_update_with_fallback; then - echo "FATAL: Failed to update package lists after adding universe repository" - exit 1 - fi if ! apt_install_with_fallback install -y --no-install-recommends \ openssh-server \ git \ diff --git a/ebssurrogate/scripts/surrogate-bootstrap-nix.sh b/ebssurrogate/scripts/surrogate-bootstrap-nix.sh index a24072b35..192b6a73b 100755 --- a/ebssurrogate/scripts/surrogate-bootstrap-nix.sh +++ b/ebssurrogate/scripts/surrogate-bootstrap-nix.sh @@ -12,87 +12,40 @@ set -o xtrace exec 1>&2 -# Mirror fallback function for resilient apt-get update -function apt_update_with_fallback { - local sources_file=/etc/apt/sources.list - local max_attempts=2 - local attempt=1 +function setup_apt_sources { + # This function assumes deb822 formatted sources are in use, which is the case in both qemu and aws images + # In aws cloud-init creates a sources file with regional mirrors for "default" Suites but keeps ubuntu for security suite + # So we grab the first (only) URI from security and append it to non-security's URI + # This ends up giving us fastest mirror for installs and falls back to ubuntu if there's an issue + # + # Note: Ubuntu amd64 mirrors have different hostnames for security vs non but aarch64 are the same, hence the amd64 specific line - # Get EC2 region if not already set - if [[ -z $REGION ]]; then - REGION=$(curl --silent --fail http://169.254.169.254/latest/meta-data/placement/availability-zone | sed -E 's|[a-z]+$||g' || echo "") - fi + # ensure deb822 format sources are in use + tail -n+1 /etc/apt/sources.list /etc/apt/sources.list.d/ubuntu.sources >&2 - # Define mirror tiers (in priority order) - local -a mirror_tiers=() + local sources defmirror ubumirror + sources=$(grep -e '^URIs\s*:' -e '^Suites\s*:' /etc/apt/sources.list.d/ubuntu.sources) + defmirror=$(grep -B1 "$CODENAME-updates" <<<"$sources" | awk '/URIs/ {print $2}') + ubumirror=$(grep -B1 "$CODENAME-security" <<<"$sources" | awk '/URIs/ {print $2}') if [[ $ARCH == amd64 ]]; then - if [[ -n $REGION ]]; then - mirror_tiers+=("$REGION.ec2.archive.ubuntu.com") - fi - mirror_tiers+=("archive.ubuntu.com") - else - if [[ -n $REGION ]]; then - mirror_tiers+=("$REGION.clouds.ports.ubuntu.com") - fi - mirror_tiers+=("ports.ubuntu.com") + # amd64 hosts use security.ubuntu.com for security but archive.ubuntu.com for everything else + ubumirror=${ubumirror/security/archive} fi - # If we couldn't get REGION, skip tier 1 - if [[ -z $REGION ]]; then - echo "Warning: Could not determine EC2 region, skipping regional mirror" - mirror_tiers=("${mirror_tiers[@]:1}") # Remove first element + if [[ $ubumirror == "$defmirror" ]]; then + # Only using one mirror so nothing to add as fallback, not running in AWS maybe? + return fi - for mirror in "${mirror_tiers[@]}"; do - echo "=========================================" - echo "Attempting apt-get update with mirror: $mirror" - echo "Attempt $attempt of $max_attempts" - echo "=========================================" - - # Update sources.list to use current mirror - if [[ $ARCH == amd64 ]]; then - sed -i "s|http://[^/]*/ubuntu/|http://$mirror/ubuntu/|g" "$sources_file" - else - sed -i "s|http://[^/]*/ubuntu-ports/|http://$mirror/ubuntu-ports/|g" "$sources_file" - sed -i "s|http://ports.ubuntu.com/ubuntu-ports|http://$mirror/ubuntu-ports|g" "$sources_file" - fi - - # Show what we're using - echo "Current sources.list configuration:" - grep -E '^deb ' "$sources_file" | head -3 - - # Attempt update with timeout (5 minutes) - if timeout 300 apt-get update 2>&1; then - echo "=========================================" - echo "✓ Successfully updated apt cache using mirror: $mirror" - echo "=========================================" - return 0 - else - local exit_code=$? - echo "=========================================" - echo "✗ Failed to update using mirror: $mirror" - echo "Exit code: $exit_code" - echo "=========================================" - - # Clean partial downloads - apt-get clean - rm -rf /var/lib/apt/lists/* - - # Exponential backoff before next attempt - if ((attempt < max_attempts)); then - local sleep_time=$((attempt * 5)) - echo "Waiting $sleep_time seconds before trying next mirror..." - sleep $sleep_time - fi - fi - - attempt=$((attempt + 1)) - done + if grep -q "^URIs:.*$defmirror.*$ubumirror" /etc/apt/sources.list.d/ubuntu.sources; then + echo "Ubuntu upstream is already a fallback, this is unexpected and needs source changes" >&2 + exit 1 + fi + sed -i "s|$defmirror|& $ubumirror|" /etc/apt/sources.list.d/ubuntu.sources +} - echo "=========================================" - echo "ERROR: All mirror tiers failed after $max_attempts attempts" - echo "=========================================" - return 1 +function apt_update_with_fallback { + timeout 300 apt-get update 2>&1 } function waitfor_boot_finished { @@ -226,9 +179,6 @@ function create_fstab { } function setup_chroot_environment { - local UBUNTU_VERSION - UBUNTU_VERSION=$(lsb_release -cs) # 'noble' for Ubuntu 24.04 - # sometimes debootstrap will get stuck on a download for a long time # the default read timeout in wget is 900s, which can cause a ~15min increase in build time # this forces the process to fail-fast and retry @@ -238,19 +188,14 @@ function setup_chroot_environment { tries = 5 EOF - # Update ec2-region - local REGION - REGION=$(curl --silent --fail http://169.254.169.254/latest/meta-data/placement/availability-zone | sed -E 's|[a-z]+$||g') - - # Bootstrap Ubuntu into /mnt using the regional mirror (avoids global mirror stalls) - if [[ $ARCH == amd64 ]]; then - debootstrap --arch "$ARCH" --variant=minbase "$UBUNTU_VERSION" /mnt "http://$REGION.ec2.archive.ubuntu.com/ubuntu" - else - debootstrap --arch "$ARCH" --variant=minbase "$UBUNTU_VERSION" /mnt "http://$REGION.clouds.ports.ubuntu.com/ubuntu-ports" - fi + # Use the preferred mirror (if multiple), which is the first URI/preferred + local mirror + mirror=$(awk '/^URIs:/{uri=$2} /^Suites:.*\<'"$CODENAME-updates"'\>/{print uri; exit}' /etc/apt/sources.list.d/ubuntu.sources) + debootstrap --arch "$ARCH" --variant=minbase "$CODENAME" /mnt "$mirror" - sed -i "s/REGION/$REGION/g" /tmp/sources.list - cp /tmp/sources.list /mnt/etc/apt/sources.list + # Copy our files in since they are updated with all the mirrors! + cp -a /etc/apt/sources.list /mnt/etc/apt/sources.list + cp -a /etc/apt/sources.list.d/ubuntu.sources /mnt/etc/apt/sources.list.d/ubuntu.sources create_fstab @@ -399,12 +344,15 @@ function umount_reset_mappings { done } -export DEBIAN_FRONTEND=noninteractive - ARCH=$(dpkg --print-architecture) : "${ARCH:?Failed to detect architecture}" +# shellcheck source=/dev/null +CODENAME=$(source /etc/os-release && echo "$VERSION_CODENAME") +: "${CODENAME:?Failed to detect OS codename}" +export DEBIAN_FRONTEND=noninteractive waitfor_boot_finished +setup_apt_sources install_packages device_partition_mappings format_and_mount_rootfs From 72206a7c85aadd8866753ffe7dfe2560518f97ac Mon Sep 17 00:00:00 2001 From: Manuel Mendez Date: Thu, 30 Jul 2026 14:01:27 -0400 Subject: [PATCH 3/8] packer: Minimize and improve apt-get calls We are wasting a bunch of time and brain power keeping track of package installs, so lets minimize them while keeping the grouping/messages intact. I did move the package installs from setup_apparmor && setup_grub into update_install_packages because why not just install everything in one place and leave the _setup_ to somewhere else. I moved them out of because package installation should all be done early and ASAP IMO so it can fail fast if its going to fail. Also I moved ec2-hibinit-agent, ec2-instance-connect, hibagent here since this is only run for AMIs and the comment isn't currently accurate. Either through `export DEBIAN_FRONTEND=noninteractive`, `disable_services` or maybe it was treated as a bug and is now fixed in the package I'm not sure but I ran this out of AWS and it was fine. --- ansible/playbook.yml | 11 -- ebssurrogate/scripts/chroot-bootstrap-nix.sh | 101 ++++++++---------- ebssurrogate/scripts/nix-provision.sh | 20 +--- ebssurrogate/scripts/qemu-bootstrap-nix.sh | 31 +++--- .../scripts/surrogate-bootstrap-nix.sh | 24 ++--- 5 files changed, 72 insertions(+), 115 deletions(-) diff --git a/ansible/playbook.yml b/ansible/playbook.yml index 5202fe987..bc32aafea 100644 --- a/ansible/playbook.yml +++ b/ansible/playbook.yml @@ -141,17 +141,6 @@ when: not stage2 import_tasks: tasks/setup-fail2ban.yml - # Install EC2 instance connect - # Only for AWS images - - name: install EC2 instance connect - when: not qemu - become: yes - apt: - pkg: - - ec2-instance-connect - tags: - - aws-only - # Install this at the end to prevent it from kicking in during the apt process, causing conflicts - name: Install security tools become: yes diff --git a/ebssurrogate/scripts/chroot-bootstrap-nix.sh b/ebssurrogate/scripts/chroot-bootstrap-nix.sh index 862f8f190..7ed28626c 100755 --- a/ebssurrogate/scripts/chroot-bootstrap-nix.sh +++ b/ebssurrogate/scripts/chroot-bootstrap-nix.sh @@ -28,33 +28,45 @@ function update_install_packages { fi apt-get "${APT_OPTIONS[@]}" --yes dist-upgrade - # Do not configure grub during package install + local packages=( + e2fsprogs + initramfs-tools + linux-aws + ) + # Install various packages needed for a booting system (with mirror fallback) if [[ $ARCH == amd64 ]]; then + # Do not configure grub during package install echo 'grub-pc grub-pc/install_devices_empty select true' | debconf-set-selections echo 'grub-pc grub-pc/install_devices select' | debconf-set-selections - # Install various packages needed for a booting system (with mirror fallback) - if ! apt_install_with_fallback install -y linux-aws grub-pc e2fsprogs; then - echo "FATAL: Failed to install boot packages" - exit 1 - fi + packages+=( + grub-pc + ) else - if ! apt_install_with_fallback install -y e2fsprogs; then - echo "FATAL: Failed to install e2fsprogs" - exit 1 - fi + packages+=( + cloud-guest-utils + dosfstools + efibootmgr + fdisk + grub-efi-arm64 + ) fi - # Install standard packages (with mirror fallback) - # Note: ec2-hibinit-agent, ec2-instance-connect, hibagent moved to stage 2 - # because their post-install scripts try to access EC2 metadata service - # which doesn't work in a chroot and causes long hangs - if ! apt_install_with_fallback install -y \ - bzip2 \ - sudo \ - wget \ - cloud-init \ - acpid \ - ncurses-term \ - ssh-import-id; then + + packages+=( + acpid + apparmor + apparmor-utils + auditd + bzip2 + cloud-init + ec2-hibinit-agent + ec2-instance-connect + hibagent + ncurses-term + ssh-import-id + sudo + wget + ) + if ! apt_install_with_fallback install -y "${packages[@]}"; then echo "FATAL: Failed to install standard packages" exit 1 fi @@ -62,32 +74,22 @@ function update_install_packages { # apt upgrade apt-get upgrade -y - if ! apt_install_with_fallback install -y --no-install-recommends \ - openssh-server \ - git \ - ufw \ - cron \ - logrotate \ - fail2ban \ - locales \ - at \ - less \ - python3-systemd; then + packages=( + at + cron + fail2ban + git + less + locales + logrotate + openssh-server + python3-systemd + ufw + ) + if ! apt_install_with_fallback install -y --no-install-recommends "${packages[@]}"; then echo "FATAL: Failed to install universe packages" exit 1 fi - - if [[ $ARCH == arm64 ]]; then - if ! apt_install_with_fallback "${APT_OPTIONS[@]}" --yes install linux-aws initramfs-tools dosfstools; then - echo "FATAL: Failed to install arm64 boot packages" - exit 1 - fi - else - if ! apt_install_with_fallback "${APT_OPTIONS[@]}" --yes install initramfs-tools; then - echo "FATAL: Failed to install amd64 boot packages" - exit 1 - fi - fi } function setup_locale { @@ -116,11 +118,6 @@ function setup_postgesql_env { } function setup_apparmor { - if ! apt_install_with_fallback install -y apparmor apparmor-utils auditd; then - echo "FATAL: Failed to install apparmor packages" - exit 1 - fi - # Copy apparmor profiles cp -rv /tmp/apparmor_profiles/* /etc/apparmor.d/ } @@ -138,10 +135,6 @@ function setup_grub { EOF if [[ $ARCH == arm64 ]]; then - if ! apt_install_with_fallback "${APT_OPTIONS[@]}" --yes install cloud-guest-utils fdisk grub-efi-arm64 efibootmgr; then - echo "FATAL: Failed to install grub packages for arm64" - exit 1 - fi rm -rf /etc/grub.d/30_os-prober sleep 1 fi diff --git a/ebssurrogate/scripts/nix-provision.sh b/ebssurrogate/scripts/nix-provision.sh index 3b576dc17..e9def7538 100755 --- a/ebssurrogate/scripts/nix-provision.sh +++ b/ebssurrogate/scripts/nix-provision.sh @@ -8,23 +8,8 @@ set -o xtrace exec 1>&2 function install_packages { - # Setup Ansible on host VM - apt-get update && apt-get install -y software-properties-common - - # Install EC2-specific packages that were deferred from stage 1 - # These packages have post-install scripts that need EC2 metadata service access - # which only works on a real running EC2 instance (not in chroot) - apt-get install -y ec2-hibinit-agent ec2-instance-connect hibagent - - # Manually add GPG key with explicit keyserver - apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 93C4A3FD7BB9C367 - - # Add repository and install - # TODO (darora): temporarily disabling while Launchpad is under ddos attack and very frequently timing out - # sudo add-apt-repository --yes ppa:ansible/ansible - # sudo apt-get update + apt-get update -y apt-get install -y ansible - ansible-galaxy collection install community.general } @@ -68,8 +53,7 @@ function execute_stage2_playbook { } function cleanup_packages { - apt-get -y remove --purge ansible - # sudo add-apt-repository --yes --remove ppa:ansible/ansible + apt-get remove --purge --yes ansible } function report_disk_usage { diff --git a/ebssurrogate/scripts/qemu-bootstrap-nix.sh b/ebssurrogate/scripts/qemu-bootstrap-nix.sh index 68619f3a3..f243864d1 100755 --- a/ebssurrogate/scripts/qemu-bootstrap-nix.sh +++ b/ebssurrogate/scripts/qemu-bootstrap-nix.sh @@ -18,22 +18,21 @@ function waitfor_boot_finished { function install_packages { apt-get update - apt-get install -y \ - arptables \ - e2fsprogs \ - ebtables \ - gpg \ - iptables \ - less \ - locales \ - logrotate \ - nfs-common \ - software-properties-common \ - ufw \ - ; - # TODO (darora): temporarily disabling while Launchpad is under ddos attack and very frequently timing out - # add-apt-repository --yes --update ppa:ansible/ansible && - apt-get install ansible -y + packages=( + ansible + arptables + e2fsprogs + ebtables + gpg + iptables + less + locales + logrotate + nfs-common + software-properties-common + ufw + ) + apt-get install -y "${packages[@]}" ansible-galaxy collection install community.general } diff --git a/ebssurrogate/scripts/surrogate-bootstrap-nix.sh b/ebssurrogate/scripts/surrogate-bootstrap-nix.sh index 192b6a73b..d4370a38c 100755 --- a/ebssurrogate/scripts/surrogate-bootstrap-nix.sh +++ b/ebssurrogate/scripts/surrogate-bootstrap-nix.sh @@ -62,23 +62,15 @@ function install_packages { exit 1 fi - apt-get install software-properties-common -y - # TODO (darora): temporarily disabling while Launchpad is under ddos attack and very frequently timing out - # add-apt-repository --yes --update ppa:ansible/ansible - - if ! apt_update_with_fallback; then - echo "FATAL: Failed to update package lists after adding Ansible PPA" - exit 1 - fi - - apt-get install ansible -y - ansible-galaxy collection install community.general - - apt-get install -y \ - gdisk \ - e2fsprogs \ - debootstrap \ + packages=( + ansible + debootstrap + e2fsprogs + gdisk nvme-cli + ) + apt-get install --yes "${packages[@]}" + ansible-galaxy collection install community.general } # Partition the new root EBS volume From 6018b6413b27b6f34b14fb280cc8a483d57f0ce3 Mon Sep 17 00:00:00 2001 From: Manuel Mendez Date: Wed, 29 Jul 2026 17:23:10 -0400 Subject: [PATCH 4/8] packer: Refactor apt usage Lets be a little cleaner about setup/update/cleanup, also drop the wrapper functions to plain old apt-get calls since they no longer do anything useful. I created setup_apt because I want to move the Install-Recommends=False into it later and configure all apt calls to use it seamlessly, this way we will have a better/more visible list of packages being installed. --- ebssurrogate/scripts/chroot-bootstrap-nix.sh | 52 ++++++------------- ebssurrogate/scripts/nix-provision.sh | 18 ++++++- ebssurrogate/scripts/qemu-bootstrap-nix.sh | 42 ++++++++++++--- .../scripts/surrogate-bootstrap-nix.sh | 18 +++---- 4 files changed, 76 insertions(+), 54 deletions(-) diff --git a/ebssurrogate/scripts/chroot-bootstrap-nix.sh b/ebssurrogate/scripts/chroot-bootstrap-nix.sh index 7ed28626c..354e7585b 100755 --- a/ebssurrogate/scripts/chroot-bootstrap-nix.sh +++ b/ebssurrogate/scripts/chroot-bootstrap-nix.sh @@ -8,26 +8,21 @@ set -o errexit set -o pipefail set -o xtrace -# The following 2 functions don'treally do much since we are now using deb822 formatted sources with fallbacks in the URIs. -# This means apt-get handles fallback on its own, much better and cleaner than we are doing. -# Leaving the functions as is for now to make the diff smaller, soon will go away. -function apt_update_with_fallback { - timeout 300 apt-get "${APT_OPTIONS[@]}" update 2>&1 +function setup_apt { + export DEBIAN_FRONTEND=noninteractive } -function apt_install_with_fallback { - apt-get "$@" +function cleanup_apt { + apt-get clean + apt-get autoremove --purge --yes } -function update_install_packages { - # Update APT with new sources (using fallback mechanism) - tail -n+1 /etc/apt/sources.list /etc/apt/sources.list.d/* - if ! apt_update_with_fallback; then - echo "FATAL: Failed to update package lists with any mirror tier" - exit 1 - fi - apt-get "${APT_OPTIONS[@]}" --yes dist-upgrade +function update_and_upgrade_apt { + apt-get update --yes + apt-get dist-upgrade --yes +} +function install_initial_packages { local packages=( e2fsprogs initramfs-tools @@ -66,14 +61,11 @@ function update_install_packages { sudo wget ) - if ! apt_install_with_fallback install -y "${packages[@]}"; then + if ! apt-get install --yes "${packages[@]}"; then echo "FATAL: Failed to install standard packages" exit 1 fi - # apt upgrade - apt-get upgrade -y - packages=( at cron @@ -86,8 +78,8 @@ function update_install_packages { python3-systemd ufw ) - if ! apt_install_with_fallback install -y --no-install-recommends "${packages[@]}"; then - echo "FATAL: Failed to install universe packages" + if ! apt-get install --yes --no-install-recommends "${packages[@]}"; then + echo "FATAL: Failed to install extra packages" exit 1 fi } @@ -205,28 +197,18 @@ function disable_services { chmod +x /usr/sbin/policy-rc.d } -# Clear apt caches -function cleanup_cache { - apt-get clean -} - # Remove policy-rc.d so services start normally on boot function enable_services { rm -f /usr/sbin/policy-rc.d } -export DEBIAN_FRONTEND=noninteractive -APT_OPTIONS=( - -oAPT::Install-Recommends=false - -oAPT::Install-Suggests=false - -oAcquire::Languages=none -) - ARCH=$(dpkg --print-architecture) : "${ARCH:?Failed to detect architecture}" disable_services -update_install_packages +setup_apt +update_and_upgrade_apt +install_initial_packages setup_locale setup_postgesql_env setup_grub @@ -237,5 +219,5 @@ set_default_target setup_eth0_interface disable_sshd_passwd_auth disable_fsck -cleanup_cache +cleanup_apt enable_services diff --git a/ebssurrogate/scripts/nix-provision.sh b/ebssurrogate/scripts/nix-provision.sh index e9def7538..8dbd54024 100755 --- a/ebssurrogate/scripts/nix-provision.sh +++ b/ebssurrogate/scripts/nix-provision.sh @@ -7,8 +7,21 @@ set -o xtrace exec 1>&2 +function setup_apt { + export DEBIAN_FRONTEND=noninteractive +} + +function cleanup_apt { + apt-get clean + apt-get autoremove --purge --yes + rm -rf /var/lib/apt/lists/* +} + +function update_apt { + apt-get update --yes +} + function install_packages { - apt-get update -y apt-get install -y ansible ansible-galaxy collection install community.general } @@ -62,8 +75,11 @@ function report_disk_usage { printf '::notice::disk_usage bytes=%s human=%s\n' "$dub" "$duh" | tee -a /tmp/ansible.log } +setup_apt +update_apt install_packages install_nix execute_stage2_playbook cleanup_packages +cleanup_apt report_disk_usage diff --git a/ebssurrogate/scripts/qemu-bootstrap-nix.sh b/ebssurrogate/scripts/qemu-bootstrap-nix.sh index f243864d1..1610fb758 100755 --- a/ebssurrogate/scripts/qemu-bootstrap-nix.sh +++ b/ebssurrogate/scripts/qemu-bootstrap-nix.sh @@ -8,6 +8,36 @@ set -o xtrace # stage1 things # ################# +function setup_apt { + export DEBIAN_FRONTEND=noninteractive + + cat /etc/apt/sources.list.d/ubuntu.sources >&2 + local sources defmirror ubumirror + sources=$(grep -e '^URIs\s*:' -e '^Suites\s*:' /etc/apt/sources.list.d/ubuntu.sources) + defmirror=$(grep -B1 noble-updates <<<"$sources" | awk '/URIs/ {print $2}') + ubumirror=$(grep -B1 noble-security <<<"$sources" | awk '/URIs/ {print $2}') + if [[ $ARCH == x86_64 ]]; then + # x86_64 hosts use security.ubuntu.com for security but archive.ubuntu.com for everything else + ubumirror=${ubumirror/security/archive} + fi + + if grep -q "^URIs:.*$defmirror.*$ubumirror" /etc/apt/sources.list.d/ubuntu.sources; then + echo "Ubuntu upstream is already a fallback, this is unexpected and needs source changes" >&2 + exit 1 + fi + sed -i "s|$defmirror|& $ubumirror|" /etc/apt/sources.list.d/ubuntu.sources +} + +function cleanup_apt { + apt-get clean + apt-get autoremove --purge --yes + rm -rf /var/lib/apt/lists/* +} + +function update_apt { + apt-get update --yes +} + function waitfor_boot_finished { # Wait for cloudinit on the surrogate to complete before making progress while [[ ! -f /var/lib/cloud/instance/boot-finished ]]; do @@ -17,7 +47,6 @@ function waitfor_boot_finished { } function install_packages { - apt-get update packages=( ansible arptables @@ -32,7 +61,7 @@ function install_packages { software-properties-common ufw ) - apt-get install -y "${packages[@]}" + apt-get install --yes "${packages[@]}" ansible-galaxy collection install community.general } @@ -121,8 +150,7 @@ function execute_stage2_playbook { function clean_legacy_things { # removes things that are bundled for legacy reasons, but we can start without for our newer artifacts apt-mark auto zlib1g* # TODO (darora): need to make sure that there aren't other things that still need this - apt-get -y purge kong - apt-get autoremove -y + apt-get purge --yes kong } function clean_system { @@ -162,7 +190,6 @@ function clean_system { mkdir /var/log/audit # unwanted files - rm -rf /var/lib/apt/lists/* rm -rf /root/.cache rm -rf /root/.vpython* rm -rf /root/go @@ -189,8 +216,8 @@ function clean_system { # stage1 things # ################# -export DEBIAN_FRONTEND=noninteractive - +setup_apt +update_apt waitfor_boot_finished install_packages setup_postgesql_env @@ -205,3 +232,4 @@ install_nix execute_stage2_playbook clean_legacy_things clean_system +cleanup_apt diff --git a/ebssurrogate/scripts/surrogate-bootstrap-nix.sh b/ebssurrogate/scripts/surrogate-bootstrap-nix.sh index d4370a38c..1905b3f3e 100755 --- a/ebssurrogate/scripts/surrogate-bootstrap-nix.sh +++ b/ebssurrogate/scripts/surrogate-bootstrap-nix.sh @@ -12,7 +12,9 @@ set -o xtrace exec 1>&2 -function setup_apt_sources { +function setup_apt { + export DEBIAN_FRONTEND=noninteractive + # This function assumes deb822 formatted sources are in use, which is the case in both qemu and aws images # In aws cloud-init creates a sources file with regional mirrors for "default" Suites but keeps ubuntu for security suite # So we grab the first (only) URI from security and append it to non-security's URI @@ -44,8 +46,8 @@ function setup_apt_sources { sed -i "s|$defmirror|& $ubumirror|" /etc/apt/sources.list.d/ubuntu.sources } -function apt_update_with_fallback { - timeout 300 apt-get update 2>&1 +function update_apt { + apt-get update --yes } function waitfor_boot_finished { @@ -56,12 +58,6 @@ function waitfor_boot_finished { } function install_packages { - # Setup Ansible on host VM - if ! apt_update_with_fallback; then - echo "FATAL: Failed to update package lists on host VM" - exit 1 - fi - packages=( ansible debootstrap @@ -341,10 +337,10 @@ ARCH=$(dpkg --print-architecture) # shellcheck source=/dev/null CODENAME=$(source /etc/os-release && echo "$VERSION_CODENAME") : "${CODENAME:?Failed to detect OS codename}" -export DEBIAN_FRONTEND=noninteractive waitfor_boot_finished -setup_apt_sources +setup_apt +update_apt install_packages device_partition_mappings format_and_mount_rootfs From eb040b2fdf39c42b9a763d366564d61c1d8e7692 Mon Sep 17 00:00:00 2001 From: Manuel Mendez Date: Thu, 30 Jul 2026 16:01:54 -0400 Subject: [PATCH 5/8] packer: Remove dead code from cleanup scripts No yum handling necessary and commented out code should be deleted. --- ebssurrogate/scripts/cleanup-qemu.sh | 92 +++++++++++----------------- ebssurrogate/scripts/cleanup.sh | 50 +++++++-------- 2 files changed, 59 insertions(+), 83 deletions(-) diff --git a/ebssurrogate/scripts/cleanup-qemu.sh b/ebssurrogate/scripts/cleanup-qemu.sh index e79cc445d..5876c7705 100755 --- a/ebssurrogate/scripts/cleanup-qemu.sh +++ b/ebssurrogate/scripts/cleanup-qemu.sh @@ -14,64 +14,44 @@ if [[ ! -d /tmp ]]; then fi chmod 1777 /tmp -if [ -n "$(command -v yum)" ]; then - yum update -y - yum clean all -elif [ -n "$(command -v apt-get)" ]; then - # Cleanup more packages - apt-get -y remove --purge \ - automake \ - autoconf \ - autotools-dev \ - cmake-data \ - cpp-9 \ - cpp-10 \ - gcc-9 \ - gcc-10 \ - git \ - git-man \ - ansible \ - libicu-dev \ - libcgal-dev \ - libgcc-9-dev \ - ansible \ - snapd - - if [[ $(uname -m) == aarch64 ]]; then - apt-get -y remove --purge libgcc-8-dev - fi - - # add-apt-repository --yes --remove ppa:ansible/ansible - - apt-mark manual libevent-2.1-7t64 - - apt-get remove -y --purge ansible-core apport appstream bash-completion bcache-tools bind9-dnsutils bind9-host bind9-libs bolt btrfs-progs byobu command-not-found console-setup distro-info eject fonts-ubuntu-console friendly-recovery ftp fwupd gawk gdisk keyboard-configuration libvolume-key1 libssl-dev lvm2 lxd-agent-loader man-db mdadm modemmanager mtd-utils nano netcat-openbsd nfs-common ntfs-3g parted pastebinit screen strace thin-provisioning-tools tmux usb-modeswitch vim vim-runtime wget whiptail xfsprogs - - apt remove -y --purge libc6-dev linux-libc-dev libevent-dev libpcre3-dev libsystemd-dev packagekit multipath-tools unattended-upgrades plymouth gnupg open-vm-tools xauth lxd-installer publicsuffix libclang-cpp18 python3-twisted python-babel-localedata libicu74 python3-pygments fonts-dejavu* python3-botocore - - apt-get remove -y --purge linux-headers* - - # remove old kernels - # CURRENT_KERNEL="$(uname -r | sed 's/-generic//')" - # INSTALLED_KERNELS=$(dpkg -l | awk '{print $2}' | grep -Eo 'linux-(image|headers|modules|tools)-[0-9]+' | sed -E 's/linux-(image|modules|tools)-//' | sort -Vu) - # REMOVE_KERNELS=$(echo "$INSTALLED_KERNELS" | grep -v -e "$CURRENT_KERNEL") - # for VER in $REMOVE_KERNELS; do - # for PREFIX in linux-image linux-modules linux-tools; do - # for PKG in $(dpkg -l | awk '{print $2}' | grep "^$PREFIX-$VER"); do - # apt-get purge -y "$PKG" - # done - # done - # done - # update-grub - - apt-get -y autoremove - apt-get -y autoclean - - apt-get -y update - apt-get -y upgrade - +# Cleanup more packages +apt-get -y remove --purge \ + automake \ + autoconf \ + autotools-dev \ + cmake-data \ + cpp-9 \ + cpp-10 \ + gcc-9 \ + gcc-10 \ + git \ + git-man \ + ansible \ + libicu-dev \ + libcgal-dev \ + libgcc-9-dev \ + ansible \ + snapd + +if [[ $(uname -m) == aarch64 ]]; then + apt-get -y remove --purge libgcc-8-dev fi +# add-apt-repository --yes --remove ppa:ansible/ansible + +apt-mark manual libevent-2.1-7t64 + +apt-get remove -y --purge ansible-core apport appstream bash-completion bcache-tools bind9-dnsutils bind9-host bind9-libs bolt btrfs-progs byobu command-not-found console-setup distro-info eject fonts-ubuntu-console friendly-recovery ftp fwupd gawk gdisk keyboard-configuration libvolume-key1 libssl-dev lvm2 lxd-agent-loader man-db mdadm modemmanager mtd-utils nano netcat-openbsd nfs-common ntfs-3g parted pastebinit screen strace thin-provisioning-tools tmux usb-modeswitch vim vim-runtime wget whiptail xfsprogs + +apt remove -y --purge libc6-dev linux-libc-dev libevent-dev libpcre3-dev libsystemd-dev packagekit multipath-tools unattended-upgrades plymouth gnupg open-vm-tools xauth lxd-installer publicsuffix libclang-cpp18 python3-twisted python-babel-localedata libicu74 python3-pygments fonts-dejavu* python3-botocore + +apt-get remove -y --purge linux-headers* + +apt-get -y autoremove +apt-get -y autoclean +apt-get -y update +apt-get -y upgrade + systemctl set-default multi-user.target systemctl disable getty@tty1.service systemctl mask getty@tty1.service diff --git a/ebssurrogate/scripts/cleanup.sh b/ebssurrogate/scripts/cleanup.sh index 8eab36881..7ca276ab9 100755 --- a/ebssurrogate/scripts/cleanup.sh +++ b/ebssurrogate/scripts/cleanup.sh @@ -14,35 +14,31 @@ if [[ ! -d /tmp ]]; then fi chmod 1777 /tmp -if [ -n "$(command -v yum)" ]; then - yum update -y - yum clean all -elif [ -n "$(command -v apt-get)" ]; then - # Cleanup more packages - apt-get -y remove --purge \ - automake \ - autoconf \ - autotools-dev \ - cmake-data \ - cpp-9 \ - cpp-10 \ - gcc-9 \ - gcc-10 \ - git \ - git-man \ - ansible \ - libicu-dev \ - libcgal-dev \ - libgcc-9-dev \ - ansible +# Cleanup more packages +apt-get -y remove --purge \ + automake \ + autoconf \ + autotools-dev \ + cmake-data \ + cpp-9 \ + cpp-10 \ + gcc-9 \ + gcc-10 \ + git \ + git-man \ + ansible \ + libicu-dev \ + libcgal-dev \ + libgcc-9-dev \ + ansible - # add-apt-repository --yes --remove ppa:ansible/ansible +# add-apt-repository --yes --remove ppa:ansible/ansible + +apt-get -y update +apt-get -y upgrade +apt-get -y autoremove +apt-get -y autoclean - apt-get -y update - apt-get -y upgrade - apt-get -y autoremove - apt-get -y autoclean -fi rm -rf /tmp/* /var/tmp/* history -c cat /dev/null >/root/.bash_history From 6d4a18cfb4cc7e557c1ffe3b2b018dd603b45ca1 Mon Sep 17 00:00:00 2001 From: Manuel Mendez Date: Thu, 30 Jul 2026 16:00:52 -0400 Subject: [PATCH 6/8] packer: Drop all but one apt-get remove invocation from cleanup scripts One is enough. --- ebssurrogate/scripts/cleanup-qemu.sh | 120 ++++++++++++++++++++------- ebssurrogate/scripts/cleanup.sh | 42 +++++----- 2 files changed, 111 insertions(+), 51 deletions(-) diff --git a/ebssurrogate/scripts/cleanup-qemu.sh b/ebssurrogate/scripts/cleanup-qemu.sh index 5876c7705..7179e0dad 100755 --- a/ebssurrogate/scripts/cleanup-qemu.sh +++ b/ebssurrogate/scripts/cleanup-qemu.sh @@ -15,42 +15,102 @@ fi chmod 1777 /tmp # Cleanup more packages -apt-get -y remove --purge \ - automake \ - autoconf \ - autotools-dev \ - cmake-data \ - cpp-9 \ - cpp-10 \ - gcc-9 \ - gcc-10 \ - git \ - git-man \ - ansible \ - libicu-dev \ - libcgal-dev \ - libgcc-9-dev \ - ansible \ +packages=( + ansible + ansible-core + apport + appstream + autoconf + automake + autotools-dev + bash-completion + bcache-tools + bind9-dnsutils + bind9-host + bind9-libs + bolt + btrfs-progs + byobu + cmake-data + command-not-found + console-setup + cpp-10 + cpp-9 + distro-info + eject + fonts-dejavu* + fonts-ubuntu-console + friendly-recovery + ftp + fwupd + gawk + gcc-10 + gcc-9 + gdisk + git + git-man + gnupg + keyboard-configuration + libc6-dev + libcgal-dev + libclang-cpp18 + libevent-dev + libgcc-9-dev + libicu-dev + libicu74 + libpcre3-dev + libssl-dev + libsystemd-dev + libvolume-key1 + linux-headers* + linux-libc-dev + lvm2 + lxd-agent-loader + lxd-installer + man-db + mdadm + modemmanager + mtd-utils + multipath-tools + nano + netcat-openbsd + nfs-common + ntfs-3g + open-vm-tools + packagekit + parted + pastebinit + plymouth + publicsuffix + python-babel-localedata + python3-botocore + python3-pygments + python3-twisted + screen snapd + strace + thin-provisioning-tools + tmux + unattended-upgrades + usb-modeswitch + vim + vim-runtime + wget + whiptail + xauth + xfsprogs +) if [[ $(uname -m) == aarch64 ]]; then - apt-get -y remove --purge libgcc-8-dev + packages+=(libgcc-8-dev) fi -# add-apt-repository --yes --remove ppa:ansible/ansible - apt-mark manual libevent-2.1-7t64 - -apt-get remove -y --purge ansible-core apport appstream bash-completion bcache-tools bind9-dnsutils bind9-host bind9-libs bolt btrfs-progs byobu command-not-found console-setup distro-info eject fonts-ubuntu-console friendly-recovery ftp fwupd gawk gdisk keyboard-configuration libvolume-key1 libssl-dev lvm2 lxd-agent-loader man-db mdadm modemmanager mtd-utils nano netcat-openbsd nfs-common ntfs-3g parted pastebinit screen strace thin-provisioning-tools tmux usb-modeswitch vim vim-runtime wget whiptail xfsprogs - -apt remove -y --purge libc6-dev linux-libc-dev libevent-dev libpcre3-dev libsystemd-dev packagekit multipath-tools unattended-upgrades plymouth gnupg open-vm-tools xauth lxd-installer publicsuffix libclang-cpp18 python3-twisted python-babel-localedata libicu74 python3-pygments fonts-dejavu* python3-botocore - -apt-get remove -y --purge linux-headers* - -apt-get -y autoremove -apt-get -y autoclean -apt-get -y update -apt-get -y upgrade +apt-get --yes remove --purge "${packages[@]}" +apt-get --yes autoremove +apt-get --yes autoclean +apt-get --yes update +apt-get --yes upgrade systemctl set-default multi-user.target systemctl disable getty@tty1.service diff --git a/ebssurrogate/scripts/cleanup.sh b/ebssurrogate/scripts/cleanup.sh index 7ca276ab9..f9a103fb0 100755 --- a/ebssurrogate/scripts/cleanup.sh +++ b/ebssurrogate/scripts/cleanup.sh @@ -15,29 +15,29 @@ fi chmod 1777 /tmp # Cleanup more packages -apt-get -y remove --purge \ - automake \ - autoconf \ - autotools-dev \ - cmake-data \ - cpp-9 \ - cpp-10 \ - gcc-9 \ - gcc-10 \ - git \ - git-man \ - ansible \ - libicu-dev \ - libcgal-dev \ - libgcc-9-dev \ +packages=( ansible + ansible + autoconf + automake + autotools-dev + cmake-data + cpp-10 + cpp-9 + gcc-10 + gcc-9 + git + git-man + libcgal-dev + libgcc-9-dev + libicu-dev +) -# add-apt-repository --yes --remove ppa:ansible/ansible - -apt-get -y update -apt-get -y upgrade -apt-get -y autoremove -apt-get -y autoclean +apt-get --yes remove --purge "${packages[@]}" +apt-get --yes autoremove +apt-get --yes autoclean +apt-get --yes update +apt-get --yes upgrade rm -rf /tmp/* /var/tmp/* history -c From e122c8aca937bb31045ed9fbe71a9441db32bbff Mon Sep 17 00:00:00 2001 From: Manuel Mendez Date: Wed, 29 Jul 2026 17:23:10 -0400 Subject: [PATCH 7/8] packer: Bookend and otherwise minimize apt-get update/upgrade There's no point run updates/upgrades unnecessarily while also not ensuring that we start with latest and end with latest. So I've dropped any unecessary updates and upgrades from middle of runs. Now we update and upgrade after start up ({surrogate,qemu}-bootstrap-nix, chroot was already good) and then once at the end (qemu-bootstrap-nix and nix-provision). Ansible's apt_repository module does update when necessary and has not changed. --- ansible/playbook.yml | 2 -- ansible/tasks/internal/install-salt.yml | 1 - ansible/tasks/setup-postgrest.yml | 1 - ansible/tasks/setup-system.yml | 8 -------- ansible/tasks/setup-tmux.yml | 1 - ansible/tasks/setup-tuned.yml | 1 - ansible/tasks/test-image.yml | 1 - ebssurrogate/scripts/nix-provision.sh | 6 ++++-- ebssurrogate/scripts/qemu-bootstrap-nix.sh | 5 +++-- ebssurrogate/scripts/surrogate-bootstrap-nix.sh | 5 +++-- 10 files changed, 10 insertions(+), 21 deletions(-) diff --git a/ansible/playbook.yml b/ansible/playbook.yml index bc32aafea..d9bb6e1c8 100644 --- a/ansible/playbook.yml +++ b/ansible/playbook.yml @@ -147,8 +147,6 @@ apt: pkg: - unattended-upgrades - update_cache: yes - cache_valid_time: 3600 - name: Run migrations when: stage2 diff --git a/ansible/tasks/internal/install-salt.yml b/ansible/tasks/internal/install-salt.yml index 8df927ef7..70ee3a797 100644 --- a/ansible/tasks/internal/install-salt.yml +++ b/ansible/tasks/internal/install-salt.yml @@ -29,7 +29,6 @@ - salt-common={{ salt_release }} - salt-minion={{ salt_release }} state: present - update_cache: yes - name: Pin salt packages at major version ansible.builtin.copy: diff --git a/ansible/tasks/setup-postgrest.yml b/ansible/tasks/setup-postgrest.yml index 33afbc2f7..6d217c554 100644 --- a/ansible/tasks/setup-postgrest.yml +++ b/ansible/tasks/setup-postgrest.yml @@ -27,7 +27,6 @@ package: - libnuma-dev - libpq5 - update_cache: true state: 'present' - name: PostgREST - grab the list of installed packages diff --git a/ansible/tasks/setup-system.yml b/ansible/tasks/setup-system.yml index fda2627b1..d23ab03de 100644 --- a/ansible/tasks/setup-system.yml +++ b/ansible/tasks/setup-system.yml @@ -1,15 +1,8 @@ - name: Execute stage1 tasks when: not stage2 block: - - name: System - apt update and apt upgrade - ansible.builtin.apt: - update_cache: true - upgrade: true - # SEE http://archive.vn/DKJjs#parameter-upgrade - - name: Install required security updates Ansible dependencies, security tools, and other useful things ansible.builtin.apt: - cache_valid_time: 3600 pkg: - acl - bwm-ng @@ -23,7 +16,6 @@ - sysstat - tzdata - vim - update_cache: true # SEE https://github.com/georchestra/ansible/issues/55#issuecomment-588313638 # Without this, a similar error is faced diff --git a/ansible/tasks/setup-tmux.yml b/ansible/tasks/setup-tmux.yml index 93b69ff73..658afdda7 100644 --- a/ansible/tasks/setup-tmux.yml +++ b/ansible/tasks/setup-tmux.yml @@ -2,7 +2,6 @@ ansible.builtin.apt: name: 'tmux' state: 'present' - update_cache: true become: true - name: tmux - Configure tmux alias diff --git a/ansible/tasks/setup-tuned.yml b/ansible/tasks/setup-tuned.yml index 1a02becc9..1d4fb16ce 100644 --- a/ansible/tasks/setup-tuned.yml +++ b/ansible/tasks/setup-tuned.yml @@ -6,7 +6,6 @@ name: 'tuned' policy_rc_d: 101 state: 'present' - update_cache: true become: true - name: 'tuned - Create a tuned profile directory' # noqa: name[casing] diff --git a/ansible/tasks/test-image.yml b/ansible/tasks/test-image.yml index 3acd25cef..493eed114 100644 --- a/ansible/tasks/test-image.yml +++ b/ansible/tasks/test-image.yml @@ -65,7 +65,6 @@ ansible.builtin.apt: name: 'python3-psycopg2' state: 'present' - update_cache: true become: true - name: Reset db stats diff --git a/ebssurrogate/scripts/nix-provision.sh b/ebssurrogate/scripts/nix-provision.sh index 8dbd54024..b879b8802 100755 --- a/ebssurrogate/scripts/nix-provision.sh +++ b/ebssurrogate/scripts/nix-provision.sh @@ -17,8 +17,9 @@ function cleanup_apt { rm -rf /var/lib/apt/lists/* } -function update_apt { +function update_and_upgrade_apt { apt-get update --yes + apt-get upgrade --yes } function install_packages { @@ -76,10 +77,11 @@ function report_disk_usage { } setup_apt -update_apt +update_and_upgrade_apt install_packages install_nix execute_stage2_playbook cleanup_packages +update_and_upgrade_apt cleanup_apt report_disk_usage diff --git a/ebssurrogate/scripts/qemu-bootstrap-nix.sh b/ebssurrogate/scripts/qemu-bootstrap-nix.sh index 1610fb758..ad4c27105 100755 --- a/ebssurrogate/scripts/qemu-bootstrap-nix.sh +++ b/ebssurrogate/scripts/qemu-bootstrap-nix.sh @@ -34,8 +34,9 @@ function cleanup_apt { rm -rf /var/lib/apt/lists/* } -function update_apt { +function update_and_upgrade_apt { apt-get update --yes + apt-get upgrade --yes } function waitfor_boot_finished { @@ -217,7 +218,7 @@ function clean_system { ################# setup_apt -update_apt +update_and_upgrade_apt waitfor_boot_finished install_packages setup_postgesql_env diff --git a/ebssurrogate/scripts/surrogate-bootstrap-nix.sh b/ebssurrogate/scripts/surrogate-bootstrap-nix.sh index 1905b3f3e..655c462fb 100755 --- a/ebssurrogate/scripts/surrogate-bootstrap-nix.sh +++ b/ebssurrogate/scripts/surrogate-bootstrap-nix.sh @@ -46,8 +46,9 @@ function setup_apt { sed -i "s|$defmirror|& $ubumirror|" /etc/apt/sources.list.d/ubuntu.sources } -function update_apt { +function update_and_upgrade_apt { apt-get update --yes + apt-get upgrade --yes } function waitfor_boot_finished { @@ -340,7 +341,7 @@ CODENAME=$(source /etc/os-release && echo "$VERSION_CODENAME") waitfor_boot_finished setup_apt -update_apt +update_and_upgrade_apt install_packages device_partition_mappings format_and_mount_rootfs From ebe201305be6af13aae081a79e593103a6d37e3f Mon Sep 17 00:00:00 2001 From: Manuel Mendez Date: Mon, 10 Aug 2026 13:20:50 -0400 Subject: [PATCH 8/8] packer: Use dist-upgrade instead of upgrade dist-upgrade ensures new packages are installed and unnecessary are removed while upgrade only upgrades packages. Using dist-upgrade gets us closer to inteded state according to metadata files. --- ebssurrogate/scripts/nix-provision.sh | 2 +- ebssurrogate/scripts/qemu-bootstrap-nix.sh | 2 +- ebssurrogate/scripts/surrogate-bootstrap-nix.sh | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ebssurrogate/scripts/nix-provision.sh b/ebssurrogate/scripts/nix-provision.sh index b879b8802..89beffb60 100755 --- a/ebssurrogate/scripts/nix-provision.sh +++ b/ebssurrogate/scripts/nix-provision.sh @@ -19,7 +19,7 @@ function cleanup_apt { function update_and_upgrade_apt { apt-get update --yes - apt-get upgrade --yes + apt-get dist-upgrade --yes } function install_packages { diff --git a/ebssurrogate/scripts/qemu-bootstrap-nix.sh b/ebssurrogate/scripts/qemu-bootstrap-nix.sh index ad4c27105..c9f62b558 100755 --- a/ebssurrogate/scripts/qemu-bootstrap-nix.sh +++ b/ebssurrogate/scripts/qemu-bootstrap-nix.sh @@ -36,7 +36,7 @@ function cleanup_apt { function update_and_upgrade_apt { apt-get update --yes - apt-get upgrade --yes + apt-get dist-upgrade --yes } function waitfor_boot_finished { diff --git a/ebssurrogate/scripts/surrogate-bootstrap-nix.sh b/ebssurrogate/scripts/surrogate-bootstrap-nix.sh index 655c462fb..ff91a5b6c 100755 --- a/ebssurrogate/scripts/surrogate-bootstrap-nix.sh +++ b/ebssurrogate/scripts/surrogate-bootstrap-nix.sh @@ -48,7 +48,7 @@ function setup_apt { function update_and_upgrade_apt { apt-get update --yes - apt-get upgrade --yes + apt-get dist-upgrade --yes } function waitfor_boot_finished {