diff --git a/amazon-amd64-nix.pkr.hcl b/amazon-amd64-nix.pkr.hcl index 445462186a..57e36e894f 100644 --- a/amazon-amd64-nix.pkr.hcl +++ b/amazon-amd64-nix.pkr.hcl @@ -170,11 +170,6 @@ source "amazon-ebssurrogate" "source" { build { sources = ["source.amazon-ebssurrogate.source"] - provisioner "file" { - source = "ebssurrogate/files/sources.cfg" - destination = "/tmp/sources.list" - } - provisioner "file" { source = "ebssurrogate/files/ebsnvme-id" destination = "/tmp/ebsnvme-id" diff --git a/amazon-arm64-nix.pkr.hcl b/amazon-arm64-nix.pkr.hcl index 7bca54bbc8..422886c0c0 100644 --- a/amazon-arm64-nix.pkr.hcl +++ b/amazon-arm64-nix.pkr.hcl @@ -170,11 +170,6 @@ source "amazon-ebssurrogate" "source" { build { sources = ["source.amazon-ebssurrogate.source"] - provisioner "file" { - source = "ebssurrogate/files/sources-arm64.cfg" - destination = "/tmp/sources.list" - } - provisioner "file" { source = "ebssurrogate/files/ebsnvme-id" destination = "/tmp/ebsnvme-id" diff --git a/ansible/playbook.yml b/ansible/playbook.yml index 613c4a99da..d9bb6e1c8d 100644 --- a/ansible/playbook.yml +++ b/ansible/playbook.yml @@ -141,25 +141,12 @@ when: not stage2 import_tasks: tasks/setup-fail2ban.yml - # Install EC2 instance connect - # Only for AWS images - - name: install EC2 instance connect - when: not qemu - become: yes - apt: - pkg: - - ec2-instance-connect - tags: - - aws-only - # Install this at the end to prevent it from kicking in during the apt process, causing conflicts - name: Install security tools become: yes apt: pkg: - unattended-upgrades - update_cache: yes - cache_valid_time: 3600 - name: Run migrations when: stage2 @@ -212,3 +199,8 @@ become: yes shell: | sudo -u ubuntu bash -c ". /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && nix-collect-garbage -d" + + - name: Setup Swap + when: stage2 and not qemu + become: yes + import_tasks: tasks/stage2/setup-swap.yml diff --git a/ansible/tasks/internal/install-salt.yml b/ansible/tasks/internal/install-salt.yml index 8df927ef73..70ee3a797c 100644 --- a/ansible/tasks/internal/install-salt.yml +++ b/ansible/tasks/internal/install-salt.yml @@ -29,7 +29,6 @@ - salt-common={{ salt_release }} - salt-minion={{ salt_release }} state: present - update_cache: yes - name: Pin salt packages at major version ansible.builtin.copy: diff --git a/ansible/tasks/setup-postgrest.yml b/ansible/tasks/setup-postgrest.yml index 33afbc2f73..6d217c5540 100644 --- a/ansible/tasks/setup-postgrest.yml +++ b/ansible/tasks/setup-postgrest.yml @@ -27,7 +27,6 @@ package: - libnuma-dev - libpq5 - update_cache: true state: 'present' - name: PostgREST - grab the list of installed packages diff --git a/ansible/tasks/setup-system.yml b/ansible/tasks/setup-system.yml index fda2627b19..d23ab03de1 100644 --- a/ansible/tasks/setup-system.yml +++ b/ansible/tasks/setup-system.yml @@ -1,15 +1,8 @@ - name: Execute stage1 tasks when: not stage2 block: - - name: System - apt update and apt upgrade - ansible.builtin.apt: - update_cache: true - upgrade: true - # SEE http://archive.vn/DKJjs#parameter-upgrade - - name: Install required security updates Ansible dependencies, security tools, and other useful things ansible.builtin.apt: - cache_valid_time: 3600 pkg: - acl - bwm-ng @@ -23,7 +16,6 @@ - sysstat - tzdata - vim - update_cache: true # SEE https://github.com/georchestra/ansible/issues/55#issuecomment-588313638 # Without this, a similar error is faced diff --git a/ansible/tasks/setup-tmux.yml b/ansible/tasks/setup-tmux.yml index 93b69ff735..658afdda7d 100644 --- a/ansible/tasks/setup-tmux.yml +++ b/ansible/tasks/setup-tmux.yml @@ -2,7 +2,6 @@ ansible.builtin.apt: name: 'tmux' state: 'present' - update_cache: true become: true - name: tmux - Configure tmux alias diff --git a/ansible/tasks/setup-tuned.yml b/ansible/tasks/setup-tuned.yml index 30f2f611d7..1d4fb16ce7 100644 --- a/ansible/tasks/setup-tuned.yml +++ b/ansible/tasks/setup-tuned.yml @@ -6,7 +6,6 @@ name: 'tuned' policy_rc_d: 101 state: 'present' - update_cache: true become: true - name: 'tuned - Create a tuned profile directory' # noqa: name[casing] @@ -185,43 +184,6 @@ value: '10' become: true - - name: 'tuned - Enable zswap if swap is present' # noqa: name[casing] - when: ansible_facts['swaptotal_mb'] > 0 - block: - - name: 'tuned - Decrease the kernel swappiness' # noqa: name[casing] - community.general.ini_file: - create: true - group: 'root' - mode: '0644' - no_extra_spaces: true - option: 'vm.swappiness' - path: '/etc/tuned/profiles/postgresql/tuned.conf' - section: 'sysctl' - state: 'present' - value: '10' - become: true - - - name: 'tuned - Load zstd compressor module' # noqa: name[casing] - community.general.modprobe: - name: 'zstd' - persistent: 'present' - state: 'present' - become: true - - - name: 'tuned - Configure and enable zswap' # noqa: name[casing] - ansible.builtin.shell: - cmd: "echo {{ zswap_item['value'] }} > /sys/module/zswap/parameters/{{ zswap_item['param'] }}" - changed_when: true - loop: - - param: 'compressor' - value: 'zstd' - - param: 'max_pool_percent' - value: '10' - - param: 'enabled' - value: 'Y' - loop_control: - loop_var: 'zswap_item' - - name: 'tuned - Activate the tuned service' # noqa: name[casing] ansible.builtin.systemd_service: daemon_reload: true diff --git a/ansible/tasks/stage2/setup-swap.yml b/ansible/tasks/stage2/setup-swap.yml new file mode 100644 index 0000000000..a1823e058d --- /dev/null +++ b/ansible/tasks/stage2/setup-swap.yml @@ -0,0 +1,49 @@ +- name: Allocate and format swapfile + ansible.builtin.shell: | + fallocate -l 1G /swapfile + chmod 600 /swapfile + mkswap /swapfile + args: + creates: /swapfile + +- name: Add swapfile to fstab + ansible.posix.mount: + src: /swapfile + path: none + fstype: swap + opts: sw + dump: "0" + passno: "0" + state: present + +- name: 'tuned - Decrease the kernel swappiness' # noqa: name[casing] + community.general.ini_file: + create: true + group: 'root' + mode: '0644' + no_extra_spaces: true + option: 'vm.swappiness' + path: '/etc/tuned/profiles/postgresql/tuned.conf' + section: 'sysctl' + state: 'present' + value: '10' + +- name: 'tuned - Load zstd compressor module' # noqa: name[casing] + community.general.modprobe: + name: 'zstd' + persistent: 'present' + state: 'present' + +- name: 'tuned - Configure and enable zswap' # noqa: name[casing] + ansible.builtin.shell: + cmd: "echo {{ zswap_item['value'] }} > /sys/module/zswap/parameters/{{ zswap_item['param'] }}" + changed_when: true + loop: + - param: 'compressor' + value: 'zstd' + - param: 'max_pool_percent' + value: '10' + - param: 'enabled' + value: 'Y' + loop_control: + loop_var: 'zswap_item' diff --git a/ansible/tasks/test-image.yml b/ansible/tasks/test-image.yml index 3acd25cefa..493eed114b 100644 --- a/ansible/tasks/test-image.yml +++ b/ansible/tasks/test-image.yml @@ -65,7 +65,6 @@ ansible.builtin.apt: name: 'python3-psycopg2' state: 'present' - update_cache: true become: true - name: Reset db stats diff --git a/ebssurrogate/files/sources-arm64.cfg b/ebssurrogate/files/sources-arm64.cfg deleted file mode 100644 index eed6c0f165..0000000000 --- a/ebssurrogate/files/sources-arm64.cfg +++ /dev/null @@ -1,10 +0,0 @@ -deb http://REGION.clouds.ports.ubuntu.com/ubuntu-ports/ noble main restricted -deb http://REGION.clouds.ports.ubuntu.com/ubuntu-ports/ noble-updates main restricted -deb http://REGION.clouds.ports.ubuntu.com/ubuntu-ports/ noble universe -deb http://REGION.clouds.ports.ubuntu.com/ubuntu-ports/ noble-updates universe -deb http://REGION.clouds.ports.ubuntu.com/ubuntu-ports/ noble multiverse -deb http://REGION.clouds.ports.ubuntu.com/ubuntu-ports/ noble-updates multiverse -deb http://REGION.clouds.ports.ubuntu.com/ubuntu-ports/ noble-backports main restricted universe multiverse -deb http://ports.ubuntu.com/ubuntu-ports noble-security main restricted -deb http://ports.ubuntu.com/ubuntu-ports noble-security universe -deb http://ports.ubuntu.com/ubuntu-ports noble-security multiverse diff --git a/ebssurrogate/files/sources.cfg b/ebssurrogate/files/sources.cfg deleted file mode 100644 index a27be052f7..0000000000 --- a/ebssurrogate/files/sources.cfg +++ /dev/null @@ -1,10 +0,0 @@ -deb http://REGION.ec2.archive.ubuntu.com/ubuntu/ noble main restricted -deb http://REGION.ec2.archive.ubuntu.com/ubuntu/ noble-updates main restricted -deb http://REGION.ec2.archive.ubuntu.com/ubuntu/ noble universe -deb http://REGION.ec2.archive.ubuntu.com/ubuntu/ noble-updates universe -deb http://REGION.ec2.archive.ubuntu.com/ubuntu/ noble multiverse -deb http://REGION.ec2.archive.ubuntu.com/ubuntu/ noble-updates multiverse -deb http://REGION.ec2.archive.ubuntu.com/ubuntu/ noble-backports main restricted universe multiverse -deb http://security.ubuntu.com/ubuntu noble-security main restricted -deb http://security.ubuntu.com/ubuntu noble-security universe -deb http://security.ubuntu.com/ubuntu noble-security multiverse diff --git a/ebssurrogate/scripts/chroot-bootstrap-nix.sh b/ebssurrogate/scripts/chroot-bootstrap-nix.sh index 140f6802dd..354e7585ba 100755 --- a/ebssurrogate/scripts/chroot-bootstrap-nix.sh +++ b/ebssurrogate/scripts/chroot-bootstrap-nix.sh @@ -8,238 +8,80 @@ set -o errexit set -o pipefail set -o xtrace -# Switch to a different mirror -function switch_mirror { - local new_mirror=$1 - local sources_file=/etc/apt/sources.list - - echo "Switching to mirror: $new_mirror" - if [[ $ARCH == amd64 ]]; then - sed -i "s|http://[^/]*/ubuntu/|http://$new_mirror/ubuntu/|g" "$sources_file" - else - sed -i "s|http://[^/]*/ubuntu-ports/|http://$new_mirror/ubuntu-ports/|g" "$sources_file" - fi - - # Show what we're using - echo "Current sources.list configuration:" - grep -E '^deb ' "$sources_file" | head -3 +function setup_apt { + export DEBIAN_FRONTEND=noninteractive } -# Get list of mirrors to try -function get_mirror_list { - local sources_file=/etc/apt/sources.list - local -a mirrors=() - - # Priority order: - # 1. Country-specific mirror (most reliable) - # 2. Regional CDN (can be inconsistent) - # 3. Global fallback - - local current_region - if [[ $ARCH == amd64 ]]; then - current_region=$(grep -oP '(?<=http://)[^.]+(?=\.ec2\.archive\.ubuntu\.com)' "$sources_file" | head -1 || echo "") - - if [[ -n $current_region ]]; then - mirrors+=("$current_region.ec2.archive.ubuntu.com") - fi - - mirrors+=("archive.ubuntu.com") - else - current_region=$(grep -oP '(?<=http://)[^.]+(?=\.clouds\.ports\.ubuntu\.com)' "$sources_file" | head -1 || echo "") - - # Singapore country mirror for ap-southeast-1 - if [[ $current_region == "ap-southeast-1" ]]; then - mirrors+=("sg.ports.ubuntu.com") - fi - - if [[ -n $current_region ]]; then - mirrors+=("$current_region.clouds.ports.ubuntu.com") - fi - mirrors+=("ports.ubuntu.com") - fi - - echo "${mirrors[@]}" +function cleanup_apt { + apt-get clean + apt-get autoremove --purge --yes } -# Mirror fallback function for resilient apt-get update -function apt_update_with_fallback { - local sources_file=/etc/apt/sources.list - local -a mirror_list - readarray mirror_list < <(get_mirror_list) - local attempt=1 - local max_attempts=${#mirror_list[@]} - - for mirror in "${mirror_list[@]}"; do - echo "=========================================" - echo "Attempting apt-get update with mirror: $mirror" - echo "Attempt $attempt of $max_attempts" - echo "=========================================" - - switch_mirror "$mirror" - - # Attempt update with timeout (5 minutes) - if timeout 300 apt-get "${APT_OPTIONS[@]}" update 2>&1; then - echo "=========================================" - echo "✓ Successfully updated apt cache using mirror: $mirror" - echo "=========================================" - return 0 - else - local ret=$? - echo "=========================================" - echo "✗ Failed to update using mirror: $mirror" - echo "Exit code: $ret" - echo "=========================================" - - # Clean partial downloads - apt-get clean - rm -rf /var/lib/apt/lists/* - - # Exponential backoff before next attempt - if [[ $attempt -lt $max_attempts ]]; then - local sleep_time=$((attempt * 5)) - echo "Waiting $sleep_time seconds before trying next mirror..." - sleep $sleep_time - fi - fi - - attempt=$((attempt + 1)) - done - - echo "=========================================" - echo "ERROR: All mirror tiers failed after $max_attempts attempts" - echo "=========================================" - return 1 +function update_and_upgrade_apt { + apt-get update --yes + apt-get dist-upgrade --yes } -# Wrapper for apt-get install with mirror fallback on 404 errors -function apt_install_with_fallback { - local -a mirror_list - readarray mirror_list < <(get_mirror_list) - local attempt=1 - local max_attempts=${#mirror_list[@]} - - for mirror in "${mirror_list[@]}"; do - echo "=========================================" - echo "Attempting apt-get install with mirror: $mirror" - echo "Attempt $attempt of $max_attempts" - echo "=========================================" - - switch_mirror "$mirror" - - # Re-run apt-get update to get package lists from new mirror - if ! timeout 300 apt-get "${APT_OPTIONS[@]}" update 2>&1; then - echo "Warning: apt-get update failed for mirror $mirror, trying next..." - attempt=$((attempt + 1)) - continue - fi - - # Run apt-get install directly (no output capture to avoid buffering/timeout issues) - if apt-get "$@"; then - echo "=========================================" - echo "✓ Successfully installed packages using mirror: ${mirror}" - echo "=========================================" - return 0 - else - local ret=$? - # On failure, check if it's a mirror issue worth retrying - echo "=========================================" - echo "✗ apt-get failed with exit code: $ret" - echo "=========================================" - fi - - # Clean apt cache before potential retry - apt-get clean - - if ((attempt < max_attempts)); then - local sleep_time=$((attempt * 5)) - echo "Waiting $sleep_time seconds before trying next mirror..." - sleep $sleep_time - fi - - attempt=$((attempt + 1)) - done - - echo "=========================================" - echo "ERROR: All mirror tiers failed for apt-get install after $max_attempts attempts" - echo "=========================================" - return 1 -} - -function update_install_packages { - # Update APT with new sources (using fallback mechanism) - cat /etc/apt/sources.list - if ! apt_update_with_fallback; then - echo "FATAL: Failed to update package lists with any mirror tier" - exit 1 - fi - apt-get "${APT_OPTIONS[@]}" --yes dist-upgrade - - # Do not configure grub during package install +function install_initial_packages { + local packages=( + e2fsprogs + initramfs-tools + linux-aws + ) + # Install various packages needed for a booting system (with mirror fallback) if [[ $ARCH == amd64 ]]; then + # Do not configure grub during package install echo 'grub-pc grub-pc/install_devices_empty select true' | debconf-set-selections echo 'grub-pc grub-pc/install_devices select' | debconf-set-selections - # Install various packages needed for a booting system (with mirror fallback) - if ! apt_install_with_fallback install -y linux-aws grub-pc e2fsprogs; then - echo "FATAL: Failed to install boot packages" - exit 1 - fi + packages+=( + grub-pc + ) else - if ! apt_install_with_fallback install -y e2fsprogs; then - echo "FATAL: Failed to install e2fsprogs" - exit 1 - fi + packages+=( + cloud-guest-utils + dosfstools + efibootmgr + fdisk + grub-efi-arm64 + ) fi - # Install standard packages (with mirror fallback) - # Note: ec2-hibinit-agent, ec2-instance-connect, hibagent moved to stage 2 - # because their post-install scripts try to access EC2 metadata service - # which doesn't work in a chroot and causes long hangs - if ! apt_install_with_fallback install -y \ - bzip2 \ - sudo \ - wget \ - cloud-init \ - acpid \ - ncurses-term \ - ssh-import-id; then + + packages+=( + acpid + apparmor + apparmor-utils + auditd + bzip2 + cloud-init + ec2-hibinit-agent + ec2-instance-connect + hibagent + ncurses-term + ssh-import-id + sudo + wget + ) + if ! apt-get install --yes "${packages[@]}"; then echo "FATAL: Failed to install standard packages" exit 1 fi - # apt upgrade - apt-get upgrade -y - - # Install OpenSSH and other packages - add-apt-repository --yes universe - if ! apt_update_with_fallback; then - echo "FATAL: Failed to update package lists after adding universe repository" + packages=( + at + cron + fail2ban + git + less + locales + logrotate + openssh-server + python3-systemd + ufw + ) + if ! apt-get install --yes --no-install-recommends "${packages[@]}"; then + echo "FATAL: Failed to install extra packages" exit 1 fi - if ! apt_install_with_fallback install -y --no-install-recommends \ - openssh-server \ - git \ - ufw \ - cron \ - logrotate \ - fail2ban \ - locales \ - at \ - less \ - python3-systemd; then - echo "FATAL: Failed to install universe packages" - exit 1 - fi - - if [[ $ARCH == arm64 ]]; then - if ! apt_install_with_fallback "${APT_OPTIONS[@]}" --yes install linux-aws initramfs-tools dosfstools; then - echo "FATAL: Failed to install arm64 boot packages" - exit 1 - fi - else - if ! apt_install_with_fallback "${APT_OPTIONS[@]}" --yes install initramfs-tools; then - echo "FATAL: Failed to install amd64 boot packages" - exit 1 - fi - fi } function setup_locale { @@ -268,11 +110,6 @@ function setup_postgesql_env { } function setup_apparmor { - if ! apt_install_with_fallback install -y apparmor apparmor-utils auditd; then - echo "FATAL: Failed to install apparmor packages" - exit 1 - fi - # Copy apparmor profiles cp -rv /tmp/apparmor_profiles/* /etc/apparmor.d/ } @@ -290,10 +127,6 @@ function setup_grub { EOF if [[ $ARCH == arm64 ]]; then - if ! apt_install_with_fallback "${APT_OPTIONS[@]}" --yes install cloud-guest-utils fdisk grub-efi-arm64 efibootmgr; then - echo "FATAL: Failed to install grub packages for arm64" - exit 1 - fi rm -rf /etc/grub.d/30_os-prober sleep 1 fi @@ -364,28 +197,18 @@ function disable_services { chmod +x /usr/sbin/policy-rc.d } -# Clear apt caches -function cleanup_cache { - apt-get clean -} - # Remove policy-rc.d so services start normally on boot function enable_services { rm -f /usr/sbin/policy-rc.d } -export DEBIAN_FRONTEND=noninteractive -APT_OPTIONS=( - -oAPT::Install-Recommends=false - -oAPT::Install-Suggests=false - -oAcquire::Languages=none -) - ARCH=$(dpkg --print-architecture) : "${ARCH:?Failed to detect architecture}" disable_services -update_install_packages +setup_apt +update_and_upgrade_apt +install_initial_packages setup_locale setup_postgesql_env setup_grub @@ -396,5 +219,5 @@ set_default_target setup_eth0_interface disable_sshd_passwd_auth disable_fsck -cleanup_cache +cleanup_apt enable_services diff --git a/ebssurrogate/scripts/cleanup-qemu.sh b/ebssurrogate/scripts/cleanup-qemu.sh index e79cc445d2..7179e0dad7 100755 --- a/ebssurrogate/scripts/cleanup-qemu.sh +++ b/ebssurrogate/scripts/cleanup-qemu.sh @@ -14,64 +14,104 @@ if [[ ! -d /tmp ]]; then fi chmod 1777 /tmp -if [ -n "$(command -v yum)" ]; then - yum update -y - yum clean all -elif [ -n "$(command -v apt-get)" ]; then - # Cleanup more packages - apt-get -y remove --purge \ - automake \ - autoconf \ - autotools-dev \ - cmake-data \ - cpp-9 \ - cpp-10 \ - gcc-9 \ - gcc-10 \ - git \ - git-man \ - ansible \ - libicu-dev \ - libcgal-dev \ - libgcc-9-dev \ - ansible \ - snapd - - if [[ $(uname -m) == aarch64 ]]; then - apt-get -y remove --purge libgcc-8-dev - fi - - # add-apt-repository --yes --remove ppa:ansible/ansible - - apt-mark manual libevent-2.1-7t64 - - apt-get remove -y --purge ansible-core apport appstream bash-completion bcache-tools bind9-dnsutils bind9-host bind9-libs bolt btrfs-progs byobu command-not-found console-setup distro-info eject fonts-ubuntu-console friendly-recovery ftp fwupd gawk gdisk keyboard-configuration libvolume-key1 libssl-dev lvm2 lxd-agent-loader man-db mdadm modemmanager mtd-utils nano netcat-openbsd nfs-common ntfs-3g parted pastebinit screen strace thin-provisioning-tools tmux usb-modeswitch vim vim-runtime wget whiptail xfsprogs - - apt remove -y --purge libc6-dev linux-libc-dev libevent-dev libpcre3-dev libsystemd-dev packagekit multipath-tools unattended-upgrades plymouth gnupg open-vm-tools xauth lxd-installer publicsuffix libclang-cpp18 python3-twisted python-babel-localedata libicu74 python3-pygments fonts-dejavu* python3-botocore - - apt-get remove -y --purge linux-headers* - - # remove old kernels - # CURRENT_KERNEL="$(uname -r | sed 's/-generic//')" - # INSTALLED_KERNELS=$(dpkg -l | awk '{print $2}' | grep -Eo 'linux-(image|headers|modules|tools)-[0-9]+' | sed -E 's/linux-(image|modules|tools)-//' | sort -Vu) - # REMOVE_KERNELS=$(echo "$INSTALLED_KERNELS" | grep -v -e "$CURRENT_KERNEL") - # for VER in $REMOVE_KERNELS; do - # for PREFIX in linux-image linux-modules linux-tools; do - # for PKG in $(dpkg -l | awk '{print $2}' | grep "^$PREFIX-$VER"); do - # apt-get purge -y "$PKG" - # done - # done - # done - # update-grub - - apt-get -y autoremove - apt-get -y autoclean - - apt-get -y update - apt-get -y upgrade - +# Cleanup more packages +packages=( + ansible + ansible-core + apport + appstream + autoconf + automake + autotools-dev + bash-completion + bcache-tools + bind9-dnsutils + bind9-host + bind9-libs + bolt + btrfs-progs + byobu + cmake-data + command-not-found + console-setup + cpp-10 + cpp-9 + distro-info + eject + fonts-dejavu* + fonts-ubuntu-console + friendly-recovery + ftp + fwupd + gawk + gcc-10 + gcc-9 + gdisk + git + git-man + gnupg + keyboard-configuration + libc6-dev + libcgal-dev + libclang-cpp18 + libevent-dev + libgcc-9-dev + libicu-dev + libicu74 + libpcre3-dev + libssl-dev + libsystemd-dev + libvolume-key1 + linux-headers* + linux-libc-dev + lvm2 + lxd-agent-loader + lxd-installer + man-db + mdadm + modemmanager + mtd-utils + multipath-tools + nano + netcat-openbsd + nfs-common + ntfs-3g + open-vm-tools + packagekit + parted + pastebinit + plymouth + publicsuffix + python-babel-localedata + python3-botocore + python3-pygments + python3-twisted + screen + snapd + strace + thin-provisioning-tools + tmux + unattended-upgrades + usb-modeswitch + vim + vim-runtime + wget + whiptail + xauth + xfsprogs +) + +if [[ $(uname -m) == aarch64 ]]; then + packages+=(libgcc-8-dev) fi +apt-mark manual libevent-2.1-7t64 +apt-get --yes remove --purge "${packages[@]}" +apt-get --yes autoremove +apt-get --yes autoclean +apt-get --yes update +apt-get --yes upgrade + systemctl set-default multi-user.target systemctl disable getty@tty1.service systemctl mask getty@tty1.service diff --git a/ebssurrogate/scripts/cleanup.sh b/ebssurrogate/scripts/cleanup.sh index 8eab36881a..f9a103fb02 100755 --- a/ebssurrogate/scripts/cleanup.sh +++ b/ebssurrogate/scripts/cleanup.sh @@ -14,35 +14,31 @@ if [[ ! -d /tmp ]]; then fi chmod 1777 /tmp -if [ -n "$(command -v yum)" ]; then - yum update -y - yum clean all -elif [ -n "$(command -v apt-get)" ]; then - # Cleanup more packages - apt-get -y remove --purge \ - automake \ - autoconf \ - autotools-dev \ - cmake-data \ - cpp-9 \ - cpp-10 \ - gcc-9 \ - gcc-10 \ - git \ - git-man \ - ansible \ - libicu-dev \ - libcgal-dev \ - libgcc-9-dev \ - ansible +# Cleanup more packages +packages=( + ansible + ansible + autoconf + automake + autotools-dev + cmake-data + cpp-10 + cpp-9 + gcc-10 + gcc-9 + git + git-man + libcgal-dev + libgcc-9-dev + libicu-dev +) - # add-apt-repository --yes --remove ppa:ansible/ansible +apt-get --yes remove --purge "${packages[@]}" +apt-get --yes autoremove +apt-get --yes autoclean +apt-get --yes update +apt-get --yes upgrade - apt-get -y update - apt-get -y upgrade - apt-get -y autoremove - apt-get -y autoclean -fi rm -rf /tmp/* /var/tmp/* history -c cat /dev/null >/root/.bash_history diff --git a/ebssurrogate/scripts/nix-provision.sh b/ebssurrogate/scripts/nix-provision.sh index 3b576dc179..89beffb605 100755 --- a/ebssurrogate/scripts/nix-provision.sh +++ b/ebssurrogate/scripts/nix-provision.sh @@ -7,24 +7,23 @@ set -o xtrace exec 1>&2 -function install_packages { - # Setup Ansible on host VM - apt-get update && apt-get install -y software-properties-common +function setup_apt { + export DEBIAN_FRONTEND=noninteractive +} - # Install EC2-specific packages that were deferred from stage 1 - # These packages have post-install scripts that need EC2 metadata service access - # which only works on a real running EC2 instance (not in chroot) - apt-get install -y ec2-hibinit-agent ec2-instance-connect hibagent +function cleanup_apt { + apt-get clean + apt-get autoremove --purge --yes + rm -rf /var/lib/apt/lists/* +} - # Manually add GPG key with explicit keyserver - apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 93C4A3FD7BB9C367 +function update_and_upgrade_apt { + apt-get update --yes + apt-get dist-upgrade --yes +} - # Add repository and install - # TODO (darora): temporarily disabling while Launchpad is under ddos attack and very frequently timing out - # sudo add-apt-repository --yes ppa:ansible/ansible - # sudo apt-get update +function install_packages { apt-get install -y ansible - ansible-galaxy collection install community.general } @@ -68,8 +67,7 @@ function execute_stage2_playbook { } function cleanup_packages { - apt-get -y remove --purge ansible - # sudo add-apt-repository --yes --remove ppa:ansible/ansible + apt-get remove --purge --yes ansible } function report_disk_usage { @@ -78,8 +76,12 @@ function report_disk_usage { printf '::notice::disk_usage bytes=%s human=%s\n' "$dub" "$duh" | tee -a /tmp/ansible.log } +setup_apt +update_and_upgrade_apt install_packages install_nix execute_stage2_playbook cleanup_packages +update_and_upgrade_apt +cleanup_apt report_disk_usage diff --git a/ebssurrogate/scripts/qemu-bootstrap-nix.sh b/ebssurrogate/scripts/qemu-bootstrap-nix.sh index 68619f3a3c..c9f62b558c 100755 --- a/ebssurrogate/scripts/qemu-bootstrap-nix.sh +++ b/ebssurrogate/scripts/qemu-bootstrap-nix.sh @@ -8,6 +8,37 @@ set -o xtrace # stage1 things # ################# +function setup_apt { + export DEBIAN_FRONTEND=noninteractive + + cat /etc/apt/sources.list.d/ubuntu.sources >&2 + local sources defmirror ubumirror + sources=$(grep -e '^URIs\s*:' -e '^Suites\s*:' /etc/apt/sources.list.d/ubuntu.sources) + defmirror=$(grep -B1 noble-updates <<<"$sources" | awk '/URIs/ {print $2}') + ubumirror=$(grep -B1 noble-security <<<"$sources" | awk '/URIs/ {print $2}') + if [[ $ARCH == x86_64 ]]; then + # x86_64 hosts use security.ubuntu.com for security but archive.ubuntu.com for everything else + ubumirror=${ubumirror/security/archive} + fi + + if grep -q "^URIs:.*$defmirror.*$ubumirror" /etc/apt/sources.list.d/ubuntu.sources; then + echo "Ubuntu upstream is already a fallback, this is unexpected and needs source changes" >&2 + exit 1 + fi + sed -i "s|$defmirror|& $ubumirror|" /etc/apt/sources.list.d/ubuntu.sources +} + +function cleanup_apt { + apt-get clean + apt-get autoremove --purge --yes + rm -rf /var/lib/apt/lists/* +} + +function update_and_upgrade_apt { + apt-get update --yes + apt-get dist-upgrade --yes +} + function waitfor_boot_finished { # Wait for cloudinit on the surrogate to complete before making progress while [[ ! -f /var/lib/cloud/instance/boot-finished ]]; do @@ -17,23 +48,21 @@ function waitfor_boot_finished { } function install_packages { - apt-get update - apt-get install -y \ - arptables \ - e2fsprogs \ - ebtables \ - gpg \ - iptables \ - less \ - locales \ - logrotate \ - nfs-common \ - software-properties-common \ - ufw \ - ; - # TODO (darora): temporarily disabling while Launchpad is under ddos attack and very frequently timing out - # add-apt-repository --yes --update ppa:ansible/ansible && - apt-get install ansible -y + packages=( + ansible + arptables + e2fsprogs + ebtables + gpg + iptables + less + locales + logrotate + nfs-common + software-properties-common + ufw + ) + apt-get install --yes "${packages[@]}" ansible-galaxy collection install community.general } @@ -122,8 +151,7 @@ function execute_stage2_playbook { function clean_legacy_things { # removes things that are bundled for legacy reasons, but we can start without for our newer artifacts apt-mark auto zlib1g* # TODO (darora): need to make sure that there aren't other things that still need this - apt-get -y purge kong - apt-get autoremove -y + apt-get purge --yes kong } function clean_system { @@ -163,7 +191,6 @@ function clean_system { mkdir /var/log/audit # unwanted files - rm -rf /var/lib/apt/lists/* rm -rf /root/.cache rm -rf /root/.vpython* rm -rf /root/go @@ -190,8 +217,8 @@ function clean_system { # stage1 things # ################# -export DEBIAN_FRONTEND=noninteractive - +setup_apt +update_and_upgrade_apt waitfor_boot_finished install_packages setup_postgesql_env @@ -206,3 +233,4 @@ install_nix execute_stage2_playbook clean_legacy_things clean_system +cleanup_apt diff --git a/ebssurrogate/scripts/surrogate-bootstrap-nix.sh b/ebssurrogate/scripts/surrogate-bootstrap-nix.sh index de44160b01..ff91a5b6c6 100755 --- a/ebssurrogate/scripts/surrogate-bootstrap-nix.sh +++ b/ebssurrogate/scripts/surrogate-bootstrap-nix.sh @@ -12,87 +12,43 @@ set -o xtrace exec 1>&2 -# Mirror fallback function for resilient apt-get update -function apt_update_with_fallback { - local sources_file=/etc/apt/sources.list - local max_attempts=2 - local attempt=1 - - # Get EC2 region if not already set - if [[ -z $REGION ]]; then - REGION=$(curl --silent --fail http://169.254.169.254/latest/meta-data/placement/availability-zone | sed -E 's|[a-z]+$||g' || echo "") - fi +function setup_apt { + export DEBIAN_FRONTEND=noninteractive + + # This function assumes deb822 formatted sources are in use, which is the case in both qemu and aws images + # In aws cloud-init creates a sources file with regional mirrors for "default" Suites but keeps ubuntu for security suite + # So we grab the first (only) URI from security and append it to non-security's URI + # This ends up giving us fastest mirror for installs and falls back to ubuntu if there's an issue + # + # Note: Ubuntu amd64 mirrors have different hostnames for security vs non but aarch64 are the same, hence the amd64 specific line - # Define mirror tiers (in priority order) - local -a mirror_tiers=() + # ensure deb822 format sources are in use + tail -n+1 /etc/apt/sources.list /etc/apt/sources.list.d/ubuntu.sources >&2 + + local sources defmirror ubumirror + sources=$(grep -e '^URIs\s*:' -e '^Suites\s*:' /etc/apt/sources.list.d/ubuntu.sources) + defmirror=$(grep -B1 "$CODENAME-updates" <<<"$sources" | awk '/URIs/ {print $2}') + ubumirror=$(grep -B1 "$CODENAME-security" <<<"$sources" | awk '/URIs/ {print $2}') if [[ $ARCH == amd64 ]]; then - if [[ -n $REGION ]]; then - mirror_tiers+=("$REGION.ec2.archive.ubuntu.com") - fi - mirror_tiers+=("archive.ubuntu.com") - else - if [[ -n $REGION ]]; then - mirror_tiers+=("$REGION.clouds.ports.ubuntu.com") - fi - mirror_tiers+=("ports.ubuntu.com") + # amd64 hosts use security.ubuntu.com for security but archive.ubuntu.com for everything else + ubumirror=${ubumirror/security/archive} fi - # If we couldn't get REGION, skip tier 1 - if [[ -z $REGION ]]; then - echo "Warning: Could not determine EC2 region, skipping regional mirror" - mirror_tiers=("${mirror_tiers[@]:1}") # Remove first element + if [[ $ubumirror == "$defmirror" ]]; then + # Only using one mirror so nothing to add as fallback, not running in AWS maybe? + return fi - for mirror in "${mirror_tiers[@]}"; do - echo "=========================================" - echo "Attempting apt-get update with mirror: $mirror" - echo "Attempt $attempt of $max_attempts" - echo "=========================================" - - # Update sources.list to use current mirror - if [[ $ARCH == amd64 ]]; then - sed -i "s|http://[^/]*/ubuntu/|http://$mirror/ubuntu/|g" "$sources_file" - else - sed -i "s|http://[^/]*/ubuntu-ports/|http://$mirror/ubuntu-ports/|g" "$sources_file" - sed -i "s|http://ports.ubuntu.com/ubuntu-ports|http://$mirror/ubuntu-ports|g" "$sources_file" - fi - - # Show what we're using - echo "Current sources.list configuration:" - grep -E '^deb ' "$sources_file" | head -3 - - # Attempt update with timeout (5 minutes) - if timeout 300 apt-get update 2>&1; then - echo "=========================================" - echo "✓ Successfully updated apt cache using mirror: $mirror" - echo "=========================================" - return 0 - else - local exit_code=$? - echo "=========================================" - echo "✗ Failed to update using mirror: $mirror" - echo "Exit code: $exit_code" - echo "=========================================" - - # Clean partial downloads - apt-get clean - rm -rf /var/lib/apt/lists/* - - # Exponential backoff before next attempt - if ((attempt < max_attempts)); then - local sleep_time=$((attempt * 5)) - echo "Waiting $sleep_time seconds before trying next mirror..." - sleep $sleep_time - fi - fi - - attempt=$((attempt + 1)) - done + if grep -q "^URIs:.*$defmirror.*$ubumirror" /etc/apt/sources.list.d/ubuntu.sources; then + echo "Ubuntu upstream is already a fallback, this is unexpected and needs source changes" >&2 + exit 1 + fi + sed -i "s|$defmirror|& $ubumirror|" /etc/apt/sources.list.d/ubuntu.sources +} - echo "=========================================" - echo "ERROR: All mirror tiers failed after $max_attempts attempts" - echo "=========================================" - return 1 +function update_and_upgrade_apt { + apt-get update --yes + apt-get dist-upgrade --yes } function waitfor_boot_finished { @@ -103,29 +59,15 @@ function waitfor_boot_finished { } function install_packages { - # Setup Ansible on host VM - if ! apt_update_with_fallback; then - echo "FATAL: Failed to update package lists on host VM" - exit 1 - fi - - apt-get install software-properties-common -y - # TODO (darora): temporarily disabling while Launchpad is under ddos attack and very frequently timing out - # add-apt-repository --yes --update ppa:ansible/ansible - - if ! apt_update_with_fallback; then - echo "FATAL: Failed to update package lists after adding Ansible PPA" - exit 1 - fi - - apt-get install ansible -y - ansible-galaxy collection install community.general - - apt-get install -y \ - gdisk \ - e2fsprogs \ - debootstrap \ + packages=( + ansible + debootstrap + e2fsprogs + gdisk nvme-cli + ) + apt-get install --yes "${packages[@]}" + ansible-galaxy collection install community.general } # Partition the new root EBS volume @@ -202,24 +144,15 @@ function format_and_mount_rootfs { mount -o defaults,discard /dev/xvdh /mnt/data } -function create_swapfile { - fallocate -l 1G /mnt/swapfile - chmod 600 /mnt/swapfile - mkswap /mnt/swapfile -} - function format_build_partition { mkfs.ext4 -O ^has_journal /dev/xvdc } # Create fstab function create_fstab { - local FMT="%-42s %-11s %-5s %-17s %-5s %s" ROOT_LINE DATA_LINE SWAP_LINE + local FMT="%-42s %-11s %-5s %-17s %-5s %s" ROOT_LINE DATA_LINE ROOT_LINE=$(findmnt -no SOURCE /mnt | xargs blkid -o export | awk -v FMT="$FMT" '/^UUID=/ { printf(FMT, $0, "/", "ext4", "defaults,discard", "0", "1" ) }') DATA_LINE=$(findmnt -no SOURCE /mnt/data | xargs blkid -o export | awk -v FMT="$FMT" '/^UUID=/ { printf(FMT, $0, "/data", "ext4", "defaults,discard", "0", "2" ) }') - # shellcheck disable=SC2059 - SWAP_LINE=$(printf "$FMT" "/swapfile" "none" "swap" "sw" "0" "0") - local EFI_LINE="" if [[ $ARCH == arm64 ]]; then EFI_LINE=$(findmnt -no SOURCE /mnt/boot/efi | xargs blkid -o export | awk -v FMT="$FMT" '/^UUID=/ { printf(FMT, $0, "/boot/efi", "vfat", "umask=0077", "0", "1" ) }') @@ -231,14 +164,10 @@ function create_fstab { echo "$ROOT_LINE" [ -n "$EFI_LINE" ] && echo "$EFI_LINE" echo "$DATA_LINE" - echo "$SWAP_LINE" } >/mnt/etc/fstab } function setup_chroot_environment { - local UBUNTU_VERSION - UBUNTU_VERSION=$(lsb_release -cs) # 'noble' for Ubuntu 24.04 - # sometimes debootstrap will get stuck on a download for a long time # the default read timeout in wget is 900s, which can cause a ~15min increase in build time # this forces the process to fail-fast and retry @@ -248,19 +177,14 @@ function setup_chroot_environment { tries = 5 EOF - # Update ec2-region - local REGION - REGION=$(curl --silent --fail http://169.254.169.254/latest/meta-data/placement/availability-zone | sed -E 's|[a-z]+$||g') + # Use the preferred mirror (if multiple), which is the first URI/preferred + local mirror + mirror=$(awk '/^URIs:/{uri=$2} /^Suites:.*\<'"$CODENAME-updates"'\>/{print uri; exit}' /etc/apt/sources.list.d/ubuntu.sources) + debootstrap --arch "$ARCH" --variant=minbase "$CODENAME" /mnt "$mirror" - # Bootstrap Ubuntu into /mnt using the regional mirror (avoids global mirror stalls) - if [[ $ARCH == amd64 ]]; then - debootstrap --arch "$ARCH" --variant=minbase "$UBUNTU_VERSION" /mnt "http://$REGION.ec2.archive.ubuntu.com/ubuntu" - else - debootstrap --arch "$ARCH" --variant=minbase "$UBUNTU_VERSION" /mnt "http://$REGION.clouds.ports.ubuntu.com/ubuntu-ports" - fi - - sed -i "s/REGION/$REGION/g" /tmp/sources.list - cp /tmp/sources.list /mnt/etc/apt/sources.list + # Copy our files in since they are updated with all the mirrors! + cp -a /etc/apt/sources.list /mnt/etc/apt/sources.list + cp -a /etc/apt/sources.list.d/ubuntu.sources /mnt/etc/apt/sources.list.d/ubuntu.sources create_fstab @@ -387,7 +311,6 @@ function clean_system { rm -rf /mnt/root/.vpython* rm -rf /mnt/root/go rm -rf /mnt/usr/share/doc - } # Unmount bind mounts @@ -410,16 +333,18 @@ function umount_reset_mappings { done } -export DEBIAN_FRONTEND=noninteractive - ARCH=$(dpkg --print-architecture) : "${ARCH:?Failed to detect architecture}" +# shellcheck source=/dev/null +CODENAME=$(source /etc/os-release && echo "$VERSION_CODENAME") +: "${CODENAME:?Failed to detect OS codename}" waitfor_boot_finished +setup_apt +update_and_upgrade_apt install_packages device_partition_mappings format_and_mount_rootfs -create_swapfile format_build_partition setup_chroot_environment execute_playbook