feat(net): per-VM egress networking (netns + TAP + nftables NAT)

.github/scripts/provision-kvm-host.sh

@@ -17,6 +17,9 @@
 #     the node)
 #   - DEBIAN_FRONTEND=noninteractive (install.md is written for an interactive
 #     operator session; CI has no tty to prompt on)
+#   - [network] is always written here (install.md presents it as an optional
+#     section) since the runner's default route interface is detected fresh
+#     each run and integration coverage wants egress networking exercised
 set -euo pipefail
 
 export DEBIAN_FRONTEND=noninteractive
@@ -30,12 +33,20 @@ sudo udevadm trigger --name-match=kvm
 
 sudo apt-get update
 sudo apt-get install -y \
-  coreutils e2fsprogs libc-bin lvm2 skopeo thin-provisioning-tools util-linux \
+  coreutils e2fsprogs iproute2 libc-bin lvm2 nftables skopeo thin-provisioning-tools util-linux \
   "linux-modules-extra-$(uname -r)"
 
 # -a is load-bearing: without it modprobe reads the 2nd+ names as module
 # PARAMETERS of the first, and the load fails.
-sudo modprobe -av dm_snapshot dm_thin_pool loop
+#
+# nf_tables + nf_nat back host-init's `hyper` nftables table and its
+# masquerade/SNAT/DNAT rules. A fresh GitHub-hosted runner does not have them
+# loaded, and the netfilter family does not always autoload on first `nft` use,
+# so `nft add table ip hyper` fails (empty stderr, non-zero exit) and the node
+# refuses to start. Load the base explicitly, as we do for the dm modules; the
+# nat expression sub-modules (nft_chain_nat, nft_masq) autoload once the base is
+# present and a rule is issued.
+sudo modprobe -av dm_snapshot dm_thin_pool loop nf_tables nf_nat
 targets="$(sudo dmsetup targets)"
 echo "dmsetup targets: ${targets}"
 grep -q thin-pool <<<"${targets}" || { echo "ERROR: thin-pool dm target missing" >&2; exit 1; }
@@ -47,8 +58,35 @@ command -v thin_dump >/dev/null || { echo "ERROR: thin_dump missing (thin-provis
 sudo install -o root -g root -m 0755 "$(readlink -f "$(command -v thin_dump)")" /usr/local/sbin/thin_dump
 [ ! -L /usr/local/sbin/thin_dump ] || { echo "ERROR: /usr/local/sbin/thin_dump is still a symlink" >&2; exit 1; }
 
+# host-init (run by the node at start) asserts ip_forward=1 rather than
+# setting it, so provisioning must turn it on and persist it across reboots.
+sudo sysctl -w net.ipv4.ip_forward=1
+echo 'net.ipv4.ip_forward=1' \
+  | sudo tee /etc/sysctl.d/99-hyper-ip-forward.conf >/dev/null
+
+# GitHub-hosted runners ship Docker, which sets the iptables `filter` FORWARD
+# policy to DROP and only admits its own bridge traffic. Hyper's guest egress is
+# *forwarded* (per-VM netns veth -> uplink), so Docker's drop silently eats
+# every packet the guest sends — the guest configures its NIC fine but nothing
+# ever returns. Admit the clone pool (172.31.0.0/16, the helper's default) both
+# directions via DOCKER-USER, which Docker evaluates before its own drops; the
+# `hyper` nftables forward chain still enforces guest isolation. Fall back to an
+# accepting FORWARD policy on a host without Docker's chain.
+clone_pool="172.31.0.0/16"
+if sudo iptables -L DOCKER-USER >/dev/null 2>&1; then
+  sudo iptables -I DOCKER-USER -s "${clone_pool}" -j ACCEPT
+  sudo iptables -I DOCKER-USER -d "${clone_pool}" -j ACCEPT
+else
+  sudo iptables -P FORWARD ACCEPT
+fi
+
+# The default-route interface is the uplink guests NAT egress through.
+uplink="$(ip route show default | awk '{print $5; exit}')"
+[ -n "${uplink}" ] || { echo "ERROR: could not detect default-route uplink interface" >&2; exit 1; }
+echo "detected uplink: ${uplink}"
+
 sudo mkdir -p /etc/hyper
-sudo tee /etc/hyper/config.toml >/dev/null <<'EOF'
+sudo tee /etc/hyper/config.toml >/dev/null <<EOF
 work_dir = "/srv/hyper"
 
 [tools]
@@ -65,6 +103,11 @@ cgroup = "hyper"
 # load is still decaying, and nothing else runs here worth protecting.
 [budget]
 cpu_max_load = 4.0
+
+# Enables per-VM egress NAT. The nft table itself is owned by host-init
+# (created idempotently at node start), not by this script.
+[network]
+uplink = "${uplink}"
 EOF
 sudo chown root:root /etc/hyper/config.toml
 sudo chmod 0644 /etc/hyper/config.toml

docs/cookbook/install.md

@@ -191,6 +191,72 @@ cgroup = "hyper"
 For more details on configuring and tuning Hyper, we suggest you see the
 [configuration guide](config.md).
 
+### VM Networking (required)
+
+VM networking is **mandatory**: a node refuses to start without a `[network]`
+table in `/etc/hyper/config.toml`, and every VM gets NAT'd egress out through a
+physical interface on the host. There is no opt-out — a node that fails this
+preflight will not boot (`{:error, :network_not_configured}`, a missing uplink,
+or IPv4 forwarding off). The `hyper` nft table itself is created (and owned) by
+`host-init`, which runs automatically at node start — you never create nft rules
+by hand. You must prepare the host first:
+
+Install `iproute2` and `nftables`:
+
+```sh
+sudo apt install -y iproute2 nftables   # dnf install -y iproute nftables
+```
+
+Enable IPv4 forwarding. `host-init` only *asserts* this is on — it refuses to
+proceed rather than setting it for you:
+
+```sh
+sudo sysctl -w net.ipv4.ip_forward=1
+```
+
+> #### Persistent Config {: .warning}
+>
+> Loading this via `sysctl -w` is ephemeral and resets on reboot. Persist it:
+>
+> ```sh
+> echo 'net.ipv4.ip_forward=1' \
+>     | sudo tee /etc/sysctl.d/99-hyper-ip-forward.conf
+> ```
+
+> #### Conflicting FORWARD firewall {: .warning}
+>
+> Guest egress is *forwarded* (per-VM netns veth → uplink). If the host already
+> runs a firewall that defaults the `filter` FORWARD chain to DROP — Docker does
+> this, and so do some hardened base images — those drops silently eat every
+> guest packet: the guest configures its NIC correctly but nothing ever returns.
+> Admit the clone pool through the firewall's user hook (Docker evaluates
+> `DOCKER-USER` before its own drops); the `hyper` nft forward chain still
+> enforces guest isolation:
+>
+> ```sh
+> sudo iptables -I DOCKER-USER -s 172.31.0.0/16 -j ACCEPT
+> sudo iptables -I DOCKER-USER -d 172.31.0.0/16 -j ACCEPT
+> ```
+>
+> On a host without Docker, ensure nothing else sets the FORWARD policy to DROP.
+
+Then set `uplink` to the physical NIC guests should NAT out through — the
+default-route interface is a reasonable choice on a single-uplink host:
+
+```sh
+ip route show default | awk '{print $5; exit}'
+```
+
+```toml
+[network]
+# **required**. The physical uplink interface guests NAT egress through.
+uplink = "eth0"
+
+# optional, defaults shown
+clone_pool = "172.31.0.0/16"
+resolver = "1.1.1.1"
+```
+
 ### Cgroups
 
 Hyper uses cgroups to impose limits on each VM. Each VM has its own cgroup,

lib/hyper/cfg/network.ex

@@ -0,0 +1,53 @@
+defmodule Hyper.Cfg.Network do
+  @moduledoc """
+  VM egress networking settings from the `[network]` table — `config.toml`-only
+  because the setuid helper reads the same `uplink` and `clone_pool` to build
+  each VM's netns, veth, and NAT rules.
+
+  Networking is **mandatory**: a node refuses to start unless `[network]` is
+  configured (see `Hyper.Node.FireVMM.Jailer.Checks.network_ready/0`). Every VM
+  gets a NIC; there is no opt-out. `configured?/0` exists only so the startup
+  preflight can fail fast with a clear message rather than crash later.
+  """
+
+  import Hyper.Cfg, only: [get_cfg: 1]
+
+  @default_clone_pool "172.31.0.0/16"
+  @default_resolver "1.1.1.1"
+
+  @doc """
+  Whether the required `[network] uplink` is present. Used only by the startup
+  preflight to fail fast; once a node is running, networking is guaranteed
+  configured, so runtime code calls `uplink/0` directly.
+  """
+  @spec configured?() :: boolean()
+  def configured?, do: not is_nil(get_cfg(toml: "network.uplink", default: nil))
+
+  @doc "Physical uplink interface guest egress is NAT'd out of. Raises if unset (a running node has already passed the preflight)."
+  @spec uplink() :: String.t()
+  def uplink do
+    case get_cfg(toml: "network.uplink", default: nil) do
+      nil ->
+        raise Hyper.Cfg.MissingError, "network.uplink is required — VM networking is mandatory"
+
+      v when is_binary(v) ->
+        v
+
+      other ->
+        raise ArgumentError, "network.uplink must be a string, got: #{inspect(other)}"
+    end
+  end
+
+  @doc "IPv4 CIDR the per-VM clone /30s are carved from. `[network] clone_pool`."
+  @spec clone_pool() :: String.t()
+  def clone_pool, do: get_cfg(toml: "network.clone_pool", default: @default_clone_pool)
+
+  @doc """
+  DNS resolver IP handed to the guest via the `hyper.resolver=` kernel cmdline
+  param, since the kernel's `ip=` autoconfig sets the guest's address/route but
+  never DNS. The PID-1 guest agent reads this back out of `/proc/cmdline` and
+  writes it to `/etc/resolv.conf`. `[network] resolver`.
+  """
+  @spec resolver() :: String.t()
+  def resolver, do: get_cfg(toml: "network.resolver", default: @default_resolver)
+end

lib/hyper/node.ex

@@ -310,11 +310,25 @@ defmodule Hyper.Node do
          :ok <- Hyper.Node.Layer.Repo.test_system(),
          :ok <- Hyper.SuidHelper.test_system(),
          {:ok, base} <- Hyper.SuidHelper.sys_test(),
-         :ok <- check_helper_base(base) do
+         :ok <- check_helper_base(base),
+         :ok <- init_network() do
       Hyper.Node.FireVMM.test_system()
     end
   end
 
+  # VM networking is mandatory: refuse to start unless `[network]` is
+  # configured, then idempotently reconcile the host-wide `hyper` nftables
+  # table. Runs after `Hyper.SuidHelper.test_system/0` so the helper is
+  # confirmed present first.
+  @spec init_network :: :ok | {:error, term()}
+  defp init_network do
+    if Hyper.Cfg.Network.configured?() do
+      Hyper.SuidHelper.Network.host_init()
+    else
+      {:error, :network_not_configured}
+    end
+  end
+
   @spec check_firecracker_bins ::
           :ok
           | {:error, {:firecracker_bin_missing | :jailer_bin_missing, Path.t()}}

lib/hyper/node/fire_vmm/boot_spec.ex

@@ -32,11 +32,22 @@ defmodule Hyper.Node.FireVMM.BootSpec do
 
   @spec resolve(Hyper.Vm.source(), Instance.t()) :: Cold.t()
   def resolve(source, type) when is_map(source) do
+    # Networking is mandatory: every VM gets a NIC on the uniform inner-world
+    # address plus the kernel `ip=`/`hyper.resolver=` cmdline. A node that
+    # reached here has already passed the startup network preflight.
+    vm_id = Map.fetch!(source, :vm_id)
+
+    boot_args =
+      Map.get(source, :boot_args, @default_boot_args) <>
+        " " <>
+        Hyper.Node.FireVMM.Net.ip_cmdline() <>
+        " " <> Hyper.Node.FireVMM.Net.resolver_cmdline(Hyper.Cfg.Network.resolver())
+
     %Cold{
       machine_config: Instance.Spec.machine_config(Instance.spec(type)),
       boot_source: %BootSource{
         kernel_image_path: Map.fetch!(source, :kernel_image_path),
-        boot_args: Map.get(source, :boot_args, @default_boot_args)
+        boot_args: boot_args
       },
       drives: [
         %Drive{
@@ -45,7 +56,8 @@ defmodule Hyper.Node.FireVMM.BootSpec do
           is_read_only: false,
           path_on_host: Map.fetch!(source, :root_drive_path)
         }
-      ]
+      ],
+      network_interfaces: [Hyper.Node.FireVMM.Net.interface(vm_id)]
     }
   end
 end

lib/hyper/node/fire_vmm/daemon.ex

@@ -57,11 +57,13 @@ defmodule Hyper.Node.FireVMM.Daemon do
     # link kill) and so `terminate/2` runs on supervisor shutdown.
     Process.flag(:trap_exit, true)
 
-    with :ok <- reset_stale_jail(id),
+    with :ok <- prelaunch(opts),
          {:ok, muontrap} <- launch(opts) do
       {:ok, %__MODULE__{opts: opts, muontrap: muontrap}}
     else
-      {:error, reason} -> {:stop, reason}
+      {:error, reason} ->
+        Logger.error("vm #{id}: init failed: #{inspect(reason)}")
+        {:stop, reason}
     end
   end
 
@@ -79,8 +81,8 @@ defmodule Hyper.Node.FireVMM.Daemon do
   # here is logged, and the `Reaper` will retry, but it must not crash teardown.
   @impl true
   @decorate with_span("Hyper.Node.FireVMM.Daemon.terminate", include: [:id])
-  def terminate(_reason, %__MODULE__{opts: %Opts{vm_id: id}}) do
-    case clear_jail(id) do
+  def terminate(_reason, %__MODULE__{opts: %Opts{vm_id: id} = opts}) do
+    case clear_jail(opts) do
       :ok ->
         :ok
 
@@ -89,12 +91,25 @@ defmodule Hyper.Node.FireVMM.Daemon do
     end
   end
 
-  @spec reset_stale_jail(Hyper.Vm.Id.t()) :: :ok | {:error, term()}
-  defp reset_stale_jail(id), do: clear_jail(id)
+  # Clear any stale jail + netns left by a prior incarnation, then prepare the
+  # fresh netns the jailer's `--netns` will enter. Must run before `launch/1`:
+  # the jailer refuses to start if the netns it is told to enter does not yet
+  # exist. Networking is mandatory, so this is unconditional.
+  @spec prelaunch(Opts.t()) :: :ok | {:error, term()}
+  defp prelaunch(%Opts{vm_id: id, uid: uid} = opts) do
+    with :ok <- clear_jail(opts) do
+      Hyper.SuidHelper.Network.prepare(id, uid)
+    end
+  end
 
-  @spec clear_jail(Hyper.Vm.Id.t()) :: :ok | {:error, term()}
-  defp clear_jail(id) do
-    SuidHelper.ChrootJail.remove(Jailer.chroot_dir(id), Jailer.cgroup_dir(id))
+  # Best-effort: both the chroot/cgroup removal and the network teardown are
+  # attempted regardless of the other's outcome (a `Reaper` backstops either
+  # failure); the first error is surfaced to the caller.
+  @spec clear_jail(Opts.t()) :: :ok | {:error, term()}
+  defp clear_jail(%Opts{vm_id: id, uid: uid}) do
+    net = Hyper.SuidHelper.Network.teardown(id, uid)
+    jail = SuidHelper.ChrootJail.remove(Jailer.chroot_dir(id), Jailer.cgroup_dir(id))
+    with :ok <- net, do: jail
   end
 
   @spec launch(Opts.t()) :: {:ok, pid()} | {:error, term()}

lib/hyper/node/fire_vmm/jailer.ex

@@ -38,7 +38,9 @@ defmodule Hyper.Node.FireVMM.Jailer do
     first failure.
     """
 
-    alias Hyper.Cfg.{Dirs, Jails}
+    alias Hyper.Cfg.{Dirs, Jails, Network}
+
+    @ip_forward_path "/proc/sys/net/ipv4/ip_forward"
 
     @doc "Run every pre-requisite check, halting at the first failure."
     @spec run() :: :ok | {:error, term()}
@@ -56,10 +58,46 @@ defmodule Hyper.Node.FireVMM.Jailer do
         &kvm_present/0,
         &cgroup_v2_available/0,
         &parent_cgroup_present/0,
-        &chroot_writable/0
+        &chroot_writable/0,
+        &network_ready/0
       ]
     end
 
+    @doc """
+    Host preflight for VM egress networking, which is mandatory: refuses to
+    start unless `[network]` is configured, the uplink interface exists, and
+    the kernel is forwarding IPv4 — all prerequisites
+    `Hyper.SuidHelper.Network.host_init/0` and per-VM `prepare/2` depend on but
+    cannot themselves provision. This is the gate that makes networking
+    non-optional: a node missing any of these will not boot.
+    """
+    @spec network_ready() :: :ok | {:error, term()}
+    def network_ready do
+      if Network.configured?() do
+        with :ok <- uplink_present(Network.uplink()) do
+          ip_forward_enabled()
+        end
+      else
+        {:error, :network_not_configured}
+      end
+    end
+
+    defp uplink_present(uplink) do
+      if File.dir?("/sys/class/net/#{uplink}"),
+        do: :ok,
+        else: {:error, {:missing_uplink, uplink}}
+    end
+
+    defp ip_forward_enabled do
+      case File.read(@ip_forward_path) do
+        {:ok, contents} ->
+          if String.trim(contents) == "1", do: :ok, else: {:error, :ip_forward_disabled}
+
+        {:error, reason} ->
+          {:error, {:ip_forward_unreadable, reason}}
+      end
+    end
+
     defp kvm_present do
       if File.exists?("/dev/kvm"), do: :ok, else: {:error, :kvm_unavailable}
     end