diff --git a/.github/scripts/provision-kvm-host.sh b/.github/scripts/provision-kvm-host.sh index 0d514c38..97fee2c4 100755 --- a/.github/scripts/provision-kvm-host.sh +++ b/.github/scripts/provision-kvm-host.sh @@ -1,10 +1,11 @@ #!/usr/bin/env bash # Provision a GitHub-hosted ubuntu runner as a single-node Firecracker host # for the `:integration` E2E suite. Mirrors docs/cookbook/install.md; keep -# the two in sync. Assumes: passwordless sudo, repo compiled (the firecracker +# the two (and setup.sh, the dev-host equivalent) in sync. Assumes: +# passwordless sudo, repo compiled (the firecracker # and suidhelper install tasks are mix tasks), MIX_ENV matching the test run. # -# Deliberate CI-only deltas from install.md — do NOT remove these in a future +# Deliberate CI-only deltas from install.md -- do NOT remove these in a future # sync pass: # - the 0666 udev kvm rule (install.md assumes a real host with a `kvm` # group an operator is added to; the ephemeral runner has neither) @@ -72,7 +73,7 @@ sudo chmod 0644 /etc/hyper/config.toml sudo mkdir -p /sys/fs/cgroup/hyper echo '+cpu +memory' | sudo tee /sys/fs/cgroup/hyper/cgroup.subtree_control >/dev/null -# The BEAM (the `runner` user — Hyper refuses to run as root) owns work_dir. +# The BEAM (the `runner` user -- Hyper refuses to run as root) owns work_dir. # layers/ must pre-exist: boot validation (Layer.Repo.test_system) checks it, # and the node only creates it lazily on first image load. sudo mkdir -p /srv/hyper diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3bb2c0f0..5b60a609 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -152,6 +152,18 @@ jobs: - name: Dialyzer run: mix dialyzer + shellcheck: + name: ShellCheck + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + + # shellcheck ships preinstalled on GitHub-hosted ubuntu runners. Lint + # every tracked shell script; `git ls-files` keeps new scripts covered + # without a hand-maintained list and skips untracked _build/deps. + - name: ShellCheck + run: git ls-files -z '*.sh' | xargs -0 -r shellcheck + rust: name: Rust (suidhelper) runs-on: ubuntu-latest @@ -271,14 +283,14 @@ jobs: integration: name: Integration (KVM E2E) # ubuntu-latest x64 exposes /dev/kvm (all Linux runners with 2+ vCPUs, - # per GitHub's Apr 2024 changelog); arm64 runners do NOT — keep this x64. + # per GitHub's Apr 2024 changelog); arm64 runners do NOT -- keep this x64. runs-on: ubuntu-latest # Worst case: three 10-minute tests (vm_lifecycle + crash_recovery + fork), # plus the gRPC contract wrapper's own 25-minute cap (20-min lifecycle # vitest test + errors suite + npm ci/gen), plus ~5 minutes of # provisioning/compile/npm setup. Budget with headroom above that sum: a # tight budget could cancel the job right as it finishes, and - # `!cancelled()` steps then skip result uploads — exactly when flake + # `!cancelled()` steps then skip result uploads -- exactly when flake # history matters most. timeout-minutes: 75 env: @@ -379,7 +391,7 @@ jobs: # Multiple --only flags OR together: this runs ONLY the gated tags. # coveralls.json wraps `mix test`, so the tag filters and # --warnings-as-errors pass through; it writes cover/excoveralls.json, - # which is this job's Codecov upload — without it the integration-only + # which is this job's Codecov upload -- without it the integration-only # modules (e.g. Hyper.Metering.Usage) report zero coverage. - name: Integration + external tests run: mix coveralls.json --only integration --only external --warnings-as-errors diff --git a/README.md b/README.md index 7b2361f1..5a1e75eb 100644 --- a/README.md +++ b/README.md @@ -26,20 +26,16 @@ same niche as [Daytona](https://github.com/daytonaio/daytona), ## Quick Start -Add Hyper to your Mix project: +On an Ubuntu/Debian machine with KVM (bare metal, or a cloud instance with +nested virtualization): -```elixir -def deps do - [ - {:hypervm, "~> 0.1"} - ] -end +```sh +git clone https://github.com/harmont-dev/hyper && cd hyper +./setup.sh # one-time host provisioning (asks for sudo) +iex -S mix # boot a Hyper node ``` -Prepare the host (KVM, device-mapper, PostgreSQL, the Firecracker binaries and -the setuid helper — the [installation -guide](https://hexdocs.pm/hypervm/install.html) walks through each step), then -load an OCI image and boot it: +Then load an OCI image and boot it: ```elixir {:ok, img_id} = Hyper.Img.OciLoader.load("docker.io/library/alpine:3.19") @@ -47,8 +43,23 @@ load an OCI image and boot it: {:ok, %{stdout: "hello\n"}} = Hyper.exec(vm, ["/bin/echo", "hello"]) ``` -Please read the [Hexdocs](https://hexdocs.pm/hypervm/) for guides on using, -deploying and integrating Hyper. +That's a real Firecracker microVM with a copy-on-write rootfs -- the +[quickstart guide](https://hexdocs.pm/hypervm/quickstart.html) has the +details. To embed Hyper in your own application instead, add it to your Mix +project: + +```elixir +def deps do + [ + {:hypervm, "~> 0.1"} + ] +end +``` + +and follow the [installation +guide](https://hexdocs.pm/hypervm/install.html). The +[Hexdocs](https://hexdocs.pm/hypervm/) cover using, deploying and +integrating Hyper. ## Features diff --git a/docs/cookbook/install.md b/docs/cookbook/install.md index f57c9c6f..d078ed64 100644 --- a/docs/cookbook/install.md +++ b/docs/cookbook/install.md @@ -1,6 +1,8 @@ -# Quick Start +# Installation -This document provides the quickest start available to get Hyper running. +This document walks through a full installation of Hyper on a node. For the +fastest dev/eval path, see the [quickstart](quickstart.md), which automates +this guide via `setup.sh`. ## Configuration @@ -67,8 +69,8 @@ sudo dnf install -y \ ### Build Toolchain -Hyper compiles two Rust components as part of `mix compile` — the setuid -helper and the in-guest agent — and generates its Firecracker/gRPC bindings +Hyper compiles two Rust components as part of `mix compile` -- the setuid +helper and the in-guest agent -- and generates its Firecracker/gRPC bindings from the shipped specs. Every machine that **compiles** Hyper (including as a Mix dependency) therefore needs, besides Elixir `~> 1.20` on OTP 28+: @@ -84,7 +86,7 @@ sudo apt install -y protobuf-compiler # dnf install protobuf-compiler mix escript.install hex protobuf 0.17.0 ``` -`~/.mix/escripts` does not need to be on your `PATH` — the build finds +`~/.mix/escripts` does not need to be on your `PATH` -- the build finds `protoc-gen-elixir` there itself. ### Device Mapper Config @@ -233,14 +235,14 @@ as this user; every operation that genuinely needs root is routed through the setuid helper (see [SUID Helper](#suid-helper)), so the node itself never holds privilege. -Create the user — system account, no login shell: +Create the user -- system account, no login shell: ```sh sudo useradd --system --shell /usr/sbin/nologin --home-dir /srv/hyper hyper ``` Start Hyper as this user (for example `sudo -u hyper ...`, or `User=hyper` in a -systemd unit). The rest of this section covers the few permissions it needs — +systemd unit). The rest of this section covers the few permissions it needs -- and the ones it deliberately does **not**. #### Working directory @@ -249,7 +251,7 @@ The node builds its entire on-disk tree (`jails`, `socks`, `scratch`, `layers`, `redist`) under `work_dir` (from `/etc/hyper/config.toml`, default `/srv/hyper`) **as this user**. It must therefore own that directory. Boot validation (`Hyper.Node.Layer.Repo.test_system/0`) refuses to start unless the `layers` -subdirectory already exists — the node only creates it lazily on first image +subdirectory already exists -- the node only creates it lazily on first image load, so pre-create it now: ```sh @@ -274,7 +276,7 @@ end ``` Then `mix deps.get && mix compile`. Alternatively, work from a source -checkout of the [repository](https://github.com/harmont-dev/hyper) — every +checkout of the [repository](https://github.com/harmont-dev/hyper) -- every step below is identical. ### Firecracker @@ -288,7 +290,7 @@ mix firecracker.install # installs to /opt/firecracker ``` The task installs the binaries under their bare basenames (`firecracker`, -`jailer` — the setuid helper rejects version-stamped names), marks them +`jailer` -- the setuid helper rejects version-stamped names), marks them executable, and prints the `[tools]` snippet for `/etc/hyper/config.toml`. After installing, make both binaries root-owned and not group- or world-writable (the task prints the exact `chown` command); the helper refuses @@ -307,7 +309,7 @@ operations. This is achieved through a side-car binary called > build produced**: `mix compile` stamps the helper with a BLAKE3 checksum and > bakes that identity into the release, and a deployed helper whose version or > checksum differs is refused. A binary from another machine or build will not -> pass — always install the helper from the same tree you compiled. +> pass -- always install the helper from the same tree you compiled. Build and install it with: @@ -328,7 +330,7 @@ sudo install -o root -g root -m 4755 \ With PostgreSQL reachable (see above) and your database credentials configured (see the [configuration guide](config.md)), create and migrate the image -database — once per cluster, from any node: +database -- once per cluster, from any node: ```sh mix ecto.create -r Hyper.Img.Db.Repo @@ -337,7 +339,7 @@ mix ecto.migrate -r Hyper.Img.Db.Repo ## Booting -Start your application as the `hyper` user — Hyper's supervision tree boots +Start your application as the `hyper` user -- Hyper's supervision tree boots with it and the node becomes a VM runner. From a source checkout, an interactive session is the quickest smoke test: @@ -348,7 +350,7 @@ sudo -u hyper iex -S mix At boot Hyper validates the node: config file ownership, the setuid helper's build identity, and the device-mapper targets. A misconfigured node refuses to start with a specific error rather than limping along. Once up, load an image -and boot a VM — see the [intro](intro.md#usage) for the walkthrough. +and boot a VM -- see the [intro](intro.md#usage) for the walkthrough. For production, run it under a supervisor of your choice (e.g. a systemd unit with `User=hyper`). Nodes that join the BEAM cluster become additional VM diff --git a/docs/cookbook/quickstart.md b/docs/cookbook/quickstart.md new file mode 100644 index 00000000..75a7d0e5 --- /dev/null +++ b/docs/cookbook/quickstart.md @@ -0,0 +1,57 @@ +# Quick Start + +Boot a real Firecracker microVM on your own Linux machine in about five +minutes. + +This is the fastest path to a running Hyper node. It is a dev/eval setup -- for +a production install (dedicated system user, managed PostgreSQL, systemd) +follow the [installation guide](install.md). + +## What you need + +- An Ubuntu/Debian host (x86_64 or aarch64) with KVM -- `stat /dev/kvm` must + succeed. (Other distros work too, via the manual [installation + guide](install.md).) +- cgroups v2 (the default on any modern distro). +- `sudo` -- host provisioning (device-mapper, the setuid helper) needs root. + Hyper itself never runs as root. +- [Docker](https://docs.docker.com/engine/install/) -- for the side-car + Postgres. +- Elixir `~> 1.20` OTP 28+ ([install](https://elixir-lang.org/install.html)). +- Rust via [rustup](https://rustup.rs) -- the build compiles the setuid helper + and the in-guest agent. + +`setup.sh` checks every requirement up front and stops with an actionable +error before touching the host. + +## Run it + +```sh +git clone https://github.com/harmont-dev/hyper && cd hyper +./setup.sh # one-time host provisioning (asks for sudo) + # NOTE: please read this script and ensure you're comfortable + # with it + +iex -S mix # boot a Hyper node +``` + +## Boot a VM + +In the `iex` shell: + +```elixir +{:ok, img_id} = Hyper.Img.OciLoader.load("docker.io/library/alpine:3.19") +{:ok, vm} = Hyper.create_vm(%Hyper.Vm.Spec{img_id: img_id, type: :micro}) +{:ok, %{stdout: "hello\n"}} = Hyper.exec(vm, ["/bin/echo", "hello"]) +``` + +That is a real Firecracker microVM with a copy-on-write rootfs. The +[intro](intro.md#usage) walks through loading images, booting VMs, and +running commands; for other languages there is a +[gRPC interface](../grpc.md). + +## Where next + +- [Intro](intro.md) -- concepts and the usage walkthrough. +- [Installation guide](install.md) -- the manual, production-grade setup. +- [Configuration guide](config.md) -- every `/etc/hyper/config.toml` knob. diff --git a/mix.exs b/mix.exs index 0dd99dff..0a2b1f37 100644 --- a/mix.exs +++ b/mix.exs @@ -118,6 +118,7 @@ defmodule Hyper.MixProject do # Narrative/guide pages rendered alongside the API reference. extras: [ "README.md", + "docs/cookbook/quickstart.md", "docs/cookbook/intro.md", "docs/cookbook/install.md", "docs/cookbook/config.md", @@ -195,7 +196,7 @@ defmodule Hyper.MixProject do # the bindings in a consumer's build (they are gitignored, not in `lib`). # priv/repo ships the migrations so a consumer can `mix ecto.migrate`. # native/ ships the Rust crate *sources* because the `:suidhelper_stamp` and - # `:guest_agent_build` compilers build them wherever hyper compiles — the + # `:guest_agent_build` compilers build them wherever hyper compiles -- the # suidhelper must be built locally anyway (`Hyper.SuidHelper.verify_version/0` # checks the deployed helper's BLAKE3 against this build's stamp, so a # prebuilt binary can never match). Crate subpaths are listed explicitly to @@ -216,7 +217,7 @@ defmodule Hyper.MixProject do ), # `lib` is included wholesale, so on a dev box the gitignored generated # outputs (bindings, expected.ex) exist on disk and would leak into the - # tarball — with a build identity from the packaging machine. Keep the + # tarball -- with a build identity from the packaging machine. Keep the # package deterministic: consumers regenerate all of these at compile. exclude_patterns: [ ~r{^lib/hyper/firecracker/api/(operations|schemas)/}, @@ -239,7 +240,7 @@ defmodule Hyper.MixProject do # zsh (it only de-selects the `$ ` prompt; everything else is plain text) # from `ExDoc.Application.start`, which runs during the `docs` task. # - # So we start :ex_doc and :makeup_syntect here first (idempotent — the later + # So we start :ex_doc and :makeup_syntect here first (idempotent -- the later # `docs` task won't re-run their `start/2`), then register our shell aliases # LAST so they win. Dev-only; runs as the `docs` alias's first step. defp register_doc_lexers(_args) do diff --git a/setup.sh b/setup.sh new file mode 100755 index 00000000..34fa796c --- /dev/null +++ b/setup.sh @@ -0,0 +1,143 @@ +#!/usr/bin/env bash +# One-time host provisioning for a Hyper dev/eval node. After this succeeds, +# `iex -S mix` boots a Hyper node. +# +# Provisioning mirrors docs/cookbook/install.md -- keep the two in sync, the +# same way .github/scripts/provision-kvm-host.sh is kept in lockstep with it. +# +# Deliberate deltas from install.md (this is a dev/eval path, not production): +# - the BEAM runs as the invoking user, not a dedicated `hyper` system user +# - PostgreSQL is a docker container (hyper-pg), --restart unless-stopped +set -euo pipefail + +cd "$(dirname "$0")" + +WORK_DIR=/srv/hyper +PG_CONTAINER=hyper-pg +PG_IMAGE=postgres:16 + +info() { printf '\033[1;34m==>\033[0m %s\n' "$*"; } +fail() { printf '\033[1;31mERROR:\033[0m %s\n' "$*" >&2; exit 1; } +have() { command -v "$1" >/dev/null 2>&1; } + +[ -e /dev/kvm ] \ + || fail "no /dev/kvm -- Hyper needs a Linux host with KVM (bare metal, or a cloud instance with nested virtualization)" +[ "$(stat -fc %T /sys/fs/cgroup)" = cgroup2fs ] \ + || fail "cgroups v2 not mounted at /sys/fs/cgroup" +have apt-get \ + || fail "setup.sh automates Ubuntu/Debian only -- follow docs/cookbook/install.md on other distros" +have sudo || fail "sudo is required for host provisioning" +have elixir \ + || fail "install Elixir ~> 1.20 on OTP 28+ first: https://elixir-lang.org/install.html" +have rustup || fail "install Rust via rustup first: https://rustup.rs" +have docker \ + || fail "install Docker first (used only for the dev postgres): https://docs.docker.com/engine/install/" +docker info >/dev/null 2>&1 \ + || fail "docker is installed but not usable -- is the daemon running, and are you in the docker group?" + +info "host provisioning needs root -- asking sudo once up front" +sudo -v + +info "OS packages" +sudo apt-get update +sudo apt-get install -y \ + coreutils e2fsprogs libc-bin lvm2 skopeo thin-provisioning-tools \ + util-linux "linux-modules-extra-$(uname -r)" + +info "device-mapper modules" +# -a is load-bearing: without it modprobe reads the 2nd+ names as module +# PARAMETERS of the first, and the load fails. +sudo modprobe -av dm_snapshot dm_thin_pool loop +printf 'dm_snapshot\ndm_thin_pool\nloop\n' \ + | sudo tee /etc/modules-load.d/hyper.conf >/dev/null +sudo dmsetup targets | grep -q thin-pool \ + || fail "thin-pool dm target missing after modprobe" + +# Ubuntu ships thin_dump as a symlink into pdata_tools; the helper's SafeBin +# rejects symlinks, so install a dereferenced hard copy under the name the +# multi-call binary dispatches on. Skip when a prior run already placed it -- +# it then wins on PATH, and install refuses to copy a file onto itself. +thin_dump_src="$(readlink -f "$(command -v thin_dump)")" +if [ "$thin_dump_src" != /usr/local/sbin/thin_dump ]; then + sudo install -o root -g root -m 0755 "$thin_dump_src" /usr/local/sbin/thin_dump +fi + +info "build toolchain extras" +rustup target add "$(uname -m)-unknown-linux-musl" +have protoc || sudo apt-get install -y protobuf-compiler +[ -x "$HOME/.mix/escripts/protoc-gen-elixir" ] \ + || mix escript.install --force hex protobuf 0.17.0 + +if [ -f /etc/hyper/config.toml ]; then + info "/etc/hyper/config.toml exists -- leaving it alone" +else + info "writing /etc/hyper/config.toml" + sudo mkdir -p /etc/hyper + sudo tee /etc/hyper/config.toml >/dev/null <<'EOF' +work_dir = "/srv/hyper" + +[tools] +firecracker = "/opt/firecracker/firecracker" +jailer = "/opt/firecracker/jailer" +thin_dump = "/usr/local/sbin/thin_dump" + +[jails] +uid_gid_range = [900000, 999999] +cgroup = "hyper" +EOF + sudo chown root:root /etc/hyper/config.toml + sudo chmod 0644 /etc/hyper/config.toml +fi + +info "cgroup subtree" +sudo mkdir -p /sys/fs/cgroup/hyper +echo '+cpu +memory' \ + | sudo tee /sys/fs/cgroup/hyper/cgroup.subtree_control >/dev/null +echo 'd /sys/fs/cgroup/hyper 0755 root root -' \ + | sudo tee /etc/tmpfiles.d/hyper-cgroup.conf >/dev/null + +info "work dir ($WORK_DIR)" +sudo mkdir -p "$WORK_DIR" +sudo chown "$(id -u):$(id -g)" "$WORK_DIR" +# layers/ must pre-exist: boot validation (Layer.Repo.test_system) checks +# it, and the node only creates it lazily on first image load. +mkdir -p "$WORK_DIR/layers" + +info "compiling hyper (first run builds two Rust crates -- takes a while)" +mix deps.get +mix compile + +info "firecracker + jailer" +sudo mkdir -p /opt/firecracker +# -R so a re-run can overwrite binaries the previous run left root-owned (the +# chmod below hands them to root); without it mix firecracker.install's cp +# hits permission denied on the second pass. +sudo chown -R "$(id -u):$(id -g)" /opt/firecracker +mix firecracker.install +sudo chown root:root /opt/firecracker/firecracker /opt/firecracker/jailer +sudo chmod 0755 /opt/firecracker/firecracker /opt/firecracker/jailer + +info "setuid helper" +mix suidhelper.install +[ -u /usr/local/bin/hyper-suidhelper ] \ + || fail "hyper-suidhelper missing or not setuid-root" + +info "postgres ($PG_CONTAINER)" +pg_ready() { docker exec "$PG_CONTAINER" pg_isready -U postgres >/dev/null 2>&1; } +if ! pg_ready; then + docker start "$PG_CONTAINER" >/dev/null 2>&1 \ + || docker run -d --restart unless-stopped --name "$PG_CONTAINER" \ + -e POSTGRES_USER=postgres -e POSTGRES_PASSWORD=postgres \ + -e POSTGRES_DB=hyper_dev -p 5432:5432 "$PG_IMAGE" >/dev/null + for _ in $(seq 1 60); do + pg_ready && break + sleep 1 + done + pg_ready || fail "postgres ($PG_CONTAINER) not ready after 60s" +fi + +info "image database" +mix ecto.create -r Hyper.Img.Db.Repo +mix ecto.migrate -r Hyper.Img.Db.Repo + +info "setup complete -- boot a node with: iex -S mix"