From 16c10b592b9ca4d797aea37e93083c0002c063c9 Mon Sep 17 00:00:00 2001 From: Tianyu Zhou Date: Mon, 24 Aug 2026 03:11:06 +0800 Subject: [PATCH 1/5] feat: add sandbox checkpoint restore API Expose a small synchronous checkpoint API in akernel-sdk with immutable checkpoint identities, restore-to-new-sandbox semantics, and explicit list and delete operations. Keep snapshot lifetime manual and default checkpoint timeouts to 180 seconds without adding TTL policy to the SDK. Adapt the default YuanRong sandbox backend, document the public contract, and add unit and integration coverage. Extend standalone GitHub CI to run checkpoint and restore scenarios for runsc and Firecracker on KVM runners. Signed-off-by: Tianyu Zhou --- .github/workflows/ci.yml | 23 +++ sdk/python/README.md | 58 ++++++++ sdk/python/akernel_sdk/__init__.py | 2 + sdk/python/akernel_sdk/_backends/base.py | 14 ++ .../_backends/openyuanrong_sandbox.py | 123 +++++++++++++++- .../akernel_sdk/_backends/openyuanrong_sdk.py | 48 ++++++- sdk/python/akernel_sdk/sandbox.py | 134 +++++++++++++++++- sdk/python/akernel_sdk/types.py | 15 ++ sdk/python/examples/checkpoint_restore.py | 39 +++++ sdk/python/pyproject.toml | 2 +- sdk/python/tests/integration/test_sandbox.py | 45 ++++++ sdk/python/tests/unit/test_backends.py | 120 ++++++++++++++++ sdk/python/tests/unit/test_sandbox.py | 84 +++++++++++ sdk/python/tests/unit/test_types.py | 17 ++- 14 files changed, 717 insertions(+), 7 deletions(-) create mode 100644 sdk/python/examples/checkpoint_restore.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5e6c682..b24c993 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -121,7 +121,11 @@ jobs: sudo modprobe loop sudo modprobe erofs sudo modprobe br_netfilter + sudo modprobe kvm + sudo modprobe kvm_intel || sudo modprobe kvm_amd || true test -c /dev/net/tun + test -c /dev/kvm + sudo chmod 0666 /dev/kvm sudo sysctl -w net.bridge.bridge-nf-call-iptables=1 - name: Build all-in-one image @@ -166,6 +170,25 @@ jobs: timeout 120s python "sdk/python/examples/${example}" done + - name: Run runsc and Firecracker checkpoint restore E2E + run: | + gateway_ip="$(docker inspect \ + --format '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' \ + akernel-traefik)" + test -n "${gateway_ip}" + token="$(cat deploy/standalone/data/token)" + export AKERNEL_TOKEN="${token}" + export AKERNEL_SERVER_ADDRESS="${gateway_ip}" + export AKERNEL_RUN_INTEGRATION=1 + export PYTHONPATH="${GITHUB_WORKSPACE}/sdk/python" + + for runtime in runsc firecracker; do + echo "=== Checkpoint/restore runtime=${runtime} ===" + AKERNEL_TEST_RUNTIME="${runtime}" timeout 300s python \ + sdk/python/tests/integration/test_sandbox.py \ + SandboxCheckpointIntegrationTest -v + done + - name: Show standalone diagnostics if: failure() run: | diff --git a/sdk/python/README.md b/sdk/python/README.md index 700f93f..0a4320e 100644 --- a/sdk/python/README.md +++ b/sdk/python/README.md @@ -22,6 +22,7 @@ It supports two backends: - [Filesystem](#filesystem) - [Interactive PTYs](#interactive-ptys) - [Port forwarding](#port-forwarding) + - [Checkpoint and restore](#checkpoint-and-restore) - [Reverse tunnels](#reverse-tunnels) - [Rootfs and mounts](#rootfs-and-mounts) - [Launch from a Dockerfile](#launch-from-a-dockerfile) @@ -352,6 +353,61 @@ with Sandbox(port_forwardings=[8080]) as sandbox: deployment operator explicitly wants the direct Traefik address instead of the public gateway. +## Checkpoint and restore + +Create an immutable checkpoint of a running sandbox and restore it as a new, +independent sandbox: + +```python +from akernel_sdk import Sandbox + +checkpoint = None +try: + with Sandbox(runtime="runsc") as source: + source.commands.run("printf before > /tmp/state && sync") + checkpoint = source.checkpoint(timeout=180) + source.commands.run("printf after > /tmp/state") + + with Sandbox.restore(checkpoint) as restored: + assert restored.id != source.id + assert restored.commands.run("cat /tmp/state").stdout == "before" +finally: + if checkpoint is not None: + Sandbox.delete_checkpoint(checkpoint) +``` + +`checkpoint()` keeps the source running by default. Set +`leave_running=False` to terminate it after the checkpoint commits. Checkpoints +do not expire and must be removed explicitly with `delete_checkpoint()`; +`list_checkpoints()` returns all checkpoint identities visible to the current +tenant. + +Each restore gets a new sandbox ID, placement, network attachment, and routes. +The runtime, root filesystem, resources, mounts, environment, network policy, +and filesystem/process state come from the checkpoint. v1 does not support +in-place rollback or restore-time resource and configuration overrides. + +The bundled backend supports checkpoints for runsc and Firecracker. A restore +must use compatible runtime binaries, architecture, kernel, and runtime +configuration. The cluster prefers the source node when it is available and +may fall back to another compatible node through the configured snapshot +storage. + +For a checkpoint created from a sandbox with a reverse tunnel, pass an +explicit `reverse_tunnel` to `restore()` using the same `reverse_port` and +`listen_port`. The target and connection timeout may change. A checkpoint made +without a tunnel rejects adding one during restore. The source tunnel is +briefly disconnected during checkpoint creation and then reconnected. + +Checkpoint/restore is available through the default `openyuanrong-sandbox` +backend. The legacy `openyuanrong-sdk` actor backend reports it as unsupported. +The current official backend package supports the default 180-second +checkpoint timeout. Custom checkpoint timeouts and checkpointing a sandbox +with an active reverse tunnel require a backend release containing the +corresponding YuanRong changes. +See [`examples/checkpoint_restore.py`](./examples/checkpoint_restore.py) for a +runnable example. + ## Reverse tunnels A reverse tunnel lets sandbox code call an HTTP or HTTPS service reachable @@ -519,6 +575,7 @@ Maintained examples are under [`examples/`](./examples): - `basic_usage.py` - `command_stdin.py` +- `checkpoint_restore.py` - `custom_image.py` - `dockerfile_launch.py` - `gpu_sandbox.py` @@ -556,6 +613,7 @@ not part of the default test suite. | `CommandInfo` | `pid`, `command`, `running` | | `EntryInfo` | `name`, `path`, `type`, `size`, `permissions`, `modified_time` | | `SandboxInfo` | `id`, `state`, `cpu`, `memory`, `image`, `xpu`, `storage_mb` | +| `CheckpointInfo` | `id` | | `NodeInfo` | `id`, `status`, `capacity`, `allocatable`, `labels` | | `S3Config` | `endpoint`, `bucket`, `object`, optional credentials | | `Mount` | `target`, one source, and `type` | diff --git a/sdk/python/akernel_sdk/__init__.py b/sdk/python/akernel_sdk/__init__.py index 5e77dd9..78fff0c 100644 --- a/sdk/python/akernel_sdk/__init__.py +++ b/sdk/python/akernel_sdk/__init__.py @@ -24,6 +24,7 @@ ) from ._backends.registry import selected_backend from .types import ( + CheckpointInfo, CommandInfo, CommandResult, EntryInfo, @@ -37,6 +38,7 @@ __all__ = [ "Sandbox", + "CheckpointInfo", "S3Config", "Mount", "NetworkPolicy", diff --git a/sdk/python/akernel_sdk/_backends/base.py b/sdk/python/akernel_sdk/_backends/base.py index e0ec3b7..6c1bd1d 100644 --- a/sdk/python/akernel_sdk/_backends/base.py +++ b/sdk/python/akernel_sdk/_backends/base.py @@ -41,6 +41,7 @@ class Capability(Enum): NODE_PLACEMENT = auto() CUSTOM_REVERSE_TUNNEL_PORTS = auto() REVERSE_WEBSOCKET = auto() + CHECKPOINT_RESTORE = auto() @dataclass(frozen=True) @@ -149,6 +150,8 @@ def is_running(self) -> bool: ... def get_info(self) -> SandboxInfo: ... + def checkpoint(self, *, timeout: int) -> str: ... + def terminate(self) -> None: ... def close(self) -> None: ... @@ -163,6 +166,17 @@ class Backend(Protocol): def create(self, spec: SandboxSpec) -> BackendSession: ... + def restore( + self, + checkpoint_id: str, + *, + reverse_tunnel: HttpReverseTunnel | None, + ) -> BackendSession: ... + + def list_checkpoints(self) -> list[str]: ... + + def delete_checkpoint(self, checkpoint_id: str) -> None: ... + def delete_named(self, name: str) -> None: ... def close(self) -> None: ... diff --git a/sdk/python/akernel_sdk/_backends/openyuanrong_sandbox.py b/sdk/python/akernel_sdk/_backends/openyuanrong_sandbox.py index 56773b5..594d037 100644 --- a/sdk/python/akernel_sdk/_backends/openyuanrong_sandbox.py +++ b/sdk/python/akernel_sdk/_backends/openyuanrong_sandbox.py @@ -16,13 +16,20 @@ from __future__ import annotations +import inspect import os from collections.abc import Mapping from typing import Any import yr_sandbox -from ..types import CommandInfo, CommandResult, EntryInfo, SandboxInfo +from ..types import ( + CommandInfo, + CommandResult, + EntryInfo, + HttpReverseTunnel, + SandboxInfo, +) from .base import ( Backend, BackendConfig, @@ -246,6 +253,40 @@ def get_info(self) -> SandboxInfo: storage_mb=self._spec.storage_mb, ) + def checkpoint(self, *, timeout: int) -> str: + if self._terminated or self._closed: + raise BackendOperationError("checkpoint requires a running sandbox") + create_snapshot = self._sandbox.create_snapshot + try: + parameters = inspect.signature(create_snapshot).parameters.values() + supports_timeout = any( + parameter.name == "timeout" + or parameter.kind is inspect.Parameter.VAR_KEYWORD + for parameter in parameters + ) + except (TypeError, ValueError): + # Unknown callable signatures are treated as current backends. If + # they reject the keyword, the normal backend error includes the + # native failure instead of silently changing timeout semantics. + supports_timeout = True + if not supports_timeout and timeout != 180: + raise UnsupportedBackendFeatureError( + "The installed openyuanrong-sandbox backend only supports " + "the default 180-second checkpoint timeout. Upgrade the " + "backend to use a custom timeout." + ) + try: + if supports_timeout: + value = create_snapshot(timeout=timeout) + else: + value = create_snapshot() + except Exception as error: + raise _convert_error("checkpoint sandbox", error) from error + checkpoint_id = str(value.snapshot_id).strip() + if not checkpoint_id: + raise BackendOperationError("checkpoint returned an empty identity") + return checkpoint_id + def terminate(self) -> None: if self._terminated: return @@ -280,6 +321,7 @@ class OpenYuanRongSandboxBackend: { Capability.S3_ROOTFS, Capability.NODE_PLACEMENT, + Capability.CHECKPOINT_RESTORE, } ) @@ -380,6 +422,85 @@ def create(self, spec: SandboxSpec) -> BackendSession: raise _convert_error("create sandbox", error) from error return _Session(sandbox, spec) + def restore( + self, + checkpoint_id: str, + *, + reverse_tunnel: HttpReverseTunnel | None, + ) -> BackendSession: + if reverse_tunnel is not None and ( + reverse_tunnel.reverse_port != reverse_tunnel.listen_port - 1 + ): + raise UnsupportedBackendFeatureError( + "Backend 'openyuanrong-sandbox' requires reverse_port to equal " + "listen_port - 1." + ) + try: + sandbox = yr_sandbox.Sandbox.create( + checkpoint_id, + upstream=( + reverse_tunnel.target if reverse_tunnel is not None else None + ), + tunnel_connect_timeout=( + reverse_tunnel.connect_timeout + if reverse_tunnel is not None + else None + ), + proxy_port=( + reverse_tunnel.listen_port + if reverse_tunnel is not None + else _DEFAULT_LISTEN_PORT + ), + ) + except Exception as error: + raise _convert_error("restore checkpoint", error) from error + restored_spec = SandboxSpec( + image=None, + rootfs=None, + runtime="runsc", + cpu=1000, + memory=4096, + cpu_limit=0, + mem_limit=0, + idle_timeout=300, + schedule_timeout=30, + env={}, + name=None, + command_cwd=None, + port_forwardings=(), + mounts=(), + reverse_tunnel=reverse_tunnel, + detached=False, + node_id=None, + xpu=None, + storage_mb=None, + network_policy=None, + extra_config={}, + ) + return _Session(sandbox, restored_spec) + + def list_checkpoints(self) -> list[str]: + checkpoint_ids: list[str] = [] + page_token: str | None = None + try: + while True: + items, next_page_token = yr_sandbox.Sandbox.list_snapshots( + page_token=page_token, + page_size=100, + ) + checkpoint_ids.extend(str(item.snapshot_id) for item in items) + if not next_page_token: + return checkpoint_ids + page_token = next_page_token + except Exception as error: + raise _convert_error("list checkpoints", error) from error + + def delete_checkpoint(self, checkpoint_id: str) -> None: + try: + yr_sandbox.Sandbox.delete_snapshot(checkpoint_id) + except Exception as error: + raise _convert_error("delete checkpoint", error) from error + def delete_named(self, name: str) -> None: sandbox_id = f"{self.namespace}-{name}" try: diff --git a/sdk/python/akernel_sdk/_backends/openyuanrong_sdk.py b/sdk/python/akernel_sdk/_backends/openyuanrong_sdk.py index 51cc488..9be4b62 100644 --- a/sdk/python/akernel_sdk/_backends/openyuanrong_sdk.py +++ b/sdk/python/akernel_sdk/_backends/openyuanrong_sdk.py @@ -20,7 +20,13 @@ from collections.abc import Mapping from typing import Any -from ..types import CommandInfo, CommandResult, EntryInfo, SandboxInfo +from ..types import ( + CommandInfo, + CommandResult, + EntryInfo, + HttpReverseTunnel, + SandboxInfo, +) from . import openyuanrong_sdk_impl as _impl from .base import ( Backend, @@ -29,7 +35,7 @@ Capability, SandboxSpec, ) -from .errors import BackendOperationError +from .errors import BackendOperationError, UnsupportedBackendFeatureError from .openyuanrong_sdk_commands import ( CommandHandle as NativeCommandHandle, ) @@ -238,6 +244,13 @@ def get_info(self) -> SandboxInfo: storage_mb=self._spec.storage_mb, ) + def checkpoint(self, *, timeout: int) -> str: + del timeout + raise UnsupportedBackendFeatureError( + "Backend 'openyuanrong-sdk' does not support reusable checkpoints. " + "Use the default 'openyuanrong-sandbox' backend." + ) + def terminate(self) -> None: if self._terminated: return @@ -265,7 +278,11 @@ class OpenYuanRongSdkBackend: name = "openyuanrong-sdk" namespace = _NAMESPACE - capabilities = frozenset(Capability) + capabilities: frozenset[Capability] = frozenset( + capability + for capability in Capability + if capability is not Capability.CHECKPOINT_RESTORE + ) def __init__(self, _config: BackendConfig) -> None: _impl.ensure_initialized() @@ -333,6 +350,31 @@ def create(self, spec: SandboxSpec) -> BackendSession: _rollback_instance(instance, "session initialization") raise _convert_error("initialize sandbox session", error) from error + def restore( + self, + checkpoint_id: str, + *, + reverse_tunnel: HttpReverseTunnel | None, + ) -> BackendSession: + del checkpoint_id, reverse_tunnel + raise UnsupportedBackendFeatureError( + "Backend 'openyuanrong-sdk' does not support reusable checkpoints. " + "Use the default 'openyuanrong-sandbox' backend." + ) + + def list_checkpoints(self) -> list[str]: + raise UnsupportedBackendFeatureError( + "Backend 'openyuanrong-sdk' does not support reusable checkpoints. " + "Use the default 'openyuanrong-sandbox' backend." + ) + + def delete_checkpoint(self, checkpoint_id: str) -> None: + del checkpoint_id + raise UnsupportedBackendFeatureError( + "Backend 'openyuanrong-sdk' does not support reusable checkpoints. " + "Use the default 'openyuanrong-sandbox' backend." + ) + def delete_named(self, name: str) -> None: try: _impl.delete_named_instance(name) diff --git a/sdk/python/akernel_sdk/sandbox.py b/sdk/python/akernel_sdk/sandbox.py index f48851a..38ff75d 100644 --- a/sdk/python/akernel_sdk/sandbox.py +++ b/sdk/python/akernel_sdk/sandbox.py @@ -32,7 +32,14 @@ from .commands import CommandHandle, Commands from .filesystem import Filesystem from .pty import Pty -from .types import HttpReverseTunnel, Mount, NetworkPolicy, S3Config, SandboxInfo +from .types import ( + CheckpointInfo, + HttpReverseTunnel, + Mount, + NetworkPolicy, + S3Config, + SandboxInfo, +) _traefik_internal_ip_cache: str | None = None logger = logging.getLogger(__name__) @@ -135,6 +142,17 @@ def _validate_integer( raise ValueError(f"{name} must be greater than or equal to {minimum}") +def _checkpoint_id(checkpoint: CheckpointInfo | str) -> str: + if isinstance(checkpoint, CheckpointInfo): + return checkpoint.id + if not isinstance(checkpoint, str): + raise TypeError("checkpoint must be a CheckpointInfo or string") + value = checkpoint.strip() + if not value: + raise ValueError("checkpoint must be a non-empty string") + return value + + def _get_traefik_internal_ip(gateway: Endpoint) -> tuple[str, int]: """Resolve Traefik's direct address for ``internal=True`` URLs.""" @@ -443,6 +461,120 @@ def reverse_tunnel(self) -> HttpReverseTunnel | None: return self._reverse_tunnel + def checkpoint( + self, + *, + timeout: int = 180, + leave_running: bool = True, + ) -> CheckpointInfo: + """Create a reusable checkpoint of this sandbox. + + The checkpoint has no TTL and remains available until explicitly + deleted with :meth:`delete_checkpoint`. A successful checkpoint is an + immutable template; each restore creates a new sandbox identity with + fresh placement and routes. + + Args: + timeout: Positive checkpoint timeout in seconds. + leave_running: Keep this source sandbox running after success. + + Returns: + The stable checkpoint identity. + """ + + _validate_integer("timeout", timeout, minimum=1) + if not isinstance(leave_running, bool): + raise TypeError("leave_running must be a boolean") + if self._closed or self._session is None or not self.is_running(): + raise RuntimeError("checkpoint requires a running sandbox") + checkpoint = CheckpointInfo(self._session.checkpoint(timeout=timeout)) + if not leave_running: + self.kill() + return checkpoint + + @classmethod + def restore( + cls, + checkpoint: CheckpointInfo | str, + *, + reverse_tunnel: HttpReverseTunnel | None = None, + ) -> Sandbox: + """Restore an independent sandbox from a reusable checkpoint. + + Runtime, root filesystem, resources, mounts, environment, network + policy, and exposed-port shape are inherited from the checkpoint. + Resource overrides and in-place rollback are intentionally not part of + v1. A source created with a reverse tunnel requires an explicit tunnel + with the same ports here; its target and connect timeout may differ. + """ + + checkpoint_id = _checkpoint_id(checkpoint) + if reverse_tunnel is not None and not isinstance( + reverse_tunnel, HttpReverseTunnel + ): + raise TypeError("reverse_tunnel must be an HttpReverseTunnel") + session = load_backend().restore( + checkpoint_id, + reverse_tunnel=reverse_tunnel, + ) + restored = cls.__new__(cls) + restored._session = session + restored._startup_command = None + restored._pty = None + restored._closed = False + restored._terminated = False + restored._reverse_tunnel = reverse_tunnel + restored._forwarded_ports = set() + restored._image = None + restored._cpu = 0 + restored._memory = 0 + restored._xpu = None + restored._storage_mb = None + restored._id = "" + try: + restored._id = session.id + restored._files = Filesystem(session.files) + restored._commands = Commands(session.commands) + restored._pty = Pty(restored._id) + info = session.get_info() + restored._image = info.image + restored._cpu = info.cpu if info.cpu is not None else 0 + restored._memory = info.memory if info.memory is not None else 0 + restored._xpu = info.xpu + restored._storage_mb = info.storage_mb + except Exception: + restored._closed = True + try: + session.terminate() + except Exception: + logger.warning( + "failed to roll back a partially initialized restore", + exc_info=True, + ) + try: + session.close() + except Exception: + logger.warning( + "failed to close a partially initialized restore session", + exc_info=True, + ) + raise + return restored + + @classmethod + def list_checkpoints(cls) -> list[CheckpointInfo]: + """List reusable checkpoints visible to the current tenant.""" + + del cls + return [CheckpointInfo(value) for value in load_backend().list_checkpoints()] + + @classmethod + def delete_checkpoint(cls, checkpoint: CheckpointInfo | str) -> None: + """Permanently delete one reusable checkpoint.""" + + del cls + load_backend().delete_checkpoint(_checkpoint_id(checkpoint)) + def get_port_url(self, port: int, *, internal: bool = False) -> str: """Return the gateway URL for a declared sandbox port. diff --git a/sdk/python/akernel_sdk/types.py b/sdk/python/akernel_sdk/types.py index 67a43cd..db137da 100644 --- a/sdk/python/akernel_sdk/types.py +++ b/sdk/python/akernel_sdk/types.py @@ -160,6 +160,21 @@ class SandboxInfo: storage_mb: int | None = None +@dataclass(frozen=True) +class CheckpointInfo: + """Stable identity of a reusable sandbox checkpoint.""" + + id: str + + def __post_init__(self) -> None: + if not isinstance(self.id, str): + raise TypeError("id must be a string") + normalized = self.id.strip() + if not normalized: + raise ValueError("id must be a non-empty string") + object.__setattr__(self, "id", normalized) + + @dataclass(frozen=True) class NodeInfo: """Capacity, allocation, and labels advertised by an AKernel node.""" diff --git a/sdk/python/examples/checkpoint_restore.py b/sdk/python/examples/checkpoint_restore.py new file mode 100644 index 0000000..b9359b7 --- /dev/null +++ b/sdk/python/examples/checkpoint_restore.py @@ -0,0 +1,39 @@ +# Copyright (c) 2026 Ant Group Corporation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Create, restore, and explicitly delete a reusable checkpoint.""" + +from akernel_sdk import CheckpointInfo, Sandbox + + +def main() -> None: + checkpoint: CheckpointInfo | None = None + try: + with Sandbox(runtime="runsc") as source: + source.commands.run("printf before > /tmp/checkpoint-state && sync") + checkpoint = source.checkpoint() + source.commands.run("printf after > /tmp/checkpoint-state") + print("source:", source.id, "checkpoint:", checkpoint.id) + + with Sandbox.restore(checkpoint) as restored: + value = restored.commands.run("cat /tmp/checkpoint-state") + print("restored:", restored.id, "state:", value.stdout) + assert value.stdout == "before" + finally: + if checkpoint is not None: + Sandbox.delete_checkpoint(checkpoint) + + +if __name__ == "__main__": + main() diff --git a/sdk/python/pyproject.toml b/sdk/python/pyproject.toml index 166f6d6..f3619e7 100644 --- a/sdk/python/pyproject.toml +++ b/sdk/python/pyproject.toml @@ -28,7 +28,7 @@ classifiers = [ "Topic :: System :: Distributed Computing", ] dependencies = [ - "openyuanrong-sandbox==0.9.9", + "openyuanrong-sandbox==0.9.10", "websockets>=10.0", "dockerfile-parse>=2.0.1", ] diff --git a/sdk/python/tests/integration/test_sandbox.py b/sdk/python/tests/integration/test_sandbox.py index 08586e3..15259b0 100644 --- a/sdk/python/tests/integration/test_sandbox.py +++ b/sdk/python/tests/integration/test_sandbox.py @@ -99,5 +99,50 @@ def test_pty_interrupts_foreground_process(self): self.assertIn(b"PTY_AFTER_INTERRUPT", output) +@unittest.skipUnless( + _ENABLED, + "set AKERNEL_RUN_INTEGRATION=1 and the AKernel SDK environment", +) +class SandboxCheckpointIntegrationTest(unittest.TestCase): + def test_checkpoint_restore_and_delete(self): + source = Sandbox(cpu=1000, memory=2048, runtime=_RUNTIME) + restored = None + checkpoint = None + try: + source_id = source.id + created = source.commands.run( + "printf checkpoint-before > /tmp/akernel-checkpoint-state && sync" + ) + self.assertEqual(created.exit_code, 0) + + checkpoint = source.checkpoint(timeout=180) + self.assertTrue(source.is_running()) + checkpoint_ids = {item.id for item in Sandbox.list_checkpoints()} + self.assertIn(checkpoint.id, checkpoint_ids) + + changed = source.commands.run( + "printf source-after > /tmp/akernel-checkpoint-state && sync" + ) + self.assertEqual(changed.exit_code, 0) + + restored = Sandbox.restore(checkpoint) + self.assertNotEqual(restored.id, source_id) + restored_value = restored.commands.run( + "cat /tmp/akernel-checkpoint-state" + ) + self.assertEqual(restored_value.exit_code, 0) + self.assertEqual(restored_value.stdout, "checkpoint-before") + self.assertEqual( + source.commands.run("cat /tmp/akernel-checkpoint-state").stdout, + "source-after", + ) + finally: + if restored is not None: + restored.kill() + source.kill() + if checkpoint is not None: + Sandbox.delete_checkpoint(checkpoint) + + if __name__ == "__main__": unittest.main() diff --git a/sdk/python/tests/unit/test_backends.py b/sdk/python/tests/unit/test_backends.py index c65924c..cc40955 100644 --- a/sdk/python/tests/unit/test_backends.py +++ b/sdk/python/tests/unit/test_backends.py @@ -464,6 +464,118 @@ def test_named_delete_uses_deterministic_sid(self): self.backend.delete_named("worker") delete.assert_called_once_with("default-worker") + def test_checkpoint_delegates_to_reusable_snapshot_api(self): + native = MagicMock() + native.id = "default-source" + native.commands = MagicMock() + native.files = MagicMock() + native.create_snapshot.return_value = SimpleNamespace( + snapshot_id="checkpoint-1" + ) + with patch.object( + openyuanrong_sandbox.yr_sandbox, + "Sandbox", + return_value=native, + ): + session = self.backend.create(_spec()) + + self.assertEqual(session.checkpoint(timeout=240), "checkpoint-1") + + native.create_snapshot.assert_called_once_with(timeout=240) + + def test_checkpoint_uses_official_backend_default_timeout(self): + native = MagicMock() + native.id = "default-source" + native.commands = MagicMock() + native.files = MagicMock() + calls = [] + + def create_snapshot(): + calls.append(True) + return SimpleNamespace(snapshot_id="checkpoint-1") + + native.create_snapshot = create_snapshot + with patch.object( + openyuanrong_sandbox.yr_sandbox, + "Sandbox", + return_value=native, + ): + session = self.backend.create(_spec()) + + self.assertEqual(session.checkpoint(timeout=180), "checkpoint-1") + + self.assertEqual(calls, [True]) + + def test_checkpoint_rejects_custom_timeout_on_official_backend(self): + native = MagicMock() + native.id = "default-source" + native.commands = MagicMock() + native.files = MagicMock() + native.create_snapshot = lambda: SimpleNamespace( + snapshot_id="checkpoint-1" + ) + with patch.object( + openyuanrong_sandbox.yr_sandbox, + "Sandbox", + return_value=native, + ): + session = self.backend.create(_spec()) + + with self.assertRaisesRegex( + UnsupportedBackendFeatureError, + "default 180-second", + ): + session.checkpoint(timeout=240) + + def test_restore_uses_snapshot_template_and_explicit_tunnel(self): + native = MagicMock() + native.id = "default-restored" + native.commands = MagicMock() + native.files = MagicMock() + tunnel = HttpReverseTunnel( + "https://new-target.example", + reverse_port=9000, + listen_port=9001, + connect_timeout=12, + ) + with patch.object( + openyuanrong_sandbox.yr_sandbox, + "Sandbox", + ) as sandbox_type: + sandbox_type.create.return_value = native + session = self.backend.restore( + "checkpoint-1", + reverse_tunnel=tunnel, + ) + + self.assertEqual(session.id, "default-restored") + sandbox_type.create.assert_called_once_with( + "checkpoint-1", + upstream="https://new-target.example", + tunnel_connect_timeout=12, + proxy_port=9001, + ) + + def test_checkpoint_catalog_pages_and_deletes(self): + first = ([SimpleNamespace(snapshot_id="checkpoint-1")], "next") + second = ([SimpleNamespace(snapshot_id="checkpoint-2")], "") + with patch.object( + openyuanrong_sandbox.yr_sandbox.Sandbox, + "list_snapshots", + side_effect=[first, second], + ) as list_snapshots, patch.object( + openyuanrong_sandbox.yr_sandbox.Sandbox, + "delete_snapshot", + ) as delete_snapshot: + self.assertEqual( + self.backend.list_checkpoints(), + ["checkpoint-1", "checkpoint-2"], + ) + self.backend.delete_checkpoint("checkpoint-1") + + self.assertEqual(list_snapshots.call_count, 2) + delete_snapshot.assert_called_once_with("checkpoint-1") + class OpenYuanRongSdkBackendTest(unittest.TestCase): def setUp(self): @@ -598,6 +710,14 @@ def test_close_finalizes_actor_sdk(self): finalize.assert_called_once_with() + def test_reusable_checkpoint_operations_are_explicitly_unsupported(self): + with self.assertRaises(UnsupportedBackendFeatureError): + self.backend.restore("checkpoint-1", reverse_tunnel=None) + with self.assertRaises(UnsupportedBackendFeatureError): + self.backend.list_checkpoints() + with self.assertRaises(UnsupportedBackendFeatureError): + self.backend.delete_checkpoint("checkpoint-1") + if __name__ == "__main__": unittest.main() diff --git a/sdk/python/tests/unit/test_sandbox.py b/sdk/python/tests/unit/test_sandbox.py index 14a1386..9d9ddc4 100644 --- a/sdk/python/tests/unit/test_sandbox.py +++ b/sdk/python/tests/unit/test_sandbox.py @@ -18,6 +18,7 @@ from unittest.mock import MagicMock, patch from akernel_sdk import ( + CheckpointInfo, DockerfileLaunch, HttpReverseTunnel, NetworkPolicy, @@ -169,6 +170,89 @@ def test_named_delete_hides_backend_namespace(self): Sandbox.delete("worker") self.backend.delete_named.assert_called_once_with("worker") + def test_checkpoint_returns_public_identity_and_keeps_source_running(self): + self.session.checkpoint.return_value = "checkpoint-1" + sandbox = Sandbox() + + checkpoint = sandbox.checkpoint(timeout=240) + + self.assertEqual(checkpoint, CheckpointInfo("checkpoint-1")) + self.session.checkpoint.assert_called_once_with(timeout=240) + self.session.terminate.assert_not_called() + sandbox.kill() + + def test_checkpoint_can_terminate_source_after_success(self): + self.session.checkpoint.return_value = "checkpoint-1" + sandbox = Sandbox() + + checkpoint = sandbox.checkpoint(leave_running=False) + + self.assertEqual(checkpoint.id, "checkpoint-1") + self.session.terminate.assert_called_once_with() + self.session.close.assert_called_once_with() + + def test_checkpoint_validates_arguments_and_running_state(self): + sandbox = Sandbox() + for timeout in (True, 0, -1, 1.5): + with self.subTest(timeout=timeout), self.assertRaises( + (TypeError, ValueError) + ): + sandbox.checkpoint(timeout=timeout) + with self.assertRaisesRegex(TypeError, "leave_running"): + sandbox.checkpoint(leave_running=1) + self.session.is_running.return_value = False + with self.assertRaisesRegex(RuntimeError, "running sandbox"): + sandbox.checkpoint() + sandbox.kill() + + def test_restore_builds_facades_around_new_backend_session(self): + restored_session = MagicMock() + restored_session.id = "restored-physical-id" + restored_session.commands = MagicMock() + restored_session.files = MagicMock() + restored_session.get_info.return_value = SandboxInfo( + id="restored-physical-id", + state="running", + cpu=2000, + memory=8192, + image="base-image", + ) + self.backend.restore.return_value = restored_session + tunnel = HttpReverseTunnel("http://127.0.0.1:9000") + + restored = Sandbox.restore( + CheckpointInfo("checkpoint-1"), reverse_tunnel=tunnel + ) + + self.backend.restore.assert_called_once_with( + "checkpoint-1", reverse_tunnel=tunnel + ) + self.assertEqual(restored.id, "restored-physical-id") + self.assertIs(restored.reverse_tunnel, tunnel) + self.assertEqual(restored.get_info().cpu, 2000) + restored.kill() + restored_session.terminate.assert_called_once_with() + restored_session.close.assert_called_once_with() + + def test_list_and_delete_checkpoints_hide_backend_details(self): + self.backend.list_checkpoints.return_value = ["checkpoint-1", "checkpoint-2"] + + self.assertEqual( + Sandbox.list_checkpoints(), + [CheckpointInfo("checkpoint-1"), CheckpointInfo("checkpoint-2")], + ) + Sandbox.delete_checkpoint(CheckpointInfo("checkpoint-1")) + Sandbox.delete_checkpoint(" checkpoint-2 ") + + self.assertEqual( + self.backend.delete_checkpoint.call_args_list, + [unittest.mock.call("checkpoint-1"), unittest.mock.call("checkpoint-2")], + ) + with self.assertRaises(ValueError): + Sandbox.delete_checkpoint(" ") + with self.assertRaises(TypeError): + Sandbox.restore(object()) # type: ignore[arg-type] + def test_rootfs_requires_s3_config(self): with self.assertRaisesRegex(TypeError, "S3Config"): Sandbox(rootfs={"type": "s3"}) diff --git a/sdk/python/tests/unit/test_types.py b/sdk/python/tests/unit/test_types.py index e9766c4..2a8f4a8 100644 --- a/sdk/python/tests/unit/test_types.py +++ b/sdk/python/tests/unit/test_types.py @@ -19,7 +19,13 @@ from pathlib import Path import akernel_sdk -from akernel_sdk import DockerContextEntry, HttpReverseTunnel, Mount, S3Config +from akernel_sdk import ( + CheckpointInfo, + DockerContextEntry, + HttpReverseTunnel, + Mount, + S3Config, +) class PublicTypesTest(unittest.TestCase): @@ -69,6 +75,7 @@ def test_public_exports_are_minimal(self): set(akernel_sdk.__all__), { "Sandbox", + "CheckpointInfo", "S3Config", "Mount", "NetworkPolicy", @@ -120,6 +127,14 @@ def test_docker_context_entry_is_public_and_immutable(self): with self.assertRaisesRegex(AttributeError, "cannot assign"): entry.mode = 0o700 # type: ignore[misc] + def test_checkpoint_info_is_public_normalized_and_immutable(self): + checkpoint = CheckpointInfo(" checkpoint-1 ") + self.assertEqual(checkpoint.id, "checkpoint-1") + with self.assertRaisesRegex(AttributeError, "cannot assign"): + checkpoint.id = "changed" # type: ignore[misc] + with self.assertRaises(ValueError): + CheckpointInfo(" ") + def test_s3_config_serialization(self): config = S3Config( endpoint="https://s3.example.com", From 11846c6e9d27352266da59bb0e6e08f044f9ac74 Mon Sep 17 00:00:00 2001 From: Tianyu Zhou Date: Mon, 24 Aug 2026 03:11:36 +0800 Subject: [PATCH 2/5] build: pin checkpoint restore components Advance sandboxd to the latest mainline checkpoint implementation and pin YuanRong to the coordinated reusable snapshot integration. This keeps the AKernel build on the current StartRequest-based restore contract instead of the obsolete implementation used by the earlier prototype. Signed-off-by: Tianyu Zhou --- src/sandboxd | 2 +- src/yuanrong | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/sandboxd b/src/sandboxd index 1918fad..9979147 160000 --- a/src/sandboxd +++ b/src/sandboxd @@ -1 +1 @@ -Subproject commit 1918fadb03b59bc6f540196e14b90a91bdf31b7d +Subproject commit 99791478526fd7628d7aba46537d3ad029c971ab diff --git a/src/yuanrong b/src/yuanrong index 177592c..3c62d3f 160000 --- a/src/yuanrong +++ b/src/yuanrong @@ -1 +1 @@ -Subproject commit 177592c7eb208698bd9b048a818b9cfb02060545 +Subproject commit 3c62d3f8ad6c7be5e11d3141d6c396a8a2d10d86 From 92578322c36783fd5e99f81ba01d8c730a12ef10 Mon Sep 17 00:00:00 2001 From: Tianyu Zhou Date: Mon, 24 Aug 2026 05:44:33 +0800 Subject: [PATCH 3/5] feat: enable standalone checkpoint restore Enable YuanRong's snapshot data plane in both node startup modes and provide a stable checkpoint staging directory. Advance sandboxd and YuanRong to the implementations validated with the SDK checkpoint contract for runsc and Firecracker. Document the standalone storage and manual checkpoint lifecycle so operators do not confuse node-local staging with an SDK TTL or automatic expiration. Signed-off-by: Tianyu Zhou --- AGENTS.md | 6 ++++++ README.md | 2 +- builder/scripts/yr_node_bootstrap.sh | 8 ++++++++ deploy/standalone/README.md | 6 ++++++ src/sandboxd | 2 +- src/yuanrong | 2 +- 6 files changed, 23 insertions(+), 3 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 7fdbfed..a1e8fec 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -368,6 +368,12 @@ quotas for runsc and Firecracker use this local-disk filestore. Without an explicit quota, runsc retains its configured memory-backed overlay while Firecracker creates its configured sparse ext4 default. +The bundled node enables YuanRong's sandbox snapshot data plane with the +DataSystem backend and uses `/home/yuanrong/checkpoints` as node-local staging. +The public SDK does not expose snapshot TTLs: reusable checkpoints remain +until explicitly deleted. Keep the DataSystem backend and checkpoint staging +configuration enabled together when changing node startup arguments. + Keep detailed SDK reference material with the SDK. The root README should contain only the project-level entry points and representative examples: diff --git a/README.md b/README.md index aa78185..9ed91ce 100644 --- a/README.md +++ b/README.md @@ -207,7 +207,7 @@ See the complete [basic usage example](./sdk/python/examples/basic_usage.py), th - [x] Optional native Linux runc runtime - [x] Sandbox network ACL - [ ] Fork-based sandbox launch based on gVisor -- [ ] Sandbox checkpoint and restore +- [x] Sandbox checkpoint and restore for runsc and Firecracker - [ ] Support for GKE and AWS - [x] Cgroup v2 node support diff --git a/builder/scripts/yr_node_bootstrap.sh b/builder/scripts/yr_node_bootstrap.sh index 52e5e21..743ccd5 100755 --- a/builder/scripts/yr_node_bootstrap.sh +++ b/builder/scripts/yr_node_bootstrap.sh @@ -41,6 +41,8 @@ resolve_node_ip() { YR_NODE_IP="$(resolve_node_ip)" echo "Using ${YR_NODE_IP} as the YuanRong node address" +CHECKPOINT_DIR="/home/yuanrong/checkpoints" +mkdir -p "${CHECKPOINT_DIR}" # Select the legacy etcd registry or the FunctionMaster HTTP provider. if [ "${TRAEFIK_MODE:-etcd}" = "etcd" ]; then @@ -107,6 +109,9 @@ if [ "x${AKS_LOCAL_MODE}" == "xtrue" ]; then --iam_local_ip 127.0.0.1 \ --frontend_lease_bypass true \ --force_low_reliability_instance true \ + --enable_sandbox_pause_resume true \ + --snapshot_storage_backend datasystem \ + --checkpoint_dir "${CHECKPOINT_DIR}" \ --enable_sandbox_router true \ --enable_direct_routing false else @@ -145,5 +150,8 @@ else --function_proxy_merge_process_enable true \ --enable_direct_routing false \ --force_low_reliability_instance true \ + --enable_sandbox_pause_resume true \ + --snapshot_storage_backend datasystem \ + --checkpoint_dir "${CHECKPOINT_DIR}" \ --block true fi diff --git a/deploy/standalone/README.md b/deploy/standalone/README.md index 28f359a..574723d 100644 --- a/deploy/standalone/README.md +++ b/deploy/standalone/README.md @@ -57,6 +57,12 @@ rather than tmpfs. Without `storage_mb`, runsc retains its configured memory-backed overlay while Firecracker uses its configured sparse ext4 default. +Sandbox checkpoints for runsc and Firecracker are published through the +embedded YuanRong DataSystem. `/home/yuanrong/checkpoints` is only the node's +local staging directory; SDK checkpoint records have no automatic TTL and are +retained until `Sandbox.delete_checkpoint()` is called. A restored sandbox is +a new sandbox and receives fresh network routes. + `start.sh` loads the host `tun` module and verifies `/dev/net/tun` before starting the pooled-TAP runtimes. Runc retains its separate veth network path. diff --git a/src/sandboxd b/src/sandboxd index 9979147..d100919 160000 --- a/src/sandboxd +++ b/src/sandboxd @@ -1 +1 @@ -Subproject commit 99791478526fd7628d7aba46537d3ad029c971ab +Subproject commit d100919a069f9025f577b8d02c6144a668b05219 diff --git a/src/yuanrong b/src/yuanrong index 3c62d3f..acd1dab 160000 --- a/src/yuanrong +++ b/src/yuanrong @@ -1 +1 @@ -Subproject commit 3c62d3f8ad6c7be5e11d3141d6c396a8a2d10d86 +Subproject commit acd1dab312199e6a0f1b4aad466d5ce3bd25f954 From 5d5c143a5ec22ab73c3de014e977d7085c571c4f Mon Sep 17 00:00:00 2001 From: Tianyu Zhou Date: Mon, 24 Aug 2026 11:49:37 +0800 Subject: [PATCH 4/5] fix(deploy): delegate resolver injection to sandboxd Remove the static /etc/resolv.conf mount from standalone and Helm base OCI configuration. Sandboxd already prepares the resolver file for every sandbox, using the host resolver without an ACL and its managed resolver when an ACL is enabled, so carrying both mounts creates duplicate OCI destinations during checkpoint restore. Signed-off-by: Tianyu Zhou --- deploy/akernel/charts/core/values.yaml | 9 --------- deploy/standalone/config/config.json | 9 --------- 2 files changed, 18 deletions(-) diff --git a/deploy/akernel/charts/core/values.yaml b/deploy/akernel/charts/core/values.yaml index 6dfd158..5703561 100644 --- a/deploy/akernel/charts/core/values.yaml +++ b/deploy/akernel/charts/core/values.yaml @@ -379,15 +379,6 @@ node: "noexec", "nodev" ] - }, - { - "destination": "/etc/resolv.conf", - "type": "bind", - "source": "/etc/resolv_akernel.conf", - "options": [ - "bind", - "ro" - ] } ], "linux": { diff --git a/deploy/standalone/config/config.json b/deploy/standalone/config/config.json index 5387596..25458c9 100644 --- a/deploy/standalone/config/config.json +++ b/deploy/standalone/config/config.json @@ -232,15 +232,6 @@ "noexec", "nodev" ] - }, - { - "destination": "/etc/resolv.conf", - "type": "bind", - "source": "/etc/resolv.conf", - "options": [ - "bind", - "ro" - ] } ], "linux": { From 4439a75e9d9c02b4f5676f753af0527e09e76336 Mon Sep 17 00:00:00 2001 From: Tianyu Zhou Date: Mon, 24 Aug 2026 11:50:20 +0800 Subject: [PATCH 5/5] build: pin checkpoint capability integration Advance sandboxd and YuanRong together so the runtime capability response, guest handoff endpoints, FunctionSystem environment configuration, and RRT restore protocol use one compatible contract in AKernel builds. Signed-off-by: Tianyu Zhou --- src/sandboxd | 2 +- src/yuanrong | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/sandboxd b/src/sandboxd index d100919..a55f382 160000 --- a/src/sandboxd +++ b/src/sandboxd @@ -1 +1 @@ -Subproject commit d100919a069f9025f577b8d02c6144a668b05219 +Subproject commit a55f3827c477d68f1b0cb494f0a573ba27eb4cd8 diff --git a/src/yuanrong b/src/yuanrong index acd1dab..883028d 160000 --- a/src/yuanrong +++ b/src/yuanrong @@ -1 +1 @@ -Subproject commit acd1dab312199e6a0f1b4aad466d5ce3bd25f954 +Subproject commit 883028de4ad7ecceba08460c567ad4426b7965a7