diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5e6c682..b24c993 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -121,7 +121,11 @@ jobs: sudo modprobe loop sudo modprobe erofs sudo modprobe br_netfilter + sudo modprobe kvm + sudo modprobe kvm_intel || sudo modprobe kvm_amd || true test -c /dev/net/tun + test -c /dev/kvm + sudo chmod 0666 /dev/kvm sudo sysctl -w net.bridge.bridge-nf-call-iptables=1 - name: Build all-in-one image @@ -166,6 +170,25 @@ jobs: timeout 120s python "sdk/python/examples/${example}" done + - name: Run runsc and Firecracker checkpoint restore E2E + run: | + gateway_ip="$(docker inspect \ + --format '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' \ + akernel-traefik)" + test -n "${gateway_ip}" + token="$(cat deploy/standalone/data/token)" + export AKERNEL_TOKEN="${token}" + export AKERNEL_SERVER_ADDRESS="${gateway_ip}" + export AKERNEL_RUN_INTEGRATION=1 + export PYTHONPATH="${GITHUB_WORKSPACE}/sdk/python" + + for runtime in runsc firecracker; do + echo "=== Checkpoint/restore runtime=${runtime} ===" + AKERNEL_TEST_RUNTIME="${runtime}" timeout 300s python \ + sdk/python/tests/integration/test_sandbox.py \ + SandboxCheckpointIntegrationTest -v + done + - name: Show standalone diagnostics if: failure() run: | diff --git a/AGENTS.md b/AGENTS.md index 7fdbfed..a1e8fec 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -368,6 +368,12 @@ quotas for runsc and Firecracker use this local-disk filestore. Without an explicit quota, runsc retains its configured memory-backed overlay while Firecracker creates its configured sparse ext4 default. +The bundled node enables YuanRong's sandbox snapshot data plane with the +DataSystem backend and uses `/home/yuanrong/checkpoints` as node-local staging. +The public SDK does not expose snapshot TTLs: reusable checkpoints remain +until explicitly deleted. Keep the DataSystem backend and checkpoint staging +configuration enabled together when changing node startup arguments. + Keep detailed SDK reference material with the SDK. The root README should contain only the project-level entry points and representative examples: diff --git a/README.md b/README.md index aa78185..9ed91ce 100644 --- a/README.md +++ b/README.md @@ -207,7 +207,7 @@ See the complete [basic usage example](./sdk/python/examples/basic_usage.py), th - [x] Optional native Linux runc runtime - [x] Sandbox network ACL - [ ] Fork-based sandbox launch based on gVisor -- [ ] Sandbox checkpoint and restore +- [x] Sandbox checkpoint and restore for runsc and Firecracker - [ ] Support for GKE and AWS - [x] Cgroup v2 node support diff --git a/builder/scripts/yr_node_bootstrap.sh b/builder/scripts/yr_node_bootstrap.sh index 52e5e21..743ccd5 100755 --- a/builder/scripts/yr_node_bootstrap.sh +++ b/builder/scripts/yr_node_bootstrap.sh @@ -41,6 +41,8 @@ resolve_node_ip() { YR_NODE_IP="$(resolve_node_ip)" echo "Using ${YR_NODE_IP} as the YuanRong node address" +CHECKPOINT_DIR="/home/yuanrong/checkpoints" +mkdir -p "${CHECKPOINT_DIR}" # Select the legacy etcd registry or the FunctionMaster HTTP provider. if [ "${TRAEFIK_MODE:-etcd}" = "etcd" ]; then @@ -107,6 +109,9 @@ if [ "x${AKS_LOCAL_MODE}" == "xtrue" ]; then --iam_local_ip 127.0.0.1 \ --frontend_lease_bypass true \ --force_low_reliability_instance true \ + --enable_sandbox_pause_resume true \ + --snapshot_storage_backend datasystem \ + --checkpoint_dir "${CHECKPOINT_DIR}" \ --enable_sandbox_router true \ --enable_direct_routing false else @@ -145,5 +150,8 @@ else --function_proxy_merge_process_enable true \ --enable_direct_routing false \ --force_low_reliability_instance true \ + --enable_sandbox_pause_resume true \ + --snapshot_storage_backend datasystem \ + --checkpoint_dir "${CHECKPOINT_DIR}" \ --block true fi diff --git a/deploy/akernel/charts/core/values.yaml b/deploy/akernel/charts/core/values.yaml index 6dfd158..5703561 100644 --- a/deploy/akernel/charts/core/values.yaml +++ b/deploy/akernel/charts/core/values.yaml @@ -379,15 +379,6 @@ node: "noexec", "nodev" ] - }, - { - "destination": "/etc/resolv.conf", - "type": "bind", - "source": "/etc/resolv_akernel.conf", - "options": [ - "bind", - "ro" - ] } ], "linux": { diff --git a/deploy/standalone/README.md b/deploy/standalone/README.md index 28f359a..574723d 100644 --- a/deploy/standalone/README.md +++ b/deploy/standalone/README.md @@ -57,6 +57,12 @@ rather than tmpfs. Without `storage_mb`, runsc retains its configured memory-backed overlay while Firecracker uses its configured sparse ext4 default. +Sandbox checkpoints for runsc and Firecracker are published through the +embedded YuanRong DataSystem. `/home/yuanrong/checkpoints` is only the node's +local staging directory; SDK checkpoint records have no automatic TTL and are +retained until `Sandbox.delete_checkpoint()` is called. A restored sandbox is +a new sandbox and receives fresh network routes. + `start.sh` loads the host `tun` module and verifies `/dev/net/tun` before starting the pooled-TAP runtimes. Runc retains its separate veth network path. diff --git a/deploy/standalone/config/config.json b/deploy/standalone/config/config.json index 5387596..25458c9 100644 --- a/deploy/standalone/config/config.json +++ b/deploy/standalone/config/config.json @@ -232,15 +232,6 @@ "noexec", "nodev" ] - }, - { - "destination": "/etc/resolv.conf", - "type": "bind", - "source": "/etc/resolv.conf", - "options": [ - "bind", - "ro" - ] } ], "linux": { diff --git a/sdk/python/README.md b/sdk/python/README.md index 700f93f..0a4320e 100644 --- a/sdk/python/README.md +++ b/sdk/python/README.md @@ -22,6 +22,7 @@ It supports two backends: - [Filesystem](#filesystem) - [Interactive PTYs](#interactive-ptys) - [Port forwarding](#port-forwarding) + - [Checkpoint and restore](#checkpoint-and-restore) - [Reverse tunnels](#reverse-tunnels) - [Rootfs and mounts](#rootfs-and-mounts) - [Launch from a Dockerfile](#launch-from-a-dockerfile) @@ -352,6 +353,61 @@ with Sandbox(port_forwardings=[8080]) as sandbox: deployment operator explicitly wants the direct Traefik address instead of the public gateway. +## Checkpoint and restore + +Create an immutable checkpoint of a running sandbox and restore it as a new, +independent sandbox: + +```python +from akernel_sdk import Sandbox + +checkpoint = None +try: + with Sandbox(runtime="runsc") as source: + source.commands.run("printf before > /tmp/state && sync") + checkpoint = source.checkpoint(timeout=180) + source.commands.run("printf after > /tmp/state") + + with Sandbox.restore(checkpoint) as restored: + assert restored.id != source.id + assert restored.commands.run("cat /tmp/state").stdout == "before" +finally: + if checkpoint is not None: + Sandbox.delete_checkpoint(checkpoint) +``` + +`checkpoint()` keeps the source running by default. Set +`leave_running=False` to terminate it after the checkpoint commits. Checkpoints +do not expire and must be removed explicitly with `delete_checkpoint()`; +`list_checkpoints()` returns all checkpoint identities visible to the current +tenant. + +Each restore gets a new sandbox ID, placement, network attachment, and routes. +The runtime, root filesystem, resources, mounts, environment, network policy, +and filesystem/process state come from the checkpoint. v1 does not support +in-place rollback or restore-time resource and configuration overrides. + +The bundled backend supports checkpoints for runsc and Firecracker. A restore +must use compatible runtime binaries, architecture, kernel, and runtime +configuration. The cluster prefers the source node when it is available and +may fall back to another compatible node through the configured snapshot +storage. + +For a checkpoint created from a sandbox with a reverse tunnel, pass an +explicit `reverse_tunnel` to `restore()` using the same `reverse_port` and +`listen_port`. The target and connection timeout may change. A checkpoint made +without a tunnel rejects adding one during restore. The source tunnel is +briefly disconnected during checkpoint creation and then reconnected. + +Checkpoint/restore is available through the default `openyuanrong-sandbox` +backend. The legacy `openyuanrong-sdk` actor backend reports it as unsupported. +The current official backend package supports the default 180-second +checkpoint timeout. Custom checkpoint timeouts and checkpointing a sandbox +with an active reverse tunnel require a backend release containing the +corresponding YuanRong changes. +See [`examples/checkpoint_restore.py`](./examples/checkpoint_restore.py) for a +runnable example. + ## Reverse tunnels A reverse tunnel lets sandbox code call an HTTP or HTTPS service reachable @@ -519,6 +575,7 @@ Maintained examples are under [`examples/`](./examples): - `basic_usage.py` - `command_stdin.py` +- `checkpoint_restore.py` - `custom_image.py` - `dockerfile_launch.py` - `gpu_sandbox.py` @@ -556,6 +613,7 @@ not part of the default test suite. | `CommandInfo` | `pid`, `command`, `running` | | `EntryInfo` | `name`, `path`, `type`, `size`, `permissions`, `modified_time` | | `SandboxInfo` | `id`, `state`, `cpu`, `memory`, `image`, `xpu`, `storage_mb` | +| `CheckpointInfo` | `id` | | `NodeInfo` | `id`, `status`, `capacity`, `allocatable`, `labels` | | `S3Config` | `endpoint`, `bucket`, `object`, optional credentials | | `Mount` | `target`, one source, and `type` | diff --git a/sdk/python/akernel_sdk/__init__.py b/sdk/python/akernel_sdk/__init__.py index 5e77dd9..78fff0c 100644 --- a/sdk/python/akernel_sdk/__init__.py +++ b/sdk/python/akernel_sdk/__init__.py @@ -24,6 +24,7 @@ ) from ._backends.registry import selected_backend from .types import ( + CheckpointInfo, CommandInfo, CommandResult, EntryInfo, @@ -37,6 +38,7 @@ __all__ = [ "Sandbox", + "CheckpointInfo", "S3Config", "Mount", "NetworkPolicy", diff --git a/sdk/python/akernel_sdk/_backends/base.py b/sdk/python/akernel_sdk/_backends/base.py index e0ec3b7..6c1bd1d 100644 --- a/sdk/python/akernel_sdk/_backends/base.py +++ b/sdk/python/akernel_sdk/_backends/base.py @@ -41,6 +41,7 @@ class Capability(Enum): NODE_PLACEMENT = auto() CUSTOM_REVERSE_TUNNEL_PORTS = auto() REVERSE_WEBSOCKET = auto() + CHECKPOINT_RESTORE = auto() @dataclass(frozen=True) @@ -149,6 +150,8 @@ def is_running(self) -> bool: ... def get_info(self) -> SandboxInfo: ... + def checkpoint(self, *, timeout: int) -> str: ... + def terminate(self) -> None: ... def close(self) -> None: ... @@ -163,6 +166,17 @@ class Backend(Protocol): def create(self, spec: SandboxSpec) -> BackendSession: ... + def restore( + self, + checkpoint_id: str, + *, + reverse_tunnel: HttpReverseTunnel | None, + ) -> BackendSession: ... + + def list_checkpoints(self) -> list[str]: ... + + def delete_checkpoint(self, checkpoint_id: str) -> None: ... + def delete_named(self, name: str) -> None: ... def close(self) -> None: ... diff --git a/sdk/python/akernel_sdk/_backends/openyuanrong_sandbox.py b/sdk/python/akernel_sdk/_backends/openyuanrong_sandbox.py index 56773b5..594d037 100644 --- a/sdk/python/akernel_sdk/_backends/openyuanrong_sandbox.py +++ b/sdk/python/akernel_sdk/_backends/openyuanrong_sandbox.py @@ -16,13 +16,20 @@ from __future__ import annotations +import inspect import os from collections.abc import Mapping from typing import Any import yr_sandbox -from ..types import CommandInfo, CommandResult, EntryInfo, SandboxInfo +from ..types import ( + CommandInfo, + CommandResult, + EntryInfo, + HttpReverseTunnel, + SandboxInfo, +) from .base import ( Backend, BackendConfig, @@ -246,6 +253,40 @@ def get_info(self) -> SandboxInfo: storage_mb=self._spec.storage_mb, ) + def checkpoint(self, *, timeout: int) -> str: + if self._terminated or self._closed: + raise BackendOperationError("checkpoint requires a running sandbox") + create_snapshot = self._sandbox.create_snapshot + try: + parameters = inspect.signature(create_snapshot).parameters.values() + supports_timeout = any( + parameter.name == "timeout" + or parameter.kind is inspect.Parameter.VAR_KEYWORD + for parameter in parameters + ) + except (TypeError, ValueError): + # Unknown callable signatures are treated as current backends. If + # they reject the keyword, the normal backend error includes the + # native failure instead of silently changing timeout semantics. + supports_timeout = True + if not supports_timeout and timeout != 180: + raise UnsupportedBackendFeatureError( + "The installed openyuanrong-sandbox backend only supports " + "the default 180-second checkpoint timeout. Upgrade the " + "backend to use a custom timeout." + ) + try: + if supports_timeout: + value = create_snapshot(timeout=timeout) + else: + value = create_snapshot() + except Exception as error: + raise _convert_error("checkpoint sandbox", error) from error + checkpoint_id = str(value.snapshot_id).strip() + if not checkpoint_id: + raise BackendOperationError("checkpoint returned an empty identity") + return checkpoint_id + def terminate(self) -> None: if self._terminated: return @@ -280,6 +321,7 @@ class OpenYuanRongSandboxBackend: { Capability.S3_ROOTFS, Capability.NODE_PLACEMENT, + Capability.CHECKPOINT_RESTORE, } ) @@ -380,6 +422,85 @@ def create(self, spec: SandboxSpec) -> BackendSession: raise _convert_error("create sandbox", error) from error return _Session(sandbox, spec) + def restore( + self, + checkpoint_id: str, + *, + reverse_tunnel: HttpReverseTunnel | None, + ) -> BackendSession: + if reverse_tunnel is not None and ( + reverse_tunnel.reverse_port != reverse_tunnel.listen_port - 1 + ): + raise UnsupportedBackendFeatureError( + "Backend 'openyuanrong-sandbox' requires reverse_port to equal " + "listen_port - 1." + ) + try: + sandbox = yr_sandbox.Sandbox.create( + checkpoint_id, + upstream=( + reverse_tunnel.target if reverse_tunnel is not None else None + ), + tunnel_connect_timeout=( + reverse_tunnel.connect_timeout + if reverse_tunnel is not None + else None + ), + proxy_port=( + reverse_tunnel.listen_port + if reverse_tunnel is not None + else _DEFAULT_LISTEN_PORT + ), + ) + except Exception as error: + raise _convert_error("restore checkpoint", error) from error + restored_spec = SandboxSpec( + image=None, + rootfs=None, + runtime="runsc", + cpu=1000, + memory=4096, + cpu_limit=0, + mem_limit=0, + idle_timeout=300, + schedule_timeout=30, + env={}, + name=None, + command_cwd=None, + port_forwardings=(), + mounts=(), + reverse_tunnel=reverse_tunnel, + detached=False, + node_id=None, + xpu=None, + storage_mb=None, + network_policy=None, + extra_config={}, + ) + return _Session(sandbox, restored_spec) + + def list_checkpoints(self) -> list[str]: + checkpoint_ids: list[str] = [] + page_token: str | None = None + try: + while True: + items, next_page_token = yr_sandbox.Sandbox.list_snapshots( + page_token=page_token, + page_size=100, + ) + checkpoint_ids.extend(str(item.snapshot_id) for item in items) + if not next_page_token: + return checkpoint_ids + page_token = next_page_token + except Exception as error: + raise _convert_error("list checkpoints", error) from error + + def delete_checkpoint(self, checkpoint_id: str) -> None: + try: + yr_sandbox.Sandbox.delete_snapshot(checkpoint_id) + except Exception as error: + raise _convert_error("delete checkpoint", error) from error + def delete_named(self, name: str) -> None: sandbox_id = f"{self.namespace}-{name}" try: diff --git a/sdk/python/akernel_sdk/_backends/openyuanrong_sdk.py b/sdk/python/akernel_sdk/_backends/openyuanrong_sdk.py index 51cc488..9be4b62 100644 --- a/sdk/python/akernel_sdk/_backends/openyuanrong_sdk.py +++ b/sdk/python/akernel_sdk/_backends/openyuanrong_sdk.py @@ -20,7 +20,13 @@ from collections.abc import Mapping from typing import Any -from ..types import CommandInfo, CommandResult, EntryInfo, SandboxInfo +from ..types import ( + CommandInfo, + CommandResult, + EntryInfo, + HttpReverseTunnel, + SandboxInfo, +) from . import openyuanrong_sdk_impl as _impl from .base import ( Backend, @@ -29,7 +35,7 @@ Capability, SandboxSpec, ) -from .errors import BackendOperationError +from .errors import BackendOperationError, UnsupportedBackendFeatureError from .openyuanrong_sdk_commands import ( CommandHandle as NativeCommandHandle, ) @@ -238,6 +244,13 @@ def get_info(self) -> SandboxInfo: storage_mb=self._spec.storage_mb, ) + def checkpoint(self, *, timeout: int) -> str: + del timeout + raise UnsupportedBackendFeatureError( + "Backend 'openyuanrong-sdk' does not support reusable checkpoints. " + "Use the default 'openyuanrong-sandbox' backend." + ) + def terminate(self) -> None: if self._terminated: return @@ -265,7 +278,11 @@ class OpenYuanRongSdkBackend: name = "openyuanrong-sdk" namespace = _NAMESPACE - capabilities = frozenset(Capability) + capabilities: frozenset[Capability] = frozenset( + capability + for capability in Capability + if capability is not Capability.CHECKPOINT_RESTORE + ) def __init__(self, _config: BackendConfig) -> None: _impl.ensure_initialized() @@ -333,6 +350,31 @@ def create(self, spec: SandboxSpec) -> BackendSession: _rollback_instance(instance, "session initialization") raise _convert_error("initialize sandbox session", error) from error + def restore( + self, + checkpoint_id: str, + *, + reverse_tunnel: HttpReverseTunnel | None, + ) -> BackendSession: + del checkpoint_id, reverse_tunnel + raise UnsupportedBackendFeatureError( + "Backend 'openyuanrong-sdk' does not support reusable checkpoints. " + "Use the default 'openyuanrong-sandbox' backend." + ) + + def list_checkpoints(self) -> list[str]: + raise UnsupportedBackendFeatureError( + "Backend 'openyuanrong-sdk' does not support reusable checkpoints. " + "Use the default 'openyuanrong-sandbox' backend." + ) + + def delete_checkpoint(self, checkpoint_id: str) -> None: + del checkpoint_id + raise UnsupportedBackendFeatureError( + "Backend 'openyuanrong-sdk' does not support reusable checkpoints. " + "Use the default 'openyuanrong-sandbox' backend." + ) + def delete_named(self, name: str) -> None: try: _impl.delete_named_instance(name) diff --git a/sdk/python/akernel_sdk/sandbox.py b/sdk/python/akernel_sdk/sandbox.py index f48851a..38ff75d 100644 --- a/sdk/python/akernel_sdk/sandbox.py +++ b/sdk/python/akernel_sdk/sandbox.py @@ -32,7 +32,14 @@ from .commands import CommandHandle, Commands from .filesystem import Filesystem from .pty import Pty -from .types import HttpReverseTunnel, Mount, NetworkPolicy, S3Config, SandboxInfo +from .types import ( + CheckpointInfo, + HttpReverseTunnel, + Mount, + NetworkPolicy, + S3Config, + SandboxInfo, +) _traefik_internal_ip_cache: str | None = None logger = logging.getLogger(__name__) @@ -135,6 +142,17 @@ def _validate_integer( raise ValueError(f"{name} must be greater than or equal to {minimum}") +def _checkpoint_id(checkpoint: CheckpointInfo | str) -> str: + if isinstance(checkpoint, CheckpointInfo): + return checkpoint.id + if not isinstance(checkpoint, str): + raise TypeError("checkpoint must be a CheckpointInfo or string") + value = checkpoint.strip() + if not value: + raise ValueError("checkpoint must be a non-empty string") + return value + + def _get_traefik_internal_ip(gateway: Endpoint) -> tuple[str, int]: """Resolve Traefik's direct address for ``internal=True`` URLs.""" @@ -443,6 +461,120 @@ def reverse_tunnel(self) -> HttpReverseTunnel | None: return self._reverse_tunnel + def checkpoint( + self, + *, + timeout: int = 180, + leave_running: bool = True, + ) -> CheckpointInfo: + """Create a reusable checkpoint of this sandbox. + + The checkpoint has no TTL and remains available until explicitly + deleted with :meth:`delete_checkpoint`. A successful checkpoint is an + immutable template; each restore creates a new sandbox identity with + fresh placement and routes. + + Args: + timeout: Positive checkpoint timeout in seconds. + leave_running: Keep this source sandbox running after success. + + Returns: + The stable checkpoint identity. + """ + + _validate_integer("timeout", timeout, minimum=1) + if not isinstance(leave_running, bool): + raise TypeError("leave_running must be a boolean") + if self._closed or self._session is None or not self.is_running(): + raise RuntimeError("checkpoint requires a running sandbox") + checkpoint = CheckpointInfo(self._session.checkpoint(timeout=timeout)) + if not leave_running: + self.kill() + return checkpoint + + @classmethod + def restore( + cls, + checkpoint: CheckpointInfo | str, + *, + reverse_tunnel: HttpReverseTunnel | None = None, + ) -> Sandbox: + """Restore an independent sandbox from a reusable checkpoint. + + Runtime, root filesystem, resources, mounts, environment, network + policy, and exposed-port shape are inherited from the checkpoint. + Resource overrides and in-place rollback are intentionally not part of + v1. A source created with a reverse tunnel requires an explicit tunnel + with the same ports here; its target and connect timeout may differ. + """ + + checkpoint_id = _checkpoint_id(checkpoint) + if reverse_tunnel is not None and not isinstance( + reverse_tunnel, HttpReverseTunnel + ): + raise TypeError("reverse_tunnel must be an HttpReverseTunnel") + session = load_backend().restore( + checkpoint_id, + reverse_tunnel=reverse_tunnel, + ) + restored = cls.__new__(cls) + restored._session = session + restored._startup_command = None + restored._pty = None + restored._closed = False + restored._terminated = False + restored._reverse_tunnel = reverse_tunnel + restored._forwarded_ports = set() + restored._image = None + restored._cpu = 0 + restored._memory = 0 + restored._xpu = None + restored._storage_mb = None + restored._id = "" + try: + restored._id = session.id + restored._files = Filesystem(session.files) + restored._commands = Commands(session.commands) + restored._pty = Pty(restored._id) + info = session.get_info() + restored._image = info.image + restored._cpu = info.cpu if info.cpu is not None else 0 + restored._memory = info.memory if info.memory is not None else 0 + restored._xpu = info.xpu + restored._storage_mb = info.storage_mb + except Exception: + restored._closed = True + try: + session.terminate() + except Exception: + logger.warning( + "failed to roll back a partially initialized restore", + exc_info=True, + ) + try: + session.close() + except Exception: + logger.warning( + "failed to close a partially initialized restore session", + exc_info=True, + ) + raise + return restored + + @classmethod + def list_checkpoints(cls) -> list[CheckpointInfo]: + """List reusable checkpoints visible to the current tenant.""" + + del cls + return [CheckpointInfo(value) for value in load_backend().list_checkpoints()] + + @classmethod + def delete_checkpoint(cls, checkpoint: CheckpointInfo | str) -> None: + """Permanently delete one reusable checkpoint.""" + + del cls + load_backend().delete_checkpoint(_checkpoint_id(checkpoint)) + def get_port_url(self, port: int, *, internal: bool = False) -> str: """Return the gateway URL for a declared sandbox port. diff --git a/sdk/python/akernel_sdk/types.py b/sdk/python/akernel_sdk/types.py index 67a43cd..db137da 100644 --- a/sdk/python/akernel_sdk/types.py +++ b/sdk/python/akernel_sdk/types.py @@ -160,6 +160,21 @@ class SandboxInfo: storage_mb: int | None = None +@dataclass(frozen=True) +class CheckpointInfo: + """Stable identity of a reusable sandbox checkpoint.""" + + id: str + + def __post_init__(self) -> None: + if not isinstance(self.id, str): + raise TypeError("id must be a string") + normalized = self.id.strip() + if not normalized: + raise ValueError("id must be a non-empty string") + object.__setattr__(self, "id", normalized) + + @dataclass(frozen=True) class NodeInfo: """Capacity, allocation, and labels advertised by an AKernel node.""" diff --git a/sdk/python/examples/checkpoint_restore.py b/sdk/python/examples/checkpoint_restore.py new file mode 100644 index 0000000..b9359b7 --- /dev/null +++ b/sdk/python/examples/checkpoint_restore.py @@ -0,0 +1,39 @@ +# Copyright (c) 2026 Ant Group Corporation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Create, restore, and explicitly delete a reusable checkpoint.""" + +from akernel_sdk import CheckpointInfo, Sandbox + + +def main() -> None: + checkpoint: CheckpointInfo | None = None + try: + with Sandbox(runtime="runsc") as source: + source.commands.run("printf before > /tmp/checkpoint-state && sync") + checkpoint = source.checkpoint() + source.commands.run("printf after > /tmp/checkpoint-state") + print("source:", source.id, "checkpoint:", checkpoint.id) + + with Sandbox.restore(checkpoint) as restored: + value = restored.commands.run("cat /tmp/checkpoint-state") + print("restored:", restored.id, "state:", value.stdout) + assert value.stdout == "before" + finally: + if checkpoint is not None: + Sandbox.delete_checkpoint(checkpoint) + + +if __name__ == "__main__": + main() diff --git a/sdk/python/pyproject.toml b/sdk/python/pyproject.toml index 166f6d6..f3619e7 100644 --- a/sdk/python/pyproject.toml +++ b/sdk/python/pyproject.toml @@ -28,7 +28,7 @@ classifiers = [ "Topic :: System :: Distributed Computing", ] dependencies = [ - "openyuanrong-sandbox==0.9.9", + "openyuanrong-sandbox==0.9.10", "websockets>=10.0", "dockerfile-parse>=2.0.1", ] diff --git a/sdk/python/tests/integration/test_sandbox.py b/sdk/python/tests/integration/test_sandbox.py index 08586e3..15259b0 100644 --- a/sdk/python/tests/integration/test_sandbox.py +++ b/sdk/python/tests/integration/test_sandbox.py @@ -99,5 +99,50 @@ def test_pty_interrupts_foreground_process(self): self.assertIn(b"PTY_AFTER_INTERRUPT", output) +@unittest.skipUnless( + _ENABLED, + "set AKERNEL_RUN_INTEGRATION=1 and the AKernel SDK environment", +) +class SandboxCheckpointIntegrationTest(unittest.TestCase): + def test_checkpoint_restore_and_delete(self): + source = Sandbox(cpu=1000, memory=2048, runtime=_RUNTIME) + restored = None + checkpoint = None + try: + source_id = source.id + created = source.commands.run( + "printf checkpoint-before > /tmp/akernel-checkpoint-state && sync" + ) + self.assertEqual(created.exit_code, 0) + + checkpoint = source.checkpoint(timeout=180) + self.assertTrue(source.is_running()) + checkpoint_ids = {item.id for item in Sandbox.list_checkpoints()} + self.assertIn(checkpoint.id, checkpoint_ids) + + changed = source.commands.run( + "printf source-after > /tmp/akernel-checkpoint-state && sync" + ) + self.assertEqual(changed.exit_code, 0) + + restored = Sandbox.restore(checkpoint) + self.assertNotEqual(restored.id, source_id) + restored_value = restored.commands.run( + "cat /tmp/akernel-checkpoint-state" + ) + self.assertEqual(restored_value.exit_code, 0) + self.assertEqual(restored_value.stdout, "checkpoint-before") + self.assertEqual( + source.commands.run("cat /tmp/akernel-checkpoint-state").stdout, + "source-after", + ) + finally: + if restored is not None: + restored.kill() + source.kill() + if checkpoint is not None: + Sandbox.delete_checkpoint(checkpoint) + + if __name__ == "__main__": unittest.main() diff --git a/sdk/python/tests/unit/test_backends.py b/sdk/python/tests/unit/test_backends.py index c65924c..cc40955 100644 --- a/sdk/python/tests/unit/test_backends.py +++ b/sdk/python/tests/unit/test_backends.py @@ -464,6 +464,118 @@ def test_named_delete_uses_deterministic_sid(self): self.backend.delete_named("worker") delete.assert_called_once_with("default-worker") + def test_checkpoint_delegates_to_reusable_snapshot_api(self): + native = MagicMock() + native.id = "default-source" + native.commands = MagicMock() + native.files = MagicMock() + native.create_snapshot.return_value = SimpleNamespace( + snapshot_id="checkpoint-1" + ) + with patch.object( + openyuanrong_sandbox.yr_sandbox, + "Sandbox", + return_value=native, + ): + session = self.backend.create(_spec()) + + self.assertEqual(session.checkpoint(timeout=240), "checkpoint-1") + + native.create_snapshot.assert_called_once_with(timeout=240) + + def test_checkpoint_uses_official_backend_default_timeout(self): + native = MagicMock() + native.id = "default-source" + native.commands = MagicMock() + native.files = MagicMock() + calls = [] + + def create_snapshot(): + calls.append(True) + return SimpleNamespace(snapshot_id="checkpoint-1") + + native.create_snapshot = create_snapshot + with patch.object( + openyuanrong_sandbox.yr_sandbox, + "Sandbox", + return_value=native, + ): + session = self.backend.create(_spec()) + + self.assertEqual(session.checkpoint(timeout=180), "checkpoint-1") + + self.assertEqual(calls, [True]) + + def test_checkpoint_rejects_custom_timeout_on_official_backend(self): + native = MagicMock() + native.id = "default-source" + native.commands = MagicMock() + native.files = MagicMock() + native.create_snapshot = lambda: SimpleNamespace( + snapshot_id="checkpoint-1" + ) + with patch.object( + openyuanrong_sandbox.yr_sandbox, + "Sandbox", + return_value=native, + ): + session = self.backend.create(_spec()) + + with self.assertRaisesRegex( + UnsupportedBackendFeatureError, + "default 180-second", + ): + session.checkpoint(timeout=240) + + def test_restore_uses_snapshot_template_and_explicit_tunnel(self): + native = MagicMock() + native.id = "default-restored" + native.commands = MagicMock() + native.files = MagicMock() + tunnel = HttpReverseTunnel( + "https://new-target.example", + reverse_port=9000, + listen_port=9001, + connect_timeout=12, + ) + with patch.object( + openyuanrong_sandbox.yr_sandbox, + "Sandbox", + ) as sandbox_type: + sandbox_type.create.return_value = native + session = self.backend.restore( + "checkpoint-1", + reverse_tunnel=tunnel, + ) + + self.assertEqual(session.id, "default-restored") + sandbox_type.create.assert_called_once_with( + "checkpoint-1", + upstream="https://new-target.example", + tunnel_connect_timeout=12, + proxy_port=9001, + ) + + def test_checkpoint_catalog_pages_and_deletes(self): + first = ([SimpleNamespace(snapshot_id="checkpoint-1")], "next") + second = ([SimpleNamespace(snapshot_id="checkpoint-2")], "") + with patch.object( + openyuanrong_sandbox.yr_sandbox.Sandbox, + "list_snapshots", + side_effect=[first, second], + ) as list_snapshots, patch.object( + openyuanrong_sandbox.yr_sandbox.Sandbox, + "delete_snapshot", + ) as delete_snapshot: + self.assertEqual( + self.backend.list_checkpoints(), + ["checkpoint-1", "checkpoint-2"], + ) + self.backend.delete_checkpoint("checkpoint-1") + + self.assertEqual(list_snapshots.call_count, 2) + delete_snapshot.assert_called_once_with("checkpoint-1") + class OpenYuanRongSdkBackendTest(unittest.TestCase): def setUp(self): @@ -598,6 +710,14 @@ def test_close_finalizes_actor_sdk(self): finalize.assert_called_once_with() + def test_reusable_checkpoint_operations_are_explicitly_unsupported(self): + with self.assertRaises(UnsupportedBackendFeatureError): + self.backend.restore("checkpoint-1", reverse_tunnel=None) + with self.assertRaises(UnsupportedBackendFeatureError): + self.backend.list_checkpoints() + with self.assertRaises(UnsupportedBackendFeatureError): + self.backend.delete_checkpoint("checkpoint-1") + if __name__ == "__main__": unittest.main() diff --git a/sdk/python/tests/unit/test_sandbox.py b/sdk/python/tests/unit/test_sandbox.py index 14a1386..9d9ddc4 100644 --- a/sdk/python/tests/unit/test_sandbox.py +++ b/sdk/python/tests/unit/test_sandbox.py @@ -18,6 +18,7 @@ from unittest.mock import MagicMock, patch from akernel_sdk import ( + CheckpointInfo, DockerfileLaunch, HttpReverseTunnel, NetworkPolicy, @@ -169,6 +170,89 @@ def test_named_delete_hides_backend_namespace(self): Sandbox.delete("worker") self.backend.delete_named.assert_called_once_with("worker") + def test_checkpoint_returns_public_identity_and_keeps_source_running(self): + self.session.checkpoint.return_value = "checkpoint-1" + sandbox = Sandbox() + + checkpoint = sandbox.checkpoint(timeout=240) + + self.assertEqual(checkpoint, CheckpointInfo("checkpoint-1")) + self.session.checkpoint.assert_called_once_with(timeout=240) + self.session.terminate.assert_not_called() + sandbox.kill() + + def test_checkpoint_can_terminate_source_after_success(self): + self.session.checkpoint.return_value = "checkpoint-1" + sandbox = Sandbox() + + checkpoint = sandbox.checkpoint(leave_running=False) + + self.assertEqual(checkpoint.id, "checkpoint-1") + self.session.terminate.assert_called_once_with() + self.session.close.assert_called_once_with() + + def test_checkpoint_validates_arguments_and_running_state(self): + sandbox = Sandbox() + for timeout in (True, 0, -1, 1.5): + with self.subTest(timeout=timeout), self.assertRaises( + (TypeError, ValueError) + ): + sandbox.checkpoint(timeout=timeout) + with self.assertRaisesRegex(TypeError, "leave_running"): + sandbox.checkpoint(leave_running=1) + self.session.is_running.return_value = False + with self.assertRaisesRegex(RuntimeError, "running sandbox"): + sandbox.checkpoint() + sandbox.kill() + + def test_restore_builds_facades_around_new_backend_session(self): + restored_session = MagicMock() + restored_session.id = "restored-physical-id" + restored_session.commands = MagicMock() + restored_session.files = MagicMock() + restored_session.get_info.return_value = SandboxInfo( + id="restored-physical-id", + state="running", + cpu=2000, + memory=8192, + image="base-image", + ) + self.backend.restore.return_value = restored_session + tunnel = HttpReverseTunnel("http://127.0.0.1:9000") + + restored = Sandbox.restore( + CheckpointInfo("checkpoint-1"), reverse_tunnel=tunnel + ) + + self.backend.restore.assert_called_once_with( + "checkpoint-1", reverse_tunnel=tunnel + ) + self.assertEqual(restored.id, "restored-physical-id") + self.assertIs(restored.reverse_tunnel, tunnel) + self.assertEqual(restored.get_info().cpu, 2000) + restored.kill() + restored_session.terminate.assert_called_once_with() + restored_session.close.assert_called_once_with() + + def test_list_and_delete_checkpoints_hide_backend_details(self): + self.backend.list_checkpoints.return_value = ["checkpoint-1", "checkpoint-2"] + + self.assertEqual( + Sandbox.list_checkpoints(), + [CheckpointInfo("checkpoint-1"), CheckpointInfo("checkpoint-2")], + ) + Sandbox.delete_checkpoint(CheckpointInfo("checkpoint-1")) + Sandbox.delete_checkpoint(" checkpoint-2 ") + + self.assertEqual( + self.backend.delete_checkpoint.call_args_list, + [unittest.mock.call("checkpoint-1"), unittest.mock.call("checkpoint-2")], + ) + with self.assertRaises(ValueError): + Sandbox.delete_checkpoint(" ") + with self.assertRaises(TypeError): + Sandbox.restore(object()) # type: ignore[arg-type] + def test_rootfs_requires_s3_config(self): with self.assertRaisesRegex(TypeError, "S3Config"): Sandbox(rootfs={"type": "s3"}) diff --git a/sdk/python/tests/unit/test_types.py b/sdk/python/tests/unit/test_types.py index e9766c4..2a8f4a8 100644 --- a/sdk/python/tests/unit/test_types.py +++ b/sdk/python/tests/unit/test_types.py @@ -19,7 +19,13 @@ from pathlib import Path import akernel_sdk -from akernel_sdk import DockerContextEntry, HttpReverseTunnel, Mount, S3Config +from akernel_sdk import ( + CheckpointInfo, + DockerContextEntry, + HttpReverseTunnel, + Mount, + S3Config, +) class PublicTypesTest(unittest.TestCase): @@ -69,6 +75,7 @@ def test_public_exports_are_minimal(self): set(akernel_sdk.__all__), { "Sandbox", + "CheckpointInfo", "S3Config", "Mount", "NetworkPolicy", @@ -120,6 +127,14 @@ def test_docker_context_entry_is_public_and_immutable(self): with self.assertRaisesRegex(AttributeError, "cannot assign"): entry.mode = 0o700 # type: ignore[misc] + def test_checkpoint_info_is_public_normalized_and_immutable(self): + checkpoint = CheckpointInfo(" checkpoint-1 ") + self.assertEqual(checkpoint.id, "checkpoint-1") + with self.assertRaisesRegex(AttributeError, "cannot assign"): + checkpoint.id = "changed" # type: ignore[misc] + with self.assertRaises(ValueError): + CheckpointInfo(" ") + def test_s3_config_serialization(self): config = S3Config( endpoint="https://s3.example.com", diff --git a/src/sandboxd b/src/sandboxd index 1918fad..a55f382 160000 --- a/src/sandboxd +++ b/src/sandboxd @@ -1 +1 @@ -Subproject commit 1918fadb03b59bc6f540196e14b90a91bdf31b7d +Subproject commit a55f3827c477d68f1b0cb494f0a573ba27eb4cd8 diff --git a/src/yuanrong b/src/yuanrong index 177592c..883028d 160000 --- a/src/yuanrong +++ b/src/yuanrong @@ -1 +1 @@ -Subproject commit 177592c7eb208698bd9b048a818b9cfb02060545 +Subproject commit 883028de4ad7ecceba08460c567ad4426b7965a7