Add extensible free-space motion benchmark

docs/source/guides/cli.md

@@ -394,9 +394,9 @@ Run the packaged benchmark suites through the same CLI:
 # RL train/evaluate/report workflow
 embodichain benchmark rl --tasks push_cube --algorithms ppo
 
-# Kinematic solver and neural planner benchmarks
+# Kinematic solver and motion-generation benchmarks
 embodichain benchmark robotics-kinematic-solver --solvers all
-embodichain benchmark planners-neural-planner --num-waypoints 1 3 5
+embodichain benchmark motion-generation --suite smoke
 
 # Atomic actions, grasp generation, and workspace analysis
 embodichain benchmark atomic-action --smoke

embodichain/lab/sim/planners/curobo/curobo_planner.py

@@ -799,6 +799,54 @@ def with_motion_context(
             options.control_part = control_part
         return options
 
+    def prepare_backend(
+        self,
+        *,
+        control_part: str,
+        batch_size: int,
+        move_type: MoveType = MoveType.EEF_MOVE,
+    ) -> dict[str, object]:
+        """Materialize and warm one lazy cuRobo backend without planning a case.
+
+        This explicit lifecycle hook lets deployment tooling and benchmarks
+        separate one-time robot/world YAML generation, collision-sphere setup,
+        CUDA graph capture, and cuRobo warmup from the first real planning call.
+        Repeated calls for the same backend key reuse the cached backend.
+
+        Args:
+            control_part: Robot control part to prepare.
+            batch_size: Goal batch size used by the future planning calls.
+            move_type: Goal type whose cuRobo buffers and graph are prepared.
+
+        Returns:
+            Metadata describing the resolved backend and actual CUDA graph mode.
+
+        Raises:
+            ValueError: If the batch size or move type is unsupported.
+        """
+        if batch_size < 1:
+            logger.log_error("batch_size must be >= 1.", ValueError)
+        if move_type not in self.supported_move_types:
+            logger.log_error(
+                f"cuRobo cannot prepare unsupported move type {move_type}.",
+                ValueError,
+            )
+        robot_batch_size = int(getattr(self.robot, "num_instances", 1))
+        if batch_size not in (1, robot_batch_size):
+            logger.log_error(
+                f"batch_size={batch_size} must be 1 or robot.num_instances="
+                f"{robot_batch_size}.",
+                ValueError,
+            )
+        backend = self._get_backend(control_part, batch_size, move_type)
+        return {
+            "control_part": backend.control_part,
+            "batch_size": backend.batch_size,
+            "move_type": backend.planning_mode.name,
+            "multi_env": bool(self.cfg.world.multi_env),
+            "use_cuda_graph": backend.use_cuda_graph,
+        }
+
     @validate_plan_options(options_cls=CuroboPlanOptions)
     def plan(
         self,

embodichain/lab/sim/planners/toppra_planner.py

@@ -396,13 +396,17 @@ def _shutdown_pool(self) -> None:
 
         self._pool = None
 
+    def close(self) -> None:
+        """Release TOPPRA worker processes owned by this planner."""
+        self._shutdown_pool()
+
     def __del__(self):
         # Only matters for in-process GC of an abandoned planner (and as a
         # non-Linux fallback).  Process-exit cleanup is handled by the kernel
         # via PR_SET_PDEATHSIG installed in each worker, which survives the
         # os._exit(0) path that SimulationManager.destroy() takes.
         try:
-            self._shutdown_pool()
+            self.close()
         except Exception:
             pass
 

pyproject.toml

@@ -43,6 +43,7 @@ dependencies = [
   "tensorboard>=2.20.0",
   "ortools",
   "prettytable",
+  "psutil>=5.9",
   "black==26.3.1",
   "fvcore",
   "h5py",

scripts/benchmark/__main__.py

@@ -21,7 +21,7 @@
     embodichain benchmark rl --tasks push_cube --algorithms ppo --suite default
     embodichain benchmark rl --rebuild-report-only
     embodichain benchmark robotics-kinematic-solver -s pytorch
-    embodichain benchmark planners-neural-planner --num-waypoints 1 3 5
+    embodichain benchmark motion-generation --suite smoke
     embodichain benchmark atomic-action --smoke
     embodichain benchmark grasp-pose-generator --device cuda
     embodichain benchmark workspace-analyzer
@@ -50,24 +50,11 @@ def _run_rl_cli(_: argparse.Namespace) -> None:
     rl_main()
 
 
-def _run_neural_planner_cli(args: argparse.Namespace) -> None:
-    """Run NeuralPlanner benchmark with forwarded CLI args."""
-    from scripts.benchmark.planners.neural_planner.run_benchmark import (
-        run_all_benchmarks,
-    )
+def _run_motion_generation_cli(args: argparse.Namespace) -> None:
+    """Run the free-space motion-generation benchmark."""
+    from scripts.benchmark.motion_generation.run_benchmark import run_from_args
 
-    run_all_benchmarks(
-        num_waypoints_list=args.num_waypoints,
-        sim_device=args.device,
-        headless=args.headless,
-        checkpoint_path=args.checkpoint_path,
-        num_trials=args.num_trials,
-        warmup_trials=args.warmup_trials,
-        sample_interval=args.sample_interval,
-        compare_ik=args.compare_ik,
-        compare_toppra=args.compare_toppra,
-        include_trial_details=args.save_trial_details,
-    )
+    run_from_args(args)
 
 
 def _run_atomic_action_cli(_: argparse.Namespace) -> None:
@@ -124,76 +111,17 @@ def main(argv: Sequence[str] | None = None) -> None:
     )
     robotics_ks_parser.set_defaults(func=_run_robotics_kinematic_solver_cli)
 
-    # -- planners-neural-planner --------------------------------------------
-    neural_planner_parser = subparsers.add_parser(
-        "planners-neural-planner",
-        help="Benchmark NeuralPlanner planning latency and quality on Franka.",
-    )
-    neural_planner_parser.add_argument(
-        "--device",
-        choices=("auto", "cpu", "cuda"),
-        default="auto",
-        help="Simulation and planner device. Auto uses CUDA when available.",
-    )
-    neural_planner_parser.add_argument(
-        "--num-waypoints",
-        nargs="+",
-        type=int,
-        default=[1, 3, 5],
-        help="Number of EEF waypoints to sweep.",
-    )
-    neural_planner_parser.add_argument(
-        "--num-trials",
-        type=int,
-        default=8,
-        help="Measured trials per (impl, num_waypoints) configuration.",
-    )
-    neural_planner_parser.add_argument(
-        "--warmup-trials",
-        type=int,
-        default=1,
-        help="Warmup trials per configuration; excluded from summary aggregation.",
+    # -- motion-generation ---------------------------------------------------
+    from scripts.benchmark.motion_generation.run_benchmark import (
+        add_parser_arguments,
     )
-    neural_planner_parser.add_argument(
-        "--sample-interval",
-        type=int,
-        default=20,
-        help="Resampled trajectory length for ik_interpolate and ik_toppra.",
-    )
-    neural_planner_parser.add_argument(
-        "--compare-ik",
-        action="store_true",
-        help="Also benchmark sequential IK plus joint interpolation.",
-    )
-    neural_planner_parser.add_argument(
-        "--compare-toppra",
-        action="store_true",
-        help="Also benchmark EEF IK interpolation followed by TOPPRA.",
-    )
-    neural_planner_parser.add_argument(
-        "--save-trial-details",
-        action="store_true",
-        help="Include per-trial rows in the markdown report.",
-    )
-    neural_planner_parser.add_argument(
-        "--checkpoint-path",
-        type=str,
-        default=None,
-        help="Local neural planner checkpoint path. Skips HuggingFace download.",
-    )
-    neural_planner_parser.add_argument(
-        "--headless",
-        action="store_true",
-        default=True,
-        help="Run simulation headlessly (default: True).",
-    )
-    neural_planner_parser.add_argument(
-        "--no-headless",
-        action="store_false",
-        dest="headless",
-        help="Open the simulation viewer window.",
+
+    motion_generation_parser = subparsers.add_parser(
+        "motion-generation",
+        help="Benchmark free-space motion generation with cuRobo as baseline.",
     )
-    neural_planner_parser.set_defaults(func=_run_neural_planner_cli)
+    add_parser_arguments(motion_generation_parser)
+    motion_generation_parser.set_defaults(func=_run_motion_generation_cli)
 
     # -- atomic-action -------------------------------------------------------
     atomic_action_parser = subparsers.add_parser(

scripts/benchmark/curobo_extraction/__init__.py

@@ -0,0 +1,21 @@
+# ----------------------------------------------------------------------------
+# Copyright (c) 2021-2026 DexForce Technology Co., Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ----------------------------------------------------------------------------
+
+"""cuRobo post-processing hot-path microbenchmark."""
+
+from __future__ import annotations
+
+__all__: list[str] = []

scripts/benchmark/planners/benchmark_curobo_extraction.py → scripts/benchmark/curobo_extraction/run_benchmark.py

@@ -29,7 +29,7 @@
 old ``_extract_segment`` does B per-row H2D copies, replaced by one bulk H2D.
 Part (2) only shows on CUDA, so the benchmark runs on both ``cuda`` and ``cpu``.
 
-Run: python -m scripts.benchmark.planners.benchmark_curobo_extraction
+Run: python -m scripts.benchmark.curobo_extraction.run_benchmark
 """
 
 from __future__ import annotations

scripts/benchmark/planners/neural_planner/BENCHMARK_DESIGN.md → scripts/benchmark/motion_generation/BENCHMARK_DESIGN.md

@@ -68,7 +68,7 @@ reported as `unsupported`, not silently converted into success or failure.
 
 ### 2.2 Gaps in the current NeuralPlanner benchmark
 
-`scripts/benchmark/planners/neural_planner/run_benchmark.py` already handles:
+`scripts/benchmark/motion_generation/run_benchmark.py` already handles:
 
 - warmup trials separately from measured trials;
 - CUDA synchronization;
@@ -153,7 +153,7 @@ Suite YAML + fixed Case Manifest
 Keep the existing CLI entry point:
 
 ```bash
-embodichain benchmark planners-neural-planner
+embodichain benchmark motion-generation
 ```
 
 Reuse established patterns from the current benchmark system:
@@ -170,8 +170,8 @@ Reuse established patterns from the current benchmark system:
 Refactor the current monolithic script incrementally into:
 
 ```text
-scripts/benchmark/planners/neural_planner/
-├── run_benchmark.py        # thin CLI and compatibility entry point
+scripts/benchmark/motion_generation/
+├── run_benchmark.py        # thin CLI entry point
 ├── config.py               # suite, planner, and scenario configuration
 ├── registry.py             # planner/scenario/metric registries
 ├── runner.py               # case matrix, warmup, trials, and resume
@@ -842,7 +842,7 @@ Aggregate by `(track, scenario, algorithm, condition_bin)`:
 ```text
 track, scenario, comparison_mode, algorithm, algorithm_role,
 constraint_information, cases, coverage_rate, success_rate,
-planning_success_rate, motion_valid_rate, execution_success_rate,
+planning_success_rate, execution_success_rate,
 task_success_rate, final_pos_err_mm, final_rot_err_deg,
 waypoint_completion_rate, joint_violation_rate, dynamic_violation_rate,
 collision_rate, min_clearance_m, path_efficiency, jerk_cost,

scripts/benchmark/motion_generation/README.md

@@ -0,0 +1,36 @@
+# Motion Generation Benchmark
+
+Free-space motion-generation suite with cuRobo as the default primary baseline.
+
+Design background and roadmap: see [`BENCHMARK_DESIGN.md`](./BENCHMARK_DESIGN.md).
+
+## Run
+
+```bash
+embodichain benchmark motion-generation --suite smoke
+embodichain benchmark motion-generation --suite coverage
+embodichain benchmark motion-generation --extra-baselines ik_interpolate toppra
+embodichain benchmark motion-generation --path-shapes direct l_turn --start-state-bins nominal near_singularity
+```
+
+Artifacts land under `outputs/benchmarks/motion_generation/<timestamp>/`
+(`resolved_suite.yaml`, `case_manifest.json`, `trials.jsonl`, `aggregates.json`,
+`report.md` with exactly three tables).
+
+## Implemented
+
+- Extensible planner/scenario registries and track-based suite YAML
+- `free-space-common` track with fixed manifests and start-state bins
+- Default matrix: cuRobo (`primary_baseline`); IK / TOPPRA optional diagnostics
+- NMG adapter stub (`candidate`, disabled until a checkpoint is ready)
+- Lifecycle timing: construct / prepare / cold / warm
+- Ordered waypoint matching and external `motion_valid` (separate from
+  `PlanResult.success`)
+- One Markdown report: Time & Memory, Success & Other Metrics, Leaderboard
+
+## Not implemented yet
+
+- Real NMG checkpoint adapter
+- `collision-deployment` and `atomic-task` tracks
+- Physics execution / task-success metrics
+- Latency-budget Pareto sweeps, confidence intervals, subprocess isolation

scripts/benchmark/motion_generation/__init__.py

@@ -0,0 +1,21 @@
+# ----------------------------------------------------------------------------
+# Copyright (c) 2021-2026 DexForce Technology Co., Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ----------------------------------------------------------------------------
+
+"""Extensible motion-generation benchmark suite."""
+
+from __future__ import annotations
+
+__all__: list[str] = []