Local control: in-SDK sidecar + desktop/browser drivers

pyproject.toml

@@ -37,10 +37,24 @@ cli = [
   "rich>=13,<16",
   "typer>=0.12,<1",
 ]
+desktop = [
+  "pyautogui>=0.9.54",
+  "pynput>=1.7",
+  "pillow>=10",
+]
+browser = [
+  "selenium>=4.20",
+  "markdownify>=0.11",
+]
 all = [
   "python-dotenv>=1.2.2,<2",
   "rich>=13,<16",
   "typer>=0.12,<1",
+  "pyautogui>=0.9.54",
+  "pynput>=1.7",
+  "pillow>=10",
+  "selenium>=4.20",
+  "markdownify>=0.11",
 ]
 
 [project.scripts]
@@ -55,6 +69,9 @@ Documentation = "https://hub.hcompany.ai/computer-use-agents"
 [tool.hatch.build.targets.wheel]
 packages = ["src/hai_agents", "src/hai_agents_cli", "src/hai_agents_common"]
 
+[tool.hatch.build.targets.wheel.force-include]
+"src/hai_agents/local/browser/js" = "hai_agents/local/browser/js"
+
 [dependency-groups]
 dev = [
   "pytest>=9.0.3,<10",

src/hai_agents/client.py

@@ -7,11 +7,14 @@
 
 from __future__ import annotations
 
+import functools
 import typing
 
 import typing_extensions
 
+from .agents.client import AgentsClient, AsyncAgentsClient
 from .base_client import AsyncBaseClient, BaseClient
+from .local.wiring import localize_agent, localize_environments
 from .polling import (
     AnswerT,
     AsyncSessionHandle,
@@ -24,9 +27,65 @@
 )
 from .polling import async_run_session as _async_run_session
 from .polling import run_session as _run_session
+from .sessions.client import AsyncSessionsClient, SessionsClient
 from .tools import ToolInput, as_tools
 
 
+def _wire_environments(kwargs: typing.Dict[str, typing.Any], get_api_key: typing.Callable[[], str]) -> None:
+    if kwargs.get("environments"):
+        kwargs["environments"] = localize_environments(kwargs["environments"], get_api_key)
+
+
+class _LocalAgentsClient(AgentsClient):
+    @functools.wraps(AgentsClient.create_agent)
+    def create_agent(self, **kwargs: typing.Any) -> typing.Any:
+        _wire_environments(kwargs, self._raw_client._client_wrapper._get_api_key)
+        return super().create_agent(**kwargs)
+
+    @functools.wraps(AgentsClient.update_agent)
+    def update_agent(self, **kwargs: typing.Any) -> typing.Any:
+        _wire_environments(kwargs, self._raw_client._client_wrapper._get_api_key)
+        return super().update_agent(**kwargs)
+
+    @functools.wraps(AgentsClient.patch_agent)
+    def patch_agent(self, **kwargs: typing.Any) -> typing.Any:
+        _wire_environments(kwargs, self._raw_client._client_wrapper._get_api_key)
+        return super().patch_agent(**kwargs)
+
+
+class _LocalSessionsClient(SessionsClient):
+    @functools.wraps(SessionsClient.create_session)
+    def create_session(self, **kwargs: typing.Any) -> typing.Any:
+        if "agent" in kwargs:
+            kwargs["agent"] = localize_agent(kwargs["agent"], self._raw_client._client_wrapper._get_api_key)
+        return super().create_session(**kwargs)
+
+
+class _LocalAsyncAgentsClient(AsyncAgentsClient):
+    @functools.wraps(AsyncAgentsClient.create_agent)
+    async def create_agent(self, **kwargs: typing.Any) -> typing.Any:
+        _wire_environments(kwargs, self._raw_client._client_wrapper._get_api_key)
+        return await super().create_agent(**kwargs)
+
+    @functools.wraps(AsyncAgentsClient.update_agent)
+    async def update_agent(self, **kwargs: typing.Any) -> typing.Any:
+        _wire_environments(kwargs, self._raw_client._client_wrapper._get_api_key)
+        return await super().update_agent(**kwargs)
+
+    @functools.wraps(AsyncAgentsClient.patch_agent)
+    async def patch_agent(self, **kwargs: typing.Any) -> typing.Any:
+        _wire_environments(kwargs, self._raw_client._client_wrapper._get_api_key)
+        return await super().patch_agent(**kwargs)
+
+
+class _LocalAsyncSessionsClient(AsyncSessionsClient):
+    @functools.wraps(AsyncSessionsClient.create_session)
+    async def create_session(self, **kwargs: typing.Any) -> typing.Any:
+        if "agent" in kwargs:
+            kwargs["agent"] = localize_agent(kwargs["agent"], self._raw_client._client_wrapper._get_api_key)
+        return await super().create_session(**kwargs)
+
+
 class Client(BaseClient):
     def run_session(
         self,
@@ -75,6 +134,18 @@ def session(self, id: str) -> SessionHandle:
         """Wrap an existing session id in a handle."""
         return SessionHandle(self, id)
 
+    @property
+    def agents(self) -> _LocalAgentsClient:
+        if self._agents is None:
+            self._agents = _LocalAgentsClient(client_wrapper=self._client_wrapper)
+        return self._agents
+
+    @property
+    def sessions(self) -> _LocalSessionsClient:
+        if self._sessions is None:
+            self._sessions = _LocalSessionsClient(client_wrapper=self._client_wrapper)
+        return self._sessions
+
 
 class AsyncClient(AsyncBaseClient):
     async def run_session(
@@ -123,3 +194,15 @@ async def start_session(
     def session(self, id: str) -> AsyncSessionHandle:
         """Wrap an existing session id in a handle."""
         return AsyncSessionHandle(self, id)
+
+    @property
+    def agents(self) -> _LocalAsyncAgentsClient:
+        if self._agents is None:
+            self._agents = _LocalAsyncAgentsClient(client_wrapper=self._client_wrapper)
+        return self._agents
+
+    @property
+    def sessions(self) -> _LocalAsyncSessionsClient:
+        if self._sessions is None:
+            self._sessions = _LocalAsyncSessionsClient(client_wrapper=self._client_wrapper)
+        return self._sessions

src/hai_agents/local/__init__.py

@@ -0,0 +1,13 @@
+from __future__ import annotations
+
+from .config import SidecarConfig, session_id_from_environment_id
+from .policy import CapabilityPolicy
+from .sidecar import SidecarBusyError, SidecarClient
+
+__all__ = [
+    "CapabilityPolicy",
+    "SidecarBusyError",
+    "SidecarClient",
+    "SidecarConfig",
+    "session_id_from_environment_id",
+]

src/hai_agents/local/browser/__init__.py

@@ -0,0 +1,5 @@
+from __future__ import annotations
+
+from .driver import SeleniumWebDriver
+
+__all__ = ["SeleniumWebDriver"]

src/hai_agents/local/browser/defuddle.py

@@ -0,0 +1,17 @@
+from __future__ import annotations
+
+from importlib.resources import files
+
+_DEFUDDLE_BUNDLE = files("hai_agents.local.browser").joinpath("js/defuddle.full.js").read_text(encoding="utf-8")
+
+DEFUDDLE_EXTRACT_MARKDOWN_SCRIPT = (
+    "(function(){"
+    "var __d=typeof define!=='undefined'?define:void 0;"
+    "var __m=typeof module!=='undefined'?module:void 0;"
+    "var __e=typeof exports!=='undefined'?exports:void 0;"
+    "define=void 0;module=void 0;exports=void 0;" + _DEFUDDLE_BUNDLE + ";if(typeof __d!=='undefined')define=__d;"
+    "if(typeof __m!=='undefined')module=__m;"
+    "if(typeof __e!=='undefined')exports=__e;"
+    "})();"
+    "return new Defuddle(document, { markdown: true }).parse().content"
+)