From fabadc4dec4db9b19cfb13aef5748fe5e92012ca Mon Sep 17 00:00:00 2001 From: Aditi Kumari Date: Mon, 22 Jun 2026 23:09:05 +0530 Subject: [PATCH 01/11] feat(governance): add LlamaIndex governance adapter Registers a BaseEventHandler on the root instrumentation dispatcher (LLMChatStartEvent -> BEFORE_MODEL, LLMChatEndEvent -> AFTER_MODEL, AgentToolCallEvent -> TOOL_CALL). Self-registers via the uipath.governance.adapters entry point; unit-tested and verified firing through the framework's real execution path. BEFORE/AFTER_AGENT remain owned by the uipath-runtime wrapper. Co-Authored-By: Claude Opus 4.8 --- packages/uipath-llamaindex/pyproject.toml | 4 + .../uipath_llamaindex/governance/__init__.py | 57 ++++ .../uipath_llamaindex/governance/adapter.py | 296 ++++++++++++++++++ .../tests/governance/__init__.py | 0 .../tests/governance/test_adapter.py | 250 +++++++++++++++ packages/uipath-llamaindex/uv.lock | 2 + 6 files changed, 609 insertions(+) create mode 100644 packages/uipath-llamaindex/src/uipath_llamaindex/governance/__init__.py create mode 100644 packages/uipath-llamaindex/src/uipath_llamaindex/governance/adapter.py create mode 100644 packages/uipath-llamaindex/tests/governance/__init__.py create mode 100644 packages/uipath-llamaindex/tests/governance/test_adapter.py diff --git a/packages/uipath-llamaindex/pyproject.toml b/packages/uipath-llamaindex/pyproject.toml index edcaff6c..51f3b7bf 100644 --- a/packages/uipath-llamaindex/pyproject.toml +++ b/packages/uipath-llamaindex/pyproject.toml @@ -12,6 +12,7 @@ dependencies = [ "llama-index-llms-azure-openai>=0.4.2", "openinference-instrumentation-llama-index>=4.3.9", "uipath>=2.10.0, <2.11.0", + "uipath-core>=0.5.18, <0.7.0", "uipath-runtime>=0.11.0, <0.12.0", ] classifiers = [ @@ -44,6 +45,9 @@ register = "uipath_llamaindex.middlewares:register_middleware" [project.entry-points."uipath.runtime.factories"] llamaindex = "uipath_llamaindex.runtime:register_runtime_factory" +[project.entry-points."uipath.governance.adapters"] +llamaindex = "uipath_llamaindex.governance:register_governance_adapter" + [project.urls] Homepage = "https://uipath.com" Repository = "https://github.com/UiPath/uipath-integrations-python/" diff --git a/packages/uipath-llamaindex/src/uipath_llamaindex/governance/__init__.py b/packages/uipath-llamaindex/src/uipath_llamaindex/governance/__init__.py new file mode 100644 index 00000000..6f10a31e --- /dev/null +++ b/packages/uipath-llamaindex/src/uipath_llamaindex/governance/__init__.py @@ -0,0 +1,57 @@ +"""Governance integration for ``uipath-llamaindex``. + +Registers :class:`LlamaIndexAdapter` with the global adapter registry in +``uipath.core.adapters`` so ``uipath.runtime.governance.GovernanceRuntime`` can +attach the LlamaIndex-specific governance (BEFORE_MODEL, AFTER_MODEL, TOOL_CALL) +when it sees a LlamaIndex workflow/agent. + +Registration is **idempotent**: calling :func:`register_governance_adapter` +twice is a no-op on the second call. + +Wiring: + 1. Importing this module triggers registration as a side-effect, so any + caller that does ``import uipath_llamaindex.governance`` is opted in. + 2. The package also exposes :func:`register_governance_adapter` as an entry + point under ``uipath.governance.adapters`` so the registry's entry-point + discovery can plug us in without an explicit import. +""" + +from __future__ import annotations + +import logging + +from uipath.core.adapters import get_adapter_registry + +from .adapter import GovernanceEventHandler, LlamaIndexAdapter + +logger = logging.getLogger(__name__) + +_registered: bool = False + + +def register_governance_adapter() -> None: + """Register :class:`LlamaIndexAdapter` with the global registry. + + Idempotent — safe to call multiple times. + """ + global _registered + if _registered: + return + registry = get_adapter_registry() + if any(a.name == "LlamaIndex" for a in registry.get_all()): + _registered = True + return + registry.register(LlamaIndexAdapter()) + _registered = True + logger.debug("Registered uipath-llamaindex governance adapter") + + +# Side-effect registration on module import. +register_governance_adapter() + + +__all__ = [ + "GovernanceEventHandler", + "LlamaIndexAdapter", + "register_governance_adapter", +] \ No newline at end of file diff --git a/packages/uipath-llamaindex/src/uipath_llamaindex/governance/adapter.py b/packages/uipath-llamaindex/src/uipath_llamaindex/governance/adapter.py new file mode 100644 index 00000000..b0d20265 --- /dev/null +++ b/packages/uipath-llamaindex/src/uipath_llamaindex/governance/adapter.py @@ -0,0 +1,296 @@ +"""LlamaIndex adapter for UiPath governance. + +Provides governance for LlamaIndex agents/workflows. Unlike the ADK / OpenAI / +Agent-Framework adapters — which install per-agent callbacks or middleware — +LlamaIndex routes everything (LLM calls, tool calls) through its global +**instrumentation dispatcher** (the same mechanism the package already uses for +OpenInference tracing). So this adapter governs by registering a +:class:`GovernanceEventHandler` on the **root dispatcher**, which receives every +event propagated from child dispatchers: + +- ``LLMChatStartEvent`` → BEFORE_MODEL (scans the latest input message) +- ``LLMChatEndEvent`` → AFTER_MODEL (scans the response) +- ``AgentToolCallEvent`` → TOOL_CALL (tool name + arguments) + +The dispatcher is process-global, so registration is process-wide — which fits +the coded-agent model (one workflow per process). :meth:`attach` therefore +returns the ``agent`` unchanged (nothing is mutated on it); the wiring lives on +the dispatcher. :meth:`detach` removes the handler. + +LlamaIndex does **not** emit a tool-*end* instrumentation event, so AFTER_TOOL +is not wired here; a tool's result is governed at the next ``LLMChatStartEvent`` +where it is fed back to the model as input (analogous to how the OpenAI adapter +handles its missing tool-args). + +Chain-level boundaries (BEFORE_AGENT / AFTER_AGENT) are owned by the runtime +wrapper layer in ``uipath-runtime`` and are intentionally not fired here. + +Contracts and the evaluator protocol come from ``uipath-core``; this package +contributes only the LlamaIndex-specific implementation and self-registers it +with the global adapter registry when ``uipath_llamaindex.governance`` is +imported. + +Audit emission and enforcement (raising :class:`GovernanceBlockException` on +DENY) are owned by the evaluator. The handler only extracts payloads and calls +the matching ``evaluate_*`` method; :class:`GovernanceBlockException` propagates +(aborting the run), anything else is logged and swallowed. +""" + +from __future__ import annotations + +import json +import logging +from typing import Any, Dict, List +from uuid import uuid4 + +from llama_index.core.instrumentation import ( + get_dispatcher, # type: ignore[attr-defined] +) +from llama_index.core.instrumentation.event_handlers.base import ( # type: ignore[attr-defined] + BaseEventHandler, +) +from llama_index.core.instrumentation.events.agent import AgentToolCallEvent +from llama_index.core.instrumentation.events.llm import ( + LLMChatEndEvent, + LLMChatStartEvent, +) +from pydantic import PrivateAttr +from uipath.core.adapters import BaseAdapter, EvaluatorProtocol +from uipath.core.governance.exceptions import GovernanceBlockException + +logger = logging.getLogger(__name__) + +# Cap on the text blob passed to BEFORE_MODEL / AFTER_MODEL governance +# evaluation. Sized to match the runtime side and the other adapters. +_BEFORE_MODEL_TEXT_CAP = 64000 + + +class LlamaIndexAdapter(BaseAdapter): + """Adapter for the LlamaIndex framework. + + Detects LlamaIndex workflows/agents and governs them by registering a + :class:`GovernanceEventHandler` on the root instrumentation dispatcher. + """ + + @property + def name(self) -> str: + return "LlamaIndex" + + def can_handle(self, agent: Any) -> bool: + """Return True if this looks like a LlamaIndex workflow/agent.""" + try: + from workflows import Workflow + + if isinstance(agent, Workflow): + return True + except ImportError: + pass + + # Duck-typed fallback: a workflow/agent exposes ``run`` and a workflow + # step surface (``_get_steps`` / ``steps``) or a Workflow-shaped name. + if hasattr(agent, "run") and ( + hasattr(agent, "_get_steps") + or hasattr(agent, "steps") + or type(agent).__name__.endswith(("Workflow", "Agent")) + ): + return True + return False + + def attach( + self, + agent: Any, + agent_id: str, + session_id: str, + evaluator: EvaluatorProtocol, + ) -> Any: + """Register the governance event handler on the root dispatcher. + + Returns the ``agent`` unchanged — LlamaIndex governance is wired on the + process-global dispatcher, not on the agent object. Idempotent: a + second attach is a no-op while a handler is already registered. + """ + dispatcher = get_dispatcher() + if any(isinstance(h, GovernanceEventHandler) for h in dispatcher.event_handlers): + return agent # idempotent — already governed + callbacks = GovernanceCallbacks( + evaluator=evaluator, agent_name=agent_id, session_id=session_id + ) + dispatcher.add_event_handler(GovernanceEventHandler(callbacks=callbacks)) + logger.debug("Registered governance event handler on LlamaIndex dispatcher") + return agent + + def detach(self, governed: Any) -> Any: + """Remove the governance event handler from the root dispatcher.""" + dispatcher = get_dispatcher() + dispatcher.event_handlers = [ + h + for h in dispatcher.event_handlers + if not isinstance(h, GovernanceEventHandler) + ] + return governed + + +class GovernanceEventHandler(BaseEventHandler): + """Routes LlamaIndex instrumentation events to a governance evaluator. + + A pydantic model (``BaseEventHandler`` is one), so the evaluator + state + are held in a private attribute. ``handle`` is called synchronously by the + dispatcher for every event; we dispatch the three governance-relevant + types and ignore the rest. + """ + + _callbacks: "GovernanceCallbacks" = PrivateAttr() + + def __init__(self, callbacks: "GovernanceCallbacks", **data: Any) -> None: + super().__init__(**data) + self._callbacks = callbacks + + @classmethod + def class_name(cls) -> str: + return "GovernanceEventHandler" + + def handle(self, event: Any, **kwargs: Any) -> Any: + if isinstance(event, LLMChatStartEvent): + self._callbacks.before_model(event.messages) + elif isinstance(event, LLMChatEndEvent): + self._callbacks.after_model(event.response) + elif isinstance(event, AgentToolCallEvent): + self._callbacks.tool_call(event.tool, event.arguments) + return None + + +class GovernanceCallbacks: + """Holds the evaluator + per-attach state, called by the event handler. + + :class:`GovernanceBlockException` is re-raised (it aborts the run); + anything else is logged and swallowed so a governance bug never breaks an + agent run. + """ + + def __init__( + self, + evaluator: EvaluatorProtocol, + agent_name: str, + session_id: str, + ) -> None: + self._evaluator = evaluator + self._agent_name = agent_name + self._session_id = session_id + self._trace_id = str(uuid4()) + self._session_state: Dict[str, Any] = {"tool_calls": 0, "llm_calls": 0} + + def before_model(self, messages: Any) -> None: + """Evaluate BEFORE_MODEL on the latest input message (see ADK rationale).""" + try: + self._session_state["llm_calls"] = ( + self._session_state.get("llm_calls", 0) + 1 + ) + self._evaluator.evaluate_before_model( + model_input=_latest_message_text(messages), + agent_name=self._agent_name, + runtime_id=self._session_id, + trace_id=self._trace_id, + ) + except GovernanceBlockException: + raise + except Exception as e: # noqa: BLE001 - governance must not break the run + logger.warning("before_model governance check failed (continuing): %s", e) + + def after_model(self, response: Any) -> None: + """Evaluate AFTER_MODEL on the chat response text.""" + try: + self._evaluator.evaluate_after_model( + model_output=_response_text(response), + agent_name=self._agent_name, + runtime_id=self._session_id, + trace_id=self._trace_id, + ) + except GovernanceBlockException: + raise + except Exception as e: # noqa: BLE001 + logger.warning("after_model governance check failed (continuing): %s", e) + + def tool_call(self, tool: Any, arguments: Any) -> None: + """Evaluate TOOL_CALL with the tool name + arguments.""" + try: + self._session_state["tool_calls"] = ( + self._session_state.get("tool_calls", 0) + 1 + ) + self._evaluator.evaluate_tool_call( + tool_name=getattr(tool, "name", None) or "unknown", + tool_args=_coerce_args(arguments), + agent_name=self._agent_name, + runtime_id=self._session_id, + trace_id=self._trace_id, + session_state=self._session_state, + ) + except GovernanceBlockException: + raise + except Exception as e: # noqa: BLE001 + logger.warning("tool_call governance check failed (continuing): %s", e) + + +# -------------------------------------------------------------------------- +# Text / argument extraction +# -------------------------------------------------------------------------- + + +def _latest_message_text(messages: Any) -> str: + """Text of the most-recent message in a chat request.""" + if not messages: + return "" + if isinstance(messages, (list, tuple)): + return _message_text(messages[-1]) + return _message_text(messages) + + +def _message_text(message: Any) -> str: + """Pull text from a ``ChatMessage`` (``.content``) or a bare string.""" + if message is None: + return "" + if isinstance(message, str): + return message[:_BEFORE_MODEL_TEXT_CAP] + content = getattr(message, "content", None) + if isinstance(content, str) and content: + return content[:_BEFORE_MODEL_TEXT_CAP] + # Newer ChatMessage carries typed blocks; fall back to str(). + return str(message)[:_BEFORE_MODEL_TEXT_CAP] + + +def _response_text(response: Any) -> str: + """Pull assistant text from a ``ChatResponse`` (``.message.content``).""" + if response is None: + return "" + message = getattr(response, "message", None) + if message is not None: + return _message_text(message) + text = getattr(response, "text", None) + if isinstance(text, str): + return text[:_BEFORE_MODEL_TEXT_CAP] + return str(response)[:_BEFORE_MODEL_TEXT_CAP] + + +def _coerce_args(arguments: Any) -> Dict[str, Any]: + """Normalise tool arguments (JSON string / Mapping / None) to a dict. + + ``AgentToolCallEvent.arguments`` is a JSON-encoded string; other call + sites may hand a dict directly. + """ + if arguments is None: + return {} + if isinstance(arguments, dict): + return arguments + if isinstance(arguments, str): + try: + parsed = json.loads(arguments) + return parsed if isinstance(parsed, dict) else {"_": parsed} + except (TypeError, ValueError): + return {} + return {} + + +__all__: List[str] = [ + "GovernanceCallbacks", + "GovernanceEventHandler", + "LlamaIndexAdapter", +] \ No newline at end of file diff --git a/packages/uipath-llamaindex/tests/governance/__init__.py b/packages/uipath-llamaindex/tests/governance/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/packages/uipath-llamaindex/tests/governance/test_adapter.py b/packages/uipath-llamaindex/tests/governance/test_adapter.py new file mode 100644 index 00000000..600f4080 --- /dev/null +++ b/packages/uipath-llamaindex/tests/governance/test_adapter.py @@ -0,0 +1,250 @@ +"""Unit tests for the LlamaIndex governance adapter. + +The adapter governs via the LlamaIndex instrumentation dispatcher, so these +tests exercise the real event types (``LLMChatStartEvent`` etc.) routed +through :class:`GovernanceEventHandler`, plus the adapter's register/detach on +the dispatcher. The dispatcher is process-global, so each dispatcher test +cleans up after itself via ``detach``. +""" + +from __future__ import annotations + +import logging +from typing import Any, List + +import pytest +from llama_index.core.base.llms.types import ChatMessage, ChatResponse +from llama_index.core.instrumentation import get_dispatcher +from llama_index.core.instrumentation.events.agent import AgentToolCallEvent +from llama_index.core.instrumentation.events.llm import ( + LLMChatEndEvent, + LLMChatStartEvent, +) +from llama_index.core.tools.types import ToolMetadata +from uipath.core.governance.exceptions import GovernanceBlockException + +from uipath_llamaindex.governance.adapter import ( + _BEFORE_MODEL_TEXT_CAP, + GovernanceCallbacks, + GovernanceEventHandler, + LlamaIndexAdapter, + _coerce_args, +) + +# -------------------------------------------------------------------------- +# Fakes +# -------------------------------------------------------------------------- + + +class FakeEvaluator: + """Records evaluate_* calls; optionally BLOCKs on a named hook.""" + + def __init__(self, block_on: str | None = None) -> None: + self.block_on = block_on + self.calls: List[tuple[str, dict]] = [] + + def _record(self, hook: str, **kwargs: Any) -> None: + self.calls.append((hook, kwargs)) + if self.block_on == hook: + raise GovernanceBlockException("blocked") # type: ignore[call-arg] + + def evaluate_before_agent(self, **kwargs: Any) -> None: + self._record("before_agent", **kwargs) + + def evaluate_after_agent(self, **kwargs: Any) -> None: + self._record("after_agent", **kwargs) + + def evaluate_before_model(self, **kwargs: Any) -> None: + self._record("before_model", **kwargs) + + def evaluate_after_model(self, **kwargs: Any) -> None: + self._record("after_model", **kwargs) + + def evaluate_tool_call(self, **kwargs: Any) -> None: + self._record("tool_call", **kwargs) + + def evaluate_after_tool(self, **kwargs: Any) -> None: + self._record("after_tool", **kwargs) + + +class FakeWorkflow: + """Duck-typed LlamaIndex workflow stand-in.""" + + async def run(self, *_a: Any, **_k: Any) -> None: + return None + + +def _make_callbacks(ev: FakeEvaluator) -> GovernanceCallbacks: + return GovernanceCallbacks(evaluator=ev, agent_name="agent-1", session_id="sess-1") + + +def _handler(ev: FakeEvaluator) -> GovernanceEventHandler: + return GovernanceEventHandler(callbacks=_make_callbacks(ev)) + + +# -------------------------------------------------------------------------- +# can_handle +# -------------------------------------------------------------------------- + + +def test_can_handle_workflow_like(): + assert LlamaIndexAdapter().can_handle(FakeWorkflow()) is True + + +def test_can_handle_rejects_plain_object(): + assert LlamaIndexAdapter().can_handle(object()) is False + + +# -------------------------------------------------------------------------- +# attach / detach (real dispatcher) +# -------------------------------------------------------------------------- + + +def _gov_handlers() -> list: + return [ + h + for h in get_dispatcher().event_handlers + if isinstance(h, GovernanceEventHandler) + ] + + +def test_attach_registers_handler_then_detach_removes(): + adapter = LlamaIndexAdapter() + agent = FakeWorkflow() + try: + returned = adapter.attach(agent, agent_id="x", session_id="s", evaluator=FakeEvaluator()) + assert returned is agent + assert len(_gov_handlers()) == 1 + finally: + adapter.detach(agent) + assert _gov_handlers() == [] + + +def test_attach_is_idempotent(): + adapter = LlamaIndexAdapter() + agent = FakeWorkflow() + ev = FakeEvaluator() + try: + adapter.attach(agent, agent_id="x", session_id="s", evaluator=ev) + adapter.attach(agent, agent_id="x", session_id="s", evaluator=ev) + assert len(_gov_handlers()) == 1 + finally: + adapter.detach(agent) + + +# -------------------------------------------------------------------------- +# event routing through the handler +# -------------------------------------------------------------------------- + + +def test_handler_routes_llm_chat_start_to_before_model(): + ev = FakeEvaluator() + h = _handler(ev) + event = LLMChatStartEvent( + messages=[ChatMessage(role="user", content="old"), + ChatMessage(role="user", content="the question")], + additional_kwargs={}, + model_dict={}, + ) + h.handle(event) + hook, kwargs = ev.calls[-1] + assert hook == "before_model" + assert kwargs["model_input"] == "the question" # latest only + + +def test_handler_routes_llm_chat_end_to_after_model(): + ev = FakeEvaluator() + h = _handler(ev) + event = LLMChatEndEvent( + messages=[ChatMessage(role="user", content="q")], + response=ChatResponse(message=ChatMessage(role="assistant", content="the answer")), + ) + h.handle(event) + hook, kwargs = ev.calls[-1] + assert hook == "after_model" + assert kwargs["model_output"] == "the answer" + + +def test_handler_routes_tool_call(): + ev = FakeEvaluator() + h = _handler(ev) + event = AgentToolCallEvent( + tool=ToolMetadata(description="d", name="transfer"), + arguments='{"amount": 50}', + ) + h.handle(event) + hook, kwargs = ev.calls[-1] + assert hook == "tool_call" + assert kwargs["tool_name"] == "transfer" + assert kwargs["tool_args"] == {"amount": 50} + assert kwargs["session_state"]["tool_calls"] == 1 + + +def test_handler_ignores_unrelated_events(): + ev = FakeEvaluator() + h = _handler(ev) + h.handle(object()) # not a governance-relevant event + assert ev.calls == [] + + +# -------------------------------------------------------------------------- +# text / arg extraction +# -------------------------------------------------------------------------- + + +def test_before_model_caps_text(): + ev = FakeEvaluator() + cb = _make_callbacks(ev) + huge = "x" * (_BEFORE_MODEL_TEXT_CAP + 5000) + cb.before_model([ChatMessage(role="user", content=huge)]) + assert len(ev.calls[-1][1]["model_input"]) <= _BEFORE_MODEL_TEXT_CAP + + +def test_before_model_empty(): + ev = FakeEvaluator() + cb = _make_callbacks(ev) + cb.before_model([]) + assert ev.calls[-1][1]["model_input"] == "" + + +def test_coerce_args_json_string(): + assert _coerce_args('{"a": 1}') == {"a": 1} + + +def test_coerce_args_dict_passthrough(): + assert _coerce_args({"a": 1}) == {"a": 1} + + +def test_coerce_args_none_and_bad(): + assert _coerce_args(None) == {} + assert _coerce_args("not json") == {} + + +# -------------------------------------------------------------------------- +# enforcement semantics +# -------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "hook,invoke", + [ + ("before_model", lambda cb: cb.before_model([ChatMessage(role="user", content="hi")])), + ("after_model", lambda cb: cb.after_model(ChatResponse(message=ChatMessage(role="assistant", content="o")))), + ("tool_call", lambda cb: cb.tool_call(ToolMetadata(description="d", name="t"), "{}")), + ], +) +def test_block_exception_propagates(hook, invoke): + cb = _make_callbacks(FakeEvaluator(block_on=hook)) + with pytest.raises(GovernanceBlockException): + invoke(cb) + + +def test_non_block_exception_is_swallowed(caplog): + class Boom: + def evaluate_before_model(self, **_: Any) -> None: + raise RuntimeError("evaluator bug") + + cb = GovernanceCallbacks(evaluator=Boom(), agent_name="a", session_id="s") # type: ignore[arg-type] + with caplog.at_level(logging.WARNING): + cb.before_model([ChatMessage(role="user", content="x")]) + assert any("governance check failed" in r.message for r in caplog.records) \ No newline at end of file diff --git a/packages/uipath-llamaindex/uv.lock b/packages/uipath-llamaindex/uv.lock index f879a865..fe5fd437 100644 --- a/packages/uipath-llamaindex/uv.lock +++ b/packages/uipath-llamaindex/uv.lock @@ -3516,6 +3516,7 @@ dependencies = [ { name = "llama-index-workflows" }, { name = "openinference-instrumentation-llama-index" }, { name = "uipath" }, + { name = "uipath-core" }, { name = "uipath-runtime" }, ] @@ -3560,6 +3561,7 @@ requires-dist = [ { name = "llama-index-workflows", specifier = ">=2.18.0,<3.0.0" }, { name = "openinference-instrumentation-llama-index", specifier = ">=4.3.9" }, { name = "uipath", specifier = ">=2.10.0,<2.11.0" }, + { name = "uipath-core", specifier = ">=0.5.18,<0.7.0" }, { name = "uipath-runtime", specifier = ">=0.11.0,<0.12.0" }, ] provides-extras = ["bedrock", "vertex"] From e3f0e0ec0048ed186b976a85887cf53c5958f01a Mon Sep 17 00:00:00 2001 From: Aditi Kumari Date: Wed, 24 Jun 2026 17:37:30 +0530 Subject: [PATCH 02/11] chore(governance): apply review feedback (no import-time registration, framework-only can_handle) Mirror radu's LangChain-adapter review across the LlamaIndex adapter: - __init__: drop the import-time registration side-effect; registration only via the uipath.governance.adapters entry point. - can_handle: claim only a real workflows.Workflow; remove the duck-typed (run / Workflow-shaped name) fallback. - docstring: 'governance host' instead of uipath-runtime internals. - tests: can_handle uses a real stepped Workflow; a duck-typed look-alike is now rejected. Co-Authored-By: Claude Opus 4.8 --- .../uipath_llamaindex/governance/__init__.py | 21 ++++++--------- .../uipath_llamaindex/governance/adapter.py | 26 +++++-------------- .../tests/governance/test_adapter.py | 17 +++++++++--- 3 files changed, 29 insertions(+), 35 deletions(-) diff --git a/packages/uipath-llamaindex/src/uipath_llamaindex/governance/__init__.py b/packages/uipath-llamaindex/src/uipath_llamaindex/governance/__init__.py index 6f10a31e..7dfcc4e7 100644 --- a/packages/uipath-llamaindex/src/uipath_llamaindex/governance/__init__.py +++ b/packages/uipath-llamaindex/src/uipath_llamaindex/governance/__init__.py @@ -1,19 +1,17 @@ """Governance integration for ``uipath-llamaindex``. -Registers :class:`LlamaIndexAdapter` with the global adapter registry in -``uipath.core.adapters`` so ``uipath.runtime.governance.GovernanceRuntime`` can -attach the LlamaIndex-specific governance (BEFORE_MODEL, AFTER_MODEL, TOOL_CALL) -when it sees a LlamaIndex workflow/agent. +Registers :class:`LlamaIndexAdapter` with the adapter registry in +``uipath.core.adapters`` so the governance host can attach the +LlamaIndex-specific governance (BEFORE_MODEL, AFTER_MODEL, TOOL_CALL) when it +sees a LlamaIndex workflow/agent. Registration is **idempotent**: calling :func:`register_governance_adapter` twice is a no-op on the second call. -Wiring: - 1. Importing this module triggers registration as a side-effect, so any - caller that does ``import uipath_llamaindex.governance`` is opted in. - 2. The package also exposes :func:`register_governance_adapter` as an entry - point under ``uipath.governance.adapters`` so the registry's entry-point - discovery can plug us in without an explicit import. +Wiring: the package exposes :func:`register_governance_adapter` as an entry +point under ``uipath.governance.adapters``. The governance adapter discovery +path calls it to register the adapter. Importing this module does not, by +itself, mutate the global registry. """ from __future__ import annotations @@ -46,9 +44,6 @@ def register_governance_adapter() -> None: logger.debug("Registered uipath-llamaindex governance adapter") -# Side-effect registration on module import. -register_governance_adapter() - __all__ = [ "GovernanceEventHandler", diff --git a/packages/uipath-llamaindex/src/uipath_llamaindex/governance/adapter.py b/packages/uipath-llamaindex/src/uipath_llamaindex/governance/adapter.py index b0d20265..c3833b7e 100644 --- a/packages/uipath-llamaindex/src/uipath_llamaindex/governance/adapter.py +++ b/packages/uipath-llamaindex/src/uipath_llamaindex/governance/adapter.py @@ -22,8 +22,8 @@ where it is fed back to the model as input (analogous to how the OpenAI adapter handles its missing tool-args). -Chain-level boundaries (BEFORE_AGENT / AFTER_AGENT) are owned by the runtime -wrapper layer in ``uipath-runtime`` and are intentionally not fired here. +Chain-level boundaries (BEFORE_AGENT / AFTER_AGENT) are owned by the +governance host and are intentionally not fired here. Contracts and the evaluator protocol come from ``uipath-core``; this package contributes only the LlamaIndex-specific implementation and self-registers it @@ -43,8 +43,8 @@ from typing import Any, Dict, List from uuid import uuid4 -from llama_index.core.instrumentation import ( - get_dispatcher, # type: ignore[attr-defined] +from llama_index.core.instrumentation import ( # type: ignore[attr-defined] + get_dispatcher, ) from llama_index.core.instrumentation.event_handlers.base import ( # type: ignore[attr-defined] BaseEventHandler, @@ -77,24 +77,12 @@ def name(self) -> str: return "LlamaIndex" def can_handle(self, agent: Any) -> bool: - """Return True if this looks like a LlamaIndex workflow/agent.""" + """Return True only for a LlamaIndex ``Workflow`` (incl. agent workflows).""" try: from workflows import Workflow - - if isinstance(agent, Workflow): - return True except ImportError: - pass - - # Duck-typed fallback: a workflow/agent exposes ``run`` and a workflow - # step surface (``_get_steps`` / ``steps``) or a Workflow-shaped name. - if hasattr(agent, "run") and ( - hasattr(agent, "_get_steps") - or hasattr(agent, "steps") - or type(agent).__name__.endswith(("Workflow", "Agent")) - ): - return True - return False + return False + return isinstance(agent, Workflow) def attach( self, diff --git a/packages/uipath-llamaindex/tests/governance/test_adapter.py b/packages/uipath-llamaindex/tests/governance/test_adapter.py index 600f4080..b9022ed2 100644 --- a/packages/uipath-llamaindex/tests/governance/test_adapter.py +++ b/packages/uipath-llamaindex/tests/governance/test_adapter.py @@ -87,11 +87,22 @@ def _handler(ev: FakeEvaluator) -> GovernanceEventHandler: # -------------------------------------------------------------------------- -def test_can_handle_workflow_like(): - assert LlamaIndexAdapter().can_handle(FakeWorkflow()) is True +def test_can_handle_real_workflow(): + from workflows import Workflow, step + from workflows.events import StartEvent, StopEvent + class _RealWorkflow(Workflow): + @step + async def go(self, ev: StartEvent) -> StopEvent: + return StopEvent() -def test_can_handle_rejects_plain_object(): + assert LlamaIndexAdapter().can_handle(_RealWorkflow()) is True + + +def test_can_handle_rejects_non_workflow(): + # A duck-typed look-alike (has run / Workflow-shaped name) must NOT be + # claimed — only a real workflows.Workflow is. + assert LlamaIndexAdapter().can_handle(FakeWorkflow()) is False assert LlamaIndexAdapter().can_handle(object()) is False From 6a82c95be3b719d09b833ac1ddb1b63c5809f61a Mon Sep 17 00:00:00 2001 From: Aditi Kumari Date: Wed, 24 Jun 2026 18:27:28 +0530 Subject: [PATCH 03/11] docs(governance): address Copilot review on the LlamaIndex adapter Module docstring: registers via the uipath.governance.adapters entry point, not at import time. Co-Authored-By: Claude Opus 4.8 --- .../src/uipath_llamaindex/governance/adapter.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/packages/uipath-llamaindex/src/uipath_llamaindex/governance/adapter.py b/packages/uipath-llamaindex/src/uipath_llamaindex/governance/adapter.py index c3833b7e..1e1e46fc 100644 --- a/packages/uipath-llamaindex/src/uipath_llamaindex/governance/adapter.py +++ b/packages/uipath-llamaindex/src/uipath_llamaindex/governance/adapter.py @@ -26,9 +26,8 @@ governance host and are intentionally not fired here. Contracts and the evaluator protocol come from ``uipath-core``; this package -contributes only the LlamaIndex-specific implementation and self-registers it -with the global adapter registry when ``uipath_llamaindex.governance`` is -imported. +contributes only the LlamaIndex-specific implementation and registers it with +the adapter registry via the ``uipath.governance.adapters`` entry point. Audit emission and enforcement (raising :class:`GovernanceBlockException` on DENY) are owned by the evaluator. The handler only extracts payloads and calls From a9275dfa2e777cb4df903635ac0447139581d9f8 Mon Sep 17 00:00:00 2001 From: Aditi Kumari Date: Wed, 24 Jun 2026 18:40:51 +0530 Subject: [PATCH 04/11] chore(governance): remove unrelated files bundled from a dirty tree These files were swept into the branch by a broad add; they are unrelated to the governance adapter. Reverting/removing them so the PR contains only governance changes. Co-Authored-By: Claude Opus 4.8 --- SETUP.MD | 141 ------------------ .../docs/llms_and_embeddings.md | 56 +++---- 2 files changed, 19 insertions(+), 178 deletions(-) delete mode 100644 SETUP.MD diff --git a/SETUP.MD b/SETUP.MD deleted file mode 100644 index d7b750a8..00000000 --- a/SETUP.MD +++ /dev/null @@ -1,141 +0,0 @@ -# SETUP.MD - -This file documents how to provision a clean development environment for the five packages in this repo (`uipath-agent-framework`, `uipath-google-adk`, `uipath-llamaindex`, `uipath-openai-agents`, `uipath-pydantic-ai`), run the build, execute the tests, and validate a sample code change end-to-end. It is intended both as a quick reference for human contributors and as a structured guide for automated environment-setup tooling. - -## Prerequisites - -- Python 3.11+ -- [uv](https://docs.astral.sh/uv/) 0.5+ - -### Supported platforms - -`uv` is shell- and OS-agnostic, so the commands below run unchanged on every supported platform: - -- [x] Linux -- [x] Windows -- [x] macOS - -## Environment Variables - -None required for environment setup, build, or unit tests. The suites under the `Test` section run fully offline and require no external authentication. - -> **All commands below must be run from the repository root.** The `uv --directory packages/` invocations resolve each subpackage relative to the current working directory. The first line of `## Setup` enforces this by `cd`-ing to the git root. - -## Setup - -```bash -cd "$(git rev-parse --show-toplevel)" -python3 -m pip install --upgrade uv - -# Sync all five packages (each is independent) -uv --directory packages/uipath-agent-framework sync --all-extras -uv --directory packages/uipath-google-adk sync --all-extras -uv --directory packages/uipath-llamaindex sync --all-extras -uv --directory packages/uipath-openai-agents sync --all-extras -uv --directory packages/uipath-pydantic-ai sync --all-extras -``` - -## Verify Setup - -```bash -uv --version -uv --directory packages/uipath-pydantic-ai run python --version -uv --directory packages/uipath-agent-framework run python -c "import uipath_agent_framework; print('uipath-agent-framework ok')" -uv --directory packages/uipath-google-adk run python -c "import uipath_google_adk; print('uipath-google-adk ok')" -uv --directory packages/uipath-llamaindex run python -c "import uipath_llamaindex; print('uipath-llamaindex ok')" -uv --directory packages/uipath-openai-agents run python -c "import uipath_openai_agents; print('uipath-openai-agents ok')" -uv --directory packages/uipath-pydantic-ai run python -c "import uipath_pydantic_ai; print('uipath-pydantic-ai ok')" -``` - -## Build - -N/A - -## Test - -```bash -uv --directory packages/uipath-agent-framework run pytest -uv --directory packages/uipath-google-adk run pytest -uv --directory packages/uipath-llamaindex run pytest -uv --directory packages/uipath-openai-agents run pytest -uv --directory packages/uipath-pydantic-ai run pytest -``` - -## Sample Code Change - -### The change - -Add a new `agent_count` property to `PydanticAiConfig` in `packages/uipath-pydantic-ai/src/uipath_pydantic_ai/runtime/config.py`, immediately after the existing `entrypoint` property: - -```python -@property -def agent_count(self) -> int: - """Number of agents defined in the configuration.""" - return len(self.agents) -``` - -Then create `packages/uipath-pydantic-ai/tests/test_config_agent_count.py` with two pytest tests: - -```python -"""Tests for PydanticAiConfig.agent_count.""" - -import json -from pathlib import Path - -from uipath_pydantic_ai.runtime.config import PydanticAiConfig - - -def test_agent_count_single(tmp_path: Path) -> None: - config_path = tmp_path / "pydantic_ai.json" - config_path.write_text(json.dumps({"agents": {"main": "main:agent"}})) - cfg = PydanticAiConfig(str(config_path)) - assert cfg.agent_count == 1 - - -def test_agent_count_multiple(tmp_path: Path) -> None: - config_path = tmp_path / "pydantic_ai.json" - config_path.write_text( - json.dumps( - { - "agents": { - "alpha": "alpha:agent", - "beta": "beta:agent", - "gamma": "gamma:agent", - } - } - ) - ) - cfg = PydanticAiConfig(str(config_path)) - assert cfg.agent_count == 3 -``` - -### Verification - -```bash -uv --directory packages/uipath-pydantic-ai run pytest tests/test_config_agent_count.py -v -``` - -## Test with a real UiPath Coded Agent - -The unit tests above are necessary but not sufficient — they don't exercise the package end-to-end through a real agent. The flow below validates changes against a live runtime: - -1. Apply the code changes locally. -2. Run the unit tests (see the `Sample Code Change` section above). -3. Scaffold a coded UiPath agent (PydanticAI / OpenAI / Google ADK / LlamaIndex / Agent Framework, matching the package you changed) that exercises the changed code path. -4. In the downstream project's `pyproject.toml`, add this local library as an editable dependency (substitute the package you changed): - - ```toml - [tool.uv.sources] - uipath-pydantic-ai = { path = "../path/to/uipath-integrations-python/packages/uipath-pydantic-ai", editable = true } - ``` - -5. Exercise the new behavior end-to-end: - - ```bash - uv run uipath run --input '{...}' - ``` - -6. (Optional) Open a PR and apply the `build:dev` label — this publishes the development version to Test PyPI. -7. The PR description is updated automatically with instructions for pointing the downstream agent at the Test PyPI dev version. -8. Push the dev version to UiPath with [`uipath push`](https://uipath.github.io/uipath-python/cli/#push), then deploy it to Orchestrator or Studio Web with [`uipath deploy`](https://uipath.github.io/uipath-python/cli/#deploy), and run it in cloud to confirm the changes behave correctly against the real platform. -9. Once validation is done, close the dev PR — these PRs are not meant to be merged; their only purpose was to publish a Test PyPI build for end-to-end validation. diff --git a/packages/uipath-llamaindex/docs/llms_and_embeddings.md b/packages/uipath-llamaindex/docs/llms_and_embeddings.md index 01416be8..15e25eb4 100644 --- a/packages/uipath-llamaindex/docs/llms_and_embeddings.md +++ b/packages/uipath-llamaindex/docs/llms_and_embeddings.md @@ -1,39 +1,7 @@ # LLMs and Embeddings -UiPath provides pre-configured LLM and embedding classes for several providers (OpenAI via `UiPathOpenAI`, Anthropic on AWS Bedrock via `UiPathChatBedrockConverse`, Google Vertex AI via `UiPathVertex`, and more), plus embeddings via `UiPathOpenAIEmbedding`. These handle authentication, routing, and configuration automatically, allowing you to focus on building your agents. You do not need to add API keys from OpenAI, AWS, or Google, usage of these models will consume `Agent Units` on your account. - -## Available models - -LLM models are served through the UiPath LLM Gateway and are subject to [AI Trust Layer](https://docs.uipath.com/automation-cloud/automation-cloud/latest/admin-guide/about-ai-trust-layer) policies, so the exact set of models available to you depends on your tenant configuration. List the models you can use with the `uipath` CLI: - -```console -$ uipath list-models - Available LLM Models -┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━┓ -┃ AwsBedrock ┃ OpenAi ┃ VertexAi ┃ -┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━┩ -│ anthropic.claude-haiku-4-5-20251001-v1:0 │ gpt-4.1-2025-04-14 │ gemini-2.5-flash │ -│ anthropic.claude-opus-4-7 │ gpt-4.1-mini-2025-04-14 │ gemini-2.5-pro │ -│ ... │ ... │ ... │ -└──────────────────────────────────────────┴─────────────────────────┴──────────────────┘ -``` - -Pick a model id from the relevant provider column and pass it (or the matching enum member) to the matching class: - -```python -from uipath_llamaindex.llms import UiPathOpenAI -from uipath_llamaindex.llms.bedrock import UiPathChatBedrockConverse -from uipath_llamaindex.llms.vertex import UiPathVertex - -# OpenAI models -llm = UiPathOpenAI(model="gpt-4.1-mini-2025-04-14") - -# AWS Bedrock (Anthropic) models -llm = UiPathChatBedrockConverse(model="anthropic.claude-sonnet-4-5-20250929-v1:0") - -# Google Vertex AI (Gemini) models -llm = UiPathVertex(model="gemini-2.5-flash") -``` +UiPath provides pre-configured LLM and embedding classes that handle authentication, routing, and configuration automatically, allowing you to focus on building your agents. +You do not need to add API keys from OpenAI, AWS, or Google, usage of these models will consume `Agent Units` on your account. ## UiPathOpenAI @@ -41,7 +9,17 @@ The `UiPathOpenAI` class is a pre-configured Azure OpenAI client that routes req ### Available Models -The OpenAI models from the `OpenAi` column of [`uipath list-models`](#available-models) can be used here, either as a model string or via the `OpenAIModel` enum. +The following OpenAI models are available through the `OpenAIModel` enum: + +- `GPT_4_1_2025_04_14` +- `GPT_4_1_MINI_2025_04_14` +- `GPT_4_1_NANO_2025_04_14` +- `GPT_4O_2024_05_13` +- `GPT_4O_2024_08_06` +- `GPT_4O_2024_11_20` +- `GPT_4O_MINI_2024_07_18` (default) +- `O3_MINI_2025_01_31` +- `TEXT_DAVINCI_003` ### Basic Usage @@ -165,7 +143,9 @@ from uipath_llamaindex.llms import BedrockModel llm = UiPathChatBedrock(model=BedrockModel.anthropic_claude_sonnet_4) ``` -The available models are the ones in the `AwsBedrock` column of [`uipath list-models`](#available-models). +Currently, the following models can be used (this list can be updated in the future): + +- `anthropic.claude-3-7-sonnet-20250219-v1:0`, `anthropic.claude-sonnet-4-20250514-v1:0`, `anthropic.claude-sonnet-4-5-20250929-v1:0`, `anthropic.claude-haiku-4-5-20251001-v1:0` ## UiPathVertex @@ -204,7 +184,9 @@ response = llm.chat(messages) print(response) ``` -The available models are the ones in the `VertexAi` column of [`uipath list-models`](#available-models). +Currently, the following models can be used (this list can be updated in the future): + +- `gemini-2.0-flash-001`, `gemini-2.5-flash`, `gemini-2.5-pro` ## Integration with LlamaIndex From a4ce20299ad31815adfce172595e942a19fb882b Mon Sep 17 00:00:00 2001 From: Aditi Kumari Date: Wed, 24 Jun 2026 19:01:06 +0530 Subject: [PATCH 05/11] chore(governance): restore SETUP.MD and llms doc that belong to main An earlier cleanup commit compared against a stale local main and wrongly removed SETUP.MD and reverted the LlamaIndex docs change. Both files come from main (PRs #352/#356), not this branch. Restore them to the main version so this PR is governance-only with no spurious deletions. Co-Authored-By: Claude Opus 4.8 --- SETUP.MD | 141 ++++++++++++++++++ .../docs/llms_and_embeddings.md | 56 ++++--- 2 files changed, 178 insertions(+), 19 deletions(-) create mode 100644 SETUP.MD diff --git a/SETUP.MD b/SETUP.MD new file mode 100644 index 00000000..d7b750a8 --- /dev/null +++ b/SETUP.MD @@ -0,0 +1,141 @@ +# SETUP.MD + +This file documents how to provision a clean development environment for the five packages in this repo (`uipath-agent-framework`, `uipath-google-adk`, `uipath-llamaindex`, `uipath-openai-agents`, `uipath-pydantic-ai`), run the build, execute the tests, and validate a sample code change end-to-end. It is intended both as a quick reference for human contributors and as a structured guide for automated environment-setup tooling. + +## Prerequisites + +- Python 3.11+ +- [uv](https://docs.astral.sh/uv/) 0.5+ + +### Supported platforms + +`uv` is shell- and OS-agnostic, so the commands below run unchanged on every supported platform: + +- [x] Linux +- [x] Windows +- [x] macOS + +## Environment Variables + +None required for environment setup, build, or unit tests. The suites under the `Test` section run fully offline and require no external authentication. + +> **All commands below must be run from the repository root.** The `uv --directory packages/` invocations resolve each subpackage relative to the current working directory. The first line of `## Setup` enforces this by `cd`-ing to the git root. + +## Setup + +```bash +cd "$(git rev-parse --show-toplevel)" +python3 -m pip install --upgrade uv + +# Sync all five packages (each is independent) +uv --directory packages/uipath-agent-framework sync --all-extras +uv --directory packages/uipath-google-adk sync --all-extras +uv --directory packages/uipath-llamaindex sync --all-extras +uv --directory packages/uipath-openai-agents sync --all-extras +uv --directory packages/uipath-pydantic-ai sync --all-extras +``` + +## Verify Setup + +```bash +uv --version +uv --directory packages/uipath-pydantic-ai run python --version +uv --directory packages/uipath-agent-framework run python -c "import uipath_agent_framework; print('uipath-agent-framework ok')" +uv --directory packages/uipath-google-adk run python -c "import uipath_google_adk; print('uipath-google-adk ok')" +uv --directory packages/uipath-llamaindex run python -c "import uipath_llamaindex; print('uipath-llamaindex ok')" +uv --directory packages/uipath-openai-agents run python -c "import uipath_openai_agents; print('uipath-openai-agents ok')" +uv --directory packages/uipath-pydantic-ai run python -c "import uipath_pydantic_ai; print('uipath-pydantic-ai ok')" +``` + +## Build + +N/A + +## Test + +```bash +uv --directory packages/uipath-agent-framework run pytest +uv --directory packages/uipath-google-adk run pytest +uv --directory packages/uipath-llamaindex run pytest +uv --directory packages/uipath-openai-agents run pytest +uv --directory packages/uipath-pydantic-ai run pytest +``` + +## Sample Code Change + +### The change + +Add a new `agent_count` property to `PydanticAiConfig` in `packages/uipath-pydantic-ai/src/uipath_pydantic_ai/runtime/config.py`, immediately after the existing `entrypoint` property: + +```python +@property +def agent_count(self) -> int: + """Number of agents defined in the configuration.""" + return len(self.agents) +``` + +Then create `packages/uipath-pydantic-ai/tests/test_config_agent_count.py` with two pytest tests: + +```python +"""Tests for PydanticAiConfig.agent_count.""" + +import json +from pathlib import Path + +from uipath_pydantic_ai.runtime.config import PydanticAiConfig + + +def test_agent_count_single(tmp_path: Path) -> None: + config_path = tmp_path / "pydantic_ai.json" + config_path.write_text(json.dumps({"agents": {"main": "main:agent"}})) + cfg = PydanticAiConfig(str(config_path)) + assert cfg.agent_count == 1 + + +def test_agent_count_multiple(tmp_path: Path) -> None: + config_path = tmp_path / "pydantic_ai.json" + config_path.write_text( + json.dumps( + { + "agents": { + "alpha": "alpha:agent", + "beta": "beta:agent", + "gamma": "gamma:agent", + } + } + ) + ) + cfg = PydanticAiConfig(str(config_path)) + assert cfg.agent_count == 3 +``` + +### Verification + +```bash +uv --directory packages/uipath-pydantic-ai run pytest tests/test_config_agent_count.py -v +``` + +## Test with a real UiPath Coded Agent + +The unit tests above are necessary but not sufficient — they don't exercise the package end-to-end through a real agent. The flow below validates changes against a live runtime: + +1. Apply the code changes locally. +2. Run the unit tests (see the `Sample Code Change` section above). +3. Scaffold a coded UiPath agent (PydanticAI / OpenAI / Google ADK / LlamaIndex / Agent Framework, matching the package you changed) that exercises the changed code path. +4. In the downstream project's `pyproject.toml`, add this local library as an editable dependency (substitute the package you changed): + + ```toml + [tool.uv.sources] + uipath-pydantic-ai = { path = "../path/to/uipath-integrations-python/packages/uipath-pydantic-ai", editable = true } + ``` + +5. Exercise the new behavior end-to-end: + + ```bash + uv run uipath run --input '{...}' + ``` + +6. (Optional) Open a PR and apply the `build:dev` label — this publishes the development version to Test PyPI. +7. The PR description is updated automatically with instructions for pointing the downstream agent at the Test PyPI dev version. +8. Push the dev version to UiPath with [`uipath push`](https://uipath.github.io/uipath-python/cli/#push), then deploy it to Orchestrator or Studio Web with [`uipath deploy`](https://uipath.github.io/uipath-python/cli/#deploy), and run it in cloud to confirm the changes behave correctly against the real platform. +9. Once validation is done, close the dev PR — these PRs are not meant to be merged; their only purpose was to publish a Test PyPI build for end-to-end validation. diff --git a/packages/uipath-llamaindex/docs/llms_and_embeddings.md b/packages/uipath-llamaindex/docs/llms_and_embeddings.md index 15e25eb4..01416be8 100644 --- a/packages/uipath-llamaindex/docs/llms_and_embeddings.md +++ b/packages/uipath-llamaindex/docs/llms_and_embeddings.md @@ -1,7 +1,39 @@ # LLMs and Embeddings -UiPath provides pre-configured LLM and embedding classes that handle authentication, routing, and configuration automatically, allowing you to focus on building your agents. -You do not need to add API keys from OpenAI, AWS, or Google, usage of these models will consume `Agent Units` on your account. +UiPath provides pre-configured LLM and embedding classes for several providers (OpenAI via `UiPathOpenAI`, Anthropic on AWS Bedrock via `UiPathChatBedrockConverse`, Google Vertex AI via `UiPathVertex`, and more), plus embeddings via `UiPathOpenAIEmbedding`. These handle authentication, routing, and configuration automatically, allowing you to focus on building your agents. You do not need to add API keys from OpenAI, AWS, or Google, usage of these models will consume `Agent Units` on your account. + +## Available models + +LLM models are served through the UiPath LLM Gateway and are subject to [AI Trust Layer](https://docs.uipath.com/automation-cloud/automation-cloud/latest/admin-guide/about-ai-trust-layer) policies, so the exact set of models available to you depends on your tenant configuration. List the models you can use with the `uipath` CLI: + +```console +$ uipath list-models + Available LLM Models +┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━┓ +┃ AwsBedrock ┃ OpenAi ┃ VertexAi ┃ +┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━┩ +│ anthropic.claude-haiku-4-5-20251001-v1:0 │ gpt-4.1-2025-04-14 │ gemini-2.5-flash │ +│ anthropic.claude-opus-4-7 │ gpt-4.1-mini-2025-04-14 │ gemini-2.5-pro │ +│ ... │ ... │ ... │ +└──────────────────────────────────────────┴─────────────────────────┴──────────────────┘ +``` + +Pick a model id from the relevant provider column and pass it (or the matching enum member) to the matching class: + +```python +from uipath_llamaindex.llms import UiPathOpenAI +from uipath_llamaindex.llms.bedrock import UiPathChatBedrockConverse +from uipath_llamaindex.llms.vertex import UiPathVertex + +# OpenAI models +llm = UiPathOpenAI(model="gpt-4.1-mini-2025-04-14") + +# AWS Bedrock (Anthropic) models +llm = UiPathChatBedrockConverse(model="anthropic.claude-sonnet-4-5-20250929-v1:0") + +# Google Vertex AI (Gemini) models +llm = UiPathVertex(model="gemini-2.5-flash") +``` ## UiPathOpenAI @@ -9,17 +41,7 @@ The `UiPathOpenAI` class is a pre-configured Azure OpenAI client that routes req ### Available Models -The following OpenAI models are available through the `OpenAIModel` enum: - -- `GPT_4_1_2025_04_14` -- `GPT_4_1_MINI_2025_04_14` -- `GPT_4_1_NANO_2025_04_14` -- `GPT_4O_2024_05_13` -- `GPT_4O_2024_08_06` -- `GPT_4O_2024_11_20` -- `GPT_4O_MINI_2024_07_18` (default) -- `O3_MINI_2025_01_31` -- `TEXT_DAVINCI_003` +The OpenAI models from the `OpenAi` column of [`uipath list-models`](#available-models) can be used here, either as a model string or via the `OpenAIModel` enum. ### Basic Usage @@ -143,9 +165,7 @@ from uipath_llamaindex.llms import BedrockModel llm = UiPathChatBedrock(model=BedrockModel.anthropic_claude_sonnet_4) ``` -Currently, the following models can be used (this list can be updated in the future): - -- `anthropic.claude-3-7-sonnet-20250219-v1:0`, `anthropic.claude-sonnet-4-20250514-v1:0`, `anthropic.claude-sonnet-4-5-20250929-v1:0`, `anthropic.claude-haiku-4-5-20251001-v1:0` +The available models are the ones in the `AwsBedrock` column of [`uipath list-models`](#available-models). ## UiPathVertex @@ -184,9 +204,7 @@ response = llm.chat(messages) print(response) ``` -Currently, the following models can be used (this list can be updated in the future): - -- `gemini-2.0-flash-001`, `gemini-2.5-flash`, `gemini-2.5-pro` +The available models are the ones in the `VertexAi` column of [`uipath list-models`](#available-models). ## Integration with LlamaIndex From 9f4a4f194b0e87a6cb8bfcb9d26f7ba380122921 Mon Sep 17 00:00:00 2001 From: Aditi Kumari Date: Tue, 30 Jun 2026 20:03:39 +0530 Subject: [PATCH 06/11] refactor(governance): migrate LlamaIndex adapter to factory-evaluator Core PR #1761 removed BaseAdapter from uipath-core. Migrate to the factory-evaluator pattern (matching #899): - governance/adapter.py -> event_handler.py: replace the BaseAdapter subclass (name/can_handle/attach/detach) with module-level install_governance() that registers the GovernanceEventHandler on the root instrumentation dispatcher; keep the handler + callbacks. File named for its seam (the event handler), like LangChain's callbacks.py. - runtime/factory.py: new_runtime reads `evaluator` from kwargs and calls install_governance. - governance/__init__.py: drop register_governance_adapter + registry import; expose install_governance. No import-time side effects. - pyproject.toml: remove the uipath.governance.adapters entry point. - tests (test_adapter.py -> test_event_handler.py): drop can_handle/ attach/detach; cover install_governance + factory wiring. ruff + mypy clean; 17 governance tests pass. Co-Authored-By: Claude Opus 4.8 --- packages/uipath-llamaindex/pyproject.toml | 3 - .../uipath_llamaindex/governance/__init__.py | 52 ++------- .../{adapter.py => event_handler.py} | 94 ++++++---------- .../src/uipath_llamaindex/runtime/factory.py | 16 +++ ...{test_adapter.py => test_event_handler.py} | 106 +++++++++++------- 5 files changed, 129 insertions(+), 142 deletions(-) rename packages/uipath-llamaindex/src/uipath_llamaindex/governance/{adapter.py => event_handler.py} (76%) rename packages/uipath-llamaindex/tests/governance/{test_adapter.py => test_event_handler.py} (73%) diff --git a/packages/uipath-llamaindex/pyproject.toml b/packages/uipath-llamaindex/pyproject.toml index 51f3b7bf..36cf900a 100644 --- a/packages/uipath-llamaindex/pyproject.toml +++ b/packages/uipath-llamaindex/pyproject.toml @@ -45,9 +45,6 @@ register = "uipath_llamaindex.middlewares:register_middleware" [project.entry-points."uipath.runtime.factories"] llamaindex = "uipath_llamaindex.runtime:register_runtime_factory" -[project.entry-points."uipath.governance.adapters"] -llamaindex = "uipath_llamaindex.governance:register_governance_adapter" - [project.urls] Homepage = "https://uipath.com" Repository = "https://github.com/UiPath/uipath-integrations-python/" diff --git a/packages/uipath-llamaindex/src/uipath_llamaindex/governance/__init__.py b/packages/uipath-llamaindex/src/uipath_llamaindex/governance/__init__.py index 7dfcc4e7..bae93038 100644 --- a/packages/uipath-llamaindex/src/uipath_llamaindex/governance/__init__.py +++ b/packages/uipath-llamaindex/src/uipath_llamaindex/governance/__init__.py @@ -1,52 +1,20 @@ """Governance integration for ``uipath-llamaindex``. -Registers :class:`LlamaIndexAdapter` with the adapter registry in -``uipath.core.adapters`` so the governance host can attach the -LlamaIndex-specific governance (BEFORE_MODEL, AFTER_MODEL, TOOL_CALL) when it -sees a LlamaIndex workflow/agent. - -Registration is **idempotent**: calling :func:`register_governance_adapter` -twice is a no-op on the second call. - -Wiring: the package exposes :func:`register_governance_adapter` as an entry -point under ``uipath.governance.adapters``. The governance adapter discovery -path calls it to register the adapter. Importing this module does not, by -itself, mutate the global registry. +Exposes :func:`install_governance` — registers a :class:`GovernanceEventHandler` +on the LlamaIndex root instrumentation dispatcher, which governs LLM/tool events +(BEFORE_MODEL, AFTER_MODEL, TOOL_CALL). Wired into a run by passing an +``evaluator`` to :class:`UiPathLlamaIndexRuntimeFactory`; the factory calls +:func:`install_governance`. + +Importing this module has no side effects: no adapter is registered, no global +state is mutated. """ from __future__ import annotations -import logging - -from uipath.core.adapters import get_adapter_registry - -from .adapter import GovernanceEventHandler, LlamaIndexAdapter - -logger = logging.getLogger(__name__) - -_registered: bool = False - - -def register_governance_adapter() -> None: - """Register :class:`LlamaIndexAdapter` with the global registry. - - Idempotent — safe to call multiple times. - """ - global _registered - if _registered: - return - registry = get_adapter_registry() - if any(a.name == "LlamaIndex" for a in registry.get_all()): - _registered = True - return - registry.register(LlamaIndexAdapter()) - _registered = True - logger.debug("Registered uipath-llamaindex governance adapter") - - +from .event_handler import GovernanceEventHandler, install_governance __all__ = [ "GovernanceEventHandler", - "LlamaIndexAdapter", - "register_governance_adapter", + "install_governance", ] \ No newline at end of file diff --git a/packages/uipath-llamaindex/src/uipath_llamaindex/governance/adapter.py b/packages/uipath-llamaindex/src/uipath_llamaindex/governance/event_handler.py similarity index 76% rename from packages/uipath-llamaindex/src/uipath_llamaindex/governance/adapter.py rename to packages/uipath-llamaindex/src/uipath_llamaindex/governance/event_handler.py index 1e1e46fc..3494ffe9 100644 --- a/packages/uipath-llamaindex/src/uipath_llamaindex/governance/adapter.py +++ b/packages/uipath-llamaindex/src/uipath_llamaindex/governance/event_handler.py @@ -1,7 +1,7 @@ -"""LlamaIndex adapter for UiPath governance. +"""LlamaIndex governance event handler for UiPath. Provides governance for LlamaIndex agents/workflows. Unlike the ADK / OpenAI / -Agent-Framework adapters — which install per-agent callbacks or middleware — +Agent-Framework integrations — which install per-agent callbacks or middleware — LlamaIndex routes everything (LLM calls, tool calls) through its global **instrumentation dispatcher** (the same mechanism the package already uses for OpenInference tracing). So this adapter governs by registering a @@ -13,9 +13,9 @@ - ``AgentToolCallEvent`` → TOOL_CALL (tool name + arguments) The dispatcher is process-global, so registration is process-wide — which fits -the coded-agent model (one workflow per process). :meth:`attach` therefore -returns the ``agent`` unchanged (nothing is mutated on it); the wiring lives on -the dispatcher. :meth:`detach` removes the handler. +the coded-agent model (one workflow per process). :func:`install_governance` +therefore returns the ``agent`` unchanged (nothing is mutated on it); the wiring +lives on the dispatcher. LlamaIndex does **not** emit a tool-*end* instrumentation event, so AFTER_TOOL is not wired here; a tool's result is governed at the next ``LLMChatStartEvent`` @@ -25,9 +25,11 @@ Chain-level boundaries (BEFORE_AGENT / AFTER_AGENT) are owned by the governance host and are intentionally not fired here. -Contracts and the evaluator protocol come from ``uipath-core``; this package -contributes only the LlamaIndex-specific implementation and registers it with -the adapter registry via the ``uipath.governance.adapters`` entry point. +The evaluator protocol comes from ``uipath-core``; this package contributes +only the LlamaIndex-specific wiring. Governance is installed by the runtime +factory: passing an ``evaluator`` to ``new_runtime`` calls +:func:`install_governance`, which registers the handler on the dispatcher. No +adapter registry, no entry point, no import-time side effects. Audit emission and enforcement (raising :class:`GovernanceBlockException` on DENY) are owned by the evaluator. The handler only extracts payloads and calls @@ -54,7 +56,7 @@ LLMChatStartEvent, ) from pydantic import PrivateAttr -from uipath.core.adapters import BaseAdapter, EvaluatorProtocol +from uipath.core.adapters import EvaluatorProtocol from uipath.core.governance.exceptions import GovernanceBlockException logger = logging.getLogger(__name__) @@ -64,57 +66,31 @@ _BEFORE_MODEL_TEXT_CAP = 64000 -class LlamaIndexAdapter(BaseAdapter): - """Adapter for the LlamaIndex framework. +def install_governance( + agent: Any, + evaluator: EvaluatorProtocol, + *, + agent_name: str, + session_id: str, +) -> Any: + """Register the governance event handler on the root dispatcher. - Detects LlamaIndex workflows/agents and governs them by registering a - :class:`GovernanceEventHandler` on the root instrumentation dispatcher. - """ - - @property - def name(self) -> str: - return "LlamaIndex" - - def can_handle(self, agent: Any) -> bool: - """Return True only for a LlamaIndex ``Workflow`` (incl. agent workflows).""" - try: - from workflows import Workflow - except ImportError: - return False - return isinstance(agent, Workflow) + Returns the ``agent`` unchanged — LlamaIndex governance is wired on the + process-global instrumentation dispatcher, not on the agent object. + Idempotent: a second call is a no-op while a handler is already registered. - def attach( - self, - agent: Any, - agent_id: str, - session_id: str, - evaluator: EvaluatorProtocol, - ) -> Any: - """Register the governance event handler on the root dispatcher. - - Returns the ``agent`` unchanged — LlamaIndex governance is wired on the - process-global dispatcher, not on the agent object. Idempotent: a - second attach is a no-op while a handler is already registered. - """ - dispatcher = get_dispatcher() - if any(isinstance(h, GovernanceEventHandler) for h in dispatcher.event_handlers): - return agent # idempotent — already governed - callbacks = GovernanceCallbacks( - evaluator=evaluator, agent_name=agent_id, session_id=session_id - ) - dispatcher.add_event_handler(GovernanceEventHandler(callbacks=callbacks)) - logger.debug("Registered governance event handler on LlamaIndex dispatcher") - return agent - - def detach(self, governed: Any) -> Any: - """Remove the governance event handler from the root dispatcher.""" - dispatcher = get_dispatcher() - dispatcher.event_handlers = [ - h - for h in dispatcher.event_handlers - if not isinstance(h, GovernanceEventHandler) - ] - return governed + Called by :class:`UiPathLlamaIndexRuntimeFactory` when an ``evaluator`` + is supplied to ``new_runtime``. + """ + dispatcher = get_dispatcher() + if any(isinstance(h, GovernanceEventHandler) for h in dispatcher.event_handlers): + return agent # idempotent — already governed + callbacks = GovernanceCallbacks( + evaluator=evaluator, agent_name=agent_name, session_id=session_id + ) + dispatcher.add_event_handler(GovernanceEventHandler(callbacks=callbacks)) + logger.debug("Registered governance event handler on LlamaIndex dispatcher") + return agent class GovernanceEventHandler(BaseEventHandler): @@ -279,5 +255,5 @@ def _coerce_args(arguments: Any) -> Dict[str, Any]: __all__: List[str] = [ "GovernanceCallbacks", "GovernanceEventHandler", - "LlamaIndexAdapter", + "install_governance", ] \ No newline at end of file diff --git a/packages/uipath-llamaindex/src/uipath_llamaindex/runtime/factory.py b/packages/uipath-llamaindex/src/uipath_llamaindex/runtime/factory.py index d9535faf..cefc2c15 100644 --- a/packages/uipath-llamaindex/src/uipath_llamaindex/runtime/factory.py +++ b/packages/uipath-llamaindex/src/uipath_llamaindex/runtime/factory.py @@ -7,6 +7,7 @@ LlamaIndexInstrumentor, get_current_span, ) +from uipath.core.adapters import EvaluatorProtocol from uipath.core.tracing import UiPathSpanUtils, UiPathTraceManager from uipath.platform.resume_triggers import UiPathResumeTriggerHandler from uipath.runtime import ( @@ -19,6 +20,7 @@ from uipath.runtime.errors import UiPathErrorCategory from workflows import Workflow +from uipath_llamaindex.governance import install_governance from uipath_llamaindex.runtime._telemetry import ( ToolCallAttributeNormalizer, ) @@ -233,6 +235,7 @@ async def _create_runtime_instance( workflow: Workflow, runtime_id: str, entrypoint: str, + evaluator: EvaluatorProtocol | None = None, ) -> UiPathRuntimeProtocol: """ Create a runtime instance from a workflow. @@ -241,10 +244,19 @@ async def _create_runtime_instance( workflow: The workflow runtime_id: Unique identifier for the runtime instance entrypoint: Workflow entrypoint name + evaluator: When supplied, governance is installed on the + instrumentation dispatcher via :func:`install_governance`. Returns: Configured runtime instance """ + if evaluator is not None: + install_governance( + workflow, + evaluator, + agent_name=entrypoint, + session_id=runtime_id, + ) storage = await self._get_storage() @@ -274,6 +286,9 @@ async def new_runtime( Args: entrypoint: Workflow name from llama_index.json runtime_id: Unique identifier for the runtime instance + **kwargs: Forwarded factory kwargs. Recognized: ``evaluator`` + (``EvaluatorProtocol``) — when present, governance is installed + on the dispatcher via :func:`install_governance`. Returns: Configured runtime instance with workflow @@ -284,6 +299,7 @@ async def new_runtime( workflow=workflow, runtime_id=runtime_id, entrypoint=entrypoint, + evaluator=kwargs.get("evaluator"), ) async def dispose(self) -> None: diff --git a/packages/uipath-llamaindex/tests/governance/test_adapter.py b/packages/uipath-llamaindex/tests/governance/test_event_handler.py similarity index 73% rename from packages/uipath-llamaindex/tests/governance/test_adapter.py rename to packages/uipath-llamaindex/tests/governance/test_event_handler.py index b9022ed2..ece81b50 100644 --- a/packages/uipath-llamaindex/tests/governance/test_adapter.py +++ b/packages/uipath-llamaindex/tests/governance/test_event_handler.py @@ -1,4 +1,4 @@ -"""Unit tests for the LlamaIndex governance adapter. +"""Unit tests for the LlamaIndex governance event handler. The adapter governs via the LlamaIndex instrumentation dispatcher, so these tests exercise the real event types (``LLMChatStartEvent`` etc.) routed @@ -10,6 +10,7 @@ from __future__ import annotations import logging +from types import SimpleNamespace from typing import Any, List import pytest @@ -23,12 +24,12 @@ from llama_index.core.tools.types import ToolMetadata from uipath.core.governance.exceptions import GovernanceBlockException -from uipath_llamaindex.governance.adapter import ( +from uipath_llamaindex.governance.event_handler import ( _BEFORE_MODEL_TEXT_CAP, GovernanceCallbacks, GovernanceEventHandler, - LlamaIndexAdapter, _coerce_args, + install_governance, ) # -------------------------------------------------------------------------- @@ -83,31 +84,7 @@ def _handler(ev: FakeEvaluator) -> GovernanceEventHandler: # -------------------------------------------------------------------------- -# can_handle -# -------------------------------------------------------------------------- - - -def test_can_handle_real_workflow(): - from workflows import Workflow, step - from workflows.events import StartEvent, StopEvent - - class _RealWorkflow(Workflow): - @step - async def go(self, ev: StartEvent) -> StopEvent: - return StopEvent() - - assert LlamaIndexAdapter().can_handle(_RealWorkflow()) is True - - -def test_can_handle_rejects_non_workflow(): - # A duck-typed look-alike (has run / Workflow-shaped name) must NOT be - # claimed — only a real workflows.Workflow is. - assert LlamaIndexAdapter().can_handle(FakeWorkflow()) is False - assert LlamaIndexAdapter().can_handle(object()) is False - - -# -------------------------------------------------------------------------- -# attach / detach (real dispatcher) +# install_governance (real dispatcher) # -------------------------------------------------------------------------- @@ -119,28 +96,81 @@ def _gov_handlers() -> list: ] -def test_attach_registers_handler_then_detach_removes(): - adapter = LlamaIndexAdapter() +def _clear_gov_handlers() -> None: + d = get_dispatcher() + d.event_handlers = [ + h for h in d.event_handlers if not isinstance(h, GovernanceEventHandler) + ] + + +def test_install_governance_registers_handler(): agent = FakeWorkflow() try: - returned = adapter.attach(agent, agent_id="x", session_id="s", evaluator=FakeEvaluator()) + returned = install_governance(agent, FakeEvaluator(), agent_name="x", session_id="s") assert returned is agent assert len(_gov_handlers()) == 1 finally: - adapter.detach(agent) + _clear_gov_handlers() assert _gov_handlers() == [] -def test_attach_is_idempotent(): - adapter = LlamaIndexAdapter() - agent = FakeWorkflow() +def test_install_governance_is_idempotent(): ev = FakeEvaluator() try: - adapter.attach(agent, agent_id="x", session_id="s", evaluator=ev) - adapter.attach(agent, agent_id="x", session_id="s", evaluator=ev) + install_governance(FakeWorkflow(), ev, agent_name="x", session_id="s") + install_governance(FakeWorkflow(), ev, agent_name="x", session_id="s") assert len(_gov_handlers()) == 1 finally: - adapter.detach(agent) + _clear_gov_handlers() + + +# -------------------------------------------------------------------------- +# Factory wiring — the evaluator kwarg drives install_governance +# -------------------------------------------------------------------------- + + +def _factory_without_init(): + """A factory instance that skips __init__ (avoids config/IO).""" + from uipath_llamaindex.runtime.factory import UiPathLlamaIndexRuntimeFactory + + f = UiPathLlamaIndexRuntimeFactory.__new__(UiPathLlamaIndexRuntimeFactory) + f.context = SimpleNamespace(command="run") # read for debug_mode + return f + + +def _stub_factory_runtime(monkeypatch, factory_mod): + """Stub the runtime constructions + storage so only the governance branch runs.""" + monkeypatch.setattr(factory_mod, "UiPathLlamaIndexRuntime", lambda **kw: SimpleNamespace(**kw)) + monkeypatch.setattr(factory_mod, "UiPathResumableRuntime", lambda **kw: SimpleNamespace(**kw)) + monkeypatch.setattr(factory_mod, "UiPathResumeTriggerHandler", lambda *a, **k: None) + + async def _no_storage(self): + return None + + monkeypatch.setattr(factory_mod.UiPathLlamaIndexRuntimeFactory, "_get_storage", _no_storage) + + +async def test_factory_installs_governance_when_evaluator_supplied(monkeypatch): + from uipath_llamaindex.runtime import factory as factory_mod + + _stub_factory_runtime(monkeypatch, factory_mod) + try: + await _factory_without_init()._create_runtime_instance( + workflow=FakeWorkflow(), runtime_id="r", entrypoint="e", evaluator=FakeEvaluator() + ) + assert len(_gov_handlers()) == 1 + finally: + _clear_gov_handlers() + + +async def test_factory_skips_governance_without_evaluator(monkeypatch): + from uipath_llamaindex.runtime import factory as factory_mod + + _stub_factory_runtime(monkeypatch, factory_mod) + await _factory_without_init()._create_runtime_instance( + workflow=FakeWorkflow(), runtime_id="r", entrypoint="e" + ) + assert _gov_handlers() == [] # -------------------------------------------------------------------------- From c6f7d17218c78085717b43968244cf792135ef2e Mon Sep 17 00:00:00 2001 From: Aditi Kumari Date: Wed, 1 Jul 2026 16:30:23 +0530 Subject: [PATCH 07/11] =?UTF-8?q?fix(llamaindex):=20address=20governance?= =?UTF-8?q?=20review=20=E2=80=94=20rebind,=20detach,=20trace=5Fid?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review findings (Viswa) for PR #360: - Re-install now rebinds the single process-global handler to the new run's evaluator / session instead of silently no-oping. The dispatcher is process-global, so a reused process serving a new runtime previously kept governing under the *first* run's evaluator; last install now wins. - Added uninstall_governance() (public detach) and wired it into the factory's dispose, so the global dispatcher does not retain the evaluator (and its resources) after the runtime is gone. Tests use it instead of mutating dispatcher.event_handlers directly. - Documented the process-global model explicitly: single active governance handler per process; two *concurrently* executing runtimes in one process would share the latest-installed evaluator — a property of LlamaIndex's global instrumentation, matching the one-workflow-per-process runtime model. - Documented that handle() is a deliberately synchronous gate: a BEFORE_MODEL / TOOL_CALL decision must complete (and be able to BLOCK) before the underlying call proceeds; an async out-of-band check could not gate it. - Drop the per-callbacks uuid trace_id (identical for every call); trace correlation is owned by the layer below, matching LangChain. Requires uipath-core >= 0.5.20 (removed trace_id from EvaluatorProtocol) — bumped. - Count llm/tool calls only after governance passes (no inflation on block). Tests: reinstall-rebinds-single-handler, uninstall-removes-handler, and made the swallow-on-error caplog assertion propagation-independent. Co-Authored-By: Claude Opus 4.8 --- .../uipath_llamaindex/governance/__init__.py | 9 +- .../governance/event_handler.py | 108 +++++++++++++++--- .../src/uipath_llamaindex/runtime/factory.py | 7 +- .../tests/governance/test_event_handler.py | 102 +++++++++++++---- packages/uipath-llamaindex/uv.lock | 6 +- 5 files changed, 187 insertions(+), 45 deletions(-) diff --git a/packages/uipath-llamaindex/src/uipath_llamaindex/governance/__init__.py b/packages/uipath-llamaindex/src/uipath_llamaindex/governance/__init__.py index bae93038..914e8096 100644 --- a/packages/uipath-llamaindex/src/uipath_llamaindex/governance/__init__.py +++ b/packages/uipath-llamaindex/src/uipath_llamaindex/governance/__init__.py @@ -12,9 +12,14 @@ from __future__ import annotations -from .event_handler import GovernanceEventHandler, install_governance +from .event_handler import ( + GovernanceEventHandler, + install_governance, + uninstall_governance, +) __all__ = [ "GovernanceEventHandler", "install_governance", -] \ No newline at end of file + "uninstall_governance", +] diff --git a/packages/uipath-llamaindex/src/uipath_llamaindex/governance/event_handler.py b/packages/uipath-llamaindex/src/uipath_llamaindex/governance/event_handler.py index 3494ffe9..e9e478c1 100644 --- a/packages/uipath-llamaindex/src/uipath_llamaindex/governance/event_handler.py +++ b/packages/uipath-llamaindex/src/uipath_llamaindex/governance/event_handler.py @@ -15,7 +15,18 @@ The dispatcher is process-global, so registration is process-wide — which fits the coded-agent model (one workflow per process). :func:`install_governance` therefore returns the ``agent`` unchanged (nothing is mutated on it); the wiring -lives on the dispatcher. +lives on the dispatcher. A second install (a reused process serving a new +runtime) **rebinds** that one handler to the new run's evaluator / session +rather than silently ignoring it — the most-recent install governs. +:func:`uninstall_governance` removes the handler so the global dispatcher does +not retain the evaluator after the runtime is gone; the factory calls it on +dispose. + +Because the dispatcher is process-global and LlamaIndex events do not carry a +stable per-run identity, this adapter does not isolate two *concurrently* +executing runtimes in the same process — they would share the latest-installed +evaluator. That is a property of LlamaIndex's global instrumentation and matches +the one-workflow-per-process runtime model. LlamaIndex does **not** emit a tool-*end* instrumentation event, so AFTER_TOOL is not wired here; a tool's result is governed at the next ``LLMChatStartEvent`` @@ -42,7 +53,6 @@ import json import logging from typing import Any, Dict, List -from uuid import uuid4 from llama_index.core.instrumentation import ( # type: ignore[attr-defined] get_dispatcher, @@ -76,15 +86,22 @@ def install_governance( """Register the governance event handler on the root dispatcher. Returns the ``agent`` unchanged — LlamaIndex governance is wired on the - process-global instrumentation dispatcher, not on the agent object. - Idempotent: a second call is a no-op while a handler is already registered. + process-global instrumentation dispatcher, not on the agent object. If a + governance handler is already registered (a reused process serving a new + runtime), it is **rebound** to this run's evaluator / session instead of + being left pointing at the previous run. Called by :class:`UiPathLlamaIndexRuntimeFactory` when an ``evaluator`` is supplied to ``new_runtime``. """ dispatcher = get_dispatcher() - if any(isinstance(h, GovernanceEventHandler) for h in dispatcher.event_handlers): - return agent # idempotent — already governed + for handler in dispatcher.event_handlers: + if isinstance(handler, GovernanceEventHandler): + handler.rebind( + evaluator=evaluator, agent_name=agent_name, session_id=session_id + ) + logger.debug("Rebound existing governance handler to the new runtime") + return agent callbacks = GovernanceCallbacks( evaluator=evaluator, agent_name=agent_name, session_id=session_id ) @@ -93,6 +110,25 @@ def install_governance( return agent +def uninstall_governance(agent: Any = None) -> Any: + """Remove the governance handler(s) from the root dispatcher. + + The instrumentation dispatcher is process-global, so a registered handler + (and the evaluator it holds) would otherwise outlive the runtime. The + factory calls this on ``dispose`` to release it. Returns ``agent`` unchanged. + Safe to call when nothing is registered. + """ + dispatcher = get_dispatcher() + handlers = dispatcher.event_handlers + remaining = [h for h in handlers if not isinstance(h, GovernanceEventHandler)] + if len(remaining) != len(handlers): + # event_handlers is a plain list; mutate in place to avoid a pydantic + # attribute re-assignment on the Dispatcher model. + handlers[:] = remaining + logger.debug("Removed governance event handler from LlamaIndex dispatcher") + return agent + + class GovernanceEventHandler(BaseEventHandler): """Routes LlamaIndex instrumentation events to a governance evaluator. @@ -112,7 +148,23 @@ def __init__(self, callbacks: "GovernanceCallbacks", **data: Any) -> None: def class_name(cls) -> str: return "GovernanceEventHandler" + def rebind( + self, + evaluator: EvaluatorProtocol, + agent_name: str, + session_id: str, + ) -> None: + """Re-point the single process-global handler at a new runtime.""" + self._callbacks.rebind( + evaluator=evaluator, agent_name=agent_name, session_id=session_id + ) + def handle(self, event: Any, **kwargs: Any) -> Any: + # The dispatcher calls ``handle`` synchronously and inline with the + # instrumented call. That is deliberate: a BEFORE_MODEL / TOOL_CALL + # governance decision must complete (and be able to BLOCK) *before* the + # underlying LLM / tool call proceeds — an async, out-of-band check + # could not gate it. The evaluator is expected to be fast. if isinstance(event, LLMChatStartEvent): self._callbacks.before_model(event.messages) elif isinstance(event, LLMChatEndEvent): @@ -139,20 +191,41 @@ def __init__( self._evaluator = evaluator self._agent_name = agent_name self._session_id = session_id - self._trace_id = str(uuid4()) + # ``trace_id`` is intentionally NOT held here. A single uuid minted at + # install time would be identical for every call. Trace correlation is + # owned by the layer below (OTel span / HTTP resolve at call time), + # matching the LangChain adapter. self._session_state: Dict[str, Any] = {"tool_calls": 0, "llm_calls": 0} + def rebind( + self, + evaluator: EvaluatorProtocol, + agent_name: str, + session_id: str, + ) -> None: + """Re-point this callback set at a new run. + + Called when the process-global handler is reused for a fresh runtime — + updates the evaluator and identifiers and resets the per-run counters so + state does not bleed across runtimes. + """ + self._evaluator = evaluator + self._agent_name = agent_name + self._session_id = session_id + self._session_state = {"tool_calls": 0, "llm_calls": 0} + def before_model(self, messages: Any) -> None: """Evaluate BEFORE_MODEL on the latest input message (see ADK rationale).""" try: - self._session_state["llm_calls"] = ( - self._session_state.get("llm_calls", 0) + 1 - ) self._evaluator.evaluate_before_model( model_input=_latest_message_text(messages), agent_name=self._agent_name, runtime_id=self._session_id, - trace_id=self._trace_id, + ) + # Count only calls that passed governance — a DENY raises above, so + # a blocked call must not inflate the counter. + self._session_state["llm_calls"] = ( + self._session_state.get("llm_calls", 0) + 1 ) except GovernanceBlockException: raise @@ -166,7 +239,6 @@ def after_model(self, response: Any) -> None: model_output=_response_text(response), agent_name=self._agent_name, runtime_id=self._session_id, - trace_id=self._trace_id, ) except GovernanceBlockException: raise @@ -176,17 +248,18 @@ def after_model(self, response: Any) -> None: def tool_call(self, tool: Any, arguments: Any) -> None: """Evaluate TOOL_CALL with the tool name + arguments.""" try: - self._session_state["tool_calls"] = ( - self._session_state.get("tool_calls", 0) + 1 - ) self._evaluator.evaluate_tool_call( tool_name=getattr(tool, "name", None) or "unknown", tool_args=_coerce_args(arguments), agent_name=self._agent_name, runtime_id=self._session_id, - trace_id=self._trace_id, session_state=self._session_state, ) + # Count only calls that passed governance; the evaluator saw the + # count of prior tool calls, and a DENY raises before this bump. + self._session_state["tool_calls"] = ( + self._session_state.get("tool_calls", 0) + 1 + ) except GovernanceBlockException: raise except Exception as e: # noqa: BLE001 @@ -256,4 +329,5 @@ def _coerce_args(arguments: Any) -> Dict[str, Any]: "GovernanceCallbacks", "GovernanceEventHandler", "install_governance", -] \ No newline at end of file + "uninstall_governance", +] diff --git a/packages/uipath-llamaindex/src/uipath_llamaindex/runtime/factory.py b/packages/uipath-llamaindex/src/uipath_llamaindex/runtime/factory.py index cefc2c15..db22f73f 100644 --- a/packages/uipath-llamaindex/src/uipath_llamaindex/runtime/factory.py +++ b/packages/uipath-llamaindex/src/uipath_llamaindex/runtime/factory.py @@ -20,7 +20,7 @@ from uipath.runtime.errors import UiPathErrorCategory from workflows import Workflow -from uipath_llamaindex.governance import install_governance +from uipath_llamaindex.governance import install_governance, uninstall_governance from uipath_llamaindex.runtime._telemetry import ( ToolCallAttributeNormalizer, ) @@ -304,6 +304,11 @@ async def new_runtime( async def dispose(self) -> None: """Cleanup factory resources.""" + # The governance handler lives on the process-global instrumentation + # dispatcher; remove it so the evaluator (and its resources) are not + # retained after the runtime is gone. + uninstall_governance() + for loader in self._workflow_loaders.values(): await loader.cleanup() diff --git a/packages/uipath-llamaindex/tests/governance/test_event_handler.py b/packages/uipath-llamaindex/tests/governance/test_event_handler.py index ece81b50..6c6c1f94 100644 --- a/packages/uipath-llamaindex/tests/governance/test_event_handler.py +++ b/packages/uipath-llamaindex/tests/governance/test_event_handler.py @@ -30,6 +30,7 @@ GovernanceEventHandler, _coerce_args, install_governance, + uninstall_governance, ) # -------------------------------------------------------------------------- @@ -97,16 +98,16 @@ def _gov_handlers() -> list: def _clear_gov_handlers() -> None: - d = get_dispatcher() - d.event_handlers = [ - h for h in d.event_handlers if not isinstance(h, GovernanceEventHandler) - ] + # Use the adapter's own public detach rather than mutating the dispatcher. + uninstall_governance() def test_install_governance_registers_handler(): agent = FakeWorkflow() try: - returned = install_governance(agent, FakeEvaluator(), agent_name="x", session_id="s") + returned = install_governance( + agent, FakeEvaluator(), agent_name="x", session_id="s" + ) assert returned is agent assert len(_gov_handlers()) == 1 finally: @@ -114,16 +115,39 @@ def test_install_governance_registers_handler(): assert _gov_handlers() == [] -def test_install_governance_is_idempotent(): - ev = FakeEvaluator() +def test_install_governance_reinstall_rebinds_single_handler(): + """The dispatcher is process-global: a second install keeps one handler but + rebinds it to the new run's evaluator / session (last install wins).""" try: - install_governance(FakeWorkflow(), ev, agent_name="x", session_id="s") - install_governance(FakeWorkflow(), ev, agent_name="x", session_id="s") - assert len(_gov_handlers()) == 1 + install_governance( + FakeWorkflow(), FakeEvaluator(), agent_name="a", session_id="s1" + ) + handlers = _gov_handlers() + assert len(handlers) == 1 + gov = handlers[0] + assert gov._callbacks._session_id == "s1" + + ev2 = FakeEvaluator() + install_governance(FakeWorkflow(), ev2, agent_name="b", session_id="s2") + handlers = _gov_handlers() + assert len(handlers) == 1 # not stacked + assert handlers[0] is gov # same handler, rebound + assert gov._callbacks._session_id == "s2" + assert gov._callbacks._evaluator is ev2 finally: _clear_gov_handlers() +def test_uninstall_governance_removes_handler(): + install_governance(FakeWorkflow(), FakeEvaluator(), agent_name="x", session_id="s") + assert len(_gov_handlers()) == 1 + uninstall_governance() + assert _gov_handlers() == [] + # safe to call again when nothing is registered + uninstall_governance() + assert _gov_handlers() == [] + + # -------------------------------------------------------------------------- # Factory wiring — the evaluator kwarg drives install_governance # -------------------------------------------------------------------------- @@ -140,14 +164,20 @@ def _factory_without_init(): def _stub_factory_runtime(monkeypatch, factory_mod): """Stub the runtime constructions + storage so only the governance branch runs.""" - monkeypatch.setattr(factory_mod, "UiPathLlamaIndexRuntime", lambda **kw: SimpleNamespace(**kw)) - monkeypatch.setattr(factory_mod, "UiPathResumableRuntime", lambda **kw: SimpleNamespace(**kw)) + monkeypatch.setattr( + factory_mod, "UiPathLlamaIndexRuntime", lambda **kw: SimpleNamespace(**kw) + ) + monkeypatch.setattr( + factory_mod, "UiPathResumableRuntime", lambda **kw: SimpleNamespace(**kw) + ) monkeypatch.setattr(factory_mod, "UiPathResumeTriggerHandler", lambda *a, **k: None) async def _no_storage(self): return None - monkeypatch.setattr(factory_mod.UiPathLlamaIndexRuntimeFactory, "_get_storage", _no_storage) + monkeypatch.setattr( + factory_mod.UiPathLlamaIndexRuntimeFactory, "_get_storage", _no_storage + ) async def test_factory_installs_governance_when_evaluator_supplied(monkeypatch): @@ -156,7 +186,10 @@ async def test_factory_installs_governance_when_evaluator_supplied(monkeypatch): _stub_factory_runtime(monkeypatch, factory_mod) try: await _factory_without_init()._create_runtime_instance( - workflow=FakeWorkflow(), runtime_id="r", entrypoint="e", evaluator=FakeEvaluator() + workflow=FakeWorkflow(), + runtime_id="r", + entrypoint="e", + evaluator=FakeEvaluator(), ) assert len(_gov_handlers()) == 1 finally: @@ -182,8 +215,10 @@ def test_handler_routes_llm_chat_start_to_before_model(): ev = FakeEvaluator() h = _handler(ev) event = LLMChatStartEvent( - messages=[ChatMessage(role="user", content="old"), - ChatMessage(role="user", content="the question")], + messages=[ + ChatMessage(role="user", content="old"), + ChatMessage(role="user", content="the question"), + ], additional_kwargs={}, model_dict={}, ) @@ -198,7 +233,9 @@ def test_handler_routes_llm_chat_end_to_after_model(): h = _handler(ev) event = LLMChatEndEvent( messages=[ChatMessage(role="user", content="q")], - response=ChatResponse(message=ChatMessage(role="assistant", content="the answer")), + response=ChatResponse( + message=ChatMessage(role="assistant", content="the answer") + ), ) h.handle(event) hook, kwargs = ev.calls[-1] @@ -269,9 +306,20 @@ def test_coerce_args_none_and_bad(): @pytest.mark.parametrize( "hook,invoke", [ - ("before_model", lambda cb: cb.before_model([ChatMessage(role="user", content="hi")])), - ("after_model", lambda cb: cb.after_model(ChatResponse(message=ChatMessage(role="assistant", content="o")))), - ("tool_call", lambda cb: cb.tool_call(ToolMetadata(description="d", name="t"), "{}")), + ( + "before_model", + lambda cb: cb.before_model([ChatMessage(role="user", content="hi")]), + ), + ( + "after_model", + lambda cb: cb.after_model( + ChatResponse(message=ChatMessage(role="assistant", content="o")) + ), + ), + ( + "tool_call", + lambda cb: cb.tool_call(ToolMetadata(description="d", name="t"), "{}"), + ), ], ) def test_block_exception_propagates(hook, invoke): @@ -286,6 +334,16 @@ def evaluate_before_model(self, **_: Any) -> None: raise RuntimeError("evaluator bug") cb = GovernanceCallbacks(evaluator=Boom(), agent_name="a", session_id="s") # type: ignore[arg-type] - with caplog.at_level(logging.WARNING): + # Attach caplog's handler directly to the module logger: other suites in the + # full run can configure an ancestor ``uipath*`` logger with + # propagate=False, which breaks caplog's default root-handler capture. + logger = logging.getLogger("uipath_llamaindex.governance.event_handler") + logger.addHandler(caplog.handler) + prev = logger.level + logger.setLevel(logging.WARNING) + try: cb.before_model([ChatMessage(role="user", content="x")]) - assert any("governance check failed" in r.message for r in caplog.records) \ No newline at end of file + finally: + logger.removeHandler(caplog.handler) + logger.setLevel(prev) + assert any("governance check failed" in r.message for r in caplog.records) diff --git a/packages/uipath-llamaindex/uv.lock b/packages/uipath-llamaindex/uv.lock index fe5fd437..04634918 100644 --- a/packages/uipath-llamaindex/uv.lock +++ b/packages/uipath-llamaindex/uv.lock @@ -3492,16 +3492,16 @@ wheels = [ [[package]] name = "uipath-core" -version = "0.5.18" +version = "0.5.28" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "opentelemetry-instrumentation" }, { name = "opentelemetry-sdk" }, { name = "pydantic" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/14/b1/d4e555a1a2ccf298195a5f2968e538b0cea8592b3e03f43fc12b178d6c69/uipath_core-0.5.18.tar.gz", hash = "sha256:63ebe8bdb818ca30a4bc9ab0ea8171315680691429931282939359ce039401ab", size = 131988, upload-time = "2026-06-08T14:04:49.688Z" } +sdist = { url = "https://files.pythonhosted.org/packages/4c/f9/8d2f1d98cbebbcf059cf4561f38f34ad4cd58423e4f15cad22bd297a2563/uipath_core-0.5.28.tar.gz", hash = "sha256:942987f6b612c64f93d612ad7b242276ed75f129fdd8f25bc71c24ec8887e388", size = 130578, upload-time = "2026-06-30T14:04:48.841Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/57/de/1a820b33f7bff4565d7649772bc54c88480ac7e70f707097f7da37d05157/uipath_core-0.5.18-py3-none-any.whl", hash = "sha256:351d6faeecfc6a0acea93182e01526f39c04a77e09fa0444be5f4fb580463f5a", size = 54572, upload-time = "2026-06-08T14:04:48.22Z" }, + { url = "https://files.pythonhosted.org/packages/e8/1e/385bb166232a57ebe938cc57ad2717f350bc922bb5d2ce31af84306b7569/uipath_core-0.5.28-py3-none-any.whl", hash = "sha256:b952a46a21710073cbc16d6d5684e9aa645c107f57a636b778cfb94aa81a1e48", size = 54980, upload-time = "2026-06-30T14:04:47.374Z" }, ] [[package]] From 2b0c10cffb940198a784fee51db86f7df3b9c94f Mon Sep 17 00:00:00 2001 From: Aditi Kumari Date: Wed, 1 Jul 2026 17:58:27 +0530 Subject: [PATCH 08/11] fix(llamaindex): address remaining review minors (pages 4-5) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Follow-up to the #360 review pass — the earlier commit covered the blockers (rebind + uninstall) + trace_id; these are the payload-extraction minors: - event_handler.py:236 — _coerce_args no longer drops non-dict args: a list-shaped arg (common with MCP tools) is preserved under `_`, and malformed JSON is kept raw under `_raw` instead of {} — a payload governance can't parse must not slip past arg-based policies. - event_handler.py:224 — _message_text now walks a multimodal ChatMessage's text blocks when `.content` is empty, instead of str(message) (which serialized a pydantic repr into the scanned blob). Acknowledged, intentionally unchanged (noted for consistency / follow-up): - event_handler.py:170 (sync handle): deliberate synchronous gate — a BEFORE_MODEL/TOOL_CALL decision must complete before the call proceeds. Async governance is a protocol-wide change (evaluator is sync) → follow-up. - event_handler.py:186 (warning vs exception): kept logger.warning for the swallow path to match LangChain #899 and the other 4 adapters. - latest-message-only scan: same documented tradeoff as #899 (avoids re-firing on prior turns); kept for parity. Tests: list-shaped/malformed _coerce_args + block-extraction fallback. 130 pass. Co-Authored-By: Claude Opus 4.8 --- .../governance/event_handler.py | 31 ++++++++++++++----- .../tests/governance/test_event_handler.py | 21 ++++++++++++- 2 files changed, 43 insertions(+), 9 deletions(-) diff --git a/packages/uipath-llamaindex/src/uipath_llamaindex/governance/event_handler.py b/packages/uipath-llamaindex/src/uipath_llamaindex/governance/event_handler.py index e9e478c1..1fdad8c1 100644 --- a/packages/uipath-llamaindex/src/uipath_llamaindex/governance/event_handler.py +++ b/packages/uipath-llamaindex/src/uipath_llamaindex/governance/event_handler.py @@ -281,7 +281,7 @@ def _latest_message_text(messages: Any) -> str: def _message_text(message: Any) -> str: - """Pull text from a ``ChatMessage`` (``.content``) or a bare string.""" + """Pull text from a ``ChatMessage`` (``.content`` / ``.blocks``) or a str.""" if message is None: return "" if isinstance(message, str): @@ -289,7 +289,17 @@ def _message_text(message: Any) -> str: content = getattr(message, "content", None) if isinstance(content, str) and content: return content[:_BEFORE_MODEL_TEXT_CAP] - # Newer ChatMessage carries typed blocks; fall back to str(). + # Multimodal ChatMessage carries typed blocks. Walk them for text (a + # TextBlock exposes ``.text``) rather than ``str(message)``, which would + # serialize the pydantic repr — dict-syntax noise that pollutes the + # regex-scanned blob. Non-text blocks (image/binary) have no scannable text. + blocks = getattr(message, "blocks", None) + if isinstance(blocks, (list, tuple)): + texts = [ + t for b in blocks if isinstance((t := getattr(b, "text", None)), str) and t + ] + if texts: + return "\n".join(texts)[:_BEFORE_MODEL_TEXT_CAP] return str(message)[:_BEFORE_MODEL_TEXT_CAP] @@ -307,10 +317,14 @@ def _response_text(response: Any) -> str: def _coerce_args(arguments: Any) -> Dict[str, Any]: - """Normalise tool arguments (JSON string / Mapping / None) to a dict. - - ``AgentToolCallEvent.arguments`` is a JSON-encoded string; other call - sites may hand a dict directly. + """Normalise tool arguments (JSON string / Mapping / list / None) to a dict. + + ``AgentToolCallEvent.arguments`` is usually a JSON-encoded string; other + call sites may hand a dict directly. Non-dict payloads are preserved (not + dropped) so an arg-based policy can still scan them: a list-shaped arg + (common with MCP tools) is wrapped under ``_``, and malformed JSON is kept + raw under ``_raw`` — a payload governance can't parse must not be a way to + slip past it. """ if arguments is None: return {} @@ -321,8 +335,9 @@ def _coerce_args(arguments: Any) -> Dict[str, Any]: parsed = json.loads(arguments) return parsed if isinstance(parsed, dict) else {"_": parsed} except (TypeError, ValueError): - return {} - return {} + return {"_raw": arguments} + # list / tuple / other structured args — preserve rather than drop to {}. + return {"_": arguments} __all__: List[str] = [ diff --git a/packages/uipath-llamaindex/tests/governance/test_event_handler.py b/packages/uipath-llamaindex/tests/governance/test_event_handler.py index 6c6c1f94..62f4a7a8 100644 --- a/packages/uipath-llamaindex/tests/governance/test_event_handler.py +++ b/packages/uipath-llamaindex/tests/governance/test_event_handler.py @@ -295,7 +295,26 @@ def test_coerce_args_dict_passthrough(): def test_coerce_args_none_and_bad(): assert _coerce_args(None) == {} - assert _coerce_args("not json") == {} + # malformed JSON is preserved raw (not dropped) so policies can still scan it + assert _coerce_args("not json") == {"_raw": "not json"} + + +def test_coerce_args_preserves_list_shaped_args(): + # list-shaped tool args (common with MCP tools) must not be dropped to {} + assert _coerce_args(["a", "b"]) == {"_": ["a", "b"]} + assert _coerce_args('["a", "b"]') == {"_": ["a", "b"]} + + +def test_message_text_walks_blocks_when_content_empty(): + # a multimodal message whose .content is empty falls back to its text + # blocks, not str(message) (which would serialize a pydantic repr) + from uipath_llamaindex.governance.event_handler import _message_text + + msg = SimpleNamespace( + content=None, + blocks=[SimpleNamespace(text="block one"), SimpleNamespace(text="block two")], + ) + assert _message_text(msg) == "block one\nblock two" # -------------------------------------------------------------------------- From 639b8c4da80c637201efb646de29878f8b8e101c Mon Sep 17 00:00:00 2001 From: Aditi Kumari Date: Wed, 1 Jul 2026 18:31:13 +0530 Subject: [PATCH 09/11] docs(llamaindex): hedge the AFTER_TOOL-via-next-BEFORE_MODEL claim Review (#360 major): the claim only holds when the tool result is the latest message in the next request. Document the caveat rather than overstating coverage. Co-Authored-By: Claude Opus 4.8 --- .../src/uipath_llamaindex/governance/event_handler.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/packages/uipath-llamaindex/src/uipath_llamaindex/governance/event_handler.py b/packages/uipath-llamaindex/src/uipath_llamaindex/governance/event_handler.py index 1fdad8c1..1b82fd5a 100644 --- a/packages/uipath-llamaindex/src/uipath_llamaindex/governance/event_handler.py +++ b/packages/uipath-llamaindex/src/uipath_llamaindex/governance/event_handler.py @@ -29,9 +29,13 @@ the one-workflow-per-process runtime model. LlamaIndex does **not** emit a tool-*end* instrumentation event, so AFTER_TOOL -is not wired here; a tool's result is governed at the next ``LLMChatStartEvent`` -where it is fed back to the model as input (analogous to how the OpenAI adapter -handles its missing tool-args). +is not wired here; a tool's result is instead governed at the next +``LLMChatStartEvent`` where it is fed back to the model as input. This holds +only when the tool result is the **latest** message in that request (the usual +case — BEFORE_MODEL scans the latest message, see +:func:`_latest_message_text`); if the framework injects later messages before +the next model call, an intervening tool result is not separately scanned. +This is the LlamaIndex analogue of the OpenAI adapter's missing tool-args. Chain-level boundaries (BEFORE_AGENT / AFTER_AGENT) are owned by the governance host and are intentionally not fired here. From 16218cede5e230a12af7f6b5fe042481163f3b83 Mon Sep 17 00:00:00 2001 From: Aditi Kumari Date: Wed, 1 Jul 2026 19:18:30 +0530 Subject: [PATCH 10/11] test(llamaindex): make test_event_handler mypy-clean (fix CI lint) Same pre-existing CI-lint failure (mypy runs over tests; this package has warn_unused_ignores=true so ignores must be exact): - FakeEvaluator evaluate_* -> (self, *args, **kwargs) -> Any so it satisfies EvaluatorProtocol; bare dict/list -> dict[str, Any]/list[Any]. - removed a now-unused call-arg ignore; added attr-defined ignore on the get_dispatcher import and an assignment ignore on the SimpleNamespace test context. (bedrock/vertex src errors are a local missing-extra artifact; CI installs --all-extras.) Co-Authored-By: Claude Opus 4.8 --- .../tests/governance/test_event_handler.py | 24 ++++++++++--------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/packages/uipath-llamaindex/tests/governance/test_event_handler.py b/packages/uipath-llamaindex/tests/governance/test_event_handler.py index 62f4a7a8..ba0e53b6 100644 --- a/packages/uipath-llamaindex/tests/governance/test_event_handler.py +++ b/packages/uipath-llamaindex/tests/governance/test_event_handler.py @@ -15,7 +15,9 @@ import pytest from llama_index.core.base.llms.types import ChatMessage, ChatResponse -from llama_index.core.instrumentation import get_dispatcher +from llama_index.core.instrumentation import ( # type: ignore[attr-defined] + get_dispatcher, +) from llama_index.core.instrumentation.events.agent import AgentToolCallEvent from llama_index.core.instrumentation.events.llm import ( LLMChatEndEvent, @@ -43,29 +45,29 @@ class FakeEvaluator: def __init__(self, block_on: str | None = None) -> None: self.block_on = block_on - self.calls: List[tuple[str, dict]] = [] + self.calls: List[tuple[str, dict[str, Any]]] = [] def _record(self, hook: str, **kwargs: Any) -> None: self.calls.append((hook, kwargs)) if self.block_on == hook: - raise GovernanceBlockException("blocked") # type: ignore[call-arg] + raise GovernanceBlockException("blocked") - def evaluate_before_agent(self, **kwargs: Any) -> None: + def evaluate_before_agent(self, *args: Any, **kwargs: Any) -> Any: self._record("before_agent", **kwargs) - def evaluate_after_agent(self, **kwargs: Any) -> None: + def evaluate_after_agent(self, *args: Any, **kwargs: Any) -> Any: self._record("after_agent", **kwargs) - def evaluate_before_model(self, **kwargs: Any) -> None: + def evaluate_before_model(self, *args: Any, **kwargs: Any) -> Any: self._record("before_model", **kwargs) - def evaluate_after_model(self, **kwargs: Any) -> None: + def evaluate_after_model(self, *args: Any, **kwargs: Any) -> Any: self._record("after_model", **kwargs) - def evaluate_tool_call(self, **kwargs: Any) -> None: + def evaluate_tool_call(self, *args: Any, **kwargs: Any) -> Any: self._record("tool_call", **kwargs) - def evaluate_after_tool(self, **kwargs: Any) -> None: + def evaluate_after_tool(self, *args: Any, **kwargs: Any) -> Any: self._record("after_tool", **kwargs) @@ -89,7 +91,7 @@ def _handler(ev: FakeEvaluator) -> GovernanceEventHandler: # -------------------------------------------------------------------------- -def _gov_handlers() -> list: +def _gov_handlers() -> list[Any]: return [ h for h in get_dispatcher().event_handlers @@ -158,7 +160,7 @@ def _factory_without_init(): from uipath_llamaindex.runtime.factory import UiPathLlamaIndexRuntimeFactory f = UiPathLlamaIndexRuntimeFactory.__new__(UiPathLlamaIndexRuntimeFactory) - f.context = SimpleNamespace(command="run") # read for debug_mode + f.context = SimpleNamespace(command="run") # type: ignore[assignment] # read for debug_mode return f From afe927469f902bbfc1c479c5d273319d7df8ecc0 Mon Sep 17 00:00:00 2001 From: Aditi Kumari Date: Wed, 1 Jul 2026 19:41:20 +0530 Subject: [PATCH 11/11] test(llamaindex): cover swallow/extraction branches (Sonar coverage) New-code coverage ~89% -> over the 90% gate. Added after_model/tool_call non-block swallow + _message_text/_response_text/_latest_message_text edges. governance/event_handler.py: 89% -> 99%. Co-Authored-By: Claude Opus 4.8 --- .../tests/governance/test_event_handler.py | 53 +++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/packages/uipath-llamaindex/tests/governance/test_event_handler.py b/packages/uipath-llamaindex/tests/governance/test_event_handler.py index ba0e53b6..e607d358 100644 --- a/packages/uipath-llamaindex/tests/governance/test_event_handler.py +++ b/packages/uipath-llamaindex/tests/governance/test_event_handler.py @@ -368,3 +368,56 @@ def evaluate_before_model(self, **_: Any) -> None: logger.removeHandler(caplog.handler) logger.setLevel(prev) assert any("governance check failed" in r.message for r in caplog.records) + + +# -------------------------------------------------------------------------- +# coverage: swallow on after_model/tool_call + extraction edges +# -------------------------------------------------------------------------- + + +class _Boom: + """Evaluator whose every evaluate_* raises a non-block error.""" + + def __getattr__(self, _name: str) -> Any: + def _raise(*_a: Any, **_k: Any) -> None: + raise RuntimeError("evaluator bug") + + return _raise + + +def test_after_model_and_tool_call_swallow_non_block_errors(caplog): + cb = GovernanceCallbacks(evaluator=_Boom(), agent_name="a", session_id="s") + logger = logging.getLogger("uipath_llamaindex.governance.event_handler") + logger.addHandler(caplog.handler) + prev = logger.level + logger.setLevel(logging.WARNING) + try: + cb.after_model(SimpleNamespace(message=SimpleNamespace(content="x"))) + cb.tool_call(SimpleNamespace(name="t"), {}) + finally: + logger.removeHandler(caplog.handler) + logger.setLevel(prev) + assert sum("governance check failed" in r.message for r in caplog.records) >= 2 + + +def test_extraction_edges(): + from uipath_llamaindex.governance.event_handler import ( + _latest_message_text, + _message_text, + _response_text, + ) + + # _message_text: None / str / object with no content or blocks -> str() + assert _message_text(None) == "" + assert _message_text("plain") == "plain" + assert isinstance(_message_text(SimpleNamespace(content=None, blocks=None)), str) + # _latest_message_text: single (non-list) message + assert _latest_message_text(SimpleNamespace(content="solo")) == "solo" + # _response_text: None / .message / .text fallback / str() fallback + assert _response_text(None) == "" + assert ( + _response_text(SimpleNamespace(message=SimpleNamespace(content="viamsg"))) + == "viamsg" + ) + assert _response_text(SimpleNamespace(message=None, text="viatext")) == "viatext" + assert isinstance(_response_text(SimpleNamespace(message=None, text=None)), str)