From 15aa1165f2fe082f00933a1527fa86bf1a6012e7 Mon Sep 17 00:00:00 2001 From: etserend Date: Mon, 24 Aug 2026 19:07:50 -0500 Subject: [PATCH 1/6] chore(examples): add healthcare-assistant from observability-workshop MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Verbatim copy of workshop/healthcare-assistant/2-app-with-instrumentation. No changes — establishes a clean baseline to diff SDK migration against. Shared docs/ (qa.csv, relational_patient.csv) copied from workshop parent. --- .../.streamlit/secrets.toml.template | 26 ++ .../agent/healthcare-assistant/Dockerfile | 41 ++++ .../healthcare-assistant/_agent_galileo.py | 167 +++++++++++++ .../_hallucination_helpers_galileo.py | 181 ++++++++++++++ .../healthcare-assistant/_validate_galileo.py | 70 ++++++ .../_validate_hallucination.py | 124 ++++++++++ .../healthcare-assistant/_validate_single.py | 28 +++ .../agent-with-instrumentation.py | 167 +++++++++++++ examples/agent/healthcare-assistant/agent.py | 167 +++++++++++++ examples/agent/healthcare-assistant/app.py | 204 ++++++++++++++++ examples/agent/healthcare-assistant/config.py | 24 ++ .../agent/healthcare-assistant/config.yaml | 35 +++ .../agent/healthcare-assistant/dataset.csv | 16 ++ .../agent/healthcare-assistant/docs/qa.csv | 226 ++++++++++++++++++ .../docs/relational_patient.csv | 31 +++ .../healthcare-assistant-config.yaml | 6 + .../healthcare-assistant/helpers/__init__.py | 0 .../helpers/hallucination_helpers.py | 196 +++++++++++++++ .../helpers/pgvector_utils.py | 73 ++++++ .../helpers/setup_vectordb.py | 116 +++++++++ .../healthcare-assistant/helpers/sql_utils.py | 173 ++++++++++++++ .../helpers/text_to_sql_utils.py | 77 ++++++ examples/agent/healthcare-assistant/k8s.yaml | 115 +++++++++ .../agent/healthcare-assistant/postgres.yaml | 100 ++++++++ examples/agent/healthcare-assistant/rag.py | 133 +++++++++++ .../healthcare-assistant/requirements.txt | 16 ++ .../agent/healthcare-assistant/setup-job.yaml | 55 +++++ .../agent/healthcare-assistant/setup_env.py | 28 +++ .../healthcare-assistant/start_vectordb.sh | 2 + .../healthcare-assistant/system_prompt.json | 3 + .../healthcare-assistant/tools/__init__.py | 0 .../agent/healthcare-assistant/tools/logic.py | 161 +++++++++++++ .../healthcare-assistant/tools/schema.json | 44 ++++ .../healthcare-assistant/validate_traces.py | 42 ++++ 34 files changed, 2847 insertions(+) create mode 100644 examples/agent/healthcare-assistant/.streamlit/secrets.toml.template create mode 100644 examples/agent/healthcare-assistant/Dockerfile create mode 100644 examples/agent/healthcare-assistant/_agent_galileo.py create mode 100644 examples/agent/healthcare-assistant/_hallucination_helpers_galileo.py create mode 100644 examples/agent/healthcare-assistant/_validate_galileo.py create mode 100644 examples/agent/healthcare-assistant/_validate_hallucination.py create mode 100644 examples/agent/healthcare-assistant/_validate_single.py create mode 100644 examples/agent/healthcare-assistant/agent-with-instrumentation.py create mode 100644 examples/agent/healthcare-assistant/agent.py create mode 100644 examples/agent/healthcare-assistant/app.py create mode 100644 examples/agent/healthcare-assistant/config.py create mode 100644 examples/agent/healthcare-assistant/config.yaml create mode 100644 examples/agent/healthcare-assistant/dataset.csv create mode 100644 examples/agent/healthcare-assistant/docs/qa.csv create mode 100644 examples/agent/healthcare-assistant/docs/relational_patient.csv create mode 100644 examples/agent/healthcare-assistant/healthcare-assistant-config.yaml create mode 100644 examples/agent/healthcare-assistant/helpers/__init__.py create mode 100644 examples/agent/healthcare-assistant/helpers/hallucination_helpers.py create mode 100644 examples/agent/healthcare-assistant/helpers/pgvector_utils.py create mode 100644 examples/agent/healthcare-assistant/helpers/setup_vectordb.py create mode 100644 examples/agent/healthcare-assistant/helpers/sql_utils.py create mode 100644 examples/agent/healthcare-assistant/helpers/text_to_sql_utils.py create mode 100644 examples/agent/healthcare-assistant/k8s.yaml create mode 100644 examples/agent/healthcare-assistant/postgres.yaml create mode 100644 examples/agent/healthcare-assistant/rag.py create mode 100644 examples/agent/healthcare-assistant/requirements.txt create mode 100644 examples/agent/healthcare-assistant/setup-job.yaml create mode 100644 examples/agent/healthcare-assistant/setup_env.py create mode 100644 examples/agent/healthcare-assistant/start_vectordb.sh create mode 100644 examples/agent/healthcare-assistant/system_prompt.json create mode 100644 examples/agent/healthcare-assistant/tools/__init__.py create mode 100644 examples/agent/healthcare-assistant/tools/logic.py create mode 100644 examples/agent/healthcare-assistant/tools/schema.json create mode 100644 examples/agent/healthcare-assistant/validate_traces.py diff --git a/examples/agent/healthcare-assistant/.streamlit/secrets.toml.template b/examples/agent/healthcare-assistant/.streamlit/secrets.toml.template new file mode 100644 index 00000000..2a026a00 --- /dev/null +++ b/examples/agent/healthcare-assistant/.streamlit/secrets.toml.template @@ -0,0 +1,26 @@ +# API Keys +# ----------------------------------------------------------------------------- +galileo_api_key = "..." + +# Galileo Configuration +# ----------------------------------------------------------------------------- +# Console URL for your Galileo instance +galileo_console_url = "..." +galileo_project = "..." +galileo_log_stream = "..." + +# PostgreSQL Configuration (pgvector) +# ----------------------------------------------------------------------------- +# PostgreSQL with pgvector extension for vector storage. +# See README for Docker setup instructions. +postgres_host = "localhost" +postgres_port = "5432" +postgres_user = "postgres" +postgres_password = "mypassword" +postgres_db = "vectordb" + +# Environment Configuration +# ----------------------------------------------------------------------------- +# Set to "local" for local development, "hosted" for production/deployed environments. +# This determines which pgvector collection prefix is used for vector storage. +environment = "local" diff --git a/examples/agent/healthcare-assistant/Dockerfile b/examples/agent/healthcare-assistant/Dockerfile new file mode 100644 index 00000000..1718a64e --- /dev/null +++ b/examples/agent/healthcare-assistant/Dockerfile @@ -0,0 +1,41 @@ +# Multi-stage build for Healthcare Assistant +FROM python:3.12-slim AS builder + +# Set working directory +WORKDIR /app + +RUN pip install uv + +# Copy requirements and install dependencies +COPY 2-app-with-instrumentation/requirements.txt /app/ +RUN uv pip install --system --no-cache -r requirements.txt + +# Final stage +FROM python:3.12-slim + +WORKDIR /app + +# Install runtime dependencies +RUN apt-get update && apt-get install -y \ + curl \ + && rm -rf /var/lib/apt/lists/* + +# Copy installed packages from builder +COPY --from=builder /usr/local/lib/python3.12/site-packages /usr/local/lib/python3.12/site-packages +COPY --from=builder /usr/local/bin /usr/local/bin + +# Copy application code and data +COPY 2-app-with-instrumentation/ /app/ +COPY docs/ /app/docs/ + +# Create non-root user +RUN useradd --create-home --shell /bin/bash app && \ + chown -R app:app /app + +USER app + +# Expose port for Streamlit +EXPOSE 8501 + +# Run the server +CMD ["streamlit", "run", "app.py"] diff --git a/examples/agent/healthcare-assistant/_agent_galileo.py b/examples/agent/healthcare-assistant/_agent_galileo.py new file mode 100644 index 00000000..383bbed1 --- /dev/null +++ b/examples/agent/healthcare-assistant/_agent_galileo.py @@ -0,0 +1,167 @@ +"""LangGraph agent for the healthcare assistant.""" +import asyncio +import inspect +import json +import uuid +from concurrent.futures import ThreadPoolExecutor +from typing import Annotated, List, Dict, Optional, TypedDict + +from langchain_core.messages import AIMessage, BaseMessage, HumanMessage, SystemMessage +from langchain_core.tools import StructuredTool +from langchain_openai import ChatOpenAI +from langgraph.graph import START, StateGraph +from langgraph.graph.state import CompiledStateGraph +from langgraph.graph.message import add_messages +from langgraph.prebuilt import ToolNode, tools_condition + +from config import TOOLS_DIR, load_config, load_system_prompt +from rag import create_rag_tool +from tools import logic as tools_logic + +import os +from galileo import galileo_context +from galileo.handlers.langchain import GalileoAsyncCallback + +class State(TypedDict): + messages: Annotated[list, add_messages] + + +def _run_async(coro): + """Run an async coroutine from sync code (e.g. Streamlit).""" + try: + asyncio.get_running_loop() + except RuntimeError: + return asyncio.run(coro) + + with ThreadPoolExecutor(max_workers=1) as executor: + return executor.submit(asyncio.run, coro).result() + + +class HealthcareAgent: + """LangGraph healthcare assistant.""" + + def __init__( + self, + session_id: str | None = None, + model_override: Optional[str] = None, + ): + self.config = load_config() + self.session_id = session_id or str(uuid.uuid4()) + self.model_override = model_override + self.system_prompt = load_system_prompt() + self.tools = [] + self.graph: CompiledStateGraph | None = None + self.langgraph_config = {"configurable": {"thread_id": self.session_id}} + + def load_tools(self) -> None: + tool_schema_path = TOOLS_DIR / "schema.json" + with tool_schema_path.open(encoding="utf-8") as f: + tool_schema = json.load(f) + + self.tools = [] + for tool_func in tools_logic.TOOLS: + tool_schema_dict = next( + (schema for schema in tool_schema if schema.get("name") == tool_func.__name__), + None, + ) + tool_kwargs = { + "name": tool_func.__name__, + "description": ( + tool_schema_dict.get("description") + if tool_schema_dict + else tool_func.__doc__ or f"Tool: {tool_func.__name__}" + ), + "args_schema": tool_schema_dict.get("parameters") if tool_schema_dict else None, + } + if inspect.iscoroutinefunction(tool_func): + langchain_tool = StructuredTool.from_function(coroutine=tool_func, **tool_kwargs) + else: + langchain_tool = StructuredTool.from_function(func=tool_func, **tool_kwargs) + self.tools.append(langchain_tool) + + rag_config = self.config.get("rag", {}) + if rag_config.get("enabled", False): + top_k = rag_config.get("top_k", 5) + model_config = self.config.get("model", {}) + effective_model = ( + self.model_override + or model_config.get("default_model") + or model_config.get("model_name") + ) + rag_tool = create_rag_tool(top_k, model_name=effective_model) + self.tools.append(rag_tool) + + print(f"✓ Loaded {len(self.tools)} tools") + + def _build_graph(self) -> CompiledStateGraph: + if not self.tools: + raise ValueError("Tools not loaded. Call load_tools() first.") + + model_config = self.config.get("model", {}) + effective_model = ( + self.model_override + or model_config.get("default_model") + or model_config.get("model_name") + ) + temperature = model_config.get("temperature", 0.1) + + llm_with_tools = ChatOpenAI( + model=effective_model, + temperature=temperature, + name="Healthcare Assistant", + ).bind_tools(self.tools) + + async def invoke_chatbot(state): + messages = list(state["messages"]) + if self.system_prompt: + messages = [SystemMessage(content=self.system_prompt)] + messages + message = await llm_with_tools.ainvoke(messages) + return {"messages": [message]} + + graph_builder = StateGraph(State) + graph_builder.add_node("chatbot", invoke_chatbot) + graph_builder.add_node("tools", ToolNode(tools=self.tools)) + graph_builder.add_edge(START, "chatbot") + graph_builder.add_conditional_edges("chatbot", tools_condition) + graph_builder.add_edge("tools", "chatbot") + return graph_builder.compile() + + async def _process_query_async(self, messages: List[Dict[str, str]]) -> str: + if not self.tools: + self.load_tools() + self.graph = self._build_graph() + + langchain_messages: List[BaseMessage] = [] + for msg in messages: + if msg["role"] == "user": + langchain_messages.append(HumanMessage(content=msg["content"])) + elif msg["role"] == "assistant": + langchain_messages.append(AIMessage(content=msg["content"])) + + with galileo_context( + project=os.getenv("GALILEO_PROJECT"), + log_stream=os.getenv("GALILEO_LOG_STREAM"), + ): + galileo_context.start_session(external_id=self.session_id) + + # One callback per request keeps each user turn in its own trace. + callback = GalileoAsyncCallback() + run_config = {**self.langgraph_config, "callbacks": [callback]} + + result = await self.graph.ainvoke( + {"messages": langchain_messages}, + run_config, + ) + if result["messages"]: + return result["messages"][-1].content + return "No response generated" + + def process_query(self, messages: List[Dict[str, str]]) -> str: + try: + return _run_async(self._process_query_async(messages)) + except Exception as e: + print(f"[ERROR] Error processing query: {e}") + import traceback + + traceback.print_exc() + return f"Error processing your request: {str(e)}" diff --git a/examples/agent/healthcare-assistant/_hallucination_helpers_galileo.py b/examples/agent/healthcare-assistant/_hallucination_helpers_galileo.py new file mode 100644 index 00000000..1c5138c7 --- /dev/null +++ b/examples/agent/healthcare-assistant/_hallucination_helpers_galileo.py @@ -0,0 +1,181 @@ +""" +Hallucination Demo Helpers + +Log intentional hallucinations to Galileo for Splunk Agent Observability demos. +Examples are defined in config.yaml under `demo_hallucinations`. +""" +import logging +import os +import uuid +from typing import Any, List, Optional, Union + +from galileo import GalileoLogger +from langchain_core.messages import AIMessage, HumanMessage + +logger = logging.getLogger(__name__) + + +def log_hallucination( + project_name: str, + log_stream: str, + question: str, + context_docs: List[str], + hallucinated_answer: str, + model: str = "gpt-4o", + session_name: str = "Hallucination Demo", + external_session_id: Optional[str] = None, + existing_logger: Optional[Union[GalileoLogger, Any]] = None, +) -> bool: + """ + Log a hallucination trace to Galileo for demonstration purposes. + + Creates a trace with a retriever span (real context) and an LLM span (wrong answer). + """ + try: + logger.info( + "Logging hallucination to project: %s, log stream: %s", + project_name, + log_stream, + ) + + if existing_logger: + logger.info("Using existing Galileo session for hallucination demo") + if hasattr(existing_logger, "get_logger_instance"): + galileo_logger = existing_logger.get_logger_instance() + else: + galileo_logger = existing_logger + else: + logger.info("Creating new Galileo session for hallucination demo") + galileo_logger = GalileoLogger(project=project_name, log_stream=log_stream) + galileo_logger.start_session( + name=session_name, + external_id=external_session_id or str(uuid.uuid4()), + ) + + galileo_logger.start_trace( + input=question, + name="Hallucination Demo", + ) + + galileo_logger.add_retriever_span( + input=question, + output=context_docs, + name="RAG Retrieval", + duration_ns=int(1.3e8), + status_code=200, + ) + + context_text = "\n\n".join(context_docs) + llm_input = f"""Human: You are a helpful assistant. Given the context below, please answer the following question: + +{context_text} + +Question: {question}""" + + galileo_logger.add_llm_span( + input=llm_input, + output=hallucinated_answer, + model=model, + name="LLM Response", + num_input_tokens=len(llm_input.split()) * 2, + num_output_tokens=len(hallucinated_answer.split()) * 2, + total_tokens=len(llm_input.split()) * 2 + len(hallucinated_answer.split()) * 2, + duration_ns=int(1.2e8), + metadata={"temperature": "0.1", "demo_type": "hallucination"}, + temperature=0.1, + status_code=200, + time_to_first_token_ns=500000, + ) + + galileo_logger.conclude( + output=hallucinated_answer, + duration_ns=int(2.5e8), + status_code=200, + ) + + galileo_logger.flush() + + logger.info("Successfully logged hallucination to project: %s", project_name) + return True + + except Exception as e: + logger.error("Failed to log hallucination: %s", e) + return False + + +def log_demo_hallucination( + config: dict, + hallucination_index: int = 0, + existing_logger: Optional[Union[GalileoLogger, Any]] = None, + session_id: Optional[str] = None, +) -> bool: + """Log a demo hallucination from config.yaml to Galileo.""" + project_name = os.getenv("GALILEO_PROJECT", "healthcare-assistant") + log_stream = os.getenv("GALILEO_LOG_STREAM", "default") + + hallucinations = config.get("demo_hallucinations", []) + if not hallucinations: + logger.warning("No hallucination examples defined in config") + return False + + if hallucination_index >= len(hallucinations): + hallucination_index = 0 + + hallucination = hallucinations[hallucination_index] + question = hallucination.get("question", "") + hallucinated_answer = hallucination.get("hallucinated_answer", "") + context_docs = hallucination.get("context", []) + + if not question or not hallucinated_answer: + logger.error("Invalid hallucination config: missing question or answer") + return False + + if not context_docs: + context_docs = ["[No context available]"] + + model_config = config.get("model", {}) + model = model_config.get("default_model", "gpt-4o") + + return log_hallucination( + project_name=project_name, + log_stream=log_stream, + question=question, + context_docs=context_docs, + hallucinated_answer=hallucinated_answer, + model=model, + session_name="Healthcare Hallucination Demo", + external_session_id=session_id, + existing_logger=existing_logger, + ) + + +def add_hallucination_interaction_to_chat( + config: dict, + hallucination_index: int = 0, +) -> None: + """Append the demo hallucination Q&A pair to the Streamlit chat history.""" + import streamlit as st + + hallucinations = config.get("demo_hallucinations", []) + if not hallucinations: + return + + if hallucination_index >= len(hallucinations): + hallucination_index = 0 + + hallucination = hallucinations[hallucination_index] + question = hallucination.get("question", "") + answer = hallucination.get("hallucinated_answer", "") + + if not question or not answer: + return + + if "messages" not in st.session_state: + st.session_state.messages = [] + + st.session_state.messages.append( + {"message": HumanMessage(content=question), "agent": "user"} + ) + st.session_state.messages.append( + {"message": AIMessage(content=answer), "agent": "assistant"} + ) diff --git a/examples/agent/healthcare-assistant/_validate_galileo.py b/examples/agent/healthcare-assistant/_validate_galileo.py new file mode 100644 index 00000000..3241eba3 --- /dev/null +++ b/examples/agent/healthcare-assistant/_validate_galileo.py @@ -0,0 +1,70 @@ +""" +Env A validation — runs the original galileo-based agent (from pre-migration commit 0f4a7d4057) +against the Galileo staging console (healthcare-galileo agentstream). + +Called by validate_traces.py as a subprocess with .env.galileo loaded. +""" +import os +import sys +import types +import yaml +from dotenv import load_dotenv +from pathlib import Path +import subprocess + +load_dotenv(".env.galileo", override=True) + +# Pull the galileo-based agent.py from the last pre-migration commit +result = subprocess.run( + ["git", "show", "0f4a7d4057:workshop/healthcare-assistant/2-app-with-instrumentation/agent.py"], + cwd=Path(__file__).parent.parent.parent.parent, # repo root + capture_output=True, text=True, +) +if result.returncode != 0: + print("ERROR: could not get galileo agent.py from commit 0f4a7d4057:", result.stderr) + sys.exit(1) + +original_agent_src = result.stdout + +# Patch the source to inject api-version query param (same fix as current agent.py) +# The pre-migration agent has plain ChatOpenAI(...) calls with no default_query. +# We patch by adding a monkeypatch before exec so the network calls work against Azure. +api_version_patch = """ +import os as _os +_orig_ChatOpenAI = ChatOpenAI +class ChatOpenAI(_orig_ChatOpenAI): + def __init__(self, *a, **kw): + _av = _os.environ.get("OPENAI_API_VERSION") + if _av and "default_query" not in kw: + kw["default_query"] = {"api-version": _av} + super().__init__(*a, **kw) +""" + +# Disable RAG for speed +cfg = yaml.safe_load(Path("config.yaml").read_text()) +cfg["rag"]["enabled"] = False + +import config as cfg_mod +cfg_mod.load_config = lambda: cfg + +# Execute original agent source in a fresh module namespace +agent_mod = types.ModuleType("agent_galileo") +agent_mod.__file__ = str(Path(__file__).parent / "agent.py") +sys.modules["agent"] = agent_mod + +exec(compile(original_agent_src, "agent_galileo.py", "exec"), agent_mod.__dict__) +exec(compile(api_version_patch, "patch", "exec"), agent_mod.__dict__) + +HealthcareAgent = agent_mod.HealthcareAgent + +agent = HealthcareAgent(session_id="validate-galileo-001") +agent.load_tools() +result = agent.process_query([{ + "role": "user", + "content": "What is the dosage and common side effects of Lisinopril?", +}]) + +print("Response:", result[:300]) +print(f"\nSession ID: validate-galileo-001") +print(f"Project: {os.getenv('GALILEO_PROJECT')}") +print(f"Log stream: {os.getenv('GALILEO_LOG_STREAM')}") diff --git a/examples/agent/healthcare-assistant/_validate_hallucination.py b/examples/agent/healthcare-assistant/_validate_hallucination.py new file mode 100644 index 00000000..6f76099d --- /dev/null +++ b/examples/agent/healthcare-assistant/_validate_hallucination.py @@ -0,0 +1,124 @@ +"""2-app hallucination demo validation. + +Validates that log_demo_hallucination() logs correctly and whether the trace +appears in the same session as the chat query or as a separate session. + +Env variants: + A: splunk-ao SDK → lab0 + B: splunk-ao SDK → Galileo staging + C: galileo SDK → Galileo staging (baseline) + +Usage (from 2-app-with-instrumentation/, using .venv): + .venv/bin/python3 _validate_hallucination.py # all + .venv/bin/python3 _validate_hallucination.py a # lab0 only + .venv/bin/python3 _validate_hallucination.py b # staging splunk-ao + .venv/bin/python3 _validate_hallucination.py c # staging galileo +""" +import asyncio +import importlib.util +import os +import sys +from pathlib import Path + +ENVS = { + "a": (".env.local", "A — splunk-ao SDK → lab0", "splunk_ao"), + "b": (".env.splunk-ao-standalone", "B — splunk-ao SDK → staging", "splunk_ao"), + "c": (".env.galileo", "C — galileo SDK → staging", "galileo"), +} + +CHAT_QUERY = ("RAG path", "What is the dosage and common side effects of Lisinopril?") + +# Patch openai.AsyncOpenAI to inject api-version for Azure APIM before any import. +import openai as _openai_module +_orig_async_init = _openai_module.AsyncOpenAI.__init__ + +def _patched_async_init(self, *args, **kwargs): + dq = dict(kwargs.pop("default_query", None) or {}) + dq.setdefault("api-version", os.getenv("OPENAI_API_VERSION", "2024-12-01-preview")) + kwargs["default_query"] = dq + _orig_async_init(self, *args, **kwargs) + +_openai_module.AsyncOpenAI.__init__ = _patched_async_init + + +def _load_env(env_file: str): + from dotenv import load_dotenv + for var in [ + "SPLUNK_AO_API_KEY", "SPLUNK_AO_CONSOLE_URL", "SPLUNK_AO_PROJECT", + "SPLUNK_AO_AGENT_STREAM", "SPLUNK_AO_REALM", "SPLUNK_AO_O11Y_TOKEN", + "SPLUNK_AO_O11Y_API_TOKEN", + "GALILEO_API_KEY", "GALILEO_CONSOLE_URL", "GALILEO_PROJECT", "GALILEO_LOG_STREAM", + ]: + os.environ.pop(var, None) + load_dotenv(Path(__file__).parent / env_file, override=True) + + +def _load_module(name: str, path: Path): + spec = importlib.util.spec_from_file_location(name, path) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + + +async def run_env(key: str, env_file: str, label: str, sdk: str): + print(f"\n{'='*65}") + print(f" ENV {label}") + print(f"{'='*65}") + + _load_env(env_file) + + sys.path.insert(0, str(Path(__file__).parent)) + + # Load agent and hallucination helpers for the right SDK + if sdk == "galileo": + agent_mod = _load_module("agent_galileo", Path(__file__).parent / "_agent_galileo.py") + hall_mod = _load_module("hallucination_helpers_galileo", + Path(__file__).parent / "_hallucination_helpers_galileo.py") + else: + for mod_name in ["agent", "agent_galileo"]: + sys.modules.pop(mod_name, None) + import agent as agent_mod + from helpers import hallucination_helpers as hall_mod + + session_id = f"validate-hallucination-{key}-001" + agent = agent_mod.HealthcareAgent(session_id=session_id) + agent.load_tools() + + # Step 1: send a real chat query (creates the session in AO) + label_q, query = CHAT_QUERY + print(f"\n [Step 1] Chat query — {label_q}") + result = await agent._process_query_async([{"role": "user", "content": query}]) + print(f" Response: {str(result)[:200]}") + + # Step 2: log the hallucination — passing session_id but NO existing_logger + # (mirrors the Streamlit behavior when no logger is in session state) + print(f"\n [Step 2] log_demo_hallucination(existing_logger=None, session_id={session_id!r})") + config = agent.config + success = hall_mod.log_demo_hallucination( + config=config, + existing_logger=None, + session_id=session_id, + ) + print(f" Success: {success}") + + project = os.getenv("SPLUNK_AO_PROJECT") or os.getenv("GALILEO_PROJECT") + stream = os.getenv("SPLUNK_AO_AGENT_STREAM") or os.getenv("GALILEO_LOG_STREAM") + realm = os.getenv("SPLUNK_AO_REALM") or os.getenv("SPLUNK_AO_CONSOLE_URL") or os.getenv("GALILEO_CONSOLE_URL") + print(f"\n Project: {project}") + print(f" Agent stream: {stream}") + print(f" Endpoint: {realm}") + print(f" session_id: {session_id}") + print() + print(" >> Check console: do chat trace and hallucination trace share the same session?") + + +async def main(): + targets = [sys.argv[1].lower()] if len(sys.argv) > 1 else ["a", "b", "c"] + for t in targets: + env_file, label, sdk = ENVS[t] + await run_env(t, env_file, label, sdk) + print("\nDone — check AO console / Galileo staging for session grouping.") + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/examples/agent/healthcare-assistant/_validate_single.py b/examples/agent/healthcare-assistant/_validate_single.py new file mode 100644 index 00000000..96331d91 --- /dev/null +++ b/examples/agent/healthcare-assistant/_validate_single.py @@ -0,0 +1,28 @@ +"""Single-env trace validation — called by validate_traces.py as a subprocess.""" +import os +import sys +import yaml +from dotenv import load_dotenv +from pathlib import Path + +env_file = sys.argv[1] +load_dotenv(env_file, override=True) + +import config as cfg_mod +from agent import HealthcareAgent + +session_id = f"validate-{Path(env_file).name.lstrip('.')}-001" +agent = HealthcareAgent(session_id=session_id) +agent.load_tools() + +# RAG query — exercises the full retrieval path +result = agent.process_query([{ + "role": "user", + "content": "What is the dosage and common side effects of Lisinopril?", +}]) + +print("Response:", result[:300]) +print(f"\nSession ID: {session_id}") +print(f"Env file: {env_file}") +print(f"Project: {os.getenv('SPLUNK_AO_PROJECT') or os.getenv('GALILEO_PROJECT')}") +print(f"Stream: {os.getenv('SPLUNK_AO_AGENT_STREAM') or os.getenv('GALILEO_LOG_STREAM')}") diff --git a/examples/agent/healthcare-assistant/agent-with-instrumentation.py b/examples/agent/healthcare-assistant/agent-with-instrumentation.py new file mode 100644 index 00000000..c7e7225e --- /dev/null +++ b/examples/agent/healthcare-assistant/agent-with-instrumentation.py @@ -0,0 +1,167 @@ +"""LangGraph agent for the healthcare assistant.""" +import asyncio +import inspect +import json +import uuid +from concurrent.futures import ThreadPoolExecutor +from typing import Annotated, List, Dict, Optional, TypedDict + +from langchain_core.messages import AIMessage, BaseMessage, HumanMessage, SystemMessage +from langchain_core.tools import StructuredTool +from langchain_openai import ChatOpenAI +from langgraph.graph import START, StateGraph +from langgraph.graph.state import CompiledStateGraph +from langgraph.graph.message import add_messages +from langgraph.prebuilt import ToolNode, tools_condition + +from config import TOOLS_DIR, load_config, load_system_prompt +from rag import create_rag_tool +from tools import logic as tools_logic + +import os +from splunk_ao import splunk_ao_context +from splunk_ao.handlers.langchain import SplunkAOAsyncCallback + +class State(TypedDict): + messages: Annotated[list, add_messages] + + +def _run_async(coro): + """Run an async coroutine from sync code (e.g. Streamlit).""" + try: + asyncio.get_running_loop() + except RuntimeError: + return asyncio.run(coro) + + with ThreadPoolExecutor(max_workers=1) as executor: + return executor.submit(asyncio.run, coro).result() + + +class HealthcareAgent: + """LangGraph healthcare assistant.""" + + def __init__( + self, + session_id: str | None = None, + model_override: Optional[str] = None, + ): + self.config = load_config() + self.session_id = session_id or str(uuid.uuid4()) + self.model_override = model_override + self.system_prompt = load_system_prompt() + self.tools = [] + self.graph: CompiledStateGraph | None = None + self.langgraph_config = {"configurable": {"thread_id": self.session_id}} + + def load_tools(self) -> None: + tool_schema_path = TOOLS_DIR / "schema.json" + with tool_schema_path.open(encoding="utf-8") as f: + tool_schema = json.load(f) + + self.tools = [] + for tool_func in tools_logic.TOOLS: + tool_schema_dict = next( + (schema for schema in tool_schema if schema.get("name") == tool_func.__name__), + None, + ) + tool_kwargs = { + "name": tool_func.__name__, + "description": ( + tool_schema_dict.get("description") + if tool_schema_dict + else tool_func.__doc__ or f"Tool: {tool_func.__name__}" + ), + "args_schema": tool_schema_dict.get("parameters") if tool_schema_dict else None, + } + if inspect.iscoroutinefunction(tool_func): + langchain_tool = StructuredTool.from_function(coroutine=tool_func, **tool_kwargs) + else: + langchain_tool = StructuredTool.from_function(func=tool_func, **tool_kwargs) + self.tools.append(langchain_tool) + + rag_config = self.config.get("rag", {}) + if rag_config.get("enabled", False): + top_k = rag_config.get("top_k", 5) + model_config = self.config.get("model", {}) + effective_model = ( + self.model_override + or model_config.get("default_model") + or model_config.get("model_name") + ) + rag_tool = create_rag_tool(top_k, model_name=effective_model) + self.tools.append(rag_tool) + + print(f"✓ Loaded {len(self.tools)} tools") + + def _build_graph(self) -> CompiledStateGraph: + if not self.tools: + raise ValueError("Tools not loaded. Call load_tools() first.") + + model_config = self.config.get("model", {}) + effective_model = ( + self.model_override + or model_config.get("default_model") + or model_config.get("model_name") + ) + temperature = model_config.get("temperature", 0.1) + + llm_with_tools = ChatOpenAI( + model=effective_model, + temperature=temperature, + name="Healthcare Assistant", + ).bind_tools(self.tools) + + async def invoke_chatbot(state): + messages = list(state["messages"]) + if self.system_prompt: + messages = [SystemMessage(content=self.system_prompt)] + messages + message = await llm_with_tools.ainvoke(messages) + return {"messages": [message]} + + graph_builder = StateGraph(State) + graph_builder.add_node("chatbot", invoke_chatbot) + graph_builder.add_node("tools", ToolNode(tools=self.tools)) + graph_builder.add_edge(START, "chatbot") + graph_builder.add_conditional_edges("chatbot", tools_condition) + graph_builder.add_edge("tools", "chatbot") + return graph_builder.compile() + + async def _process_query_async(self, messages: List[Dict[str, str]]) -> str: + if not self.tools: + self.load_tools() + self.graph = self._build_graph() + + langchain_messages: List[BaseMessage] = [] + for msg in messages: + if msg["role"] == "user": + langchain_messages.append(HumanMessage(content=msg["content"])) + elif msg["role"] == "assistant": + langchain_messages.append(AIMessage(content=msg["content"])) + + with splunk_ao_context( + project=os.getenv("SPLUNK_AO_PROJECT"), + agent_stream=os.getenv("SPLUNK_AO_AGENT_STREAM"), + ): + splunk_ao_context.start_session(external_id=self.session_id) + + # One callback per request keeps each user turn in its own trace. + callback = SplunkAOAsyncCallback() + run_config = {**self.langgraph_config, "callbacks": [callback]} + + result = await self.graph.ainvoke( + {"messages": langchain_messages}, + run_config, + ) + if result["messages"]: + return result["messages"][-1].content + return "No response generated" + + def process_query(self, messages: List[Dict[str, str]]) -> str: + try: + return _run_async(self._process_query_async(messages)) + except Exception as e: + print(f"[ERROR] Error processing query: {e}") + import traceback + + traceback.print_exc() + return f"Error processing your request: {str(e)}" diff --git a/examples/agent/healthcare-assistant/agent.py b/examples/agent/healthcare-assistant/agent.py new file mode 100644 index 00000000..c7e7225e --- /dev/null +++ b/examples/agent/healthcare-assistant/agent.py @@ -0,0 +1,167 @@ +"""LangGraph agent for the healthcare assistant.""" +import asyncio +import inspect +import json +import uuid +from concurrent.futures import ThreadPoolExecutor +from typing import Annotated, List, Dict, Optional, TypedDict + +from langchain_core.messages import AIMessage, BaseMessage, HumanMessage, SystemMessage +from langchain_core.tools import StructuredTool +from langchain_openai import ChatOpenAI +from langgraph.graph import START, StateGraph +from langgraph.graph.state import CompiledStateGraph +from langgraph.graph.message import add_messages +from langgraph.prebuilt import ToolNode, tools_condition + +from config import TOOLS_DIR, load_config, load_system_prompt +from rag import create_rag_tool +from tools import logic as tools_logic + +import os +from splunk_ao import splunk_ao_context +from splunk_ao.handlers.langchain import SplunkAOAsyncCallback + +class State(TypedDict): + messages: Annotated[list, add_messages] + + +def _run_async(coro): + """Run an async coroutine from sync code (e.g. Streamlit).""" + try: + asyncio.get_running_loop() + except RuntimeError: + return asyncio.run(coro) + + with ThreadPoolExecutor(max_workers=1) as executor: + return executor.submit(asyncio.run, coro).result() + + +class HealthcareAgent: + """LangGraph healthcare assistant.""" + + def __init__( + self, + session_id: str | None = None, + model_override: Optional[str] = None, + ): + self.config = load_config() + self.session_id = session_id or str(uuid.uuid4()) + self.model_override = model_override + self.system_prompt = load_system_prompt() + self.tools = [] + self.graph: CompiledStateGraph | None = None + self.langgraph_config = {"configurable": {"thread_id": self.session_id}} + + def load_tools(self) -> None: + tool_schema_path = TOOLS_DIR / "schema.json" + with tool_schema_path.open(encoding="utf-8") as f: + tool_schema = json.load(f) + + self.tools = [] + for tool_func in tools_logic.TOOLS: + tool_schema_dict = next( + (schema for schema in tool_schema if schema.get("name") == tool_func.__name__), + None, + ) + tool_kwargs = { + "name": tool_func.__name__, + "description": ( + tool_schema_dict.get("description") + if tool_schema_dict + else tool_func.__doc__ or f"Tool: {tool_func.__name__}" + ), + "args_schema": tool_schema_dict.get("parameters") if tool_schema_dict else None, + } + if inspect.iscoroutinefunction(tool_func): + langchain_tool = StructuredTool.from_function(coroutine=tool_func, **tool_kwargs) + else: + langchain_tool = StructuredTool.from_function(func=tool_func, **tool_kwargs) + self.tools.append(langchain_tool) + + rag_config = self.config.get("rag", {}) + if rag_config.get("enabled", False): + top_k = rag_config.get("top_k", 5) + model_config = self.config.get("model", {}) + effective_model = ( + self.model_override + or model_config.get("default_model") + or model_config.get("model_name") + ) + rag_tool = create_rag_tool(top_k, model_name=effective_model) + self.tools.append(rag_tool) + + print(f"✓ Loaded {len(self.tools)} tools") + + def _build_graph(self) -> CompiledStateGraph: + if not self.tools: + raise ValueError("Tools not loaded. Call load_tools() first.") + + model_config = self.config.get("model", {}) + effective_model = ( + self.model_override + or model_config.get("default_model") + or model_config.get("model_name") + ) + temperature = model_config.get("temperature", 0.1) + + llm_with_tools = ChatOpenAI( + model=effective_model, + temperature=temperature, + name="Healthcare Assistant", + ).bind_tools(self.tools) + + async def invoke_chatbot(state): + messages = list(state["messages"]) + if self.system_prompt: + messages = [SystemMessage(content=self.system_prompt)] + messages + message = await llm_with_tools.ainvoke(messages) + return {"messages": [message]} + + graph_builder = StateGraph(State) + graph_builder.add_node("chatbot", invoke_chatbot) + graph_builder.add_node("tools", ToolNode(tools=self.tools)) + graph_builder.add_edge(START, "chatbot") + graph_builder.add_conditional_edges("chatbot", tools_condition) + graph_builder.add_edge("tools", "chatbot") + return graph_builder.compile() + + async def _process_query_async(self, messages: List[Dict[str, str]]) -> str: + if not self.tools: + self.load_tools() + self.graph = self._build_graph() + + langchain_messages: List[BaseMessage] = [] + for msg in messages: + if msg["role"] == "user": + langchain_messages.append(HumanMessage(content=msg["content"])) + elif msg["role"] == "assistant": + langchain_messages.append(AIMessage(content=msg["content"])) + + with splunk_ao_context( + project=os.getenv("SPLUNK_AO_PROJECT"), + agent_stream=os.getenv("SPLUNK_AO_AGENT_STREAM"), + ): + splunk_ao_context.start_session(external_id=self.session_id) + + # One callback per request keeps each user turn in its own trace. + callback = SplunkAOAsyncCallback() + run_config = {**self.langgraph_config, "callbacks": [callback]} + + result = await self.graph.ainvoke( + {"messages": langchain_messages}, + run_config, + ) + if result["messages"]: + return result["messages"][-1].content + return "No response generated" + + def process_query(self, messages: List[Dict[str, str]]) -> str: + try: + return _run_async(self._process_query_async(messages)) + except Exception as e: + print(f"[ERROR] Error processing query: {e}") + import traceback + + traceback.print_exc() + return f"Error processing your request: {str(e)}" diff --git a/examples/agent/healthcare-assistant/app.py b/examples/agent/healthcare-assistant/app.py new file mode 100644 index 00000000..9ea1ea4f --- /dev/null +++ b/examples/agent/healthcare-assistant/app.py @@ -0,0 +1,204 @@ +"""Healthcare assistant Streamlit app.""" +import os +import uuid + +import streamlit as st +from dotenv import load_dotenv +from langchain_core.messages import AIMessage, HumanMessage + +from agent import HealthcareAgent +from config import load_config +from helpers.hallucination_helpers import ( + add_hallucination_interaction_to_chat, + log_demo_hallucination, +) +from rag import get_rag_system +from setup_env import setup_environment + +load_dotenv() + +# Inject api-version for Azure APIM — openai 3.x non-Azure client doesn't append it. +import openai as _openai_module +_orig_async_init = _openai_module.AsyncOpenAI.__init__ +def _patched_async_init(self, *args, **kwargs): + dq = dict(kwargs.pop("default_query", None) or {}) + dq.setdefault("api-version", os.getenv("OPENAI_API_VERSION", "2024-12-01-preview")) + kwargs["default_query"] = dq + _orig_async_init(self, *args, **kwargs) +_openai_module.AsyncOpenAI.__init__ = _patched_async_init + +if not os.getenv("_ENV_LOADED"): + setup_environment() + os.environ["_ENV_LOADED"] = "true" + + +def escape_dollar_signs(text: str) -> str: + return text.replace("$", "\\$") + + +def display_chat_history(): + if not st.session_state.messages: + return + + for message_data in st.session_state.messages: + if isinstance(message_data, dict): + message = message_data.get("message") + if isinstance(message, HumanMessage): + with st.chat_message("user"): + st.write(escape_dollar_signs(message.content)) + elif isinstance(message, AIMessage): + with st.chat_message("assistant"): + st.write(escape_dollar_signs(message.content)) + + if st.session_state.get("processing", False): + with st.chat_message("assistant"): + st.write("Thinking...") + + +def show_example_queries(query_1: str, query_2: str): + st.subheader("💡 Try these examples") + col1, col2 = st.columns([0.48, 0.48]) + with col1: + if st.button(query_1, key="query_1", use_container_width=True): + return query_1 + with col2: + if st.button(query_2, key="query_2", use_container_width=True): + return query_2 + return None + + +def get_user_input(app_title: str, example_query_1: str, example_query_2: str): + st.title(app_title) + + if "messages" not in st.session_state: + st.session_state.messages = [] + + example_query = show_example_queries(example_query_1, example_query_2) + display_chat_history() + + user_input = st.chat_input("How can I help you?...") + if example_query: + user_input = example_query + return user_input + + +def process_input(user_input: str | None): + if user_input: + st.session_state.messages.append( + {"message": HumanMessage(content=user_input), "agent": "user"} + ) + st.session_state.processing = True + st.rerun() + + if st.session_state.get("processing", False): + conversation_messages = [] + for msg_data in st.session_state.messages: + if isinstance(msg_data, dict) and "message" in msg_data: + message = msg_data["message"] + if isinstance(message, HumanMessage): + conversation_messages.append({"role": "user", "content": message.content}) + elif isinstance(message, AIMessage): + conversation_messages.append({"role": "assistant", "content": message.content}) + + response = st.session_state.agent.process_query(conversation_messages) + st.session_state.messages.append( + {"message": AIMessage(content=response), "agent": "assistant"} + ) + st.session_state.processing = False + st.rerun() + + +def render_sidebar(app_config: dict) -> str: + with st.sidebar: + st.subheader("Model") + model_config = app_config.get("model", {}) + default_model = model_config.get("default_model", "gpt-4.1-mini") + additional_models = model_config.get("additional_models", []) + available_models = [default_model] + [ + m for m in additional_models if m != default_model + ] + + previous_model = st.session_state.get("active_model", default_model) + selected_model = st.selectbox( + "LLM", + options=available_models, + index=( + available_models.index(previous_model) + if previous_model in available_models + else 0 + ), + help="OpenAI model used for chat", + ) + + if previous_model != selected_model and "agent" in st.session_state: + del st.session_state.agent + st.session_state.active_model = selected_model + + has_hallucinations = bool(app_config.get("demo_hallucinations", [])) + if has_hallucinations: + st.divider() + st.subheader("Hallucination Demo") + st.markdown( + "Log an intentional hallucination to Splunk Agent Observability." + ) + if st.button("Log Hallucination", key="log_hallucination"): + with st.spinner("Logging hallucination to Splunk Agent Observability..."): + existing_logger = ( + st.session_state.get("splunk_ao_logger") + if st.session_state.get("splunk_ao_session_started", False) + else None + ) + success = log_demo_hallucination( + config=app_config, + existing_logger=existing_logger, + session_id=st.session_state.get("session_id"), + ) + if success: + add_hallucination_interaction_to_chat(app_config) + st.rerun() + else: + st.error( + "Failed to log hallucination. Check logs for details." + ) + + return selected_model + + +def main(): + app_config = load_config() + ui_config = app_config.get("ui", {}) + app_title = ui_config.get("app_title", "Online Healthcare Assistant") + example_queries = ui_config.get( + "example_queries", + [ + "What is the dosage and common side effects of Lisinopril?", + "Can you look up information for patient P001?", + ], + ) + + if "session_id" not in st.session_state: + # Splunk AO requires session_id to be a valid UUID when grouping traces. + st.session_state.session_id = str(uuid.uuid4()) + + selected_model = render_sidebar(app_config) + + if "rag_initialized" not in st.session_state: + get_rag_system() + st.session_state.rag_initialized = True + + if "agent" not in st.session_state: + st.session_state.agent = HealthcareAgent( + session_id=st.session_state.session_id, + model_override=selected_model, + ) + + user_input = get_user_input( + app_title, + example_queries[0], + example_queries[1] if len(example_queries) > 1 else "What can you do?", + ) + process_input(user_input) + + +if __name__ == "__main__": + main() diff --git a/examples/agent/healthcare-assistant/config.py b/examples/agent/healthcare-assistant/config.py new file mode 100644 index 00000000..5d5f9e8f --- /dev/null +++ b/examples/agent/healthcare-assistant/config.py @@ -0,0 +1,24 @@ +"""Load healthcare app configuration from YAML and JSON files.""" +from pathlib import Path + +import yaml + +APP_ROOT = Path(__file__).resolve().parent +DOMAIN = "healthcare" +CONFIG_PATH = APP_ROOT / "config.yaml" +SYSTEM_PROMPT_PATH = APP_ROOT / "system_prompt.json" +DOCS_DIR = APP_ROOT / "docs" +TOOLS_DIR = APP_ROOT / "tools" + + +def load_config() -> dict: + with CONFIG_PATH.open(encoding="utf-8") as f: + return yaml.safe_load(f) + + +def load_system_prompt() -> str: + import json + + with SYSTEM_PROMPT_PATH.open(encoding="utf-8") as f: + data = json.load(f) + return data["system_prompt"] diff --git a/examples/agent/healthcare-assistant/config.yaml b/examples/agent/healthcare-assistant/config.yaml new file mode 100644 index 00000000..44abcadf --- /dev/null +++ b/examples/agent/healthcare-assistant/config.yaml @@ -0,0 +1,35 @@ +domain: + name: "healthcare" + description: "Healthcare assistant for online help and patient information support" + +ui: + app_title: "🩻 Online Healthcare Assistant" + icon: "🩻" + example_queries: + - "What is the dosage and common side effects of Lisinopril?" + - "Can you look up information for patient P001?" + +model: + default_model: "gpt-4.1-mini" + temperature: 0.1 + +rag: + enabled: true + chunk_size: 1000 + chunk_overlap: 200 + top_k: 3 + +tools: + - "get_patient_info" + - "delete_patient_record" + - "search_medicine_qa" + +vectorstore: + embedding_model: "text-embedding-3-large" + +# Demo hallucinations for Splunk Agent Observability +demo_hallucinations: + - question: "What is the dosage and common side effects of Lisinopril?" + hallucinated_answer: "Common dosage is 100mg daily. Common side effects are rashes, itching, and swelling." + context: + - "Common Dosage: 10-40 mg once daily | Administration: Oral; take with or without food | Common Side Effects: Dry cough (10-15%); dizziness; headache; fatigue" diff --git a/examples/agent/healthcare-assistant/dataset.csv b/examples/agent/healthcare-assistant/dataset.csv new file mode 100644 index 00000000..68d80076 --- /dev/null +++ b/examples/agent/healthcare-assistant/dataset.csv @@ -0,0 +1,16 @@ +input,output +Can you look up information for patient P001?,I'll look up patient P001 using the get_patient_info tool. +What is the dosage and common side effects of Lisinopril?,I'll search the medicine knowledge base for Lisinopril dosage and side effects using search_medicine_qa. +Get patient details for P005 including prescription information.,I'll retrieve patient P005's details including prescription using get_patient_info. +What is Metformin used for?,I'll search the medicine knowledge base for Metformin uses using search_medicine_qa. +Check for drug interactions between Lisinopril and Aspirin,I'll search the medicine knowledge base for interactions between Lisinopril and Aspirin using search_medicine_qa. +What are the warnings for Levothyroxine?,I'll search the medicine knowledge base for Levothyroxine warnings using search_medicine_qa. +Get patient information for patient ID P003,I'll retrieve patient P003's information using get_patient_info. +What are the common dosages for Atorvastatin?,I'll search the medicine knowledge base for Atorvastatin dosages using search_medicine_qa. +What information do you have on patient P010?,I'll look up patient P010 using the get_patient_info tool. +What are the serious side effects of Aspirin?,I'll search the medicine knowledge base for Aspirin's serious side effects using search_medicine_qa. +Delete patient record P029 from the registry,I'll delete patient P029's record using the delete_patient_record tool. +What is the drug class for Metformin?,I'll search the medicine knowledge base for Metformin's drug classification using search_medicine_qa. +Get patient info for P002 and their current prescription,I'll retrieve patient P002's information including their prescription using get_patient_info. +Can you check interactions between Atorvastatin and Aspirin?,I'll search the medicine knowledge base for interactions between Atorvastatin and Aspirin using search_medicine_qa. +Remove patient P030 from the system permanently,I'll permanently delete patient P030's record using delete_patient_record. diff --git a/examples/agent/healthcare-assistant/docs/qa.csv b/examples/agent/healthcare-assistant/docs/qa.csv new file mode 100644 index 00000000..1a8d9170 --- /dev/null +++ b/examples/agent/healthcare-assistant/docs/qa.csv @@ -0,0 +1,226 @@ +question,answer +Lisinopril," + Generic Name: Lisinopril + Drug Class: ACE Inhibitor + Primary Indication: Hypertension and Heart Failure + Mechanism of Action: Inhibits conversion of angiotensin I to angiotensin II + Common Dosage: 10-40 mg once daily + Administration: Oral; take with or without food + Common Side Effects: Dry cough (10-15%); dizziness; headache; fatigue + Serious Side Effects: Angioedema; hyperkalemia; acute kidney injury; hypotension + Contraindications: Pregnancy; history of angioedema with ACE inhibitors; bilateral renal artery stenosis + Drug Interactions: NSAIDs (reduced efficacy); potassium supplements (hyperkalemia); lithium (increased levels) + Monitoring Requirements: Blood pressure; serum creatinine and potassium at baseline and periodically; monitor for signs of angioedema + Pregnancy Category: Category D - Contraindicated + Cost Tier: Low (generic available) + " +Metformin," + Generic Name: Metformin + Drug Class: Biguanide + Primary Indication: Type 2 Diabetes + Mechanism of Action: Decreases hepatic glucose production; increases insulin sensitivity + Common Dosage: 500-2000 mg daily (divided doses) + Administration: Oral; take with meals to reduce GI upset + Common Side Effects: Nausea; diarrhea; abdominal discomfort; metallic taste; vitamin B12 deficiency + Serious Side Effects: Lactic acidosis (rare but serious); severe hypoglycemia when combined with insulin/sulfonylureas + Contraindications: Severe renal impairment (eGFR <30); metabolic acidosis; acute heart failure + Drug Interactions: Contrast dyes (hold 48 hours before/after); alcohol (increased lactic acidosis risk) + Monitoring Requirements: Renal function (eGFR) before starting and annually; vitamin B12 levels annually; glucose monitoring + Pregnancy Category: Category B - Generally safe + Cost Tier: Low (generic available) + " +Atorvastatin," + Generic Name: Atorvastatin + Drug Class: Statin (HMG-CoA Reductase Inhibitor) + Primary Indication: Hyperlipidemia and ASCVD prevention + Mechanism of Action: Inhibits cholesterol synthesis in the liver + Common Dosage: 10-80 mg once daily + Administration: Oral; take any time of day with or without food + Common Side Effects: Muscle aches; headache; nausea; diarrhea; elevated liver enzymes + Serious Side Effects: Rhabdomyolysis; liver failure; new-onset diabetes; memory problems + Contraindications: Active liver disease; pregnancy; breastfeeding + Drug Interactions: Strong CYP3A4 inhibitors increase levels (clarithromycin erythromycin); grapefruit juice; fibrates increase myopathy risk + Monitoring Requirements: Lipid panel at baseline and 4-12 weeks; liver enzymes at baseline; monitor for muscle symptoms + Pregnancy Category: Category X - Contraindicated + Cost Tier: Low (generic available) + " +Amlodipine," + Generic Name: Amlodipine + Drug Class: Calcium Channel Blocker (Dihydropyridine) + Primary Indication: Hypertension and Angina + Mechanism of Action: Blocks calcium entry into vascular smooth muscle causing vasodilation + Common Dosage: 2.5-10 mg once daily + Administration: Oral; take with or without food + Common Side Effects: Peripheral edema (ankle swelling); headache; dizziness; flushing; palpitations + Serious Side Effects: Severe hypotension; worsening heart failure; MI (rare) + Contraindications: Severe aortic stenosis; cardiogenic shock + Drug Interactions: CYP3A4 inhibitors increase levels; simvastatin (limit simvastatin to 20 mg daily) + Monitoring Requirements: Blood pressure monitoring; heart rate; signs of peripheral edema + Pregnancy Category: Category C - Use with caution + Cost Tier: Low (generic available) + " +Levothyroxine," + Generic Name: Levothyroxine Sodium + Drug Class: Thyroid Hormone + Primary Indication: Hypothyroidism + Mechanism of Action: Synthetic T4; converted to active T3 in peripheral tissues + Common Dosage: 25-200 mcg once daily (individualized) + Administration: Oral; take on empty stomach 30-60 min before breakfast + Common Side Effects: Weight changes; headache; insomnia; nervousness; heat intolerance + Serious Side Effects: Cardiac arrhythmias; angina; MI; bone loss with excessive doses + Contraindications: Untreated thyrotoxicosis; acute MI; uncorrected adrenal insufficiency + Drug Interactions: Decreases absorption: calcium iron antacids soy PPIs; increases warfarin effect; decreases effect of diabetes medications + Monitoring Requirements: TSH at baseline 6-8 weeks after dose changes then annually; free T4 if indicated; heart rate and blood pressure + Pregnancy Category: Category A - Safe in pregnancy + Cost Tier: Low (generic available) + " +Aspirin," + Generic Name: Acetylsalicylic Acid + Drug Class: NSAID/Antiplatelet + Primary Indication: Pain; fever; cardiovascular disease prevention + Mechanism of Action: Irreversibly inhibits COX-1 and COX-2; inhibits platelet aggregation + Common Dosage: 81-325 mg daily (low-dose); 325-650 mg q4-6h PRN (analgesic) + Administration: Oral; enteric-coated formulations available + Common Side Effects: Dyspepsia; nausea; stomach upset; easy bruising + Serious Side Effects: GI bleeding; hemorrhagic stroke; allergic reactions; Reye's syndrome (children) + Contraindications: Active GI bleeding; hemophilia; aspirin allergy; children with viral infections + Drug Interactions: Anticoagulants (increased bleeding); NSAIDs (increased GI toxicity); methotrexate (increased toxicity) + Monitoring Requirements: No routine monitoring for low-dose; monitor for signs of bleeding; annual CBC if long-term use + Pregnancy Category: Category D in 3rd trimester + Cost Tier: Low (OTC available) + " +Losartan," + Generic Name: Losartan + Drug Class: Angiotensin Receptor Blocker (ARB) + Primary Indication: Hypertension; diabetic nephropathy + Mechanism of Action: Blocks angiotensin II at AT1 receptors causing vasodilation + Common Dosage: 25-100 mg once or twice daily + Administration: Oral; take with or without food + Common Side Effects: Dizziness; upper respiratory infection; fatigue; back pain + Serious Side Effects: Hyperkalemia; acute kidney injury; hypotension; angioedema (rare) + Contraindications: Pregnancy; bilateral renal artery stenosis + Drug Interactions: NSAIDs (reduced efficacy); potassium supplements (hyperkalemia); lithium (increased levels) + Monitoring Requirements: Blood pressure; serum creatinine and potassium at baseline and periodically + Pregnancy Category: Category D - Contraindicated + Cost Tier: Low (generic available) + " +Metoprolol," + Generic Name: Metoprolol + Drug Class: Beta-Blocker (Selective Beta-1) + Primary Indication: Hypertension; angina; heart failure; MI + Mechanism of Action: Blocks beta-1 adrenergic receptors; reduces heart rate and contractility + Common Dosage: 25-200 mg twice daily (tartrate); 25-400 mg daily (succinate ER) + Administration: Oral; take with or at same time relative to meals consistently + Common Side Effects: Fatigue; dizziness; bradycardia; cold extremities; depression + Serious Side Effects: Severe bradycardia; heart block; severe hypotension; bronchospasm; worsening heart failure + Contraindications: Sinus bradycardia; 2nd/3rd degree heart block; cardiogenic shock; severe peripheral arterial disease; untreated pheochromocytoma + Drug Interactions: Calcium channel blockers (bradycardia hypotension); insulin (masks hypoglycemia); CYP2D6 inhibitors increase levels + Monitoring Requirements: Heart rate and blood pressure; EKG if indicated; glucose in diabetics; signs/symptoms of heart failure + Pregnancy Category: Category C - Use with caution + Cost Tier: Low (generic available) + " +Omeprazole," + Generic Name: Omeprazole + Drug Class: Proton Pump Inhibitor (PPI) + Primary Indication: GERD; peptic ulcers; Zollinger-Ellison syndrome + Mechanism of Action: Irreversibly inhibits gastric H+/K+ ATPase (proton pump) + Common Dosage: 20-40 mg once daily + Administration: Oral; take 30-60 min before breakfast; do not crush or chew delayed-release capsules + Common Side Effects: Headache; nausea; diarrhea; abdominal pain; vitamin B12 deficiency + Serious Side Effects: C. difficile infection; bone fractures (long-term use); hypomagnesemia; kidney disease + Contraindications: Hypersensitivity to PPIs; concurrent use with rilpivirine + Drug Interactions: Clopidogrel (reduced activation); warfarin (increased INR); methotrexate (increased levels) + Monitoring Requirements: Magnesium levels if prolonged use or with diuretics/digoxin; vitamin B12 if long-term use; assess need for continued therapy periodically + Pregnancy Category: Category C - Use with caution + Cost Tier: Low (generic and OTC available) + " +Albuterol," + Generic Name: Albuterol Sulfate + Drug Class: Short-Acting Beta-2 Agonist (SABA) + Primary Indication: Asthma; COPD (acute bronchospasm) + Mechanism of Action: Relaxes bronchial smooth muscle by stimulating beta-2 receptors + Common Dosage: 2 puffs (90 mcg/puff) q4-6h PRN; nebulizer 2.5 mg q4-6h PRN + Administration: Inhalation; shake MDI before use; rinse mouth after + Common Side Effects: Tremor; nervousness; tachycardia; palpitations; headache + Serious Side Effects: Paradoxical bronchospasm; severe hypokalemia; cardiac arrhythmias + Contraindications: Hypersensitivity to albuterol or any component + Drug Interactions: Beta-blockers (reduced bronchodilator effect); diuretics (increased hypokalemia); MAO inhibitors (cardiovascular effects) + Monitoring Requirements: Heart rate and blood pressure; frequency of use (>2 times/week indicates poor control); potassium if high doses + Pregnancy Category: Category C - Use with caution + Cost Tier: Low (generic available) + " +Gabapentin," + Generic Name: Gabapentin + Drug Class: Anticonvulsant/Neuropathic Pain Agent + Primary Indication: Neuropathic pain; partial seizures; postherpetic neuralgia + Mechanism of Action: Unknown; structurally related to GABA but doesn't bind GABA receptors + Common Dosage: 300-3600 mg daily in 3 divided doses + Administration: Oral; take with or without food; requires renal dose adjustment + Common Side Effects: Dizziness; somnolence; ataxia; fatigue; peripheral edema + Serious Side Effects: Respiratory depression (with opioids); suicidal thoughts; severe skin reactions (rare) + Contraindications: Hypersensitivity to gabapentin + Drug Interactions: Opioids (increased respiratory depression); antacids (reduced absorption - separate by 2 hours) + Monitoring Requirements: No routine lab monitoring required; assess for suicidal ideation; renal function for dose adjustment + Pregnancy Category: Category C - Use with caution + Cost Tier: Low (generic available) + " +Sertraline," + Generic Name: Sertraline + Drug Class: Selective Serotonin Reuptake Inhibitor (SSRI) + Primary Indication: Depression; anxiety; OCD; PTSD + Mechanism of Action: Inhibits serotonin reuptake in CNS increasing synaptic serotonin + Common Dosage: 25-200 mg once daily + Administration: Oral; take with or without food; morning or evening dosing + Common Side Effects: Nausea; diarrhea; insomnia; sexual dysfunction; increased sweating + Serious Side Effects: Serotonin syndrome; suicidal ideation (especially young adults); bleeding; hyponatremia; seizures + Contraindications: Concurrent use with MAO inhibitors (14 day washout required); concurrent use with pimozide + Drug Interactions: MAO inhibitors (serotonin syndrome); NSAIDs/anticoagulants (increased bleeding); tamoxifen (reduced efficacy); other serotonergic drugs + Monitoring Requirements: Mental status at each visit; suicidal ideation especially first 1-2 months; sodium if risk factors for hyponatremia + Pregnancy Category: Category C - Use with caution + Cost Tier: Low (generic available) + " +Hydrochlorothiazide," + Generic Name: Hydrochlorothiazide (HCTZ) + Drug Class: Thiazide Diuretic + Primary Indication: Hypertension; edema + Mechanism of Action: Inhibits sodium reabsorption in distal tubule increasing urine output + Common Dosage: 12.5-50 mg once daily + Administration: Oral; take in morning to avoid nocturia + Common Side Effects: Hypokalemia; dizziness; muscle cramps; increased urination + Serious Side Effects: Severe electrolyte abnormalities; hypotension; hyperglycemia; hyperuricemia (gout); photosensitivity + Contraindications: Anuria; sulfonamide allergy + Drug Interactions: NSAIDs (reduced antihypertensive effect); lithium (increased levels); digoxin (hypokalemia increases toxicity) + Monitoring Requirements: Blood pressure; electrolytes (sodium potassium) and renal function at baseline and periodically; glucose and lipids + Pregnancy Category: Category B - Generally safe + Cost Tier: Low (generic available) + " +Warfarin," + Generic Name: Warfarin + Drug Class: Vitamin K Antagonist (Anticoagulant) + Primary Indication: DVT/PE; atrial fibrillation; mechanical heart valves + Mechanism of Action: Inhibits vitamin K-dependent clotting factors (II VII IX X) + Common Dosage: Individualized dosing based on INR (typically 2-10 mg daily) + Administration: Oral; take same time daily; consistent vitamin K intake + Common Side Effects: Bleeding; bruising; nausea + Serious Side Effects: Major hemorrhage; skin necrosis; purple toe syndrome + Contraindications: Pregnancy; active major bleeding; severe hypertension; recent surgery + Drug Interactions: EXTENSIVE - antibiotics NSAIDs acetaminophen many drugs affect INR + Monitoring Requirements: INR: daily initially then 2-3 times/week then weekly then every 4 weeks when stable; CBC; assess for bleeding + Pregnancy Category: Category X - Contraindicated + Cost Tier: Low (generic available) + " +Prednisone," + Generic Name: Prednisone + Drug Class: Corticosteroid + Primary Indication: Inflammatory conditions; autoimmune diseases; asthma + Mechanism of Action: Broad anti-inflammatory and immunosuppressive effects + Common Dosage: 5-60 mg daily (dose varies widely by indication) + Administration: Oral; take with food; taper slowly when discontinuing after >2 weeks use + Common Side Effects: Increased appetite; weight gain; insomnia; mood changes; hyperglycemia + Serious Side Effects: Adrenal suppression; infections; osteoporosis; peptic ulcers; cataracts; hyperglycemia + Contraindications: Systemic fungal infections + Drug Interactions: NSAIDs (increased GI bleed risk); vaccines (reduced efficacy live vaccines contraindicated); diabetes drugs (antagonizes effect) + Monitoring Requirements: Blood pressure and glucose regularly especially if diabetic; bone density if long-term use; growth in children; eye exams + Pregnancy Category: Category C - Use with caution + Cost Tier: Low (generic available) + " diff --git a/examples/agent/healthcare-assistant/docs/relational_patient.csv b/examples/agent/healthcare-assistant/docs/relational_patient.csv new file mode 100644 index 00000000..7a10efc7 --- /dev/null +++ b/examples/agent/healthcare-assistant/docs/relational_patient.csv @@ -0,0 +1,31 @@ +patient_id,patient_name,phone_number,address,patient_type,prescription +"P001", "George Rivera", "+1-213-555-0142", "4821 Sunset Blvd, Los Angeles, CA 90027", "inpatient", "Lisinopril 10mg" +"P002", "Theresa Greenleaf", "+1-312-555-0278", "1103 W Armitage Ave, Chicago, IL 60614", "outpatient", "Metformin 500mg" +"P003", "Marcus LaGrange", "+1-512-555-0391", "800 Congress Ave, Suite 300, Austin, TX 78701", "inpatient", "Atorvastatin 10mg" +"P004", "Herbert Richards", "+1-206-555-0467", "2250 Harbor Ave SW, Seattle, WA 98126", "outpatient", "Levothyroxine 25mcg" +"P005", "Joanne Brown", "+1-404-555-0583", "3340 Peachtree Rd NE, Atlanta, GA 30326", "inpatient", "Aspirin 100mg" +"P006", "Gloria Florian", "+1-415-555-0619", "598 Castro St, San Francisco, CA 94114", "outpatient", "Losartan 25mg" +"P007", "Tony Lakewood", "+1-303-555-0734", "1720 S Bellaire St, Denver, CO 80222", "inpatient", "Metoprolol 25mg" +"P008", "Erika Atlantic", "+1-617-555-0852", "200 State St, Boston, MA 02109", "outpatient", "Omeprazole 20mg" +"P009", "Daniel Whitmore", "+1-702-555-0921", "3900 Las Vegas Blvd S, Las Vegas, NV 89119", "inpatient", "Amlodipine 5mg" +"P010", "Patricia Nguyen", "+1-503-555-1037", "1200 SW Morrison St, Portland, OR 97205", "outpatient", "Hydrochlorothiazide 25mg" +"P011", "Robert Chen", "+1-214-555-1148", "2800 Main St, Dallas, TX 75226", "inpatient", "Gabapentin 300mg" +"P012", "Maria Santos", "+1-305-555-1259", "1450 Brickell Ave, Miami, FL 33131", "outpatient", "Sertraline 50mg" +"P013", "James O'Brien", "+1-215-555-1364", "1500 Market St, Philadelphia, PA 19102", "inpatient", "Warfarin 5mg" +"P014", "Linda Patterson", "+1-602-555-1475", "4400 N Central Ave, Phoenix, AZ 85012", "outpatient", "Albuterol 90mcg" +"P015", "Kevin Morrison", "+1-615-555-1586", "501 Broadway, Nashville, TN 37203", "inpatient", "Prednisone 10mg" +"P016", "Susan Keller", "+1-704-555-1697", "300 South Tryon St, Charlotte, NC 28202", "outpatient", "Lisinopril 20mg" +"P017", "Michael Torres", "+1-713-555-1708", "1200 Smith St, Houston, TX 77002", "inpatient", "Metformin 850mg" +"P018", "Angela Brooks", "+1-216-555-1819", "200 Public Sq, Cleveland, OH 44114", "outpatient", "Atorvastatin 20mg" +"P019", "Richard Hammond", "+1-414-555-1920", "777 N Water St, Milwaukee, WI 53202", "inpatient", "Metoprolol 50mg" +"P020", "Catherine Walsh", "+1-801-555-2031", "400 S Main St, Salt Lake City, UT 84111", "outpatient", "Levothyroxine 50mcg" +"P021", "Thomas Nguyen", "+1-901-555-2142", "100 Peabody Pl, Memphis, TN 38103", "inpatient", "Losartan 50mg" +"P022", "Diane Foster", "+1-916-555-2253", "1100 J St, Sacramento, CA 95814", "outpatient", "Omeprazole 40mg" +"P023", "William Hayes", "+1-816-555-2364", "1200 Main St, Kansas City, MO 64105", "inpatient", "Aspirin 81mg" +"P024", "Rachel Kim", "+1-612-555-2475", "350 Nicollet Mall, Minneapolis, MN 55401", "outpatient", "Amlodipine 10mg" +"P025", "Charles Evans", "+1-502-555-2586", "400 W Main St, Louisville, KY 40202", "inpatient", "Hydrochlorothiazide 12.5mg" +"P026", "Emily Rodriguez", "+1-505-555-2697", "500 Marquette Ave NW, Albuquerque, NM 87102", "outpatient", "Gabapentin 600mg" +"P027", "Frank Delaney", "+1-410-555-2708", "100 Light St, Baltimore, MD 21202", "inpatient", "Sertraline 100mg" +"P028", "Helen Park", "+1-317-555-2819", "200 S Meridian St, Indianapolis, IN 46225", "outpatient", "Albuterol 90mcg" +"P029", "Gregory Shaw", "+1-804-555-2920", "1000 E Broad St, Richmond, VA 23219", "inpatient", "Prednisone 20mg" +"P030", "Nancy Collins", "+1-405-555-3031", "200 N Walker Ave, Oklahoma City, OK 73102", "outpatient", "Metformin 1000mg" diff --git a/examples/agent/healthcare-assistant/healthcare-assistant-config.yaml b/examples/agent/healthcare-assistant/healthcare-assistant-config.yaml new file mode 100644 index 00000000..cf2f6cb4 --- /dev/null +++ b/examples/agent/healthcare-assistant/healthcare-assistant-config.yaml @@ -0,0 +1,6 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: healthcare-assistant-config +data: + ENVIRONMENT: "hosted" diff --git a/examples/agent/healthcare-assistant/helpers/__init__.py b/examples/agent/healthcare-assistant/helpers/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/examples/agent/healthcare-assistant/helpers/hallucination_helpers.py b/examples/agent/healthcare-assistant/helpers/hallucination_helpers.py new file mode 100644 index 00000000..7c899fe2 --- /dev/null +++ b/examples/agent/healthcare-assistant/helpers/hallucination_helpers.py @@ -0,0 +1,196 @@ +""" +Hallucination Demo Helpers + +Log intentional hallucinations to Splunk Agent Observability for demos. +Examples are defined in config.yaml under `demo_hallucinations`. +""" +import logging +import os +import uuid +from typing import Any, List, Optional, Union + +from splunk_ao import SplunkAOLogger +from langchain_core.messages import AIMessage, HumanMessage + +logger = logging.getLogger(__name__) + + +def log_hallucination( + project_name: str, + agent_stream: str, + question: str, + context_docs: List[str], + hallucinated_answer: str, + model: str = "gpt-4o", + session_name: str = "Hallucination Demo", + external_session_id: Optional[str] = None, + existing_logger: Optional[Union[SplunkAOLogger, Any]] = None, +) -> bool: + """ + Log a hallucination trace to Splunk AO for demonstration purposes. + + Creates a trace with a retriever span (real context) and an LLM span (wrong answer). + """ + try: + logger.info( + "Logging hallucination to project: %s, agent stream: %s", + project_name, + agent_stream, + ) + + if existing_logger: + logger.info("Using existing Splunk AO session for hallucination demo") + if hasattr(existing_logger, "get_logger_instance"): + splunk_ao_logger = existing_logger.get_logger_instance() + else: + splunk_ao_logger = existing_logger + else: + logger.info("Creating new Splunk AO session for hallucination demo") + splunk_ao_logger = SplunkAOLogger(project=project_name, agent_stream=agent_stream) + splunk_ao_logger.start_session( + name=session_name, + external_id=external_session_id or str(uuid.uuid4()), + ) + + splunk_ao_logger.start_trace( + input=question, + name="Hallucination Demo", + ) + + # Wrap spans in a workflow so the console shows "Hallucination Demo" as the + # root name — splunk-ao OTel converter derives span names from type+model for + # LLM spans, ignoring name=, so a workflow span is needed as the visible root. + splunk_ao_logger.add_workflow_span( + input=question, + name="Hallucination Demo", + ) + + splunk_ao_logger.add_retriever_span( + input=question, + output=context_docs, + name="RAG Retrieval", + duration_ns=int(1.3e8), + status_code=200, + ) + + context_text = "\n\n".join(context_docs) + llm_input = f"""Human: You are a helpful assistant. Given the context below, please answer the following question: + +{context_text} + +Question: {question}""" + + splunk_ao_logger.add_llm_span( + input=llm_input, + output=hallucinated_answer, + model=model, + name="LLM Response", + num_input_tokens=len(llm_input.split()) * 2, + num_output_tokens=len(hallucinated_answer.split()) * 2, + total_tokens=len(llm_input.split()) * 2 + len(hallucinated_answer.split()) * 2, + duration_ns=int(1.2e8), + metadata={"temperature": "0.1", "demo_type": "hallucination"}, + temperature=0.1, + status_code=200, + time_to_first_token_ns=500000, + ) + + # Conclude the workflow span, then the trace + splunk_ao_logger.conclude( + output=hallucinated_answer, + duration_ns=int(2.5e8), + status_code=200, + ) + + splunk_ao_logger.conclude( + output=hallucinated_answer, + duration_ns=int(2.5e8), + status_code=200, + ) + + splunk_ao_logger.flush() + + logger.info("Successfully logged hallucination to project: %s", project_name) + return True + + except Exception as e: + logger.error("Failed to log hallucination: %s", e) + return False + + +def log_demo_hallucination( + config: dict, + hallucination_index: int = 0, + existing_logger: Optional[Union[SplunkAOLogger, Any]] = None, + session_id: Optional[str] = None, +) -> bool: + """Log a demo hallucination from config.yaml to Splunk AO.""" + project_name = os.getenv("SPLUNK_AO_PROJECT", "healthcare-assistant") + agent_stream = os.getenv("SPLUNK_AO_AGENT_STREAM", "default") + + hallucinations = config.get("demo_hallucinations", []) + if not hallucinations: + logger.warning("No hallucination examples defined in config") + return False + + if hallucination_index >= len(hallucinations): + hallucination_index = 0 + + hallucination = hallucinations[hallucination_index] + question = hallucination.get("question", "") + hallucinated_answer = hallucination.get("hallucinated_answer", "") + context_docs = hallucination.get("context", []) + + if not question or not hallucinated_answer: + logger.error("Invalid hallucination config: missing question or answer") + return False + + if not context_docs: + context_docs = ["[No context available]"] + + model_config = config.get("model", {}) + model = model_config.get("default_model", "gpt-4o") + + return log_hallucination( + project_name=project_name, + agent_stream=agent_stream, + question=question, + context_docs=context_docs, + hallucinated_answer=hallucinated_answer, + model=model, + session_name="Healthcare Hallucination Demo", + external_session_id=session_id, + existing_logger=existing_logger, + ) + + +def add_hallucination_interaction_to_chat( + config: dict, + hallucination_index: int = 0, +) -> None: + """Append the demo hallucination Q&A pair to the Streamlit chat history.""" + import streamlit as st + + hallucinations = config.get("demo_hallucinations", []) + if not hallucinations: + return + + if hallucination_index >= len(hallucinations): + hallucination_index = 0 + + hallucination = hallucinations[hallucination_index] + question = hallucination.get("question", "") + answer = hallucination.get("hallucinated_answer", "") + + if not question or not answer: + return + + if "messages" not in st.session_state: + st.session_state.messages = [] + + st.session_state.messages.append( + {"message": HumanMessage(content=question), "agent": "user"} + ) + st.session_state.messages.append( + {"message": AIMessage(content=answer), "agent": "assistant"} + ) diff --git a/examples/agent/healthcare-assistant/helpers/pgvector_utils.py b/examples/agent/healthcare-assistant/helpers/pgvector_utils.py new file mode 100644 index 00000000..cc42ef4a --- /dev/null +++ b/examples/agent/healthcare-assistant/helpers/pgvector_utils.py @@ -0,0 +1,73 @@ +"""Shared PostgreSQL/pgvector utilities for vector storage and retrieval.""" +import os +from typing import Optional, Tuple + +from langchain_openai import OpenAIEmbeddings +from langchain_postgres import PGVector +from sqlalchemy import create_engine, text + + +def get_postgres_connection_string() -> str: + """Build SQLAlchemy connection string from environment variables.""" + host = os.environ.get("POSTGRES_HOST", "localhost") + port = os.environ.get("POSTGRES_PORT", "5432") + user = os.environ.get("POSTGRES_USER", "postgres") + password = os.environ.get("POSTGRES_PASSWORD", "") + database = os.environ.get("POSTGRES_DB", "vectordb") + return f"postgresql+psycopg://{user}:{password}@{host}:{port}/{database}" + + +def get_collection_name(domain_name: str, environment: Optional[str] = None) -> str: + """SQL-safe collection name for a domain/environment pair.""" + env = environment or os.environ.get("ENVIRONMENT", "local") + return f"{domain_name}_{env}_index" + + +def collection_exists(domain_name: str, environment: Optional[str] = None) -> bool: + """Return True if the pgvector collection has been created.""" + collection_name = get_collection_name(domain_name, environment) + engine = create_engine(get_postgres_connection_string()) + with engine.connect() as conn: + row = conn.execute( + text("SELECT 1 FROM langchain_pg_collection WHERE name = :name LIMIT 1"), + {"name": collection_name}, + ).fetchone() + return row is not None + + +def create_pgvector_store( + embeddings: OpenAIEmbeddings, + domain_name: str, + environment: Optional[str] = None, + *, + pre_delete_collection: bool = False, +) -> Tuple[PGVector, str]: + """Create or connect to a PGVector store for the given domain.""" + collection_name = get_collection_name(domain_name, environment) + vector_store = PGVector( + embeddings=embeddings, + collection_name=collection_name, + connection=get_postgres_connection_string(), + use_jsonb=True, + pre_delete_collection=pre_delete_collection, + ) + return vector_store, collection_name + + +def get_pgvector_store( + domain_name: str, + embedding_model: str = "text-embedding-3-large", + environment: Optional[str] = None, +) -> Tuple[PGVector, str]: + """Return a PGVector store for retrieval, raising if the collection is missing.""" + env = environment or os.environ.get("ENVIRONMENT", "local") + collection_name = get_collection_name(domain_name, env) + + if not collection_exists(domain_name, env): + raise ValueError( + f"PostgreSQL collection not found: {collection_name}. " + f"Run: python helpers/setup_vectordb.py {env}" + ) + + embeddings = OpenAIEmbeddings(model=embedding_model) + return create_pgvector_store(embeddings, domain_name, env) diff --git a/examples/agent/healthcare-assistant/helpers/setup_vectordb.py b/examples/agent/healthcare-assistant/helpers/setup_vectordb.py new file mode 100644 index 00000000..efd1e475 --- /dev/null +++ b/examples/agent/healthcare-assistant/helpers/setup_vectordb.py @@ -0,0 +1,116 @@ +""" +Healthcare vector database setup using PostgreSQL/pgvector. + +Usage: + python helpers/setup_vectordb.py local + python helpers/setup_vectordb.py hosted +""" +import argparse +import getpass +import os +import sys +import uuid +from pathlib import Path +from typing import List + +import pandas as pd + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from config import DOCS_DIR, DOMAIN, load_config +from langchain_core.documents import Document +from langchain_openai import OpenAIEmbeddings +from setup_env import setup_environment +from helpers.pgvector_utils import create_pgvector_store, get_collection_name +from helpers.sql_utils import load_domain_relational_csvs + + +def setup_vectordb(environment: str) -> bool: + """Set up vector database and relational tables for the healthcare app.""" + print(f"Setting up vector database for healthcare in {environment} environment") + + setup_environment() + os.environ["ENVIRONMENT"] = environment + + app_config = load_config() + rag_config = app_config.get("rag", {}) + vectorstore_config = app_config.get("vectorstore", {}) + + chunk_size = rag_config.get("chunk_size", 1000) + chunk_overlap = rag_config.get("chunk_overlap", 200) + embedding_model = vectorstore_config.get("embedding_model", "text-embedding-3-large") + + print(f"Using chunk_size: {chunk_size}, chunk_overlap: {chunk_overlap}") + print(f"Using embedding model: {embedding_model}") + + docs_dir = DOCS_DIR + if not docs_dir.exists(): + print(f"❌ Docs directory not found: {docs_dir}") + return False + + if not os.environ.get("OPENAI_API_KEY"): + os.environ["OPENAI_API_KEY"] = getpass.getpass("Enter API key for OpenAI: ") + + if not os.environ.get("POSTGRES_PASSWORD"): + os.environ["POSTGRES_PASSWORD"] = getpass.getpass("Enter PostgreSQL password: ") + + embeddings = OpenAIEmbeddings(model=embedding_model) + collection_name = get_collection_name(DOMAIN, environment) + print(f"Creating PostgreSQL/pgvector collection: {collection_name}") + vector_store, collection_name = create_pgvector_store( + embeddings, + DOMAIN, + environment, + pre_delete_collection=True, + ) + + csv_path = docs_dir / "qa.csv" + df = pd.read_csv(csv_path) + doc_list: List[Document] = [] + uuid_list = [] + for _, row in df.iterrows(): + question = str(row.get("question", "") or "").strip() + answer = str(row.get("answer", "") or "") + body = ( + f"[FAQ] Healthcare FAQ. " + f"Medication: {question}. " + f"Information: {answer}. " + ) + meta = { + "doc_family": "healthcare", + "question": question, + "answer": answer, + } + doc_list.append(Document(page_content=body, metadata=meta)) + uuid_list.append(uuid.uuid4()) + + print("Adding documents to vector store...") + vector_store.add_documents(documents=doc_list, ids=uuid_list) + embedded_count = len(doc_list) + + print("Loading relational tables for healthcare...") + load_domain_relational_csvs(docs_dir, DOMAIN) + + print("✅ Successfully created vector database for healthcare") + print(f"📊 Total documents embedded: {embedded_count}") + print(f"🔗 PostgreSQL collection: {collection_name}") + return True + + +def main(): + parser = argparse.ArgumentParser( + description="Set up PostgreSQL/pgvector for the healthcare assistant" + ) + parser.add_argument( + "environment", + choices=["local", "hosted"], + help="Environment to use ('local' or 'hosted')", + ) + args = parser.parse_args() + + if not setup_vectordb(args.environment): + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/examples/agent/healthcare-assistant/helpers/sql_utils.py b/examples/agent/healthcare-assistant/helpers/sql_utils.py new file mode 100644 index 00000000..5cb138f6 --- /dev/null +++ b/examples/agent/healthcare-assistant/helpers/sql_utils.py @@ -0,0 +1,173 @@ +"""PostgreSQL utilities for relational demo tables and SQL execution.""" +import re +from pathlib import Path +from typing import Any, Dict, List, Optional, Tuple + +import pandas as pd +from sqlalchemy import create_engine, inspect, text +from sqlalchemy.engine import Engine + +from helpers.pgvector_utils import get_postgres_connection_string + + +def relational_table_name(domain_name: str, table_suffix: str) -> str: + """Build a domain-scoped, SQL-safe table name.""" + safe_domain = re.sub(r"[^a-z0-9_]", "_", domain_name.lower()) + safe_suffix = re.sub(r"[^a-z0-9_]", "_", table_suffix.lower()) + return f"{safe_domain}_{safe_suffix}" + + +def parse_relational_csv_name(csv_path: str | Path) -> Optional[str]: + """Extract the table suffix from relational_.csv.""" + stem = Path(csv_path).stem + prefix = "relational_" + if not stem.startswith(prefix): + return None + suffix = stem[len(prefix) :].strip() + return suffix or None + + +def _infer_pg_type(series: pd.Series) -> str: + if pd.api.types.is_integer_dtype(series): + return "BIGINT" + if pd.api.types.is_float_dtype(series): + return "NUMERIC" + return "TEXT" + + +def _guess_primary_key(columns: List[str]) -> str: + for col in columns: + if col.endswith("_id"): + return col + return columns[0] + + +def _sanitize_identifier(name: str) -> str: + return re.sub(r"[^a-zA-Z0-9_]", "_", name) + + +def load_relational_csv( + engine: Engine, + csv_path: str | Path, + domain_name: str, +) -> Tuple[str, int]: + """Load a relational_.csv file into PostgreSQL.""" + csv_path = Path(csv_path) + table_suffix = parse_relational_csv_name(csv_path) + if not table_suffix: + raise ValueError(f"Not a relational CSV file: {csv_path}") + + table_name = relational_table_name(domain_name, table_suffix) + df = pd.read_csv(csv_path, skipinitialspace=True) + if df.empty: + raise ValueError(f"No rows found in {csv_path}") + + for col in df.columns: + if df[col].dtype == object: + df[col] = df[col].astype(str).str.strip() + + columns = [_sanitize_identifier(str(c)) for c in df.columns] + df.columns = columns + pk_col = _guess_primary_key(columns) + + col_defs = [] + for col in columns: + pg_type = _infer_pg_type(df[col]) + col_defs.append(f'"{col}" {pg_type}') + + create_sql = ( + f'CREATE TABLE "{table_name}" (\n ' + + ",\n ".join(col_defs) + + f',\n PRIMARY KEY ("{pk_col}")\n)' + ) + + with engine.begin() as conn: + conn.execute(text(f'DROP TABLE IF EXISTS "{table_name}" CASCADE')) + conn.execute(text(create_sql)) + + column_list = ", ".join(f'"{c}"' for c in columns) + placeholders = ", ".join(f":{col}" for col in columns) + insert_sql = ( + f'INSERT INTO "{table_name}" ({column_list}) ' + f"VALUES ({placeholders})" + ) + conn.execute(text(insert_sql), df.to_dict(orient="records")) + + for col in columns: + if col == pk_col: + continue + conn.execute( + text( + f'CREATE INDEX IF NOT EXISTS "{table_name}_{col}_idx" ' + f'ON "{table_name}" ("{col}")' + ) + ) + + return table_name, len(df) + + +def load_domain_relational_csvs(docs_dir: str | Path, domain_name: str) -> List[Tuple[str, int]]: + """Load every relational_*.csv file in the docs directory.""" + docs_path = Path(docs_dir) + engine = create_engine(get_postgres_connection_string()) + results: List[Tuple[str, int]] = [] + + for csv_path in sorted(docs_path.glob("relational_*.csv")): + table_name, row_count = load_relational_csv(engine, csv_path, domain_name) + print(f"✓ Loaded relational table {table_name} ({row_count} rows) from {csv_path.name}") + results.append((table_name, row_count)) + + return results + + +def get_table_schema_description(engine: Engine, table_name: str) -> str: + """Return a human-readable schema snippet for Text-to-SQL prompts.""" + inspector = inspect(engine) + if table_name not in inspector.get_table_names(): + raise ValueError(f"Table not found: {table_name}") + + pk = inspector.get_pk_constraint(table_name).get("constrained_columns") or [] + lines = [f'Table "{table_name}" columns:'] + for col in inspector.get_columns(table_name): + name = col["name"] + col_type = str(col["type"]) + extras = [] + if name in pk: + extras.append("PRIMARY KEY") + if col.get("nullable") is False and name not in pk: + extras.append("NOT NULL") + suffix = f" ({', '.join(extras)})" if extras else "" + lines.append(f" - {name}: {col_type}{suffix}") + return "\n".join(lines) + + +def _sql_operation(sql: str) -> str: + cleaned = (sql or "").strip().lstrip("(").upper() + for keyword in ("SELECT", "DELETE", "INSERT", "UPDATE"): + if cleaned.startswith(keyword): + return keyword.lower() + return "unknown" + + +def execute_sql(sql: str) -> Dict[str, Any]: + """Execute a SQL statement and return a JSON-serializable result.""" + sql_clean = (sql or "").strip().rstrip(";") + operation = _sql_operation(sql_clean) + engine = create_engine(get_postgres_connection_string()) + + with engine.begin() as conn: + result = conn.execute(text(sql_clean)) + if operation == "select": + rows = [dict(row) for row in result.mappings()] + count = len(rows) + else: + rows = [] + count = result.rowcount + + return { + "sql": sql_clean, + "rows": rows, + "count": count, + "source": "postgres", + "operation": operation, + } diff --git a/examples/agent/healthcare-assistant/helpers/text_to_sql_utils.py b/examples/agent/healthcare-assistant/helpers/text_to_sql_utils.py new file mode 100644 index 00000000..940c9aa5 --- /dev/null +++ b/examples/agent/healthcare-assistant/helpers/text_to_sql_utils.py @@ -0,0 +1,77 @@ +"""Text-to-SQL helpers for patient lookup and delete tools.""" +from typing import Literal + +from langchain_core.messages import HumanMessage, SystemMessage +from langchain_openai import ChatOpenAI +from sqlalchemy import create_engine + +from helpers.pgvector_utils import get_postgres_connection_string +from helpers.sql_utils import get_table_schema_description, relational_table_name + +SqlOperation = Literal["select", "delete"] + + +def _strip_sql_fences(text: str) -> str: + cleaned = (text or "").strip() + if cleaned.startswith("```"): + lines = cleaned.splitlines() + if lines and lines[0].startswith("```"): + lines = lines[1:] + if lines and lines[-1].strip() == "```": + lines = lines[:-1] + cleaned = "\n".join(lines).strip() + return cleaned.rstrip(";") + + +async def generate_sql( + *, + domain_name: str, + table_suffix: str, + id_column: str, + record_id: str, + operation: SqlOperation = "select", + model: str = "gpt-4o-mini", + temperature: float = 0.0, + use_case_identifier: str = "", + use_case_value: str = "", +) -> str: + """Use an LLM to produce a SELECT or DELETE statement for a relational table.""" + table_name = relational_table_name(domain_name, table_suffix) + engine = create_engine(get_postgres_connection_string()) + schema = get_table_schema_description(engine, table_name) + + if operation == "delete": + system_prompt = ( + "You are a PostgreSQL expert. Generate exactly one DELETE statement " + "to remove the requested record. Rules:\n" + f'- Use DELETE FROM "{table_name}" with a WHERE clause on {id_column}.\n' + "- Use only the provided table and columns.\n" + "- Match the identifier exactly (case-sensitive).\n" + "- Do not use JOINs, subqueries, CTEs, RETURNING, or semicolons.\n" + "- Output only the SQL statement with no explanation." + ) + user_prompt = ( + f"{schema}\n\n" + f"Delete request: remove the row where {use_case_identifier} equals '{use_case_value}'." + ) + else: + system_prompt = ( + "You are a PostgreSQL expert. Generate exactly one SELECT statement " + "to answer the user's lookup request. Rules:\n" + "- Use only the provided table and columns.\n" + "- Return all columns for the matching record.\n" + "- Query does not need to use the primary key column.\n" + "- Write the SQL statement to use uppercase: WHERE UPPER(column) = UPPER('value').\n" + "- Do not use JOINs, subqueries, CTEs, or semicolons.\n" + "- Output only the SQL statement with no explanation." + ) + user_prompt = ( + f"{schema}\n\n" + f"Lookup request: {use_case_identifier}='{use_case_value}'\n\n" + ) + + llm = ChatOpenAI(model=model, temperature=temperature) + response = await llm.ainvoke( + [SystemMessage(content=system_prompt), HumanMessage(content=user_prompt)] + ) + return _strip_sql_fences(str(response.content)) diff --git a/examples/agent/healthcare-assistant/k8s.yaml b/examples/agent/healthcare-assistant/k8s.yaml new file mode 100644 index 00000000..6f5181f2 --- /dev/null +++ b/examples/agent/healthcare-assistant/k8s.yaml @@ -0,0 +1,115 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: healthcare-assistant + labels: + app: healthcare-assistant +spec: + replicas: 1 + selector: + matchLabels: + app: healthcare-assistant + template: + metadata: + labels: + app: healthcare-assistant + spec: + containers: + - name: healthcare-assistant + image: ghcr.io/splunk/healthcare-assistant:app-with-instrumentation + imagePullPolicy: Always + ports: + - containerPort: 8501 + name: http + envFrom: + - configMapRef: + name: healthcare-assistant-config + - configMapRef: + name: postgres-config + env: + - name: OPENAI_BASE_URL + valueFrom: + secretKeyRef: + name: openai-secrets + key: base-url + - name: OPENAI_API_KEY + valueFrom: + secretKeyRef: + name: openai-secrets + key: api-key + - name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + name: postgres-credentials + key: POSTGRES_PASSWORD + - name: SPLUNK_AO_REALM + valueFrom: + secretKeyRef: + name: splunk-ao-secrets + key: realm + - name: SPLUNK_AO_O11Y_TOKEN + valueFrom: + secretKeyRef: + name: splunk-ao-secrets + key: o11y-token + - name: SPLUNK_AO_O11Y_API_TOKEN + valueFrom: + secretKeyRef: + name: splunk-ao-secrets + key: o11y-api-token + - name: SPLUNK_AO_PROJECT + valueFrom: + secretKeyRef: + name: splunk-ao-secrets + key: project + - name: SPLUNK_AO_AGENT_STREAM + valueFrom: + secretKeyRef: + name: splunk-ao-secrets + key: agent-stream + resources: + requests: + memory: "512Mi" + cpu: "250m" + limits: + memory: "2Gi" + cpu: "1000m" + securityContext: + allowPrivilegeEscalation: false + runAsNonRoot: true + runAsUser: 1000 + capabilities: + drop: + - ALL + readOnlyRootFilesystem: false +--- +apiVersion: v1 +kind: Service +metadata: + name: healthcare-assistant-service +spec: + selector: + app: healthcare-assistant + ports: + - port: 8501 + protocol: TCP + type: ClusterIP +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: healthcare-assistant-ingress + annotations: + traefik.ingress.kubernetes.io/router.entrypoints: web +spec: + ingressClassName: traefik + rules: + - http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: healthcare-assistant-service + port: + number: 8501 diff --git a/examples/agent/healthcare-assistant/postgres.yaml b/examples/agent/healthcare-assistant/postgres.yaml new file mode 100644 index 00000000..f36db05c --- /dev/null +++ b/examples/agent/healthcare-assistant/postgres.yaml @@ -0,0 +1,100 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: postgres-config +data: + POSTGRES_HOST: "postgres" + POSTGRES_PORT: "5432" + POSTGRES_USER: "postgres" + POSTGRES_DB: "vectordb" +--- +apiVersion: v1 +kind: Secret +metadata: + name: postgres-credentials +type: Opaque +stringData: + POSTGRES_PASSWORD: "mypassword" +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: postgres-init +data: + init.sql: | + CREATE EXTENSION IF NOT EXISTS vector; +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: postgres-pvc +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 5Gi +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: postgres + labels: + app: postgres +spec: + replicas: 1 + selector: + matchLabels: + app: postgres + template: + metadata: + labels: + app: postgres + spec: + containers: + - name: postgres + image: pgvector/pgvector:pg16 + ports: + - containerPort: 5432 + name: postgres + envFrom: + - configMapRef: + name: postgres-config + env: + - name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + name: postgres-credentials + key: POSTGRES_PASSWORD + volumeMounts: + - name: postgres-data + mountPath: /var/lib/postgresql/data + - name: postgres-init + mountPath: /docker-entrypoint-initdb.d + resources: + requests: + memory: "256Mi" + cpu: "250m" + limits: + memory: "1Gi" + cpu: "500m" + volumes: + - name: postgres-data + persistentVolumeClaim: + claimName: postgres-pvc + - name: postgres-init + configMap: + name: postgres-init +--- +apiVersion: v1 +kind: Service +metadata: + name: postgres +spec: + selector: + app: postgres + ports: + - port: 5432 + protocol: TCP + targetPort: 5432 + type: ClusterIP diff --git a/examples/agent/healthcare-assistant/rag.py b/examples/agent/healthcare-assistant/rag.py new file mode 100644 index 00000000..aaa3e07d --- /dev/null +++ b/examples/agent/healthcare-assistant/rag.py @@ -0,0 +1,133 @@ +"""RAG retrieval for the healthcare assistant using PostgreSQL/pgvector.""" +import asyncio +import os +from typing import Optional + +from langchain_classic.chains import create_retrieval_chain +from langchain_classic.chains.combine_documents import create_stuff_documents_chain +from langchain_core.prompts import ChatPromptTemplate, MessagesPlaceholder +from langchain_core.tools import tool +from langchain_openai import ChatOpenAI, OpenAIEmbeddings + +from config import DOMAIN, load_config +from helpers.pgvector_utils import collection_exists, create_pgvector_store +from setup_env import setup_environment + +_rag_cache = {} + + +class HealthcareRAGSystem: + """RAG system with eager initialization.""" + + def __init__(self, top_k: int = 5, model_name: Optional[str] = None): + self.top_k = top_k + self.model_name = model_name + self.retrieval_chain = None + self._initialized = False + self.initialize() + + def initialize(self): + if self._initialized: + return + + try: + app_config = load_config() + rag_config = app_config.get("rag", {}) + vectorstore_config = app_config.get("vectorstore", {}) + model_config = app_config.get("model", {}) + + embedding_model = vectorstore_config.get("embedding_model", "text-embedding-3-large") + llm_model = ( + self.model_name + or model_config.get("default_model") + or model_config.get("model_name", "gpt-4o") + ) + + setup_environment() + environment = os.environ.get("ENVIRONMENT", "local") + + if not os.environ.get("POSTGRES_PASSWORD"): + raise ValueError( + "POSTGRES_PASSWORD not found. Please add it to .streamlit/secrets.toml" + ) + + if not collection_exists(DOMAIN, environment): + collection_name = f"{DOMAIN}_{environment}_index" + raise ValueError( + f"PostgreSQL collection not found: {collection_name}. " + f"Please run: python helpers/setup_vectordb.py {environment}" + ) + + embeddings = OpenAIEmbeddings(model=embedding_model) + vector_store, _ = create_pgvector_store(embeddings, DOMAIN, environment) + retriever = vector_store.as_retriever(search_kwargs={"k": self.top_k}) + + llm = ChatOpenAI( + model=llm_model, + temperature=0.1, + name="Healthcare RAG Assistant", + ) + + retrieval_qa_chat_prompt = ChatPromptTemplate.from_messages( + [ + ( + "system", + "Answer any use questions based solely on the context below:\n\n" + "\n{context}\n", + ), + MessagesPlaceholder("chat_history", optional=True), + ("human", "{input}"), + ] + ) + combine_docs_chain = create_stuff_documents_chain(llm, retrieval_qa_chat_prompt) + self.retrieval_chain = create_retrieval_chain(retriever, combine_docs_chain) + self._initialized = True + print(f"✅ RAG system initialized (model: {llm_model})") + except Exception as e: + print(f"❌ Error initializing RAG system: {e}") + import traceback + + traceback.print_exc() + self._initialized = False + + async def search(self, query: str) -> str: + if not self.retrieval_chain: + return ( + "❌ RAG system not initialized. " + "Please check your vector database setup." + ) + + try: + result = await asyncio.to_thread( + self.retrieval_chain.invoke, {"input": query} + ) + return result["answer"] + except Exception as e: + return f"❌ Error during RAG search: {str(e)}" + + +def get_rag_system(top_k: int | None = None, model_name: Optional[str] = None) -> HealthcareRAGSystem: + if top_k is None: + app_config = load_config() + top_k = app_config.get("rag", {}).get("top_k", 5) + + cache_key = f"{top_k}_{model_name or 'default'}" + if cache_key not in _rag_cache: + _rag_cache[cache_key] = HealthcareRAGSystem(top_k, model_name=model_name) + return _rag_cache[cache_key] + + +def create_rag_tool(top_k: int | None = None, model_name: Optional[str] = None): + """Create a LangChain retrieval chain tool for the agent.""" + rag_system = get_rag_system(top_k, model_name=model_name) + + @tool + async def retrieve_healthcare_documents(query: str) -> str: + """Retrieve information related to a query from the healthcare knowledge base.""" + return await rag_system.search(query) + + retrieve_healthcare_documents.name = "retrieve_healthcare_documents" + retrieve_healthcare_documents.description = ( + "Retrieve information from the healthcare knowledge base" + ) + return retrieve_healthcare_documents diff --git a/examples/agent/healthcare-assistant/requirements.txt b/examples/agent/healthcare-assistant/requirements.txt new file mode 100644 index 00000000..c4fc51e9 --- /dev/null +++ b/examples/agent/healthcare-assistant/requirements.txt @@ -0,0 +1,16 @@ +streamlit +openai +python-dotenv +langchain +langchain-core +langchain-openai +langgraph +langchain-postgres +psycopg[binary] +langchain-text-splitters +langchain-community +langchain-classic +pyyaml +toml +pandas +splunk-ao diff --git a/examples/agent/healthcare-assistant/setup-job.yaml b/examples/agent/healthcare-assistant/setup-job.yaml new file mode 100644 index 00000000..a7ecd3d1 --- /dev/null +++ b/examples/agent/healthcare-assistant/setup-job.yaml @@ -0,0 +1,55 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: vectordb-setup +spec: + backoffLimit: 4 + template: + spec: + restartPolicy: OnFailure + initContainers: + - name: wait-for-postgres + image: pgvector/pgvector:pg16 + command: + - sh + - -c + - | + until pg_isready -h $POSTGRES_HOST -p $POSTGRES_PORT -U $POSTGRES_USER; do + echo "Waiting for postgres..."; sleep 2 + done + envFrom: + - configMapRef: + name: postgres-config + containers: + - name: setup + image: ghcr.io/splunk/healthcare-assistant:base-app + imagePullPolicy: Always + command: ["python", "helpers/setup_vectordb.py", "hosted"] + envFrom: + - configMapRef: + name: postgres-config + - configMapRef: + name: healthcare-assistant-config + env: + - name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + name: postgres-credentials + key: POSTGRES_PASSWORD + - name: OPENAI_API_KEY + valueFrom: + secretKeyRef: + name: openai-api + key: openai-api-key + - name: OPENAI_BASE_URL + valueFrom: + secretKeyRef: + name: openai-api + key: openai-api-endpoint + resources: + requests: + memory: "256Mi" + cpu: "250m" + limits: + memory: "1Gi" + cpu: "500m" diff --git a/examples/agent/healthcare-assistant/setup_env.py b/examples/agent/healthcare-assistant/setup_env.py new file mode 100644 index 00000000..610b592e --- /dev/null +++ b/examples/agent/healthcare-assistant/setup_env.py @@ -0,0 +1,28 @@ +"""Validate required environment variables are set.""" +import os + +REQUIRED_ENV_VARS = [ + "OPENAI_API_KEY", + "OPENAI_BASE_URL", + "POSTGRES_HOST", + "POSTGRES_PORT", + "POSTGRES_USER", + "POSTGRES_PASSWORD", + "POSTGRES_DB", + "ENVIRONMENT", + "SPLUNK_AO_REALM", + "SPLUNK_AO_O11Y_TOKEN", + "SPLUNK_AO_PROJECT", + "SPLUNK_AO_AGENT_STREAM", +] + +def setup_environment(): + missing = [var for var in REQUIRED_ENV_VARS if not os.getenv(var)] + for var in missing: + print(f"⚠️ {var} not set") + if not missing: + print("🔧 Environment setup complete") + + +if __name__ == "__main__": + setup_environment() diff --git a/examples/agent/healthcare-assistant/start_vectordb.sh b/examples/agent/healthcare-assistant/start_vectordb.sh new file mode 100644 index 00000000..5394750b --- /dev/null +++ b/examples/agent/healthcare-assistant/start_vectordb.sh @@ -0,0 +1,2 @@ +#!/bin/bash +python helpers/setup_vectordb.py local diff --git a/examples/agent/healthcare-assistant/system_prompt.json b/examples/agent/healthcare-assistant/system_prompt.json new file mode 100644 index 00000000..981d705f --- /dev/null +++ b/examples/agent/healthcare-assistant/system_prompt.json @@ -0,0 +1,3 @@ +{ + "system_prompt": "You are a knowledgeable call center assistant for an Online Healthcare system, supporting patients and internal staff.\n\nYou have three tools available:\n- search_medicine_qa: Use this to answer any questions about medicine — dosage, side effects, interactions, etc.\n- get_patient_info: Use this to look up a patient's details by their patient ID, including their name, address, phone number, patient type, and prescription.\n- delete_patient_record: Use this only when the user explicitly asks to delete or remove a patient record. Requires the patient ID.\n\nGuidelines:\n- For medicine questions, always call search_medicine_qa to retrieve accurate information before answering.\n- For patient lookups, call get_patient_info with the patient ID. Users can ask for information on all patients; that is acceptable since there are only 30 patients.\n- For delete requests, call delete_patient_record with the patient ID. Do not use get_patient_info for deletions.\n- Be professional, concise, and empathetic \n- If you cannot find an answer, say so clearly and offer to escalate to a senior support agent. Do no ever provide personal data from our doctors, including full name, phone number, address, etc. Sharing patient data is allowed, not doctor data. Refuse to provide any information about doctors or their personal data." +} diff --git a/examples/agent/healthcare-assistant/tools/__init__.py b/examples/agent/healthcare-assistant/tools/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/examples/agent/healthcare-assistant/tools/logic.py b/examples/agent/healthcare-assistant/tools/logic.py new file mode 100644 index 00000000..78244576 --- /dev/null +++ b/examples/agent/healthcare-assistant/tools/logic.py @@ -0,0 +1,161 @@ +""" +Healthcare domain tools. + +- get_patient_info: Text-to-SQL lookup against the patient registry in PostgreSQL +- delete_patient_record: Text-to-SQL delete against the patient registry in PostgreSQL +- search_medicine_qa: semantic vector search against the QA knowledge base +""" +import json +import logging +from typing import Optional, Tuple + +from langchain_postgres import PGVector + +from config import DOMAIN, load_config +from helpers.pgvector_utils import get_pgvector_store +from helpers.sql_utils import execute_sql, relational_table_name +from helpers.text_to_sql_utils import generate_sql +from rag import get_rag_system + +_TABLE_SUFFIX = "patient" +_ID_COLUMN = "patient_id" + +_vector_store: Optional[PGVector] = None +_embedding_model: Optional[str] = None +_collection_name_cached: Optional[str] = None + + +def _get_vector_store() -> Tuple[PGVector, str]: + global _vector_store, _embedding_model, _collection_name_cached + + app_config = load_config() + embedding_model = ( + app_config.get("vectorstore", {}).get("embedding_model") or "text-embedding-3-large" + ) + + if ( + _vector_store is not None + and _collection_name_cached is not None + and _embedding_model == embedding_model + ): + return _vector_store, _collection_name_cached + + _vector_store, collection_name = get_pgvector_store(DOMAIN, embedding_model) + _embedding_model = embedding_model + _collection_name_cached = collection_name + return _vector_store, collection_name + + +async def _execute_patient_sql(sql: str) -> str: + """Execute a SQL lookup against the patient registry.""" + try: + result = execute_sql(sql) + return json.dumps(result) + except Exception as e: + return json.dumps({"error": str(e), "sql": sql}) + + +async def _execute_patient_delete_sql(sql: str) -> str: + """Execute a SQL delete against the patient registry.""" + try: + result = execute_sql(sql) + return json.dumps(result) + except Exception as e: + return json.dumps({"error": str(e), "sql": sql}) + + +async def get_patient_info(patient_id: str) -> str: + """Retrieve patient information by their patient ID.""" + patient_id = patient_id.strip().upper() + q = (patient_id or "").strip() + if not q: + return json.dumps({"error": "patient_id is required"}) + + app_config = load_config() + model = app_config.get("model", {}).get("default_model", "gpt-4o-mini") + table_name = relational_table_name(DOMAIN, _TABLE_SUFFIX) + + try: + sql = await generate_sql( + domain_name=DOMAIN, + table_suffix=_TABLE_SUFFIX, + id_column=_ID_COLUMN, + record_id=q, + operation="select", + model=model, + use_case_identifier="patient_id", + use_case_value=patient_id, + ) + except Exception as e: + return json.dumps({"error": str(e), "patient_id": q}) + + raw = await _execute_patient_sql(sql) + try: + result = json.loads(raw) + except json.JSONDecodeError: + result = {"error": "Invalid SQL execution response", "raw": raw} + + if "error" not in result: + result["query"] = q + result["table"] = table_name + + return json.dumps(result) + + +async def delete_patient_record(patient_id: str) -> str: + """Permanently delete a patient record from the registry by patient ID.""" + patient_id = patient_id.strip().upper() + q = (patient_id or "").strip() + if not q: + return json.dumps({"error": "patient_id is required"}) + + app_config = load_config() + model = app_config.get("model", {}).get("default_model", "gpt-4o-mini") + table_name = relational_table_name(DOMAIN, _TABLE_SUFFIX) + + try: + sql = await generate_sql( + domain_name=DOMAIN, + table_suffix=_TABLE_SUFFIX, + id_column=_ID_COLUMN, + record_id=q, + operation="delete", + model=model, + use_case_identifier="patient_id", + use_case_value=patient_id, + ) + except Exception as e: + return json.dumps({"error": str(e), "patient_id": q}) + + raw = await _execute_patient_delete_sql(sql) + try: + result = json.loads(raw) + except json.JSONDecodeError: + result = {"error": "Invalid SQL execution response", "raw": raw} + + if "error" not in result: + result["query"] = q + result["table"] = table_name + + return json.dumps(result) + + +async def search_medicine_qa(query: str) -> str: + """Search the Medicine knowledge base using semantic vector search.""" + q = query + try: + _get_vector_store() + except Exception as e: + return json.dumps({"error": str(e), "query": q}) + + try: + rag_system = get_rag_system(top_k=1) + raw = await rag_system.search(q) + except Exception as e: + logging.exception("search_medicine_qa search failed") + return json.dumps({"error": str(e), "query": q}) + + return json.dumps([raw]) + + +TOOLS = [get_patient_info, delete_patient_record, search_medicine_qa] diff --git a/examples/agent/healthcare-assistant/tools/schema.json b/examples/agent/healthcare-assistant/tools/schema.json new file mode 100644 index 00000000..bad8d897 --- /dev/null +++ b/examples/agent/healthcare-assistant/tools/schema.json @@ -0,0 +1,44 @@ +[ + { + "name": "get_patient_info", + "description": "Retrieve patient's details by their ID, including name, address, phone number, patient type, and prescription", + "parameters": { + "type": "object", + "properties": { + "patient_id": { + "type": "string", + "description": "The patient's unique identifier (e.g., 'P001', 'P005')" + } + }, + "required": ["patient_id"] + } + }, + { + "name": "delete_patient_record", + "description": "Permanently delete a patient record from the registry by their patient ID", + "parameters": { + "type": "object", + "properties": { + "patient_id": { + "type": "string", + "description": "The patient's unique identifier to delete (e.g., 'P001', 'P005')" + } + }, + "required": ["patient_id"] + } + }, + { + "name": "search_medicine_qa", + "description": "Search the Medicine knowledge base to answer questions about medications, including dosage, side effects, and interactions", + "parameters": { + "type": "object", + "properties": { + "query": { + "type": "string", + "description": "The question or topic to search for (e.g., 'Lisinopril', 'Metformin', 'Aspirin')" + } + }, + "required": ["query"] + } + } + ] diff --git a/examples/agent/healthcare-assistant/validate_traces.py b/examples/agent/healthcare-assistant/validate_traces.py new file mode 100644 index 00000000..ab4f1678 --- /dev/null +++ b/examples/agent/healthcare-assistant/validate_traces.py @@ -0,0 +1,42 @@ +""" +Validation script — fires one LLM trace per env and reports success/failure. + + A: original galileo SDK → agentstream: healthcare-galileo (.env.galileo) + B: splunk-ao SDK (standalone) → agentstream: healthcare-splunk-ao (.env.splunk-ao-standalone) + C: splunk-ao SDK (O11y/realm) → agentstream: healthcare-assistant (.env.local) + +Usage (from 2-app-with-instrumentation/, venv active): + python validate_traces.py # all three + python validate_traces.py a # just env A + python validate_traces.py b # just env B + python validate_traces.py c # just env C +""" +import os +import sys +import subprocess +from pathlib import Path + +RUNNER = Path(__file__).parent / "_validate_single.py" +RUNNER_GALILEO = Path(__file__).parent / "_validate_galileo.py" + +ENVS = { + "a": (None, "A — galileo SDK → healthcare-galileo", ".env.galileo"), + "b": (".env.splunk-ao-standalone", "B — splunk-ao standalone → healthcare-splunk-ao", None), + "c": (".env.local", "C — splunk-ao O11y/realm → healthcare-assistant", None), +} + +if __name__ == "__main__": + targets = [sys.argv[1].lower()] if len(sys.argv) > 1 else ["a", "b", "c"] + for t in targets: + env_file, label, galileo_env = ENVS[t] + print(f"\n{'='*60}") + print(f" ENV {label}") + print(f"{'='*60}") + if t == "a": + cmd = [sys.executable, str(RUNNER_GALILEO)] + else: + cmd = [sys.executable, str(RUNNER), env_file] + result = subprocess.run(cmd, cwd=Path(__file__).parent) + if result.returncode != 0: + print(f" !! FAILED (exit {result.returncode})") + print("\nDone — check erden-framework-testing project in AO console.") From 8aae00fb0970bb0ba85d6d197624ee8c6defeadf Mon Sep 17 00:00:00 2001 From: etserend Date: Mon, 24 Aug 2026 19:12:11 -0500 Subject: [PATCH 2/6] feat(examples): migrate healthcare-assistant to splunk-ao Python SDK - Replace galileo imports with splunk_ao throughout all modules - Add create_chat_llm() / create_embeddings() factory functions in config.py to support both Azure OpenAI and direct OpenAI via AZURE_OPENAI_ENDPOINT - Use splunk_ao_context + SplunkAOAsyncCallback for LangChain instrumentation - Fix hallucination_helpers: set_session() instead of start_session() to skip CRUD REST call (no SPLUNK_AO_O11Y_API_TOKEN required) - Fix agent-with-instrumentation: wrap start_session() in try/except so CRUD failures are non-fatal - Fix Dockerfile: use COPY . /app/ instead of workshop-relative paths - Add .gitignore (.env.* pattern), .env.example, and pyproject.toml (uv workspace pointing to local splunk-ao source for dev) Co-Authored-By: Claude Opus 4.7 --- .../agent/healthcare-assistant/.env.example | 37 +++++++++++++++++++ .../agent/healthcare-assistant/.gitignore | 6 +++ .../agent/healthcare-assistant/Dockerfile | 19 ++-------- .../agent-with-instrumentation.py | 10 +++-- examples/agent/healthcare-assistant/agent.py | 32 ++++------------ examples/agent/healthcare-assistant/app.py | 10 ----- examples/agent/healthcare-assistant/config.py | 32 ++++++++++++++++ .../helpers/hallucination_helpers.py | 5 +-- .../helpers/setup_vectordb.py | 7 ++-- .../agent/healthcare-assistant/pyproject.toml | 35 ++++++++++++++++++ examples/agent/healthcare-assistant/rag.py | 12 ++---- 11 files changed, 134 insertions(+), 71 deletions(-) create mode 100644 examples/agent/healthcare-assistant/.env.example create mode 100644 examples/agent/healthcare-assistant/.gitignore create mode 100644 examples/agent/healthcare-assistant/pyproject.toml diff --git a/examples/agent/healthcare-assistant/.env.example b/examples/agent/healthcare-assistant/.env.example new file mode 100644 index 00000000..9a9e74c4 --- /dev/null +++ b/examples/agent/healthcare-assistant/.env.example @@ -0,0 +1,37 @@ +# LLM provider — set ONE of the two blocks below + +# Option A: OpenAI +# OPENAI_API_KEY= + +# Option B: Azure OpenAI +AZURE_OPENAI_API_KEY= +AZURE_OPENAI_ENDPOINT= # e.g. https://your-resource.openai.azure.com/ +AZURE_OPENAI_API_VERSION= # e.g. 2024-12-01-preview +AZURE_CHAT_DEPLOYMENT= # deployment name for chat, e.g. gpt-4o-mini +AZURE_EMBEDDING_DEPLOYMENT= # deployment name for embeddings, e.g. text-embedding-3-large + +# Splunk AO environment variables — set ONE of the two deployment blocks below + +# Option A: Splunk Observability (O11y) Cloud +SPLUNK_AO_REALM= # e.g. us0, eu0, lab0 +SPLUNK_AO_O11Y_TOKEN= # O11y ingest token (required for telemetry) +# SPLUNK_AO_O11Y_API_TOKEN= # Optional: dedicated API token for CRUD operations + +# Option B: On-premises / standalone deployment +# SPLUNK_AO_API_KEY= +# SPLUNK_AO_CONSOLE_URL= # e.g. https://console.yourcompany.com +# SPLUNK_AO_API_ENDPOINT= # Optional, only set for custom deployments + +# Routing (shared by both deployments) +SPLUNK_AO_PROJECT= +SPLUNK_AO_AGENT_STREAM= + +# PostgreSQL (pgvector) +POSTGRES_HOST=localhost +POSTGRES_PORT=5432 +POSTGRES_USER=postgres +POSTGRES_PASSWORD= +POSTGRES_DB=vectordb + +OTEL_SERVICE_NAME=healthcare-assistant +OTEL_RESOURCE_ATTRIBUTES=deployment.environment.name=your-project-name diff --git a/examples/agent/healthcare-assistant/.gitignore b/examples/agent/healthcare-assistant/.gitignore new file mode 100644 index 00000000..bacc7f06 --- /dev/null +++ b/examples/agent/healthcare-assistant/.gitignore @@ -0,0 +1,6 @@ +.env.* +!.env.example +.venv/ +__pycache__/ +*.pyc +.streamlit/secrets.toml diff --git a/examples/agent/healthcare-assistant/Dockerfile b/examples/agent/healthcare-assistant/Dockerfile index 1718a64e..1b7f2d68 100644 --- a/examples/agent/healthcare-assistant/Dockerfile +++ b/examples/agent/healthcare-assistant/Dockerfile @@ -1,41 +1,28 @@ -# Multi-stage build for Healthcare Assistant FROM python:3.12-slim AS builder -# Set working directory WORKDIR /app RUN pip install uv -# Copy requirements and install dependencies -COPY 2-app-with-instrumentation/requirements.txt /app/ +COPY requirements.txt /app/ RUN uv pip install --system --no-cache -r requirements.txt -# Final stage FROM python:3.12-slim WORKDIR /app -# Install runtime dependencies -RUN apt-get update && apt-get install -y \ - curl \ - && rm -rf /var/lib/apt/lists/* +RUN apt-get update && apt-get install -y --no-install-recommends curl && rm -rf /var/lib/apt/lists/* -# Copy installed packages from builder COPY --from=builder /usr/local/lib/python3.12/site-packages /usr/local/lib/python3.12/site-packages COPY --from=builder /usr/local/bin /usr/local/bin -# Copy application code and data -COPY 2-app-with-instrumentation/ /app/ -COPY docs/ /app/docs/ +COPY . /app/ -# Create non-root user RUN useradd --create-home --shell /bin/bash app && \ chown -R app:app /app USER app -# Expose port for Streamlit EXPOSE 8501 -# Run the server CMD ["streamlit", "run", "app.py"] diff --git a/examples/agent/healthcare-assistant/agent-with-instrumentation.py b/examples/agent/healthcare-assistant/agent-with-instrumentation.py index c7e7225e..326672a5 100644 --- a/examples/agent/healthcare-assistant/agent-with-instrumentation.py +++ b/examples/agent/healthcare-assistant/agent-with-instrumentation.py @@ -8,13 +8,12 @@ from langchain_core.messages import AIMessage, BaseMessage, HumanMessage, SystemMessage from langchain_core.tools import StructuredTool -from langchain_openai import ChatOpenAI from langgraph.graph import START, StateGraph from langgraph.graph.state import CompiledStateGraph from langgraph.graph.message import add_messages from langgraph.prebuilt import ToolNode, tools_condition -from config import TOOLS_DIR, load_config, load_system_prompt +from config import TOOLS_DIR, load_config, load_system_prompt, create_chat_llm from rag import create_rag_tool from tools import logic as tools_logic @@ -105,7 +104,7 @@ def _build_graph(self) -> CompiledStateGraph: ) temperature = model_config.get("temperature", 0.1) - llm_with_tools = ChatOpenAI( + llm_with_tools = create_chat_llm( model=effective_model, temperature=temperature, name="Healthcare Assistant", @@ -142,7 +141,10 @@ async def _process_query_async(self, messages: List[Dict[str, str]]) -> str: project=os.getenv("SPLUNK_AO_PROJECT"), agent_stream=os.getenv("SPLUNK_AO_AGENT_STREAM"), ): - splunk_ao_context.start_session(external_id=self.session_id) + try: + splunk_ao_context.start_session(external_id=self.session_id) + except Exception as e: + print(f"[WARN] Session CRUD failed (non-fatal): {e}") # One callback per request keeps each user turn in its own trace. callback = SplunkAOAsyncCallback() diff --git a/examples/agent/healthcare-assistant/agent.py b/examples/agent/healthcare-assistant/agent.py index c7e7225e..70c59f9f 100644 --- a/examples/agent/healthcare-assistant/agent.py +++ b/examples/agent/healthcare-assistant/agent.py @@ -1,20 +1,18 @@ """LangGraph agent for the healthcare assistant.""" import asyncio import inspect -import json import uuid from concurrent.futures import ThreadPoolExecutor from typing import Annotated, List, Dict, Optional, TypedDict from langchain_core.messages import AIMessage, BaseMessage, HumanMessage, SystemMessage from langchain_core.tools import StructuredTool -from langchain_openai import ChatOpenAI from langgraph.graph import START, StateGraph from langgraph.graph.state import CompiledStateGraph from langgraph.graph.message import add_messages from langgraph.prebuilt import ToolNode, tools_condition -from config import TOOLS_DIR, load_config, load_system_prompt +from config import load_config, load_system_prompt, create_chat_llm from rag import create_rag_tool from tools import logic as tools_logic @@ -54,29 +52,12 @@ def __init__( self.langgraph_config = {"configurable": {"thread_id": self.session_id}} def load_tools(self) -> None: - tool_schema_path = TOOLS_DIR / "schema.json" - with tool_schema_path.open(encoding="utf-8") as f: - tool_schema = json.load(f) - self.tools = [] for tool_func in tools_logic.TOOLS: - tool_schema_dict = next( - (schema for schema in tool_schema if schema.get("name") == tool_func.__name__), - None, - ) - tool_kwargs = { - "name": tool_func.__name__, - "description": ( - tool_schema_dict.get("description") - if tool_schema_dict - else tool_func.__doc__ or f"Tool: {tool_func.__name__}" - ), - "args_schema": tool_schema_dict.get("parameters") if tool_schema_dict else None, - } if inspect.iscoroutinefunction(tool_func): - langchain_tool = StructuredTool.from_function(coroutine=tool_func, **tool_kwargs) + langchain_tool = StructuredTool.from_function(coroutine=tool_func) else: - langchain_tool = StructuredTool.from_function(func=tool_func, **tool_kwargs) + langchain_tool = StructuredTool.from_function(func=tool_func) self.tools.append(langchain_tool) rag_config = self.config.get("rag", {}) @@ -105,7 +86,7 @@ def _build_graph(self) -> CompiledStateGraph: ) temperature = model_config.get("temperature", 0.1) - llm_with_tools = ChatOpenAI( + llm_with_tools = create_chat_llm( model=effective_model, temperature=temperature, name="Healthcare Assistant", @@ -142,7 +123,10 @@ async def _process_query_async(self, messages: List[Dict[str, str]]) -> str: project=os.getenv("SPLUNK_AO_PROJECT"), agent_stream=os.getenv("SPLUNK_AO_AGENT_STREAM"), ): - splunk_ao_context.start_session(external_id=self.session_id) + try: + splunk_ao_context.start_session(external_id=self.session_id) + except Exception as e: + print(f"[WARN] Session CRUD failed (non-fatal): {e}") # One callback per request keeps each user turn in its own trace. callback = SplunkAOAsyncCallback() diff --git a/examples/agent/healthcare-assistant/app.py b/examples/agent/healthcare-assistant/app.py index 9ea1ea4f..b1021ba4 100644 --- a/examples/agent/healthcare-assistant/app.py +++ b/examples/agent/healthcare-assistant/app.py @@ -17,16 +17,6 @@ load_dotenv() -# Inject api-version for Azure APIM — openai 3.x non-Azure client doesn't append it. -import openai as _openai_module -_orig_async_init = _openai_module.AsyncOpenAI.__init__ -def _patched_async_init(self, *args, **kwargs): - dq = dict(kwargs.pop("default_query", None) or {}) - dq.setdefault("api-version", os.getenv("OPENAI_API_VERSION", "2024-12-01-preview")) - kwargs["default_query"] = dq - _orig_async_init(self, *args, **kwargs) -_openai_module.AsyncOpenAI.__init__ = _patched_async_init - if not os.getenv("_ENV_LOADED"): setup_environment() os.environ["_ENV_LOADED"] = "true" diff --git a/examples/agent/healthcare-assistant/config.py b/examples/agent/healthcare-assistant/config.py index 5d5f9e8f..3f44a8e8 100644 --- a/examples/agent/healthcare-assistant/config.py +++ b/examples/agent/healthcare-assistant/config.py @@ -1,5 +1,7 @@ """Load healthcare app configuration from YAML and JSON files.""" +import os from pathlib import Path +from typing import Any import yaml @@ -22,3 +24,33 @@ def load_system_prompt() -> str: with SYSTEM_PROMPT_PATH.open(encoding="utf-8") as f: data = json.load(f) return data["system_prompt"] + + +def create_chat_llm(model: str, temperature: float = 0.1, **kwargs: Any): + """Return AzureChatOpenAI when AZURE_OPENAI_ENDPOINT is set, otherwise ChatOpenAI.""" + if os.environ.get("AZURE_OPENAI_ENDPOINT"): + from langchain_openai import AzureChatOpenAI + return AzureChatOpenAI( + azure_deployment=model, + azure_endpoint=os.environ["AZURE_OPENAI_ENDPOINT"], + api_key=os.environ.get("AZURE_OPENAI_API_KEY"), + api_version=os.environ.get("AZURE_OPENAI_API_VERSION", "2024-12-01-preview"), + temperature=temperature, + **kwargs, + ) + from langchain_openai import ChatOpenAI + return ChatOpenAI(model=model, temperature=temperature, **kwargs) + + +def create_embeddings(model: str): + """Return AzureOpenAIEmbeddings when AZURE_OPENAI_ENDPOINT is set, otherwise OpenAIEmbeddings.""" + if os.environ.get("AZURE_OPENAI_ENDPOINT"): + from langchain_openai import AzureOpenAIEmbeddings + return AzureOpenAIEmbeddings( + azure_deployment=os.environ.get("AZURE_EMBEDDING_DEPLOYMENT", model), + azure_endpoint=os.environ["AZURE_OPENAI_ENDPOINT"], + api_key=os.environ.get("AZURE_OPENAI_API_KEY"), + api_version=os.environ.get("AZURE_OPENAI_API_VERSION", "2024-12-01-preview"), + ) + from langchain_openai import OpenAIEmbeddings + return OpenAIEmbeddings(model=model) diff --git a/examples/agent/healthcare-assistant/helpers/hallucination_helpers.py b/examples/agent/healthcare-assistant/helpers/hallucination_helpers.py index 7c899fe2..51b4525b 100644 --- a/examples/agent/healthcare-assistant/helpers/hallucination_helpers.py +++ b/examples/agent/healthcare-assistant/helpers/hallucination_helpers.py @@ -47,10 +47,7 @@ def log_hallucination( else: logger.info("Creating new Splunk AO session for hallucination demo") splunk_ao_logger = SplunkAOLogger(project=project_name, agent_stream=agent_stream) - splunk_ao_logger.start_session( - name=session_name, - external_id=external_session_id or str(uuid.uuid4()), - ) + splunk_ao_logger.set_session(external_session_id or str(uuid.uuid4())) splunk_ao_logger.start_trace( input=question, diff --git a/examples/agent/healthcare-assistant/helpers/setup_vectordb.py b/examples/agent/healthcare-assistant/helpers/setup_vectordb.py index efd1e475..fca9900b 100644 --- a/examples/agent/healthcare-assistant/helpers/setup_vectordb.py +++ b/examples/agent/healthcare-assistant/helpers/setup_vectordb.py @@ -17,9 +17,8 @@ sys.path.insert(0, str(Path(__file__).resolve().parents[1])) -from config import DOCS_DIR, DOMAIN, load_config +from config import DOCS_DIR, DOMAIN, create_embeddings, load_config from langchain_core.documents import Document -from langchain_openai import OpenAIEmbeddings from setup_env import setup_environment from helpers.pgvector_utils import create_pgvector_store, get_collection_name from helpers.sql_utils import load_domain_relational_csvs @@ -48,13 +47,13 @@ def setup_vectordb(environment: str) -> bool: print(f"❌ Docs directory not found: {docs_dir}") return False - if not os.environ.get("OPENAI_API_KEY"): + if not os.environ.get("AZURE_OPENAI_ENDPOINT") and not os.environ.get("OPENAI_API_KEY"): os.environ["OPENAI_API_KEY"] = getpass.getpass("Enter API key for OpenAI: ") if not os.environ.get("POSTGRES_PASSWORD"): os.environ["POSTGRES_PASSWORD"] = getpass.getpass("Enter PostgreSQL password: ") - embeddings = OpenAIEmbeddings(model=embedding_model) + embeddings = create_embeddings(model=embedding_model) collection_name = get_collection_name(DOMAIN, environment) print(f"Creating PostgreSQL/pgvector collection: {collection_name}") vector_store, collection_name = create_pgvector_store( diff --git a/examples/agent/healthcare-assistant/pyproject.toml b/examples/agent/healthcare-assistant/pyproject.toml new file mode 100644 index 00000000..28e79cb0 --- /dev/null +++ b/examples/agent/healthcare-assistant/pyproject.toml @@ -0,0 +1,35 @@ +[project] +name = "healthcare-assistant" +version = "0.1.0" +description = "Healthcare assistant demo app — uses local splunk-ao source for debugging" +requires-python = ">=3.11,<3.15" +dependencies = [ + "streamlit", + "openai", + "python-dotenv", + "langchain", + "langchain-core", + "langchain-openai", + "langgraph", + "langchain-postgres", + "psycopg[binary]", + "langchain-text-splitters", + "langchain-community", + "langchain-classic", + "pyyaml", + "toml", + "pandas", + "splunk-ao", +] + +[tool.uv] +# Use local splunk-ao source instead of PyPI +[tool.uv.sources] +splunk-ao = { path = "../../..", editable = true } + +[tool.hatch.build.targets.wheel] +packages = ["."] + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" diff --git a/examples/agent/healthcare-assistant/rag.py b/examples/agent/healthcare-assistant/rag.py index aaa3e07d..b2ebd923 100644 --- a/examples/agent/healthcare-assistant/rag.py +++ b/examples/agent/healthcare-assistant/rag.py @@ -7,9 +7,7 @@ from langchain_classic.chains.combine_documents import create_stuff_documents_chain from langchain_core.prompts import ChatPromptTemplate, MessagesPlaceholder from langchain_core.tools import tool -from langchain_openai import ChatOpenAI, OpenAIEmbeddings - -from config import DOMAIN, load_config +from config import DOMAIN, load_config, create_chat_llm, create_embeddings from helpers.pgvector_utils import collection_exists, create_pgvector_store from setup_env import setup_environment @@ -58,15 +56,11 @@ def initialize(self): f"Please run: python helpers/setup_vectordb.py {environment}" ) - embeddings = OpenAIEmbeddings(model=embedding_model) + embeddings = create_embeddings(model=embedding_model) vector_store, _ = create_pgvector_store(embeddings, DOMAIN, environment) retriever = vector_store.as_retriever(search_kwargs={"k": self.top_k}) - llm = ChatOpenAI( - model=llm_model, - temperature=0.1, - name="Healthcare RAG Assistant", - ) + llm = create_chat_llm(model=llm_model, temperature=0.1, name="Healthcare RAG Assistant") retrieval_qa_chat_prompt = ChatPromptTemplate.from_messages( [ From 6bb47fb7e8818cb339e6d7638c066d682bd7e1d0 Mon Sep 17 00:00:00 2001 From: etserend Date: Mon, 24 Aug 2026 19:12:58 -0500 Subject: [PATCH 3/6] feat(examples): add hosted/ one-shot deployment for healthcare-assistant MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds a self-contained hosted/ subdirectory for running the agent as a scheduled load generator (CronJob) against a Splunk O11y backend: - run_demo_session.py: entrypoint that fires example_queries from config.yaml through the instrumented agent, then logs one hallucination, all under a single session ID per run - Dockerfile: multi-stage build; installs splunk-ao from repo src/ so the image always uses the same SDK version as the example (build from repo root) - k8s.yaml: ConfigMap + CronJob manifest (hourly schedule); all credentials via secretKeyRef — no hardcoded values Co-Authored-By: Claude Opus 4.7 --- .../healthcare-assistant/hosted/Dockerfile | 26 ++++++ .../healthcare-assistant/hosted/README.md | 86 +++++++++++++++++++ .../healthcare-assistant/hosted/k8s.yaml | 85 ++++++++++++++++++ .../hosted/run_demo_session.py | 79 +++++++++++++++++ 4 files changed, 276 insertions(+) create mode 100644 examples/agent/healthcare-assistant/hosted/Dockerfile create mode 100644 examples/agent/healthcare-assistant/hosted/README.md create mode 100644 examples/agent/healthcare-assistant/hosted/k8s.yaml create mode 100644 examples/agent/healthcare-assistant/hosted/run_demo_session.py diff --git a/examples/agent/healthcare-assistant/hosted/Dockerfile b/examples/agent/healthcare-assistant/hosted/Dockerfile new file mode 100644 index 00000000..5488e522 --- /dev/null +++ b/examples/agent/healthcare-assistant/hosted/Dockerfile @@ -0,0 +1,26 @@ +FROM python:3.12-slim + +WORKDIR /app + +RUN pip install uv --no-cache-dir + +# Install splunk-ao from local source (main branch — no unreleased local changes). +# Build context is repo root. +COPY pyproject.toml poetry.lock* README.md /sdk/ +COPY src/ /sdk/src/ +RUN uv pip install --system --no-cache /sdk + +# Install example dependencies, pinning splunk-ao to the local version above. +COPY examples/agent/healthcare-assistant/requirements.txt /app/ +RUN uv pip install --system --no-cache -r requirements.txt --override /dev/stdin <<'EOF' +splunk-ao +EOF + +COPY examples/agent/healthcare-assistant/ /app/ + +RUN useradd --create-home --shell /bin/bash app && \ + chown -R app:app /app + +USER app + +CMD ["python", "hosted/run_demo_session.py"] diff --git a/examples/agent/healthcare-assistant/hosted/README.md b/examples/agent/healthcare-assistant/hosted/README.md new file mode 100644 index 00000000..9836cdbe --- /dev/null +++ b/examples/agent/healthcare-assistant/hosted/README.md @@ -0,0 +1,86 @@ +# Healthcare Assistant — Hosted / K8s Deployment + +Runs the instrumented healthcare assistant as a Kubernetes CronJob that sends telemetry to Splunk Observability Cloud (lab0 / any O11y realm). + +Each run fires two agent queries and one intentional hallucination — all under a single session ID — producing three traces in the configured agent stream. + +## What runs + +`run_demo_session.py` is the entrypoint. It: + +1. Loads `agent-with-instrumentation.py` (LangGraph + `SplunkAOAsyncCallback`) +2. Reads the two example queries from `../config.yaml` (`ui.example_queries`) +3. Fires both queries through the agent in the same session +4. Logs one hallucination from `../config.yaml` (`demo_hallucinations[0]`) + +Telemetry is routed via `SPLUNK_AO_REALM` + `SPLUNK_AO_O11Y_TOKEN` to: +``` +https://ingest..observability.splunkcloud.com/v2/trace/otlp +``` + +## Prerequisites + +Existing k8s namespace `healthcare-assistant` with: + +| Secret | Keys used | +|---|---| +| `splunk-ao-secrets` | `realm`, `o11y-token`, `o11y-api-token` | +| `openai-secrets` | `api-key` | +| `healthcare-assistant-lab0-openai` | `azure-openai-endpoint` | +| `postgres-credentials` | `POSTGRES_PASSWORD` | + +| ConfigMap | Used for | +|---|---| +| `postgres-config` | `POSTGRES_HOST`, `POSTGRES_PORT`, `POSTGRES_USER`, `POSTGRES_DB` | + +## Configuration + +All runtime config is in the `healthcare-assistant-instrumented-config` ConfigMap inside `k8s.yaml`. Override these to point at a different project or agent stream: + +| Variable | Default | Description | +|---|---|---| +| `SPLUNK_AO_PROJECT` | `demo-healthcare` | Splunk AO project name | +| `SPLUNK_AO_AGENT_STREAM` | `assistant` | Agent stream name | +| `OTEL_SERVICE_NAME` | `healthcare-assistant-instrumented` | OTel service name | +| `AZURE_CHAT_DEPLOYMENT` | `gpt-4.1-mini` | Azure OpenAI chat deployment | +| `AZURE_EMBEDDING_DEPLOYMENT` | `text-embedding-3-large` | Azure OpenAI embedding deployment | +| `QUERY_DELAY_SECONDS` | `3` | Delay between queries | + +## Build + +Build context is the repo root. The SDK is installed from `src/` (local source, not PyPI). + +```bash +# from repo root +docker buildx build \ + --platform linux/amd64 \ + -f examples/agent/healthcare-assistant/hosted/Dockerfile \ + -t ertserendavga918/healthcare-assistant-agent-loadgen:v0.0.1 \ + --push \ + . +``` + +## Deploy + +```bash +kubectl apply -f examples/agent/healthcare-assistant/hosted/k8s.yaml +``` + +## Trigger manually (one-shot test) + +```bash +kubectl create job healthcare-assistant-instrumented-test \ + --from=cronjob/healthcare-assistant-instrumented \ + -n healthcare-assistant + +# Follow logs +kubectl logs -n healthcare-assistant -l job-name=healthcare-assistant-instrumented-test --follow +``` + +## Validate + +Check Splunk Observability Cloud → Agent Observability → project `demo-healthcare` → agent stream `assistant`. + +Each run produces: +- 2 traces with `invoke_agent Agent` root span (real LLM + tool calls) +- 1 trace with hallucinated answer for the Lisinopril question diff --git a/examples/agent/healthcare-assistant/hosted/k8s.yaml b/examples/agent/healthcare-assistant/hosted/k8s.yaml new file mode 100644 index 00000000..a508cc30 --- /dev/null +++ b/examples/agent/healthcare-assistant/hosted/k8s.yaml @@ -0,0 +1,85 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: healthcare-assistant-instrumented-config + namespace: healthcare-assistant +data: + SPLUNK_AO_PROJECT: "erden-framework-testing" + SPLUNK_AO_AGENT_STREAM: "healthcare-assistant-instrumented" + OTEL_SERVICE_NAME: "healthcare-assistant-instrumented" + OTEL_RESOURCE_ATTRIBUTES: "deployment.environment.name=agent-observability" + AZURE_OPENAI_API_VERSION: "2024-12-01-preview" + AZURE_CHAT_DEPLOYMENT: "gpt-4.1-mini" + AZURE_EMBEDDING_DEPLOYMENT: "text-embedding-3-large" + QUERY_DELAY_SECONDS: "3" + ENVIRONMENT: "hosted" +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + name: healthcare-assistant-instrumented + namespace: healthcare-assistant +spec: + schedule: "0 * * * *" + suspend: false + concurrencyPolicy: Forbid + successfulJobsHistoryLimit: 3 + failedJobsHistoryLimit: 3 + jobTemplate: + spec: + template: + spec: + restartPolicy: Never + containers: + - name: demo-session + image: ertserendavga918/healthcare-assistant-agent-loadgen:v0.0.1 + imagePullPolicy: Always + command: ["python", "hosted/run_demo_session.py"] + envFrom: + - configMapRef: + name: healthcare-assistant-instrumented-config + - configMapRef: + name: postgres-config + env: + - name: SPLUNK_AO_REALM + valueFrom: + secretKeyRef: + name: splunk-ao-secrets + key: realm + - name: SPLUNK_AO_O11Y_TOKEN + valueFrom: + secretKeyRef: + name: splunk-ao-secrets + key: o11y-token + - name: SPLUNK_AO_O11Y_API_TOKEN + valueFrom: + secretKeyRef: + name: splunk-ao-secrets + key: o11y-api-token + - name: AZURE_OPENAI_ENDPOINT + valueFrom: + secretKeyRef: + name: healthcare-assistant-lab0-openai + key: azure-openai-endpoint + - name: AZURE_OPENAI_API_KEY + valueFrom: + secretKeyRef: + name: openai-secrets + key: api-key + - name: OPENAI_API_KEY + valueFrom: + secretKeyRef: + name: openai-secrets + key: api-key + - name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + name: postgres-credentials + key: POSTGRES_PASSWORD + resources: + requests: + memory: "512Mi" + cpu: "250m" + limits: + memory: "1Gi" + cpu: "500m" diff --git a/examples/agent/healthcare-assistant/hosted/run_demo_session.py b/examples/agent/healthcare-assistant/hosted/run_demo_session.py new file mode 100644 index 00000000..c021cf3c --- /dev/null +++ b/examples/agent/healthcare-assistant/hosted/run_demo_session.py @@ -0,0 +1,79 @@ +""" +Demo session runner — CronJob entrypoint. + +Fires the two example queries from config.yaml through the instrumented agent, +then logs one hallucination, all under a single session ID per run. +""" +import asyncio +import importlib.util +import logging +import os +import sys +import time +import uuid + +# Ensure the app root (parent of hosted/) is on the path so config, helpers, etc. resolve. +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) + +logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s", stream=sys.stdout) +log = logging.getLogger(__name__) + + +def _load_instrumented_agent(): + spec = importlib.util.spec_from_file_location( + "agent_with_instrumentation", + os.path.join(os.path.dirname(__file__), "..", "agent-with-instrumentation.py"), + ) + mod = importlib.util.module_from_spec(spec) + sys.modules["agent_with_instrumentation"] = mod + spec.loader.exec_module(mod) + return mod.HealthcareAgent + + +async def run(HealthcareAgent, config, session_id): + queries = config["ui"]["example_queries"] + query_delay = float(os.getenv("QUERY_DELAY_SECONDS", "3")) + + agent = HealthcareAgent(session_id=session_id) + agent.load_tools() + + for i, question in enumerate(queries, 1): + log.info("[%d/%d] query=%s", i, len(queries), question[:80]) + try: + result = await agent._process_query_async([{"role": "user", "content": question}]) + log.info(" → %s", str(result)[:120]) + except Exception: + log.exception(" query failed") + if i < len(queries): + await asyncio.sleep(query_delay) + + return len(queries) + + +if __name__ == "__main__": + import config as cfg_mod + from helpers.hallucination_helpers import log_demo_hallucination + + config = cfg_mod.load_config() + session_id = f"demo-{uuid.uuid4().hex[:8]}" + + log.info( + "Demo session starting — project=%s stream=%s session=%s", + os.getenv("SPLUNK_AO_PROJECT"), + os.getenv("SPLUNK_AO_AGENT_STREAM"), + session_id, + ) + + HealthcareAgent = _load_instrumented_agent() + n = asyncio.run(run(HealthcareAgent, config, session_id)) + + time.sleep(float(os.getenv("QUERY_DELAY_SECONDS", "3"))) + + log.info("[%d/%d] logging hallucination", n + 1, n + 1) + try: + success = log_demo_hallucination(config=config, hallucination_index=0, session_id=session_id) + log.info(" hallucination logged: %s", success) + except Exception: + log.exception(" hallucination failed (non-fatal)") + + log.info("Demo session complete — session=%s", session_id) From fa2ad7555e8c84d1351dcd0a88cfd4b1f91dc490 Mon Sep 17 00:00:00 2001 From: etserend Date: Tue, 25 Aug 2026 14:40:08 -0500 Subject: [PATCH 4/6] docs(examples): add healthcare-assistant README and fix Azure-aware local run Co-Authored-By: Claude Opus 4.7 --- examples/README.md | 6 + .../agent/healthcare-assistant/.env.example | 29 ++- examples/agent/healthcare-assistant/README.md | 130 +++++++++++++ .../healthcare-assistant/_agent_galileo.py | 167 ---------------- .../_hallucination_helpers_galileo.py | 181 ------------------ .../healthcare-assistant/_validate_galileo.py | 70 ------- .../_validate_hallucination.py | 124 ------------ .../healthcare-assistant/_validate_single.py | 28 --- examples/agent/healthcare-assistant/app.py | 19 +- .../helpers/hallucination_helpers.py | 5 +- .../helpers/pgvector_utils.py | 6 +- .../helpers/text_to_sql_utils.py | 4 +- .../healthcare-assistant/hosted/README.md | 13 +- .../healthcare-assistant/hosted/k8s.yaml | 8 +- .../healthcare-assistant/validate_traces.py | 42 ---- 15 files changed, 187 insertions(+), 645 deletions(-) create mode 100644 examples/agent/healthcare-assistant/README.md delete mode 100644 examples/agent/healthcare-assistant/_agent_galileo.py delete mode 100644 examples/agent/healthcare-assistant/_hallucination_helpers_galileo.py delete mode 100644 examples/agent/healthcare-assistant/_validate_galileo.py delete mode 100644 examples/agent/healthcare-assistant/_validate_hallucination.py delete mode 100644 examples/agent/healthcare-assistant/_validate_single.py delete mode 100644 examples/agent/healthcare-assistant/validate_traces.py diff --git a/examples/README.md b/examples/README.md index 6130d3a7..3385426e 100644 --- a/examples/README.md +++ b/examples/README.md @@ -1,5 +1,11 @@ # Splunk Agent Observability Python SDK examples +## Agent examples + +| Example | Framework | Description | +|---|---|---| +| [healthcare-assistant](agent/healthcare-assistant/README.md) | LangGraph + Streamlit | Full-stack chat app with RAG, text-to-SQL, hallucination demo, and Splunk AO tracing | + ## Preconditions Install `uv`, we use inline dependency inside scripts. diff --git a/examples/agent/healthcare-assistant/.env.example b/examples/agent/healthcare-assistant/.env.example index 9a9e74c4..3877efd6 100644 --- a/examples/agent/healthcare-assistant/.env.example +++ b/examples/agent/healthcare-assistant/.env.example @@ -1,28 +1,23 @@ -# LLM provider — set ONE of the two blocks below - -# Option A: OpenAI -# OPENAI_API_KEY= - -# Option B: Azure OpenAI +# Azure OpenAI AZURE_OPENAI_API_KEY= AZURE_OPENAI_ENDPOINT= # e.g. https://your-resource.openai.azure.com/ -AZURE_OPENAI_API_VERSION= # e.g. 2024-12-01-preview -AZURE_CHAT_DEPLOYMENT= # deployment name for chat, e.g. gpt-4o-mini -AZURE_EMBEDDING_DEPLOYMENT= # deployment name for embeddings, e.g. text-embedding-3-large +AZURE_OPENAI_API_VERSION=2024-12-01-preview +AZURE_CHAT_DEPLOYMENT=gpt-4.1-mini +AZURE_EMBEDDING_DEPLOYMENT=text-embedding-3-large -# Splunk AO environment variables — set ONE of the two deployment blocks below +# OpenAI (alternative — comment out Azure block above and uncomment these) +# OPENAI_API_KEY= -# Option A: Splunk Observability (O11y) Cloud +# Splunk AO — Splunk Observability (O11y) Cloud SPLUNK_AO_REALM= # e.g. us0, eu0, lab0 SPLUNK_AO_O11Y_TOKEN= # O11y ingest token (required for telemetry) -# SPLUNK_AO_O11Y_API_TOKEN= # Optional: dedicated API token for CRUD operations +# SPLUNK_AO_O11Y_API_TOKEN= # Dedicated API token for CRUD / evaluators (optional) -# Option B: On-premises / standalone deployment +# Splunk AO — standalone deployment (alternative — comment out O11y block above) # SPLUNK_AO_API_KEY= # SPLUNK_AO_CONSOLE_URL= # e.g. https://console.yourcompany.com -# SPLUNK_AO_API_ENDPOINT= # Optional, only set for custom deployments -# Routing (shared by both deployments) +# Routing SPLUNK_AO_PROJECT= SPLUNK_AO_AGENT_STREAM= @@ -33,5 +28,9 @@ POSTGRES_USER=postgres POSTGRES_PASSWORD= POSTGRES_DB=vectordb +# OTel OTEL_SERVICE_NAME=healthcare-assistant OTEL_RESOURCE_ATTRIBUTES=deployment.environment.name=your-project-name + +# Runtime +ENVIRONMENT=local diff --git a/examples/agent/healthcare-assistant/README.md b/examples/agent/healthcare-assistant/README.md new file mode 100644 index 00000000..95c64cac --- /dev/null +++ b/examples/agent/healthcare-assistant/README.md @@ -0,0 +1,130 @@ +# Healthcare Assistant + +A healthcare-domain Streamlit chat app built with **LangGraph**, **PostgreSQL/pgvector**, and the **Splunk Agent Observability Python SDK**. Demonstrates real-time agent tracing, RAG retrieval, text-to-SQL, and intentional hallucination logging. + +## What's inside + +| File / Dir | Purpose | +|---|---| +| `app.py` | Streamlit UI and chat loop | +| `agent-with-instrumentation.py` | LangGraph agent with `SplunkAOAsyncCallback` | +| `agent.py` | Plain LangGraph agent (no instrumentation) | +| `rag.py` | RAG retrieval chain (pgvector) | +| `config.py` | Azure OpenAI / OpenAI factory functions | +| `config.yaml` | App settings — model, RAG, UI queries, hallucination examples | +| `system_prompt.json` | Agent system prompt | +| `tools/logic.py` | `get_patient_info`, `delete_patient_record`, `search_medicine_qa` | +| `tools/schema.json` | Tool JSON schemas | +| `helpers/` | pgvector, SQL, text-to-SQL, hallucination, and setup utilities | +| `docs/` | Source data — `qa.csv` (medicine FAQ), `relational_patient.csv` | +| `hosted/` | Kubernetes CronJob deployment — see [hosted/README.md](hosted/README.md) | + +## Prerequisites + +- Python 3.11+ +- [uv](https://docs.astral.sh/uv/) — package manager and runner +- [Docker](https://docs.docker.com/get-docker/) — for the local PostgreSQL container +- Azure OpenAI or OpenAI API key +- Splunk Agent Observability account (O11y Cloud or standalone) + +### Splunk AO authentication + +This example targets **Splunk Observability (O11y) Cloud**. Two tokens are involved: + +| Variable | Required | Purpose | +|---|---|---| +| `SPLUNK_AO_REALM` | ✅ | Your O11y Cloud realm (`us0`, `eu0`, `lab0`, …) | +| `SPLUNK_AO_O11Y_TOKEN` | ✅ | Ingest token — exports telemetry via OTLP | +| `SPLUNK_AO_O11Y_API_TOKEN` | optional | Dedicated CRUD token — enables evaluators (Correctness, Context Adherence) | + +`SPLUNK_AO_O11Y_TOKEN` is used for both telemetry ingest and CRUD when no API token is set. Set `SPLUNK_AO_O11Y_API_TOKEN` separately if your ingest token is read-only. See the [SDK authentication docs](https://github.com/splunk/splunk-ao-python#splunk-observability-o11y-cloud) for full details. + +## Setup + +Run all commands from the `healthcare-assistant/` directory. + +### 1. Start PostgreSQL + +```bash +docker run \ + -e POSTGRES_USER=postgres \ + -e POSTGRES_PASSWORD=mypassword \ + -e POSTGRES_DB=vectordb \ + --name healthcare-postgres \ + -p 5432:5432 \ + -d pgvector/pgvector:pg16 + +docker exec healthcare-postgres \ + psql -U postgres -d vectordb \ + -c "CREATE EXTENSION IF NOT EXISTS vector;" +``` + +### 2. Install dependencies + +```bash +uv venv +uv pip install -r requirements.txt +``` + +Or with the editable local SDK (repo root): + +```bash +uv venv +uv sync +``` + +### 3. Configure environment + +Copy the example and fill in your values: + +```bash +cp .env.example .env +``` + +Minimum required values: + +```bash +AZURE_OPENAI_API_KEY= +AZURE_OPENAI_ENDPOINT=https://your-resource.openai.azure.com/ +AZURE_OPENAI_API_VERSION=2024-12-01-preview +AZURE_CHAT_DEPLOYMENT=gpt-4.1-mini +AZURE_EMBEDDING_DEPLOYMENT=text-embedding-3-large + +SPLUNK_AO_REALM=us0 +SPLUNK_AO_O11Y_TOKEN= +SPLUNK_AO_PROJECT= +SPLUNK_AO_AGENT_STREAM=healthcare-assistant + +POSTGRES_PASSWORD= +``` + +### 4. Load vector and relational data + +```bash +uv run python helpers/setup_vectordb.py local +``` + +Or: + +```bash +./start_vectordb.sh +``` + +### 5. Run the app + +```bash +uv run streamlit run app.py +``` + +Open [http://localhost:8501](http://localhost:8501). + +## Example queries + +- **"What is the dosage and common side effects of Lisinopril?"** — RAG over medicine FAQ (`search_medicine_qa`) +- **"Can you look up information for patient P001?"** — text-to-SQL patient lookup (`get_patient_info`) + +Use **Log Hallucination** in the sidebar to intentionally log a wrong Lisinopril answer for demo purposes. + +## Deployment + +See [hosted/README.md](hosted/README.md) for the Kubernetes CronJob deployment that runs the demo session automatically. diff --git a/examples/agent/healthcare-assistant/_agent_galileo.py b/examples/agent/healthcare-assistant/_agent_galileo.py deleted file mode 100644 index 383bbed1..00000000 --- a/examples/agent/healthcare-assistant/_agent_galileo.py +++ /dev/null @@ -1,167 +0,0 @@ -"""LangGraph agent for the healthcare assistant.""" -import asyncio -import inspect -import json -import uuid -from concurrent.futures import ThreadPoolExecutor -from typing import Annotated, List, Dict, Optional, TypedDict - -from langchain_core.messages import AIMessage, BaseMessage, HumanMessage, SystemMessage -from langchain_core.tools import StructuredTool -from langchain_openai import ChatOpenAI -from langgraph.graph import START, StateGraph -from langgraph.graph.state import CompiledStateGraph -from langgraph.graph.message import add_messages -from langgraph.prebuilt import ToolNode, tools_condition - -from config import TOOLS_DIR, load_config, load_system_prompt -from rag import create_rag_tool -from tools import logic as tools_logic - -import os -from galileo import galileo_context -from galileo.handlers.langchain import GalileoAsyncCallback - -class State(TypedDict): - messages: Annotated[list, add_messages] - - -def _run_async(coro): - """Run an async coroutine from sync code (e.g. Streamlit).""" - try: - asyncio.get_running_loop() - except RuntimeError: - return asyncio.run(coro) - - with ThreadPoolExecutor(max_workers=1) as executor: - return executor.submit(asyncio.run, coro).result() - - -class HealthcareAgent: - """LangGraph healthcare assistant.""" - - def __init__( - self, - session_id: str | None = None, - model_override: Optional[str] = None, - ): - self.config = load_config() - self.session_id = session_id or str(uuid.uuid4()) - self.model_override = model_override - self.system_prompt = load_system_prompt() - self.tools = [] - self.graph: CompiledStateGraph | None = None - self.langgraph_config = {"configurable": {"thread_id": self.session_id}} - - def load_tools(self) -> None: - tool_schema_path = TOOLS_DIR / "schema.json" - with tool_schema_path.open(encoding="utf-8") as f: - tool_schema = json.load(f) - - self.tools = [] - for tool_func in tools_logic.TOOLS: - tool_schema_dict = next( - (schema for schema in tool_schema if schema.get("name") == tool_func.__name__), - None, - ) - tool_kwargs = { - "name": tool_func.__name__, - "description": ( - tool_schema_dict.get("description") - if tool_schema_dict - else tool_func.__doc__ or f"Tool: {tool_func.__name__}" - ), - "args_schema": tool_schema_dict.get("parameters") if tool_schema_dict else None, - } - if inspect.iscoroutinefunction(tool_func): - langchain_tool = StructuredTool.from_function(coroutine=tool_func, **tool_kwargs) - else: - langchain_tool = StructuredTool.from_function(func=tool_func, **tool_kwargs) - self.tools.append(langchain_tool) - - rag_config = self.config.get("rag", {}) - if rag_config.get("enabled", False): - top_k = rag_config.get("top_k", 5) - model_config = self.config.get("model", {}) - effective_model = ( - self.model_override - or model_config.get("default_model") - or model_config.get("model_name") - ) - rag_tool = create_rag_tool(top_k, model_name=effective_model) - self.tools.append(rag_tool) - - print(f"✓ Loaded {len(self.tools)} tools") - - def _build_graph(self) -> CompiledStateGraph: - if not self.tools: - raise ValueError("Tools not loaded. Call load_tools() first.") - - model_config = self.config.get("model", {}) - effective_model = ( - self.model_override - or model_config.get("default_model") - or model_config.get("model_name") - ) - temperature = model_config.get("temperature", 0.1) - - llm_with_tools = ChatOpenAI( - model=effective_model, - temperature=temperature, - name="Healthcare Assistant", - ).bind_tools(self.tools) - - async def invoke_chatbot(state): - messages = list(state["messages"]) - if self.system_prompt: - messages = [SystemMessage(content=self.system_prompt)] + messages - message = await llm_with_tools.ainvoke(messages) - return {"messages": [message]} - - graph_builder = StateGraph(State) - graph_builder.add_node("chatbot", invoke_chatbot) - graph_builder.add_node("tools", ToolNode(tools=self.tools)) - graph_builder.add_edge(START, "chatbot") - graph_builder.add_conditional_edges("chatbot", tools_condition) - graph_builder.add_edge("tools", "chatbot") - return graph_builder.compile() - - async def _process_query_async(self, messages: List[Dict[str, str]]) -> str: - if not self.tools: - self.load_tools() - self.graph = self._build_graph() - - langchain_messages: List[BaseMessage] = [] - for msg in messages: - if msg["role"] == "user": - langchain_messages.append(HumanMessage(content=msg["content"])) - elif msg["role"] == "assistant": - langchain_messages.append(AIMessage(content=msg["content"])) - - with galileo_context( - project=os.getenv("GALILEO_PROJECT"), - log_stream=os.getenv("GALILEO_LOG_STREAM"), - ): - galileo_context.start_session(external_id=self.session_id) - - # One callback per request keeps each user turn in its own trace. - callback = GalileoAsyncCallback() - run_config = {**self.langgraph_config, "callbacks": [callback]} - - result = await self.graph.ainvoke( - {"messages": langchain_messages}, - run_config, - ) - if result["messages"]: - return result["messages"][-1].content - return "No response generated" - - def process_query(self, messages: List[Dict[str, str]]) -> str: - try: - return _run_async(self._process_query_async(messages)) - except Exception as e: - print(f"[ERROR] Error processing query: {e}") - import traceback - - traceback.print_exc() - return f"Error processing your request: {str(e)}" diff --git a/examples/agent/healthcare-assistant/_hallucination_helpers_galileo.py b/examples/agent/healthcare-assistant/_hallucination_helpers_galileo.py deleted file mode 100644 index 1c5138c7..00000000 --- a/examples/agent/healthcare-assistant/_hallucination_helpers_galileo.py +++ /dev/null @@ -1,181 +0,0 @@ -""" -Hallucination Demo Helpers - -Log intentional hallucinations to Galileo for Splunk Agent Observability demos. -Examples are defined in config.yaml under `demo_hallucinations`. -""" -import logging -import os -import uuid -from typing import Any, List, Optional, Union - -from galileo import GalileoLogger -from langchain_core.messages import AIMessage, HumanMessage - -logger = logging.getLogger(__name__) - - -def log_hallucination( - project_name: str, - log_stream: str, - question: str, - context_docs: List[str], - hallucinated_answer: str, - model: str = "gpt-4o", - session_name: str = "Hallucination Demo", - external_session_id: Optional[str] = None, - existing_logger: Optional[Union[GalileoLogger, Any]] = None, -) -> bool: - """ - Log a hallucination trace to Galileo for demonstration purposes. - - Creates a trace with a retriever span (real context) and an LLM span (wrong answer). - """ - try: - logger.info( - "Logging hallucination to project: %s, log stream: %s", - project_name, - log_stream, - ) - - if existing_logger: - logger.info("Using existing Galileo session for hallucination demo") - if hasattr(existing_logger, "get_logger_instance"): - galileo_logger = existing_logger.get_logger_instance() - else: - galileo_logger = existing_logger - else: - logger.info("Creating new Galileo session for hallucination demo") - galileo_logger = GalileoLogger(project=project_name, log_stream=log_stream) - galileo_logger.start_session( - name=session_name, - external_id=external_session_id or str(uuid.uuid4()), - ) - - galileo_logger.start_trace( - input=question, - name="Hallucination Demo", - ) - - galileo_logger.add_retriever_span( - input=question, - output=context_docs, - name="RAG Retrieval", - duration_ns=int(1.3e8), - status_code=200, - ) - - context_text = "\n\n".join(context_docs) - llm_input = f"""Human: You are a helpful assistant. Given the context below, please answer the following question: - -{context_text} - -Question: {question}""" - - galileo_logger.add_llm_span( - input=llm_input, - output=hallucinated_answer, - model=model, - name="LLM Response", - num_input_tokens=len(llm_input.split()) * 2, - num_output_tokens=len(hallucinated_answer.split()) * 2, - total_tokens=len(llm_input.split()) * 2 + len(hallucinated_answer.split()) * 2, - duration_ns=int(1.2e8), - metadata={"temperature": "0.1", "demo_type": "hallucination"}, - temperature=0.1, - status_code=200, - time_to_first_token_ns=500000, - ) - - galileo_logger.conclude( - output=hallucinated_answer, - duration_ns=int(2.5e8), - status_code=200, - ) - - galileo_logger.flush() - - logger.info("Successfully logged hallucination to project: %s", project_name) - return True - - except Exception as e: - logger.error("Failed to log hallucination: %s", e) - return False - - -def log_demo_hallucination( - config: dict, - hallucination_index: int = 0, - existing_logger: Optional[Union[GalileoLogger, Any]] = None, - session_id: Optional[str] = None, -) -> bool: - """Log a demo hallucination from config.yaml to Galileo.""" - project_name = os.getenv("GALILEO_PROJECT", "healthcare-assistant") - log_stream = os.getenv("GALILEO_LOG_STREAM", "default") - - hallucinations = config.get("demo_hallucinations", []) - if not hallucinations: - logger.warning("No hallucination examples defined in config") - return False - - if hallucination_index >= len(hallucinations): - hallucination_index = 0 - - hallucination = hallucinations[hallucination_index] - question = hallucination.get("question", "") - hallucinated_answer = hallucination.get("hallucinated_answer", "") - context_docs = hallucination.get("context", []) - - if not question or not hallucinated_answer: - logger.error("Invalid hallucination config: missing question or answer") - return False - - if not context_docs: - context_docs = ["[No context available]"] - - model_config = config.get("model", {}) - model = model_config.get("default_model", "gpt-4o") - - return log_hallucination( - project_name=project_name, - log_stream=log_stream, - question=question, - context_docs=context_docs, - hallucinated_answer=hallucinated_answer, - model=model, - session_name="Healthcare Hallucination Demo", - external_session_id=session_id, - existing_logger=existing_logger, - ) - - -def add_hallucination_interaction_to_chat( - config: dict, - hallucination_index: int = 0, -) -> None: - """Append the demo hallucination Q&A pair to the Streamlit chat history.""" - import streamlit as st - - hallucinations = config.get("demo_hallucinations", []) - if not hallucinations: - return - - if hallucination_index >= len(hallucinations): - hallucination_index = 0 - - hallucination = hallucinations[hallucination_index] - question = hallucination.get("question", "") - answer = hallucination.get("hallucinated_answer", "") - - if not question or not answer: - return - - if "messages" not in st.session_state: - st.session_state.messages = [] - - st.session_state.messages.append( - {"message": HumanMessage(content=question), "agent": "user"} - ) - st.session_state.messages.append( - {"message": AIMessage(content=answer), "agent": "assistant"} - ) diff --git a/examples/agent/healthcare-assistant/_validate_galileo.py b/examples/agent/healthcare-assistant/_validate_galileo.py deleted file mode 100644 index 3241eba3..00000000 --- a/examples/agent/healthcare-assistant/_validate_galileo.py +++ /dev/null @@ -1,70 +0,0 @@ -""" -Env A validation — runs the original galileo-based agent (from pre-migration commit 0f4a7d4057) -against the Galileo staging console (healthcare-galileo agentstream). - -Called by validate_traces.py as a subprocess with .env.galileo loaded. -""" -import os -import sys -import types -import yaml -from dotenv import load_dotenv -from pathlib import Path -import subprocess - -load_dotenv(".env.galileo", override=True) - -# Pull the galileo-based agent.py from the last pre-migration commit -result = subprocess.run( - ["git", "show", "0f4a7d4057:workshop/healthcare-assistant/2-app-with-instrumentation/agent.py"], - cwd=Path(__file__).parent.parent.parent.parent, # repo root - capture_output=True, text=True, -) -if result.returncode != 0: - print("ERROR: could not get galileo agent.py from commit 0f4a7d4057:", result.stderr) - sys.exit(1) - -original_agent_src = result.stdout - -# Patch the source to inject api-version query param (same fix as current agent.py) -# The pre-migration agent has plain ChatOpenAI(...) calls with no default_query. -# We patch by adding a monkeypatch before exec so the network calls work against Azure. -api_version_patch = """ -import os as _os -_orig_ChatOpenAI = ChatOpenAI -class ChatOpenAI(_orig_ChatOpenAI): - def __init__(self, *a, **kw): - _av = _os.environ.get("OPENAI_API_VERSION") - if _av and "default_query" not in kw: - kw["default_query"] = {"api-version": _av} - super().__init__(*a, **kw) -""" - -# Disable RAG for speed -cfg = yaml.safe_load(Path("config.yaml").read_text()) -cfg["rag"]["enabled"] = False - -import config as cfg_mod -cfg_mod.load_config = lambda: cfg - -# Execute original agent source in a fresh module namespace -agent_mod = types.ModuleType("agent_galileo") -agent_mod.__file__ = str(Path(__file__).parent / "agent.py") -sys.modules["agent"] = agent_mod - -exec(compile(original_agent_src, "agent_galileo.py", "exec"), agent_mod.__dict__) -exec(compile(api_version_patch, "patch", "exec"), agent_mod.__dict__) - -HealthcareAgent = agent_mod.HealthcareAgent - -agent = HealthcareAgent(session_id="validate-galileo-001") -agent.load_tools() -result = agent.process_query([{ - "role": "user", - "content": "What is the dosage and common side effects of Lisinopril?", -}]) - -print("Response:", result[:300]) -print(f"\nSession ID: validate-galileo-001") -print(f"Project: {os.getenv('GALILEO_PROJECT')}") -print(f"Log stream: {os.getenv('GALILEO_LOG_STREAM')}") diff --git a/examples/agent/healthcare-assistant/_validate_hallucination.py b/examples/agent/healthcare-assistant/_validate_hallucination.py deleted file mode 100644 index 6f76099d..00000000 --- a/examples/agent/healthcare-assistant/_validate_hallucination.py +++ /dev/null @@ -1,124 +0,0 @@ -"""2-app hallucination demo validation. - -Validates that log_demo_hallucination() logs correctly and whether the trace -appears in the same session as the chat query or as a separate session. - -Env variants: - A: splunk-ao SDK → lab0 - B: splunk-ao SDK → Galileo staging - C: galileo SDK → Galileo staging (baseline) - -Usage (from 2-app-with-instrumentation/, using .venv): - .venv/bin/python3 _validate_hallucination.py # all - .venv/bin/python3 _validate_hallucination.py a # lab0 only - .venv/bin/python3 _validate_hallucination.py b # staging splunk-ao - .venv/bin/python3 _validate_hallucination.py c # staging galileo -""" -import asyncio -import importlib.util -import os -import sys -from pathlib import Path - -ENVS = { - "a": (".env.local", "A — splunk-ao SDK → lab0", "splunk_ao"), - "b": (".env.splunk-ao-standalone", "B — splunk-ao SDK → staging", "splunk_ao"), - "c": (".env.galileo", "C — galileo SDK → staging", "galileo"), -} - -CHAT_QUERY = ("RAG path", "What is the dosage and common side effects of Lisinopril?") - -# Patch openai.AsyncOpenAI to inject api-version for Azure APIM before any import. -import openai as _openai_module -_orig_async_init = _openai_module.AsyncOpenAI.__init__ - -def _patched_async_init(self, *args, **kwargs): - dq = dict(kwargs.pop("default_query", None) or {}) - dq.setdefault("api-version", os.getenv("OPENAI_API_VERSION", "2024-12-01-preview")) - kwargs["default_query"] = dq - _orig_async_init(self, *args, **kwargs) - -_openai_module.AsyncOpenAI.__init__ = _patched_async_init - - -def _load_env(env_file: str): - from dotenv import load_dotenv - for var in [ - "SPLUNK_AO_API_KEY", "SPLUNK_AO_CONSOLE_URL", "SPLUNK_AO_PROJECT", - "SPLUNK_AO_AGENT_STREAM", "SPLUNK_AO_REALM", "SPLUNK_AO_O11Y_TOKEN", - "SPLUNK_AO_O11Y_API_TOKEN", - "GALILEO_API_KEY", "GALILEO_CONSOLE_URL", "GALILEO_PROJECT", "GALILEO_LOG_STREAM", - ]: - os.environ.pop(var, None) - load_dotenv(Path(__file__).parent / env_file, override=True) - - -def _load_module(name: str, path: Path): - spec = importlib.util.spec_from_file_location(name, path) - mod = importlib.util.module_from_spec(spec) - spec.loader.exec_module(mod) - return mod - - -async def run_env(key: str, env_file: str, label: str, sdk: str): - print(f"\n{'='*65}") - print(f" ENV {label}") - print(f"{'='*65}") - - _load_env(env_file) - - sys.path.insert(0, str(Path(__file__).parent)) - - # Load agent and hallucination helpers for the right SDK - if sdk == "galileo": - agent_mod = _load_module("agent_galileo", Path(__file__).parent / "_agent_galileo.py") - hall_mod = _load_module("hallucination_helpers_galileo", - Path(__file__).parent / "_hallucination_helpers_galileo.py") - else: - for mod_name in ["agent", "agent_galileo"]: - sys.modules.pop(mod_name, None) - import agent as agent_mod - from helpers import hallucination_helpers as hall_mod - - session_id = f"validate-hallucination-{key}-001" - agent = agent_mod.HealthcareAgent(session_id=session_id) - agent.load_tools() - - # Step 1: send a real chat query (creates the session in AO) - label_q, query = CHAT_QUERY - print(f"\n [Step 1] Chat query — {label_q}") - result = await agent._process_query_async([{"role": "user", "content": query}]) - print(f" Response: {str(result)[:200]}") - - # Step 2: log the hallucination — passing session_id but NO existing_logger - # (mirrors the Streamlit behavior when no logger is in session state) - print(f"\n [Step 2] log_demo_hallucination(existing_logger=None, session_id={session_id!r})") - config = agent.config - success = hall_mod.log_demo_hallucination( - config=config, - existing_logger=None, - session_id=session_id, - ) - print(f" Success: {success}") - - project = os.getenv("SPLUNK_AO_PROJECT") or os.getenv("GALILEO_PROJECT") - stream = os.getenv("SPLUNK_AO_AGENT_STREAM") or os.getenv("GALILEO_LOG_STREAM") - realm = os.getenv("SPLUNK_AO_REALM") or os.getenv("SPLUNK_AO_CONSOLE_URL") or os.getenv("GALILEO_CONSOLE_URL") - print(f"\n Project: {project}") - print(f" Agent stream: {stream}") - print(f" Endpoint: {realm}") - print(f" session_id: {session_id}") - print() - print(" >> Check console: do chat trace and hallucination trace share the same session?") - - -async def main(): - targets = [sys.argv[1].lower()] if len(sys.argv) > 1 else ["a", "b", "c"] - for t in targets: - env_file, label, sdk = ENVS[t] - await run_env(t, env_file, label, sdk) - print("\nDone — check AO console / Galileo staging for session grouping.") - - -if __name__ == "__main__": - asyncio.run(main()) diff --git a/examples/agent/healthcare-assistant/_validate_single.py b/examples/agent/healthcare-assistant/_validate_single.py deleted file mode 100644 index 96331d91..00000000 --- a/examples/agent/healthcare-assistant/_validate_single.py +++ /dev/null @@ -1,28 +0,0 @@ -"""Single-env trace validation — called by validate_traces.py as a subprocess.""" -import os -import sys -import yaml -from dotenv import load_dotenv -from pathlib import Path - -env_file = sys.argv[1] -load_dotenv(env_file, override=True) - -import config as cfg_mod -from agent import HealthcareAgent - -session_id = f"validate-{Path(env_file).name.lstrip('.')}-001" -agent = HealthcareAgent(session_id=session_id) -agent.load_tools() - -# RAG query — exercises the full retrieval path -result = agent.process_query([{ - "role": "user", - "content": "What is the dosage and common side effects of Lisinopril?", -}]) - -print("Response:", result[:300]) -print(f"\nSession ID: {session_id}") -print(f"Env file: {env_file}") -print(f"Project: {os.getenv('SPLUNK_AO_PROJECT') or os.getenv('GALILEO_PROJECT')}") -print(f"Stream: {os.getenv('SPLUNK_AO_AGENT_STREAM') or os.getenv('GALILEO_LOG_STREAM')}") diff --git a/examples/agent/healthcare-assistant/app.py b/examples/agent/healthcare-assistant/app.py index b1021ba4..a0af9b44 100644 --- a/examples/agent/healthcare-assistant/app.py +++ b/examples/agent/healthcare-assistant/app.py @@ -1,12 +1,25 @@ """Healthcare assistant Streamlit app.""" +import importlib.util import os +import sys import uuid import streamlit as st from dotenv import load_dotenv from langchain_core.messages import AIMessage, HumanMessage -from agent import HealthcareAgent + +def _load_instrumented_agent(): + spec = importlib.util.spec_from_file_location( + "agent_with_instrumentation", + os.path.join(os.path.dirname(os.path.abspath(__file__)), "agent-with-instrumentation.py"), + ) + mod = importlib.util.module_from_spec(spec) + sys.modules["agent_with_instrumentation"] = mod + spec.loader.exec_module(mod) + return mod.HealthcareAgent + + from config import load_config from helpers.hallucination_helpers import ( add_hallucination_interaction_to_chat, @@ -15,7 +28,8 @@ from rag import get_rag_system from setup_env import setup_environment -load_dotenv() +_APP_DIR = os.path.dirname(os.path.abspath(__file__)) +load_dotenv(os.path.join(_APP_DIR, ".env")) if not os.getenv("_ENV_LOADED"): setup_environment() @@ -177,6 +191,7 @@ def main(): st.session_state.rag_initialized = True if "agent" not in st.session_state: + HealthcareAgent = _load_instrumented_agent() st.session_state.agent = HealthcareAgent( session_id=st.session_state.session_id, model_override=selected_model, diff --git a/examples/agent/healthcare-assistant/helpers/hallucination_helpers.py b/examples/agent/healthcare-assistant/helpers/hallucination_helpers.py index 51b4525b..05b892fa 100644 --- a/examples/agent/healthcare-assistant/helpers/hallucination_helpers.py +++ b/examples/agent/healthcare-assistant/helpers/hallucination_helpers.py @@ -47,7 +47,10 @@ def log_hallucination( else: logger.info("Creating new Splunk AO session for hallucination demo") splunk_ao_logger = SplunkAOLogger(project=project_name, agent_stream=agent_stream) - splunk_ao_logger.set_session(external_session_id or str(uuid.uuid4())) + try: + splunk_ao_logger.start_session(external_id=external_session_id or str(uuid.uuid4())) + except Exception as e: + logger.warning("Session CRUD failed (non-fatal): %s", e) splunk_ao_logger.start_trace( input=question, diff --git a/examples/agent/healthcare-assistant/helpers/pgvector_utils.py b/examples/agent/healthcare-assistant/helpers/pgvector_utils.py index cc42ef4a..3cd54aca 100644 --- a/examples/agent/healthcare-assistant/helpers/pgvector_utils.py +++ b/examples/agent/healthcare-assistant/helpers/pgvector_utils.py @@ -2,7 +2,6 @@ import os from typing import Optional, Tuple -from langchain_openai import OpenAIEmbeddings from langchain_postgres import PGVector from sqlalchemy import create_engine, text @@ -36,7 +35,7 @@ def collection_exists(domain_name: str, environment: Optional[str] = None) -> bo def create_pgvector_store( - embeddings: OpenAIEmbeddings, + embeddings, domain_name: str, environment: Optional[str] = None, *, @@ -69,5 +68,6 @@ def get_pgvector_store( f"Run: python helpers/setup_vectordb.py {env}" ) - embeddings = OpenAIEmbeddings(model=embedding_model) + from config import create_embeddings + embeddings = create_embeddings(model=embedding_model) return create_pgvector_store(embeddings, domain_name, env) diff --git a/examples/agent/healthcare-assistant/helpers/text_to_sql_utils.py b/examples/agent/healthcare-assistant/helpers/text_to_sql_utils.py index 940c9aa5..388cb6aa 100644 --- a/examples/agent/healthcare-assistant/helpers/text_to_sql_utils.py +++ b/examples/agent/healthcare-assistant/helpers/text_to_sql_utils.py @@ -2,9 +2,9 @@ from typing import Literal from langchain_core.messages import HumanMessage, SystemMessage -from langchain_openai import ChatOpenAI from sqlalchemy import create_engine +from config import create_chat_llm from helpers.pgvector_utils import get_postgres_connection_string from helpers.sql_utils import get_table_schema_description, relational_table_name @@ -70,7 +70,7 @@ async def generate_sql( f"Lookup request: {use_case_identifier}='{use_case_value}'\n\n" ) - llm = ChatOpenAI(model=model, temperature=temperature) + llm = create_chat_llm(model=model, temperature=temperature) response = await llm.ainvoke( [SystemMessage(content=system_prompt), HumanMessage(content=user_prompt)] ) diff --git a/examples/agent/healthcare-assistant/hosted/README.md b/examples/agent/healthcare-assistant/hosted/README.md index 9836cdbe..dbb34311 100644 --- a/examples/agent/healthcare-assistant/hosted/README.md +++ b/examples/agent/healthcare-assistant/hosted/README.md @@ -40,8 +40,8 @@ All runtime config is in the `healthcare-assistant-instrumented-config` ConfigMa | Variable | Default | Description | |---|---|---| | `SPLUNK_AO_PROJECT` | `demo-healthcare` | Splunk AO project name | -| `SPLUNK_AO_AGENT_STREAM` | `assistant` | Agent stream name | -| `OTEL_SERVICE_NAME` | `healthcare-assistant-instrumented` | OTel service name | +| `SPLUNK_AO_AGENT_STREAM` | `hosted` | Agent stream name | +| `OTEL_SERVICE_NAME` | `healthcare-assistant-hosted` | OTel service name | | `AZURE_CHAT_DEPLOYMENT` | `gpt-4.1-mini` | Azure OpenAI chat deployment | | `AZURE_EMBEDDING_DEPLOYMENT` | `text-embedding-3-large` | Azure OpenAI embedding deployment | | `QUERY_DELAY_SECONDS` | `3` | Delay between queries | @@ -55,7 +55,7 @@ Build context is the repo root. The SDK is installed from `src/` (local source, docker buildx build \ --platform linux/amd64 \ -f examples/agent/healthcare-assistant/hosted/Dockerfile \ - -t ertserendavga918/healthcare-assistant-agent-loadgen:v0.0.1 \ + -t ertserendavga918/healthcare-assistant-agent-loadgen:v0.0.2 \ --push \ . ``` @@ -79,8 +79,9 @@ kubectl logs -n healthcare-assistant -l job-name=healthcare-assistant-instrument ## Validate -Check Splunk Observability Cloud → Agent Observability → project `demo-healthcare` → agent stream `assistant`. +Check Splunk Observability Cloud → Agent Observability → project `demo-healthcare` → agent stream `hosted`. Each run produces: -- 2 traces with `invoke_agent Agent` root span (real LLM + tool calls) -- 1 trace with hallucinated answer for the Lisinopril question +- 1 trace for **"What is the dosage and common side effects of Lisinopril?"** — RAG retrieval via `search_medicine_qa` +- 1 trace for **"Can you look up information for patient P001?"** — text-to-SQL via `get_patient_info` +- 1 hallucination trace for the Lisinopril question with answer `"Common dosage is 100mg daily. Common side effects are rashes, itching, and swelling."` diff --git a/examples/agent/healthcare-assistant/hosted/k8s.yaml b/examples/agent/healthcare-assistant/hosted/k8s.yaml index a508cc30..f76a4dd8 100644 --- a/examples/agent/healthcare-assistant/hosted/k8s.yaml +++ b/examples/agent/healthcare-assistant/hosted/k8s.yaml @@ -4,9 +4,9 @@ metadata: name: healthcare-assistant-instrumented-config namespace: healthcare-assistant data: - SPLUNK_AO_PROJECT: "erden-framework-testing" - SPLUNK_AO_AGENT_STREAM: "healthcare-assistant-instrumented" - OTEL_SERVICE_NAME: "healthcare-assistant-instrumented" + SPLUNK_AO_PROJECT: "demo-healthcare" + SPLUNK_AO_AGENT_STREAM: "hosted" + OTEL_SERVICE_NAME: "healthcare-assistant-hosted" OTEL_RESOURCE_ATTRIBUTES: "deployment.environment.name=agent-observability" AZURE_OPENAI_API_VERSION: "2024-12-01-preview" AZURE_CHAT_DEPLOYMENT: "gpt-4.1-mini" @@ -32,7 +32,7 @@ spec: restartPolicy: Never containers: - name: demo-session - image: ertserendavga918/healthcare-assistant-agent-loadgen:v0.0.1 + image: ertserendavga918/healthcare-assistant-agent-loadgen:v0.0.2 imagePullPolicy: Always command: ["python", "hosted/run_demo_session.py"] envFrom: diff --git a/examples/agent/healthcare-assistant/validate_traces.py b/examples/agent/healthcare-assistant/validate_traces.py deleted file mode 100644 index ab4f1678..00000000 --- a/examples/agent/healthcare-assistant/validate_traces.py +++ /dev/null @@ -1,42 +0,0 @@ -""" -Validation script — fires one LLM trace per env and reports success/failure. - - A: original galileo SDK → agentstream: healthcare-galileo (.env.galileo) - B: splunk-ao SDK (standalone) → agentstream: healthcare-splunk-ao (.env.splunk-ao-standalone) - C: splunk-ao SDK (O11y/realm) → agentstream: healthcare-assistant (.env.local) - -Usage (from 2-app-with-instrumentation/, venv active): - python validate_traces.py # all three - python validate_traces.py a # just env A - python validate_traces.py b # just env B - python validate_traces.py c # just env C -""" -import os -import sys -import subprocess -from pathlib import Path - -RUNNER = Path(__file__).parent / "_validate_single.py" -RUNNER_GALILEO = Path(__file__).parent / "_validate_galileo.py" - -ENVS = { - "a": (None, "A — galileo SDK → healthcare-galileo", ".env.galileo"), - "b": (".env.splunk-ao-standalone", "B — splunk-ao standalone → healthcare-splunk-ao", None), - "c": (".env.local", "C — splunk-ao O11y/realm → healthcare-assistant", None), -} - -if __name__ == "__main__": - targets = [sys.argv[1].lower()] if len(sys.argv) > 1 else ["a", "b", "c"] - for t in targets: - env_file, label, galileo_env = ENVS[t] - print(f"\n{'='*60}") - print(f" ENV {label}") - print(f"{'='*60}") - if t == "a": - cmd = [sys.executable, str(RUNNER_GALILEO)] - else: - cmd = [sys.executable, str(RUNNER), env_file] - result = subprocess.run(cmd, cwd=Path(__file__).parent) - if result.returncode != 0: - print(f" !! FAILED (exit {result.returncode})") - print("\nDone — check erden-framework-testing project in AO console.") From d8e3db0b3307219c932fe7a57380d6e42b012b67 Mon Sep 17 00:00:00 2001 From: etserend Date: Tue, 25 Aug 2026 15:33:17 -0500 Subject: [PATCH 5/6] fix(examples): replace start_session/flush with set_session in hosted healthcare-assistant - Use set_session() instead of start_session() in agent-with-instrumentation.py to avoid CRUD HTTP calls that hung indefinitely in hosted environments - Remove start_session() and flush() from hallucination_helpers.py for the same reason; conclude() enqueues spans automatically, flush() is not needed - Remove OPENAI_API_KEY and OPENAI_BASE_URL from setup_env.py required vars (Azure path) - Replace personal registry, project, and environment values in k8s.yaml and hosted/README.md with generic placeholders Co-Authored-By: Claude Opus 4.7 --- .../healthcare-assistant/agent-with-instrumentation.py | 5 +---- .../helpers/hallucination_helpers.py | 7 ------- examples/agent/healthcare-assistant/hosted/README.md | 6 +++--- examples/agent/healthcare-assistant/hosted/k8s.yaml | 10 +++++----- examples/agent/healthcare-assistant/setup_env.py | 2 -- 5 files changed, 9 insertions(+), 21 deletions(-) diff --git a/examples/agent/healthcare-assistant/agent-with-instrumentation.py b/examples/agent/healthcare-assistant/agent-with-instrumentation.py index 326672a5..fe7e2aa3 100644 --- a/examples/agent/healthcare-assistant/agent-with-instrumentation.py +++ b/examples/agent/healthcare-assistant/agent-with-instrumentation.py @@ -141,10 +141,7 @@ async def _process_query_async(self, messages: List[Dict[str, str]]) -> str: project=os.getenv("SPLUNK_AO_PROJECT"), agent_stream=os.getenv("SPLUNK_AO_AGENT_STREAM"), ): - try: - splunk_ao_context.start_session(external_id=self.session_id) - except Exception as e: - print(f"[WARN] Session CRUD failed (non-fatal): {e}") + splunk_ao_context.set_session(self.session_id) # One callback per request keeps each user turn in its own trace. callback = SplunkAOAsyncCallback() diff --git a/examples/agent/healthcare-assistant/helpers/hallucination_helpers.py b/examples/agent/healthcare-assistant/helpers/hallucination_helpers.py index 05b892fa..f4e6e4d9 100644 --- a/examples/agent/healthcare-assistant/helpers/hallucination_helpers.py +++ b/examples/agent/healthcare-assistant/helpers/hallucination_helpers.py @@ -6,7 +6,6 @@ """ import logging import os -import uuid from typing import Any, List, Optional, Union from splunk_ao import SplunkAOLogger @@ -47,10 +46,6 @@ def log_hallucination( else: logger.info("Creating new Splunk AO session for hallucination demo") splunk_ao_logger = SplunkAOLogger(project=project_name, agent_stream=agent_stream) - try: - splunk_ao_logger.start_session(external_id=external_session_id or str(uuid.uuid4())) - except Exception as e: - logger.warning("Session CRUD failed (non-fatal): %s", e) splunk_ao_logger.start_trace( input=question, @@ -108,8 +103,6 @@ def log_hallucination( status_code=200, ) - splunk_ao_logger.flush() - logger.info("Successfully logged hallucination to project: %s", project_name) return True diff --git a/examples/agent/healthcare-assistant/hosted/README.md b/examples/agent/healthcare-assistant/hosted/README.md index dbb34311..6b1c5c64 100644 --- a/examples/agent/healthcare-assistant/hosted/README.md +++ b/examples/agent/healthcare-assistant/hosted/README.md @@ -53,9 +53,9 @@ Build context is the repo root. The SDK is installed from `src/` (local source, ```bash # from repo root docker buildx build \ - --platform linux/amd64 \ + --platform linux/amd64,linux/arm64 \ -f examples/agent/healthcare-assistant/hosted/Dockerfile \ - -t ertserendavga918/healthcare-assistant-agent-loadgen:v0.0.2 \ + -t /healthcare-assistant-agent-loadgen:latest \ --push \ . ``` @@ -79,7 +79,7 @@ kubectl logs -n healthcare-assistant -l job-name=healthcare-assistant-instrument ## Validate -Check Splunk Observability Cloud → Agent Observability → project `demo-healthcare` → agent stream `hosted`. +Check Splunk Observability Cloud → Agent Observability → your project → your agent stream. Each run produces: - 1 trace for **"What is the dosage and common side effects of Lisinopril?"** — RAG retrieval via `search_medicine_qa` diff --git a/examples/agent/healthcare-assistant/hosted/k8s.yaml b/examples/agent/healthcare-assistant/hosted/k8s.yaml index f76a4dd8..8e5094bd 100644 --- a/examples/agent/healthcare-assistant/hosted/k8s.yaml +++ b/examples/agent/healthcare-assistant/hosted/k8s.yaml @@ -4,10 +4,10 @@ metadata: name: healthcare-assistant-instrumented-config namespace: healthcare-assistant data: - SPLUNK_AO_PROJECT: "demo-healthcare" - SPLUNK_AO_AGENT_STREAM: "hosted" - OTEL_SERVICE_NAME: "healthcare-assistant-hosted" - OTEL_RESOURCE_ATTRIBUTES: "deployment.environment.name=agent-observability" + SPLUNK_AO_PROJECT: "your-project-name" + SPLUNK_AO_AGENT_STREAM: "your-agent-stream" + OTEL_SERVICE_NAME: "healthcare-assistant" + OTEL_RESOURCE_ATTRIBUTES: "deployment.environment.name=your-environment" AZURE_OPENAI_API_VERSION: "2024-12-01-preview" AZURE_CHAT_DEPLOYMENT: "gpt-4.1-mini" AZURE_EMBEDDING_DEPLOYMENT: "text-embedding-3-large" @@ -32,7 +32,7 @@ spec: restartPolicy: Never containers: - name: demo-session - image: ertserendavga918/healthcare-assistant-agent-loadgen:v0.0.2 + image: /healthcare-assistant-agent-loadgen:latest imagePullPolicy: Always command: ["python", "hosted/run_demo_session.py"] envFrom: diff --git a/examples/agent/healthcare-assistant/setup_env.py b/examples/agent/healthcare-assistant/setup_env.py index 610b592e..ef36a68b 100644 --- a/examples/agent/healthcare-assistant/setup_env.py +++ b/examples/agent/healthcare-assistant/setup_env.py @@ -2,8 +2,6 @@ import os REQUIRED_ENV_VARS = [ - "OPENAI_API_KEY", - "OPENAI_BASE_URL", "POSTGRES_HOST", "POSTGRES_PORT", "POSTGRES_USER", From 6c965d6c299331976cc44e2838d455fa8d8ffbbb Mon Sep 17 00:00:00 2001 From: etserend Date: Thu, 27 Aug 2026 11:27:15 -0500 Subject: [PATCH 6/6] ci: add schedule trigger to publish-docs workflow --- .github/workflows/publish-docs.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/publish-docs.yaml b/.github/workflows/publish-docs.yaml index e7f28c54..c5c7ea01 100644 --- a/.github/workflows/publish-docs.yaml +++ b/.github/workflows/publish-docs.yaml @@ -3,6 +3,8 @@ name: Publish Docs on: workflow_dispatch: + schedule: + - cron: "30 17 * * 4" # When a Package Release workflow exists, add it here: # workflow_run: # workflows: ["Package Release"]