diff --git a/.github/workflows/publish-docs.yaml b/.github/workflows/publish-docs.yaml new file mode 100644 index 00000000..d12f2289 --- /dev/null +++ b/.github/workflows/publish-docs.yaml @@ -0,0 +1,100 @@ +# Publishes SDK docs to the Splunk AO docs repo. +name: Publish Docs + +on: + workflow_dispatch: + # When a Package Release workflow exists, add it here: + # workflow_run: + # workflows: ["Package Release"] + # types: [completed] + +jobs: + publish-docs: + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v5 + with: + path: ./splunk-ao-python + + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: "3.11" + + - name: Install Docstring Parser + run: pip install 'docstring-parser>=0.17.0' + + - name: Build Docs + working-directory: ./splunk-ao-python + run: python ./scripts/create_docs.py + + - name: Verify Docs Output + working-directory: ./splunk-ao-python + run: | + count=$(find .generated_docs/reference -name "*.mdx" | wc -l) + echo "Generated $count .mdx files" + if [ "$count" -lt 1 ]; then + echo "ERROR: No .mdx files generated" + exit 1 + fi + + - name: Check out Docs repo + uses: actions/checkout@v5 + with: + repository: splunk/agent-observability-docs + path: ./splunk-ao-docs + token: ${{ secrets.PAT_AO_DOC_AUTOMATION }} + + - name: Create a branch + working-directory: ./splunk-ao-docs + run: git checkout -b feat/update-docs-${GITHUB_REF_NAME}-${{ github.run_id }} + + - name: Update SDK docs + run: | + rm -rf ./splunk-ao-docs/sdk-api/python/reference + mkdir -p ./splunk-ao-docs/sdk-api/python + cp -r ./splunk-ao-python/.generated_docs/reference ./splunk-ao-docs/sdk-api/python/ + + - name: Commit changes + id: commit + working-directory: ./splunk-ao-docs + env: + PAT: ${{ secrets.PAT_AO_DOC_AUTOMATION }} + run: | + git config --global user.email "github-actions[bot]@users.noreply.github.com" + git config --global user.name "github-actions[bot]" + git remote set-url origin https://x-access-token:${PAT}@github.com/splunk/agent-observability-docs.git + git add . + if git diff --cached --quiet; then + echo "No doc changes to publish" + echo "has_changes=false" >> "$GITHUB_OUTPUT" + exit 0 + fi + echo "has_changes=true" >> "$GITHUB_OUTPUT" + git commit -m "Updating docs from ${GITHUB_REF_NAME}-${{ github.run_id }}" + git push -u origin feat/update-docs-${GITHUB_REF_NAME}-${{ github.run_id }} + + - name: Create PR + if: steps.commit.outputs.has_changes == 'true' + working-directory: ./splunk-ao-docs + env: + GH_TOKEN: ${{ secrets.PAT_AO_DOC_AUTOMATION }} + PR_BODY: | + This PR contains the latest updates to the Splunk AO Python SDK documentation. + + The SDK doc generation will not change docs.json for new or removed files, so you + will need to manually commit a change to this PR for any file changes. + + When reviewing this PR, please do the following: + + - [ ] If there are any new files added by the PR, add them to the docs.json file in the right location + - [ ] If there are any files removed by the PR, remove them from the docs.json + - [ ] Launch the docs and check for any file path errors + - [ ] If any SDK docs are removed, check why — this should not happen. If the docstrings are wrong they should be fixed, not removed + - [ ] For added docs, check they are correct (e.g. match the function signature) + + **DO NOT** approve this PR until you have completed these checks. + run: | + gh pr create --title "Updated Python SDK docs" --body "$PR_BODY" diff --git a/.github/workflows/regenerate-api-client.yaml b/.github/workflows/regenerate-api-client.yaml new file mode 100644 index 00000000..f226459a --- /dev/null +++ b/.github/workflows/regenerate-api-client.yaml @@ -0,0 +1,74 @@ +# Regenerates the OpenAPI Python client from the Splunk AO backend spec. +name: Update API Client + +on: + workflow_dispatch: + schedule: + - cron: "0 13 * * 1-5" # Runs at 9am ET M-F + +jobs: + update_api_client: + runs-on: ubuntu-latest + + permissions: + contents: "write" + + env: + CI_ENV: dev + + steps: + - name: Update ubuntu, install dependencies + run: | + sudo apt-get update -y + sudo apt-get install socat wget jq -y + + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.13" + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install poetry + poetry install --all-extras --with dev + + - name: Get current date + id: date + run: echo "date=$(date +'%Y-%m-%d')" >> "$GITHUB_OUTPUT" + + - name: Fetch API types and regenerate + env: + PAT: ${{ secrets.PAT_AO_DOC_AUTOMATION }} + API_URL: ${{ secrets.SPLUNK_AO_API_URL }} + BRANCH_NAME: chore/splunk-ao-automation/update-api-client-${{ steps.date.outputs.date }}-${{ github.run_id }} + run: | + git config --global user.email "github-actions[bot]@users.noreply.github.com" + git config --global user.name "github-actions[bot]" + git remote set-url origin https://x-access-token:${PAT}@github.com/splunk/splunk-ao-python.git + git checkout -b $BRANCH_NAME + poetry run ./scripts/import-openapi-yaml.sh "$API_URL" + poetry run ./scripts/auto-generate-api-client.sh + git add . + if git diff --cached --quiet; then + echo "No changes to commit — spec is unchanged" + exit 0 + fi + git commit -m "Update API Client" + git push origin $BRANCH_NAME + + - name: Create pull request + env: + BRANCH_NAME: chore/splunk-ao-automation/update-api-client-${{ steps.date.outputs.date }}-${{ github.run_id }} + GH_TOKEN: ${{ secrets.PAT_AO_DOC_AUTOMATION }} + run: | + if ! git ls-remote --exit-code origin refs/heads/$BRANCH_NAME > /dev/null 2>&1; then + echo "No branch to create PR from — nothing was committed" + exit 0 + fi + gh pr create -B main -H $BRANCH_NAME --label 'dependencies' --title 'Update API Client' --body 'Fix any breaking changes if this pull request fails to deploy' diff --git a/pydoc-markdown-handlers.yml b/pydoc-markdown-handlers.yml index c666cc51..79e2fde9 100644 --- a/pydoc-markdown-handlers.yml +++ b/pydoc-markdown-handlers.yml @@ -1,4 +1,4 @@ -# Special PyDoc markdown configuration for the Galileo handlers module as this isn't created by default for some reason +# Special PyDoc markdown configuration for the Splunk AO handlers module as this isn't created by default for some reason loaders: - type: python search_path: [./src/splunk_ao/handlers] diff --git a/scripts/create_docs.py b/scripts/create_docs.py index 1b90f52e..4a2fc7b1 100644 --- a/scripts/create_docs.py +++ b/scripts/create_docs.py @@ -142,21 +142,24 @@ def _parse_text(txt: str | None) -> dict[str, Any] | None: examples = [] for e in getattr(parsed, "examples", []) or []: code = "" + had_snippet = False snippet = getattr(e, "snippet", None) if snippet: snippet = snippet.replace("\n>>> ", "\n") description = getattr(e, "description", None) if snippet and description: + had_snippet = True if snippet.startswith(">>>"): code = f"{snippet}\n{description.replace('... ', '').replace('>>> ', '')}" else: code = f"{snippet}\n{description}" elif snippet: + had_snippet = True code = f"{snippet}" elif description: code = f"{description}" if code: - examples.append({"code": code}) + examples.append({"code": code, "had_snippet": had_snippet}) if examples: out["examples"] = examples @@ -501,6 +504,59 @@ def build_docs(files_docs: list[FileDoc], source_root: str, output_root: str) -> print(f"Documentation generation complete. Output written to {out_root}") +def _escape_curly_braces(text: str) -> str: + """Escape { and } outside fenced and inline code blocks so MDX doesn't treat them as JSX.""" + triple_segments = re.split(r"(```.*?```)", text, flags=re.DOTALL) + for i, triple_seg in enumerate(triple_segments): + if triple_seg.startswith("```"): + continue + single_segments = re.split(r"(`[^`]*`)", triple_seg) + for j, single_seg in enumerate(single_segments): + if not single_seg.startswith("`"): + single_seg = single_seg.replace("{", r"\{").replace("}", r"\}") + single_segments[j] = single_seg + triple_segments[i] = "".join(single_segments) + return "".join(triple_segments) + + +def _convert_rst_code_blocks(text: str) -> str: + """Convert RST :: indented code blocks to fenced ```python blocks.""" + lines = text.split("\n") + out: list[str] = [] + i = 0 + while i < len(lines): + line = lines[i] + if line.rstrip().endswith("::"): + # Emit the line with :: stripped (or removed if standalone) + stripped = line.rstrip()[:-2].rstrip() + out.append(stripped) + i += 1 + # Collect the indented block that follows + block: list[str] = [] + while i < len(lines) and (lines[i].strip() == "" or lines[i].startswith(" ")): + block.append(lines[i]) + i += 1 + # Strip trailing blank lines from block + while block and block[-1].strip() == "": + block.pop() + if block: + out.append("```python") + for bl in block: + out.append(bl[4:] if bl.startswith(" ") else bl) + out.append("```") + else: + out.append(line) + i += 1 + return "\n".join(out) + + +def _sanitize_description(text: str) -> str: + """Convert RST code blocks and escape MDX-unsafe curly braces in description text.""" + text = _convert_rst_code_blocks(text) + text = _escape_curly_braces(text) + return text + + def write_function(parts: list[str], fn: Any, heading_level: int = 3) -> None: """ Append formatted documentation for a function to the provided parts list. @@ -537,7 +593,7 @@ def write_function(parts: list[str], fn: Any, heading_level: int = 3) -> None: fdoc = fn.doc if isinstance(fdoc, dict): if fdoc.get("description"): - parts.append(f"{fdoc.get('description')}\n") + parts.append(f"{_sanitize_description(fdoc.get('description'))}\n") if fdoc.get("params"): parts.append("**Arguments**\n") for p in fdoc.get("params", []): @@ -545,26 +601,7 @@ def write_function(parts: list[str], fn: Any, heading_level: int = 3) -> None: t = p.get("type") desc = p.get("description") or "" desc = desc.replace("\n", "\n ") - - # Escape curly braces outside code blocks (triple or single backticks) - def escape_curly_braces(text: str) -> str: - # Split by triple backtick code blocks first - triple_segments = re.split(r"(```.*?```)", text, flags=re.DOTALL) - for i, triple_seg in enumerate(triple_segments): - if triple_seg.startswith("```"): - # Inside triple backtick code block, leave as is - continue - # Now split by single backtick code blocks - single_segments = re.split(r"(`[^`]*`)", triple_seg) - for j, single_seg in enumerate(single_segments): - if not single_seg.startswith("`"): - single_seg = single_seg.replace("{", r"\{").replace("}", r"\}") - single_segments[j] = single_seg - triple_segments[i] = "".join(single_segments) - return "".join(triple_segments) - - desc = escape_curly_braces(desc) - + desc = _escape_curly_braces(desc) if t: parts.append(f"- `{name}` (`{t}`): {desc}") else: @@ -574,7 +611,7 @@ def escape_curly_braces(text: str) -> str: parts.append("**Raises**\n") for r in fdoc.get("raises", []): typ = r.get("type") or "Exception" - desc = r.get("description") or "" + desc = _escape_curly_braces(r.get("description") or "") parts.append(f"- `{typ}`: {desc}") parts.append("") if fdoc.get("returns"): @@ -582,29 +619,29 @@ def escape_curly_braces(text: str) -> str: ret = fdoc.get("returns") if ret: rtyp = ret.get("type") or "" - rdesc = ret.get("description") or "" + rdesc = _escape_curly_braces(ret.get("description") or "") if rtyp: parts.append(f"- `{rtyp}`: {rdesc}\n") else: parts.append(f"- {rdesc}\n") if fdoc.get("examples"): parts.append("**Examples**\n") - is_in_code_block = False for ex in fdoc.get("examples", []): code = ex.get("code") - if code: - if not is_in_code_block and code.startswith(">>>"): - is_in_code_block = True - parts.append("```python") - if is_in_code_block and not code.startswith(">>>") and not code.startswith("..."): - is_in_code_block = False - parts.append("```\n") - if not code.startswith(">>>") and not code.startswith("..."): - parts.append(code) - else: - parts.append(code[4:]) - if is_in_code_block: - parts.append("```\n") + if not code: + continue + if "```" in code: + parts.append(code) + elif code.startswith(">>>") or code.startswith("..."): + parts.append("```python") + parts.append(code[4:]) + parts.append("```") + elif ex.get("had_snippet"): + parts.append("```python") + parts.append(code) + parts.append("```") + else: + parts.append(_escape_curly_braces(code)) parts.append("") if fdoc.get("notes"): parts.append("**Notes**\n") @@ -651,7 +688,7 @@ def write_class(parts: list[str], cls: Any) -> None: cdoc = cls.doc if isinstance(cdoc, dict): if cdoc.get("description"): - parts.append(f"{cdoc.get('description')}\n") + parts.append(f"{_sanitize_description(cdoc.get('description'))}\n") if cdoc.get("params"): parts.append("**Arguments**\n") for p in cdoc.get("params", []): @@ -659,6 +696,7 @@ def write_class(parts: list[str], cls: Any) -> None: t = p.get("type") desc = p.get("description") or "" desc = desc.replace("\n", "\n ") + desc = _escape_curly_braces(desc) if t: parts.append(f"- `{name}` (`{t}`): {desc}\n") else: @@ -666,29 +704,29 @@ def write_class(parts: list[str], cls: Any) -> None: if cdoc.get("examples"): parts.append("**Examples**\n") - is_in_code_block = False for ex in cdoc.get("examples", []): code = ex.get("code") - if code: - if not is_in_code_block and code.startswith(">>>"): - is_in_code_block = True - parts.append("```python") - if is_in_code_block and not code.startswith(">>>") and not code.startswith("..."): - is_in_code_block = False - parts.append("```\n") - if not code.startswith(">>>") and not code.startswith("..."): - parts.append(code) - else: - parts.append(code[4:]) - if is_in_code_block: - parts.append("```\n") + if not code: + continue + if "```" in code: + parts.append(code) + elif code.startswith(">>>") or code.startswith("..."): + parts.append("```python") + parts.append(code[4:]) + parts.append("```") + elif ex.get("had_snippet"): + parts.append("```python") + parts.append(code) + parts.append("```") + else: + parts.append(_escape_curly_braces(code)) parts.append("") if cdoc.get("returns"): parts.append("**Returns**\n") ret = cdoc.get("returns") if ret: rtyp = ret.get("type") or "" - rdesc = ret.get("description") or "" + rdesc = _escape_curly_braces(ret.get("description") or "") if rtyp: parts.append(f"- `{rtyp}`: {rdesc}\n") else: @@ -736,17 +774,15 @@ def write_module(fd: FileDoc, parts: list[str]) -> None: md = fd.module.doc if isinstance(md, dict): if md.get("description"): - parts.append(f"{md.get('description')}\n") + parts.append(f"{_sanitize_description(md.get('description'))}\n") if md.get("params"): parts.append("**Arguments**\n") for p in md.get("params", []): name = p.get("name") or "" t = p.get("type") desc = p.get("description") or "" - - # If description is multi-line, indent subsequent lines for better Markdown rendering desc = desc.replace("\n", "\n ") - + desc = _escape_curly_braces(desc) if t: parts.append(f"- `{name}` (`{t}`): {desc}\n") else: @@ -755,14 +791,14 @@ def write_module(fd: FileDoc, parts: list[str]) -> None: parts.append("**Raises**\n") for r in md.get("raises", []): typ = r.get("type") or "Exception" - desc = r.get("description") or "" + desc = _escape_curly_braces(r.get("description") or "") parts.append(f"- `{typ}`: {desc}\n") if md.get("returns"): parts.append("**Returns**\n") ret = md.get("returns") if ret: rtyp = ret.get("type") or "" - rdesc = ret.get("description") or "" + rdesc = _escape_curly_braces(ret.get("description") or "") if rtyp: parts.append(f"- `{rtyp}`: {rdesc}\n") else: