diff --git a/automation/source-repo-templates/README.md b/automation/source-repo-templates/README.md index 055fe976..3186317a 100644 --- a/automation/source-repo-templates/README.md +++ b/automation/source-repo-templates/README.md @@ -65,7 +65,7 @@ it ships its own rustdoc pipeline today. | -------------------------- | ------ | ------------------------ | ---------------------------- | | `resq-software/npm` | TS | TypeDoc + markdown plug. | `api-docs.typescript.yml` | | `resq-software/dotnet-sdk` | C# | DefaultDocumentation | `api-docs.dotnet.yml` | -| `resq-software/pypi` | Python | mkdocstrings | _TODO_ | +| `resq-software/pypi` | Python | lazydocs | `api-docs.python.yml` | | `resq-software/programs` | Rust | rustdoc + cargo-readme | _TODO_ | | `resq-software/vcpkg` | C++ | Doxygen + moxygen | _TODO_ | | `resq-software/viz` | C#/web | DefaultDocumentation | _TODO_ | diff --git a/automation/source-repo-templates/api-docs.python.yml b/automation/source-repo-templates/api-docs.python.yml new file mode 100644 index 00000000..04d79c08 --- /dev/null +++ b/automation/source-repo-templates/api-docs.python.yml @@ -0,0 +1,349 @@ +# Copyright 2026 ResQ Software +# SPDX-License-Identifier: Apache-2.0 +# +# Source-of-truth template lives in resq-software/docs: +# automation/source-repo-templates/api-docs.python.yml +# +# Copy this file to resq-software/pypi at: +# .github/workflows/api-docs.yml +# +# Renders Markdown API reference for every Python package in the +# monorepo and opens a PR in resq-software/docs with generated +# content under sdks/python/api/. +# +# Pipeline: +# 1. uv pip install -e + lazydocs +# 2. lazydocs -> per-module Markdown files +# 3. Mintlify-safety post-processing (./ prefix on bare links; +# MDX curly-brace escape outside code regions) +# 4. peter-evans/create-pull-request opens a PR in the docs repo + +name: api-docs + +on: + push: + # pypi monorepo uses single-versioned bare semver tags (v1.3.4, + # v1.3.3, ...) covering both packages at once, so a single + # tag pattern is enough. + tags: + - 'v*' + workflow_dispatch: + inputs: + ref: + description: 'Source ref to document (defaults to current ref)' + required: false + type: string + +permissions: + contents: read + +concurrency: + # Each run force-pushes the same auto/python-api- branch in + # the docs repo. Cancel any earlier run still in flight so we do + # not race on that push. + group: api-docs-${{ inputs.ref || github.ref }} + cancel-in-progress: true + +jobs: + generate: + runs-on: ubuntu-latest + timeout-minutes: 15 + + env: + PYTHON_VERSION: '3.13' + OUTPUT_DIR: generated-docs + DOCS_TARGET: sdks/python/api + # Public packages to document. Format: :. + # The pyproject directory is the path under packages/; the + # import name is what gets passed to `lazydocs` (typically the + # underscore form of the distribution name). + PUBLIC_PACKAGES: >- + resq-mcp:resq_mcp + resq-dsa:resq_dsa + + steps: + - name: Resolve ref metadata + # Single source of truth for the ref this run documents. + # workflow_dispatch can pass an alternate ref via inputs.ref; + # fall back to github.ref_name (already stripped of refs/...). + # DOCS_REF_SLUG is branch-safe for use in PR/branch names. + run: | + raw='${{ inputs.ref || github.ref_name }}' + raw="${raw#refs/tags/}" + raw="${raw#refs/heads/}" + slug="${raw//\//-}" + echo "DOCS_REF_NAME=$raw" >> "$GITHUB_ENV" + echo "DOCS_REF_SLUG=$slug" >> "$GITHUB_ENV" + + - name: Checkout source repo + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + ref: ${{ inputs.ref || github.ref }} + persist-credentials: false + + - name: Setup Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: ${{ env.PYTHON_VERSION }} + + - name: Install uv + uses: astral-sh/setup-uv@c7f87aa956e4c323abf06d5dec078e358f6b4d04 # v6.0.0 + + - name: Install packages + lazydocs into a venv + # lazydocs imports the packages it documents, so they must + # be installed in the active Python environment. uv pip + # install -e per package; the editable install resolves all + # transitive runtime deps the same way `pip install` would. + run: | + set -euo pipefail + uv venv .venv --python "$PYTHON_VERSION" + # shellcheck disable=SC1091 + source .venv/bin/activate + for entry in $PUBLIC_PACKAGES; do + pkg_dir="${entry%%:*}" + uv pip install -e "packages/$pkg_dir" + done + # pydoc-markdown rather than lazydocs: lazydocs uses + # pkgutil's deprecated find_module(), which Python 3.12 + # removed. Every submodule walk fails with + # AttributeError: 'FileFinder' object has no attribute 'find_module' + # on the runner's Python 3.13. pydoc-markdown is on the + # find_spec() API and produces Markdown directly. + uv pip install "pydoc-markdown>=4" + + - name: Generate per-package Markdown + # `pydoc-markdown -p ` only documents the top-level + # module — it does NOT recurse into submodules, which left + # both overview.md files containing nothing but + # `# Table of Contents`. Walk the import graph ourselves + # via pkgutil and pass every submodule explicitly via an + # inline YAML config; this also lets us bump the renderer + # to enable module headers / class signatures so the output + # matches what lazydocs used to produce. + run: | + set -euo pipefail + # shellcheck disable=SC1091 + source .venv/bin/activate + rm -rf "$OUTPUT_DIR" + mkdir -p "$OUTPUT_DIR" + missing=0 + generated=0 + for entry in $PUBLIC_PACKAGES; do + pkg_dir="${entry%%:*}" + mod_name="${entry##*:}" + out="$OUTPUT_DIR/$pkg_dir" + mkdir -p "$out" + if ! python -c "import ${mod_name}" 2>/dev/null; then + echo "::error ::cannot import ${mod_name} (entry ${entry})" + missing=1 + continue + fi + # Discover submodules via Python's import system, then + # build a YAML config that names every one explicitly. + # The python loader's `modules:` form imports each name + # rather than scanning the filesystem, so editable + # installs work correctly. + modules=$(python - "$mod_name" <<'PY' + import importlib, pkgutil, sys + name = sys.argv[1] + mod = importlib.import_module(name) + out = [name] + if hasattr(mod, "__path__"): + for info in pkgutil.walk_packages(mod.__path__, name + "."): + if "._" in info.name: + continue + out.append(info.name) + print("\n".join(out)) + PY + ) + cfg=$(mktemp --suffix=.yml) + { + echo "loaders:" + echo " - type: python" + echo " modules:" + echo "$modules" | while IFS= read -r m; do + [ -n "$m" ] && echo " - ${m}" + done + echo "processors:" + echo " - type: filter" + echo " expression: not name.startswith('_') or name == '__init__'" + echo " - type: smart" + echo " - type: crossref" + echo "renderer:" + echo " type: markdown" + echo " render_toc: true" + echo " render_module_header: true" + echo " descriptive_class_title: true" + echo " add_method_class_prefix: true" + } > "$cfg" + pydoc-markdown "$cfg" > "$out/overview.md" + rm -f "$cfg" + # Empty / TOC-only output means the renderer found no + # documentable members. Fail fast rather than syncing. + lines=$(wc -l < "$out/overview.md") + if [ "$lines" -lt 5 ]; then + echo "::error ::pydoc-markdown produced near-empty output for ${mod_name} (${lines} lines)" + missing=1 + continue + fi + generated=$((generated + 1)) + done + if [ "$missing" -ne 0 ] || [ "$generated" -eq 0 ]; then + echo "::error ::aborting to avoid syncing partial/empty docs output" + exit 1 + fi + + - name: Write top-level index + # pydoc-markdown emits an overview.md per package. Stitch a + # small README.md at the top of the api/ folder linking to + # each so a single nav entry can drive into all packages. + # Package list is sorted alphabetically by directory name so + # it stays in sync with `_pages.json` (which `find ... | sort` + # produces below) regardless of PUBLIC_PACKAGES env order. + run: | + { + echo "# ResQ Python SDK" + echo "" + echo "You can use this auto-generated reference for the public packages in" + echo "[\`${{ github.repository }}\`](https://github.com/${{ github.repository }})." + echo "" + echo "## Packages" + echo "" + for entry in $PUBLIC_PACKAGES; do + echo "${entry%%:*}" + done | sort | while read -r pkg_dir; do + if [ -d "$OUTPUT_DIR/$pkg_dir" ]; then + echo "- [\`$pkg_dir\`]($pkg_dir/overview)" + fi + done + } > "$OUTPUT_DIR/README.md" + + - name: Prefix bare-filename intra-page links with ./ + # Mintlify rejects bare-filename .md links as broken; + # prefixing with ./ makes the resolver treat them as + # relative paths. Same pattern as the TypeScript template. + working-directory: ${{ env.OUTPUT_DIR }} + run: | + find . -type f -name '*.md' -print0 | while IFS= read -r -d '' f; do + sed -E -i \ + 's|\]\(([A-Za-z0-9][^)/]*\.md(#[^)]*)?)\)|](./\1)|g' \ + "$f" + done + + - name: Escape curly braces outside code regions (MDX safety) + # Mintlify parses .md as MDX. Any literal `{ ... }` in prose + # is interpreted as a JSX expression and trips the + # broken-links check. + # + # CommonMark inline code: a span opens with N consecutive + # backticks and closes with the same number. The earlier + # implementation toggled state on every single backtick, + # which mishandled spans like ``code with ` inside`` (used + # to embed a literal backtick) — the inner backtick flipped + # state mid-span and the closing pair landed wrong. The + # walker below counts the opening run, scans ahead for a + # matching-length closing run, and copies the entire span + # verbatim. Triple-backtick fences are still detected at + # line level above the per-character loop. + working-directory: ${{ env.OUTPUT_DIR }} + run: | + find . -type f -name '*.md' -print0 | while IFS= read -r -d '' f; do + awk ' + BEGIN { in_fence = 0 } + /^```/ { in_fence = !in_fence; print; next } + { + if (in_fence) { print; next } + s = $0 + out = "" + n = length(s) + i = 1 + while (i <= n) { + c = substr(s, i, 1) + if (c == "`") { + runlen = 0 + while (i + runlen <= n && substr(s, i + runlen, 1) == "`") runlen++ + j = i + runlen + close_at = 0 + while (j <= n) { + if (substr(s, j, 1) == "`") { + k = 0 + while (j + k <= n && substr(s, j + k, 1) == "`") k++ + if (k == runlen) { close_at = j; break } + j += k + } else { + j++ + } + } + if (close_at > 0) { + end = close_at + runlen - 1 + out = out substr(s, i, end - i + 1) + i = end + 1 + } else { + out = out substr(s, i, runlen) + i += runlen + } + } else if (c == "{") { + out = out "{"; i++ + } else if (c == "}") { + out = out "}"; i++ + } else { + out = out c; i++ + } + } + print out + } + ' "$f" > "$f.tmp" && mv "$f.tmp" "$f" + done + + - name: Build pages index + # Flat JSON array of generated Markdown paths (without + # extension) so the docs repo can later splice them into + # docs.json automatically. + working-directory: ${{ env.OUTPUT_DIR }} + run: | + find . -name '*.md' -type f \ + | sed 's|^\./||; s|\.md$||' \ + | sort > _pages.txt + jq -R -s 'split("\n") | map(select(length > 0))' _pages.txt > _pages.json + rm _pages.txt + + - name: Checkout docs repo + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + repository: resq-software/docs + path: docs-checkout + token: ${{ secrets.DOCS_REPO_PR_TOKEN }} + persist-credentials: false + + - name: Sync generated Markdown into docs checkout + run: | + target="docs-checkout/${DOCS_TARGET}" + mkdir -p "$target" + rm -rf "${target:?}"/* + cp -R "${OUTPUT_DIR}/." "$target/" + + - name: Open PR in docs repo + uses: peter-evans/create-pull-request@271a8d0340265f705b14b6d32b9829c1cb33d45e # v7.0.8 + with: + path: docs-checkout + token: ${{ secrets.DOCS_REPO_PR_TOKEN }} + author: 'resq-sw ' + committer: 'resq-sw ' + commit-message: | + docs(python): sync API reference for ${{ env.DOCS_REF_NAME }} + title: 'docs(python): API reference ${{ env.DOCS_REF_NAME }}' + body: | + Auto-generated by `${{ github.workflow }}` in + `${{ github.repository }}` for ref `${{ env.DOCS_REF_NAME }}` + (run: ${{ github.run_id }}). + + Regenerated files under `sdks/python/api/`. Review the + diff for unintended exports and merge to publish. + branch: auto/python-api-${{ env.DOCS_REF_SLUG }} + base: main + delete-branch: true + add-paths: sdks/python/api/** + labels: | + automated + docs:api-ref + language:python