diff --git a/.github/workflows/build-kotlin-docs-local.yaml b/.github/workflows/build-kotlin-docs-local.yaml new file mode 100644 index 000000000..10812886e --- /dev/null +++ b/.github/workflows/build-kotlin-docs-local.yaml @@ -0,0 +1,340 @@ +name: Build Kotlin Docs (Local) + +# Local-filesystem counterpart of build-kotlin-docs.yaml: same five steps +# (find_missing_assets -> populate_db -> insert_optimized_media -> +# build-stdlib-json-docs -> sync_kdoc_json_to_db), same ADFA-4737 blacklist, +# but reads its documentation.db/webHelpImages.zip inputs from paths on the +# runner's own disk (db_path / images_zip_path) instead of Google Drive, and +# writes its outputs (the updated database, the missing-assets report) back +# to disk (output_dir / db_path) instead of uploading them to Drive. No GCP +# Workload Identity Federation, Drive API, or associated secrets are used +# anywhere in this file. +# +# Since a GitHub-hosted runner is a fresh, disposable VM with no access to +# anyone's actual local disk, db_path/images_zip_path/output_dir only make +# sense here against a self-hosted runner, or when this workflow is run +# locally (e.g. via https://github.com/nektos/act) with those host paths +# bind-mounted into the job's container at the paths you pass as inputs. +# +# KNOWN LIMITATION: populate_db.py requires Writerside's own image export +# ("webHelpImages.zip"), which JetBrains only produces via IntelliJ IDEA's +# Writerside plugin build/export action - there is no headless/CLI way to +# generate it (see ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/README.md, +# "Inputs you need before starting"). So this workflow expects that export +# to already exist on disk at images_zip_path rather than generating it +# itself. Use skip_website_docs to bypass this entirely and only refresh the +# kotlin-stdlib/-reflect/-test JSON content. +# +# Optional secret (Slack notifications are skipped with a warning if unset) - +# same as build-kotlin-docs.yaml: +# SLACK_WEBHOOK_URL - Incoming Webhook URL for the "Notify Slack" steps +# below ("Grabbing baton" on start, "...Dropping +# baton" on finish - org shorthand for lock +# acquire/release, since this workflow mutates a +# single shared local file, db_path). + +permissions: + contents: read + +# This workflow overwrites a single shared local file (db_path) - never let +# two runs race to write it at the same time. +concurrency: + group: build-kotlin-docs-local + cancel-in-progress: false + +on: + workflow_dispatch: + inputs: + kotlin_web_site_ref: + description: >- + Branch/tag/commit of JetBrains/kotlin-web-site to check out for the + "docs" tree (topics/, images/, kr.tree, v.list). Leave empty to use + the repo's default branch. + required: false + default: '' + kotlin_ref: + description: >- + Branch/tag/commit of JetBrains/kotlin to check out for the + kotlin-stdlib-docs build. Leave empty to use the repo's default + branch. Pin this to a real release tag for a reproducible build. + required: false + default: '' + db_path: + description: >- + Path on this runner's disk to the input documentation.db. Read + directly (no download/unzip) and, unless dry_run is true, written + back to this same path when the run finishes. + required: true + images_zip_path: + description: >- + Path on this runner's disk to Writerside's webHelpImages.zip + export matching kotlin_web_site_ref (see KNOWN LIMITATION above). + Required unless skip_website_docs is true. + required: false + default: '' + output_dir: + description: >- + Directory on this runner's disk to write outputs into: the + missing-assets QA report and a run-numbered copy of the built + database (documentation-db-.db). Created if it + doesn't already exist. + required: false + default: 'build-kotlin-docs-output' + skip_website_docs: + description: 'Skip the kotlin-web-site steps and only refresh kotlin-stdlib/-reflect/-test JSON content.' + required: false + default: false + type: boolean + dry_run: + description: >- + If true, build and verify everything but do NOT write the result + back to db_path - the input file on disk is left untouched. Set to + false only once you trust a given ref/path combination (see this + workflow's testing notes). + required: false + default: true + type: boolean + +jobs: + build-kotlin-docs: + runs-on: ubuntu-latest + timeout-minutes: 180 + env: + KOTLIN_WEB_SITE_REF: ${{ inputs.kotlin_web_site_ref }} + KOTLIN_REF: ${{ inputs.kotlin_ref }} + DB_PATH: ${{ inputs.db_path }} + IMAGES_ZIP_PATH: ${{ inputs.images_zip_path }} + OUTPUT_DIR: ${{ inputs.output_dir }} + SKIP_WEBSITE_DOCS: ${{ inputs.skip_website_docs }} + steps: + - name: Checkout OfflineDocumentationTools + uses: actions/checkout@v4 + + - name: Resolve local file paths + run: | + if [ ! -f "$DB_PATH" ]; then + echo "Error: db_path '$DB_PATH' does not exist on this runner - for a self-hosted runner this must be a path on that machine; for act, bind-mount it into the container so it's visible at this exact path" >&2 + exit 1 + fi + if [ "$SKIP_WEBSITE_DOCS" != "true" ]; then + if [ -z "$IMAGES_ZIP_PATH" ]; then + echo "Error: images_zip_path is required unless skip_website_docs is true" >&2 + exit 1 + fi + if [ ! -f "$IMAGES_ZIP_PATH" ]; then + echo "Error: images_zip_path '$IMAGES_ZIP_PATH' does not exist on this runner" >&2 + exit 1 + fi + fi + mkdir -p "$OUTPUT_DIR" + echo "Resolved DB_PATH: $DB_PATH" + echo "Resolved IMAGES_ZIP_PATH: ${IMAGES_ZIP_PATH:-(skipped)}" + echo "Resolved OUTPUT_DIR: $OUTPUT_DIR" + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Set up JDK (for the kdoc-to-json / kotlin-stdlib-docs Gradle builds) + uses: actions/setup-java@v4 + with: + distribution: temurin + # kdoc-to-json's own Gradle wrapper is pinned to Gradle 9.1.0, which + # needs JDK 17+. Bump this if the kotlin checkout's own wrapper + # (invoked by build-stdlib-json-docs.sh against kotlin-stdlib-docs) + # turns out to need something newer - verify on first real run. + java-version: '17' + + - name: Install system dependencies + run: | + sudo apt-get update -y + # brotli: the CLI, not the Python package. populate_db.py's + # DictionaryCompressor and sync_kdoc_json_to_db.py shell out to it because + # no Python binding exposes a custom dictionary (ADFA-5153). + sudo apt-get install -y pngquant unzip sqlite3 brotli + + - name: Install Python dependencies + run: | + pip install -r requirements.txt + # markdown-it-py: ProcessKotlinWebsiteJSON's own requirement (see + # its README); scour/cairosvg are in requirements.txt already. + pip install markdown-it-py + + - name: Copy documentation.db from local disk + run: | + cp "$DB_PATH" documentation.db + sqlite3 documentation.db "SELECT 1;" > /dev/null + echo "DB_SIZE=$(stat -c%s documentation.db 2>/dev/null || stat -f%z documentation.db)" >> "$GITHUB_ENV" + + - name: 'Notify Slack: build started' + env: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} + run: | + if [ -z "$SLACK_WEBHOOK_URL" ]; then + echo "SLACK_WEBHOOK_URL not set - skipping Slack notification" >&2 + else + curl -sS -X POST -H 'Content-type: application/json' \ + --data '{"text": "Grabbing baton"}' \ + "$SLACK_WEBHOOK_URL" || echo "warning: Slack notification failed" >&2 + fi + + - name: Clone kotlin-web-site + if: ${{ !inputs.skip_website_docs }} + run: | + ARGS=(--depth 1) + [ -n "$KOTLIN_WEB_SITE_REF" ] && ARGS+=(--branch "$KOTLIN_WEB_SITE_REF") + git clone "${ARGS[@]}" https://github.com/JetBrains/kotlin-web-site.git kotlin-web-site + + - name: Copy Writerside image export from local disk + if: ${{ !inputs.skip_website_docs }} + run: cp "$IMAGES_ZIP_PATH" webHelpImages.zip + + - name: 'Step 1/5: find_missing_assets.py (source QA report)' + if: ${{ !inputs.skip_website_docs }} + run: | + python3 ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/find_missing_assets.py \ + kotlin-web-site/docs missing-assets-report.md + + - name: Write missing-assets report to output_dir + if: ${{ !inputs.skip_website_docs }} + run: cp missing-assets-report.md "$OUTPUT_DIR/missing-assets-report.md" + + - name: 'Step 2/5: populate_db.py (convert docs, prune blacklist, insert into db)' + if: ${{ !inputs.skip_website_docs }} + run: | + # Same three blacklist entries as run_e2e_pipeline_test.sh + # (ADFA-4737) - re-derive these from kotlin-web-site/docs/kr.tree + # if its nav structure has changed since this was written. + python3 ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/populate_db.py \ + kotlin-web-site/docs \ + ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/config.json \ + webHelpImages.zip \ + documentation.db \ + --blacklisted-element-titles \ + 'Development\/Web development' \ + 'Interoperability\/Swift/Objective-C and C interop' \ + 'Interoperability\/JavaScript interop' + + - name: 'Step 3/5: insert_optimized_media.py (re-optimize + reinsert images)' + if: ${{ !inputs.skip_website_docs }} + run: | + # --webp requires an "image/webp" ContentTypes row. The current + # production database already has one, so this is normally a no-op; + # it stays for older copies that predate it (idempotent either way). + sqlite3 documentation.db \ + "INSERT OR IGNORE INTO ContentTypes (value, compression) VALUES ('image/webp', 'brotli');" + mkdir -p media + unzip -q webHelpImages.zip -d media + python3 ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/insert_optimized_media.py \ + media documentation.db \ + --jpeg-quality 85 --webp --webp-quality 90 --verbose + + - name: Clone kotlin (for kotlin-stdlib-docs) + run: | + ARGS=(--depth 1) + [ -n "$KOTLIN_REF" ] && ARGS+=(--branch "$KOTLIN_REF") + git clone "${ARGS[@]}" https://github.com/JetBrains/kotlin.git kotlin-repo + + - name: 'Step 4/5: build-stdlib-json-docs.sh (fresh plugin build -> kotlin-stdlib/-reflect/-test JSON)' + id: stdlib_docs + run: | + OUTPUT="$(Dokka-plugin-kdoc2json/scripts/kotlin/build-stdlib-json-docs.sh kotlin-repo stdlib-json-build)" + echo "Generated JSON docs at $OUTPUT" + echo "all_libs_dir=$OUTPUT" >> "$GITHUB_OUTPUT" + + - name: 'Step 5/5: sync_kdoc_json_to_db.py (overwrite kotlin-stdlib/-reflect/-test content)' + run: | + python3 scripts/sync_kotlin_stdlib_docs/sync_kdoc_json_to_db.py \ + "${{ steps.stdlib_docs.outputs.all_libs_dir }}" --db documentation.db + + - name: Summary + run: | + python3 - documentation.db <<'PYEOF' + import sqlite3 + import sys + + conn = sqlite3.connect(sys.argv[1]) + + def count(where, params=()): + return conn.execute(f"SELECT count(*) FROM Content WHERE {where}", params).fetchone()[0] + + print(f"Database: {sys.argv[1]}") + print(f" k/html/* rows: {count('path LIKE ?', ('k/html/%',))}") + print(f" k/html/images/* rows: {count('path LIKE ?', ('k/html/images/%',))}") + print(f" k/html/images/*.webp rows: {count('path LIKE ?', ('k/html/images/%.webp%',))}") + print(f" k/kotlin-stdlib/* rows: {count('path LIKE ? OR path = ?', ('k/kotlin-stdlib/%', 'k/kotlin-stdlib'))}") + print(f" k/kotlin-reflect/* rows: {count('path LIKE ? OR path = ?', ('k/kotlin-reflect/%', 'k/kotlin-reflect'))}") + print(f" k/kotlin-test/* rows: {count('path LIKE ? OR path = ?', ('k/kotlin-test/%', 'k/kotlin-test'))}") + conn.close() + PYEOF + + - name: Blacklist pruning verification + if: ${{ !inputs.skip_website_docs }} + run: | + python3 - ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON kotlin-web-site/docs documentation.db \ + 'Development\/Web development' \ + 'Interoperability\/Swift/Objective-C and C interop' \ + 'Interoperability\/JavaScript interop' <<'PYEOF' + import sqlite3 + import sys + import xml.etree.ElementTree as ET + from pathlib import Path + + process_dir, docs_root, db_path, *blacklist_raw = sys.argv[1:] + sys.path.insert(0, process_dir) + import populate_db # noqa: E402 + + root = ET.parse(Path(docs_root) / "kr.tree").getroot() + blacklisted_paths = {populate_db.parse_blacklist_path(raw) for raw in blacklist_raw} + blacklisted_stems, unmatched_paths = populate_db.prune_blacklisted_elements(root, blacklisted_paths) + + conn = sqlite3.connect(db_path) + leftover = [] + for stem in sorted(blacklisted_stems): + path = f"k/html/{stem}.html" + if conn.execute("SELECT 1 FROM Content WHERE path = ?", (path,)).fetchone(): + leftover.append(path) + conn.close() + + print(f"Blacklisted toc-element path(s) checked: {len(blacklisted_paths)}") + for path in sorted(blacklisted_paths): + status = "unmatched (no such element in kr.tree)" if path in unmatched_paths else "matched" + print(f" {' > '.join(path)}: {status}") + print(f"Topic page(s) expected removed: {len(blacklisted_stems)}") + + if unmatched_paths: + print(f"FAIL: {len(unmatched_paths)} blacklist path(s) never matched a .") + sys.exit(1) + if leftover: + print(f"FAIL: {len(leftover)} blacklisted page(s) still present in the database:") + for path in leftover: + print(f" {path}") + sys.exit(1) + + print(f"PASS: all {len(blacklisted_stems)} blacklisted topic page(s) confirmed absent from {db_path}.") + PYEOF + + - name: Write built database to output_dir + run: | + cp documentation.db "$OUTPUT_DIR/documentation-db-${{ github.run_number }}.db" + echo "Wrote $OUTPUT_DIR/documentation-db-${{ github.run_number }}.db" + + - name: Write updated database back to db_path + if: ${{ !inputs.dry_run }} + run: | + cp documentation.db "$DB_PATH" + echo "Wrote updated documentation.db back to $DB_PATH" + + - name: 'Notify Slack: build complete' + if: ${{ !inputs.dry_run }} + env: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} + run: | + if [ -z "$SLACK_WEBHOOK_URL" ]; then + echo "SLACK_WEBHOOK_URL not set - skipping Slack notification" >&2 + else + curl -sS -X POST -H 'Content-type: application/json' \ + --data '{"text": "Updated Kotlin documentation. Dropping baton"}' \ + "$SLACK_WEBHOOK_URL" || echo "warning: Slack notification failed" >&2 + fi diff --git a/.github/workflows/build-kotlin-docs.yaml b/.github/workflows/build-kotlin-docs.yaml new file mode 100644 index 000000000..d72dfa47e --- /dev/null +++ b/.github/workflows/build-kotlin-docs.yaml @@ -0,0 +1,382 @@ +name: Build Kotlin Docs + +# CI counterpart of ProcessDocs/ProcessKotlinDocs/run_e2e_pipeline_test.sh - +# same five steps (find_missing_assets -> populate_db -> insert_optimized_media +# -> build-stdlib-json-docs -> sync_kdoc_json_to_db), same ADFA-4737 blacklist, +# but sourcing its inputs from fresh git checkouts instead of a developer's +# local machine, and reading/writing the real database on Google Drive +# (GOOGLE_DRIVE_FILE_ID) instead of a local SOURCE_DB copy. +# +# KNOWN LIMITATION: populate_db.py requires Writerside's own image export +# ("webHelpImages.zip"), which JetBrains only produces via IntelliJ IDEA's +# Writerside plugin build/export action - there is no headless/CLI way to +# generate it (see ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/README.md, +# "Inputs you need before starting"). So this workflow downloads it from +# Google Drive rather than generating it itself; someone has to run that IDE +# export, upload the zip to Drive, and supply its file ID (see +# images_zip_file_id / GOOGLE_DRIVE_IMAGES_ZIP_FILE_ID below) before +# triggering a run that touches the website docs. Use skip_website_docs to +# bypass this entirely and only refresh the kotlin-stdlib/-reflect/-test +# JSON content. +# +# Required secrets (already configured - see docdb-regression-test.yaml for +# their other use in this repo): +# GCP_WIF_PROVIDER - Workload Identity Federation provider name +# GCP_WIF_SERVICE_ACCOUNT - Service account email for WIF (needs read +# access to the images-zip file below, and +# write access - not just view - to the +# database file, since this workflow +# overwrites it) +# GOOGLE_DRIVE_FILE_ID - File ID of the production documentation.db +# (stored on Drive as a zip) +# +# Optional secret (falls back to the images_zip_file_id input if unset; see +# also the hard-coded TEST_*_FILE_ID overrides below for one-off testing): +# GOOGLE_DRIVE_IMAGES_ZIP_FILE_ID - File ID of Writerside's webHelpImages.zip export +# +# Optional secret (Slack notifications are skipped with a warning if unset): +# SLACK_WEBHOOK_URL - Incoming Webhook URL for the "Notify Slack" steps +# below ("Grabbing baton" on start, "...Dropping +# baton" on finish - org shorthand for lock +# acquire/release, since this workflow mutates a +# single shared Drive file). + +permissions: + contents: read + id-token: write + +# This workflow overwrites a single shared Drive file - never let two runs +# race to upload against each other. +concurrency: + group: build-kotlin-docs + cancel-in-progress: false + +on: + workflow_dispatch: + inputs: + kotlin_web_site_ref: + description: >- + Branch/tag/commit of JetBrains/kotlin-web-site to check out for the + "docs" tree (topics/, images/, kr.tree, v.list). Leave empty to use + the repo's default branch. + required: false + default: '' + kotlin_ref: + description: >- + Branch/tag/commit of JetBrains/kotlin to check out for the + kotlin-stdlib-docs build. Leave empty to use the repo's default + branch. Pin this to a real release tag for a reproducible build. + required: false + default: '' + images_zip_file_id: + description: >- + Google Drive file ID for Writerside's webHelpImages.zip export + matching kotlin_web_site_ref (see KNOWN LIMITATION above). Falls + back to the GOOGLE_DRIVE_IMAGES_ZIP_FILE_ID secret if left empty. + Ignored if skip_website_docs is true. + required: false + default: '' + skip_website_docs: + description: 'Skip the kotlin-web-site steps and only refresh kotlin-stdlib/-reflect/-test JSON content.' + required: false + default: false + type: boolean + dry_run: + description: >- + If true, build and verify everything but do NOT upload the result + back to Google Drive - the production database is left untouched. + Set to false only once you trust a given ref/URL combination (see + this workflow's testing notes). + required: false + default: true + type: boolean + +jobs: + build-kotlin-docs: + runs-on: ubuntu-latest + timeout-minutes: 180 + env: + KOTLIN_WEB_SITE_REF: ${{ inputs.kotlin_web_site_ref }} + KOTLIN_REF: ${{ inputs.kotlin_ref }} + DB_FILE_ID_SECRET: ${{ secrets.GOOGLE_DRIVE_FILE_ID }} + IMAGES_ZIP_FILE_ID_INPUT: ${{ inputs.images_zip_file_id }} + IMAGES_ZIP_FILE_ID_SECRET: ${{ secrets.GOOGLE_DRIVE_IMAGES_ZIP_FILE_ID }} + # --- Hard-coded overrides for one-off manual testing ----------------- + # Fill in either of these with a literal Google Drive file ID to + # bypass the secret/input resolution above for a quick, repeatable + # test run (e.g. against scratch copies of the database/images zip on + # Drive). Leave both empty ('') for normal operation. + TEST_DB_FILE_ID: '' + TEST_IMAGES_ZIP_FILE_ID: '' + steps: + - name: Checkout OfflineDocumentationTools + uses: actions/checkout@v4 + + - name: Resolve Google Drive file IDs + run: | + DB_FILE_ID="${TEST_DB_FILE_ID:-$DB_FILE_ID_SECRET}" + IMG_FILE_ID="${TEST_IMAGES_ZIP_FILE_ID:-${IMAGES_ZIP_FILE_ID_INPUT:-$IMAGES_ZIP_FILE_ID_SECRET}}" + if [ -z "$DB_FILE_ID" ]; then + echo "Error: no database file ID resolved - set the GOOGLE_DRIVE_FILE_ID secret, or TEST_DB_FILE_ID above for a test run" >&2 + exit 1 + fi + echo "Resolved DB_FILE_ID: ${DB_FILE_ID:+(set)}" + echo "Resolved IMG_FILE_ID: ${IMG_FILE_ID:+(set)}" + echo "DB_FILE_ID=$DB_FILE_ID" >> "$GITHUB_ENV" + echo "IMG_FILE_ID=$IMG_FILE_ID" >> "$GITHUB_ENV" + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Set up JDK (for the kdoc-to-json / kotlin-stdlib-docs Gradle builds) + uses: actions/setup-java@v4 + with: + distribution: temurin + # kdoc-to-json's own Gradle wrapper is pinned to Gradle 9.1.0, which + # needs JDK 17+. Bump this if the kotlin checkout's own wrapper + # (invoked by build-stdlib-json-docs.sh against kotlin-stdlib-docs) + # turns out to need something newer - verify on first real run. + java-version: '17' + + - name: Install system dependencies + run: | + sudo apt-get update -y + # brotli: the CLI, not the Python package. populate_db.py's + # DictionaryCompressor and sync_kdoc_json_to_db.py shell out to it because + # no Python binding exposes a custom dictionary (ADFA-5153). + sudo apt-get install -y pngquant unzip zip sqlite3 brotli + + - name: Install Python dependencies + run: | + pip install -r requirements.txt + # markdown-it-py: ProcessKotlinWebsiteJSON's own requirement (see + # its README); scour/cairosvg are in requirements.txt already. + # google-api-python-client & friends: Drive download/upload, same + # libraries check-tools/download_database.py already depends on. + pip install markdown-it-py \ + google-api-python-client google-auth-httplib2 google-auth-oauthlib + + - name: Authenticate to Google Cloud using Workload Identity Federation + uses: google-github-actions/auth@v2 + with: + workload_identity_provider: ${{ secrets.GCP_WIF_PROVIDER }} + service_account: ${{ secrets.GCP_WIF_SERVICE_ACCOUNT }} + access_token_scopes: | + https://www.googleapis.com/auth/drive.file + + - name: Download current documentation.db from Google Drive + run: | + python3 check-tools/download_database.py "$DB_FILE_ID" documentation.zip + unzip -o documentation.zip + if [ ! -f documentation.db ]; then + found="$(find . -maxdepth 2 -name documentation.db | head -n1)" + [ -n "$found" ] && mv "$found" documentation.db + fi + test -f documentation.db + sqlite3 documentation.db "SELECT 1;" > /dev/null + rm -f documentation.zip + echo "DB_SIZE=$(stat -c%s documentation.db 2>/dev/null || stat -f%z documentation.db)" >> "$GITHUB_ENV" + + - name: 'Notify Slack: build started' + env: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} + run: | + if [ -z "$SLACK_WEBHOOK_URL" ]; then + echo "SLACK_WEBHOOK_URL not set - skipping Slack notification" >&2 + else + curl -sS -X POST -H 'Content-type: application/json' \ + --data '{"text": "Grabbing baton"}' \ + "$SLACK_WEBHOOK_URL" || echo "warning: Slack notification failed" >&2 + fi + + - name: Clone kotlin-web-site + if: ${{ !inputs.skip_website_docs }} + run: | + ARGS=(--depth 1) + [ -n "$KOTLIN_WEB_SITE_REF" ] && ARGS+=(--branch "$KOTLIN_WEB_SITE_REF") + git clone "${ARGS[@]}" https://github.com/JetBrains/kotlin-web-site.git kotlin-web-site + + - name: Download Writerside image export from Google Drive + if: ${{ !inputs.skip_website_docs }} + run: | + if [ -z "$IMG_FILE_ID" ]; then + echo "Error: no images-zip file ID resolved - set images_zip_file_id, the GOOGLE_DRIVE_IMAGES_ZIP_FILE_ID secret, or TEST_IMAGES_ZIP_FILE_ID above (see KNOWN LIMITATION in this workflow's header comment). Required unless skip_website_docs is true." >&2 + exit 1 + fi + # download_database.py is a generic Drive-file-by-ID downloader + # despite its name - reused here rather than duplicating the + # WIF/Drive-API download logic for a second file type. + python3 check-tools/download_database.py "$IMG_FILE_ID" webHelpImages.zip + + - name: 'Step 1/5: find_missing_assets.py (source QA report)' + if: ${{ !inputs.skip_website_docs }} + run: | + python3 ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/find_missing_assets.py \ + kotlin-web-site/docs missing-assets-report.md + + - name: Upload missing-assets report + if: ${{ !inputs.skip_website_docs }} + uses: actions/upload-artifact@v4 + with: + name: missing-assets-report + path: missing-assets-report.md + + - name: 'Step 2/5: populate_db.py (convert docs, prune blacklist, insert into db)' + if: ${{ !inputs.skip_website_docs }} + run: | + # Same three blacklist entries as run_e2e_pipeline_test.sh + # (ADFA-4737) - re-derive these from kotlin-web-site/docs/kr.tree + # if its nav structure has changed since this was written. + python3 ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/populate_db.py \ + kotlin-web-site/docs \ + ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/config.json \ + webHelpImages.zip \ + documentation.db \ + --blacklisted-element-titles \ + 'Development\/Web development' \ + 'Interoperability\/Swift/Objective-C and C interop' \ + 'Interoperability\/JavaScript interop' + + - name: 'Step 3/5: insert_optimized_media.py (re-optimize + reinsert images)' + if: ${{ !inputs.skip_website_docs }} + run: | + # --webp requires an "image/webp" ContentTypes row. The current + # production database already has one, so this is normally a no-op; + # it stays for older copies that predate it (idempotent either way). + sqlite3 documentation.db \ + "INSERT OR IGNORE INTO ContentTypes (value, compression) VALUES ('image/webp', 'brotli');" + mkdir -p media + unzip -q webHelpImages.zip -d media + python3 ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/insert_optimized_media.py \ + media documentation.db \ + --jpeg-quality 85 --webp --webp-quality 90 --verbose + + - name: Clone kotlin (for kotlin-stdlib-docs) + run: | + ARGS=(--depth 1) + [ -n "$KOTLIN_REF" ] && ARGS+=(--branch "$KOTLIN_REF") + git clone "${ARGS[@]}" https://github.com/JetBrains/kotlin.git kotlin-repo + + - name: 'Step 4/5: build-stdlib-json-docs.sh (fresh plugin build -> kotlin-stdlib/-reflect/-test JSON)' + id: stdlib_docs + run: | + OUTPUT="$(Dokka-plugin-kdoc2json/scripts/kotlin/build-stdlib-json-docs.sh kotlin-repo stdlib-json-build)" + echo "Generated JSON docs at $OUTPUT" + echo "all_libs_dir=$OUTPUT" >> "$GITHUB_OUTPUT" + + - name: 'Step 5/5: sync_kdoc_json_to_db.py (overwrite kotlin-stdlib/-reflect/-test content)' + run: | + python3 scripts/sync_kotlin_stdlib_docs/sync_kdoc_json_to_db.py \ + "${{ steps.stdlib_docs.outputs.all_libs_dir }}" --db documentation.db + + - name: Summary + run: | + python3 - documentation.db <<'PYEOF' + import sqlite3 + import sys + + conn = sqlite3.connect(sys.argv[1]) + + def count(where, params=()): + return conn.execute(f"SELECT count(*) FROM Content WHERE {where}", params).fetchone()[0] + + print(f"Database: {sys.argv[1]}") + print(f" k/html/* rows: {count('path LIKE ?', ('k/html/%',))}") + print(f" k/html/images/* rows: {count('path LIKE ?', ('k/html/images/%',))}") + print(f" k/html/images/*.webp rows: {count('path LIKE ?', ('k/html/images/%.webp%',))}") + print(f" k/kotlin-stdlib/* rows: {count('path LIKE ? OR path = ?', ('k/kotlin-stdlib/%', 'k/kotlin-stdlib'))}") + print(f" k/kotlin-reflect/* rows: {count('path LIKE ? OR path = ?', ('k/kotlin-reflect/%', 'k/kotlin-reflect'))}") + print(f" k/kotlin-test/* rows: {count('path LIKE ? OR path = ?', ('k/kotlin-test/%', 'k/kotlin-test'))}") + conn.close() + PYEOF + + - name: Blacklist pruning verification + if: ${{ !inputs.skip_website_docs }} + run: | + python3 - ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON kotlin-web-site/docs documentation.db \ + 'Development\/Web development' \ + 'Interoperability\/Swift/Objective-C and C interop' \ + 'Interoperability\/JavaScript interop' <<'PYEOF' + import sqlite3 + import sys + import xml.etree.ElementTree as ET + from pathlib import Path + + process_dir, docs_root, db_path, *blacklist_raw = sys.argv[1:] + sys.path.insert(0, process_dir) + import populate_db # noqa: E402 + + root = ET.parse(Path(docs_root) / "kr.tree").getroot() + blacklisted_paths = {populate_db.parse_blacklist_path(raw) for raw in blacklist_raw} + blacklisted_stems, unmatched_paths = populate_db.prune_blacklisted_elements(root, blacklisted_paths) + + conn = sqlite3.connect(db_path) + leftover = [] + for stem in sorted(blacklisted_stems): + path = f"k/html/{stem}.html" + if conn.execute("SELECT 1 FROM Content WHERE path = ?", (path,)).fetchone(): + leftover.append(path) + conn.close() + + print(f"Blacklisted toc-element path(s) checked: {len(blacklisted_paths)}") + for path in sorted(blacklisted_paths): + status = "unmatched (no such element in kr.tree)" if path in unmatched_paths else "matched" + print(f" {' > '.join(path)}: {status}") + print(f"Topic page(s) expected removed: {len(blacklisted_stems)}") + + if unmatched_paths: + print(f"FAIL: {len(unmatched_paths)} blacklist path(s) never matched a .") + sys.exit(1) + if leftover: + print(f"FAIL: {len(leftover)} blacklisted page(s) still present in the database:") + for path in leftover: + print(f" {path}") + sys.exit(1) + + print(f"PASS: all {len(blacklisted_stems)} blacklisted topic page(s) confirmed absent from {db_path}.") + PYEOF + + - name: Upload built database as workflow artifact + uses: actions/upload-artifact@v4 + with: + name: documentation-db-${{ github.run_number }} + path: documentation.db + retention-days: 14 + + - name: Zip updated database for upload + if: ${{ !inputs.dry_run }} + run: zip -j documentation.zip documentation.db + + - name: Upload updated database to Google Drive + if: ${{ !inputs.dry_run }} + run: | + python3 - <<'PYEOF' + import os + from google.auth import default + from googleapiclient.discovery import build + from googleapiclient.http import MediaFileUpload + + file_id = os.environ["DB_FILE_ID"] + credentials, _ = default() + service = build("drive", "v3", credentials=credentials) + media = MediaFileUpload("documentation.zip", mimetype="application/zip", resumable=True) + updated = service.files().update( + fileId=file_id, media_body=media, fields="id, modifiedTime, md5Checksum" + ).execute() + print(f"Uploaded new revision of {file_id}: {updated}") + PYEOF + + - name: 'Notify Slack: build complete' + if: ${{ !inputs.dry_run }} + env: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} + run: | + if [ -z "$SLACK_WEBHOOK_URL" ]; then + echo "SLACK_WEBHOOK_URL not set - skipping Slack notification" >&2 + else + curl -sS -X POST -H 'Content-type: application/json' \ + --data '{"text": "Updated Kotlin documentation. Dropping baton"}' \ + "$SLACK_WEBHOOK_URL" || echo "warning: Slack notification failed" >&2 + fi diff --git a/.github/workflows/docdb-regression-test.yaml b/.github/workflows/docdb-regression-test.yaml index fa5346e04..d9222dfa8 100644 --- a/.github/workflows/docdb-regression-test.yaml +++ b/.github/workflows/docdb-regression-test.yaml @@ -98,7 +98,9 @@ jobs: echo "Extracting database from zip file..." # Install unzip if not available - sudo apt-get update -qq && sudo apt-get install -y unzip + # brotli: docdb_studio reads dictionary-compressed Content rows through the + # CLI (ADFA-5153); the downloaded production database is one of those. + sudo apt-get update -qq && sudo apt-get install -y unzip brotli # Extract the zip file if ! unzip -o documentation.zip; then diff --git a/.gitignore b/.gitignore index 59cc3acb6..794d48977 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,11 @@ __pycache__/ *$py.class *.db *.sqlite +run_e2e_pipeline_test.local.sh +grep_content_blobs.local.py + +# Timestamped safety backups written by populate_db.py / +# insert_optimized_media.py / sync_kdoc_json_to_db.py before they modify a +# database ("*.db" above does not match these - the timestamp comes last). +*.db.backup-* +*.db.bak.* diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 000000000..5f16792f3 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,219 @@ +# CLAUDE.md + +Guidance for Claude (and anyone else) working in this repository. + +## What this repository is + +App Dev For All builds **Code on the Go**, an Android IDE aimed at users with no or limited +internet access +(code: [appdevforall/CodeOnTheGo](https://github.com/appdevforall/CodeOnTheGo)). To support that, +Java/Kotlin/Android API documentation is bundled into the app as a single SQLite file — the +**documentation database** — rather than fetched from the web. + +The documentation database serves two distinct features in the IDE: + +1. **Tooltips (Tier 1/2).** When a user selects a keyword/symbol in the code editor, a dialog + shows short (Tier 1) and detailed (Tier 2) tooltip text if the selection matches an entry in + the DB. This lookup happens elsewhere in the CodeOnTheGo Android code (not in this repo, and + not in `WebServer.kt` — see below). +2. **Content pages (Tier 3).** From a tooltip, the user can click through to a full documentation + page. Those pages (and other static content — HTML, images, PDFs) are served over HTTP by + **`WebServer.kt`** + ([CodeOnTheGo/app/src/main/java/com/itsaky/androidide/localWebServer/WebServer.kt](https://github.com/appdevforall/CodeOnTheGo/blob/stage/app/src/main/java/com/itsaky/androidide/localWebServer/WebServer.kt)), + which runs inside the app and reads directly from the `Content` (and, as of recently, + `Templates`/`Bookshelf`/`BookCategories`) tables of the same database. + +**This repository (`OfflineDocumentationTools`) is the collection of offline tools that build and +edit that database** — it contains no part of the production Android app itself. + +> **Alex's standing caveat, worth repeating at the top of every session:** nothing in this +> repository is guaranteed to work against the *current* production database. The schema has moved +> forward (in the app / by hand) faster than the tooling in this repo has been updated. See +> "Schema: current vs. what this repo expects" below — that gap is the most important thing to +> understand before making changes here. + +## Schema: current vs. what this repo expects + +> ### Schema 2.0.0: every Brotli row uses a shared dictionary +> +> As of 2026-08-20 `~/documentation.db` reports **2.0.0** in `DocumentationDatabaseVersion` +> ("Add CompressionDictionary table", David) — a deliberately *incompatible* major bump. Every +> `brotli` Content row is compressed against the shared 256 KiB raw LZ77 dictionary held in +> `CompressionDictionary` (id 1). Measured on that database: **0 of 24 sampled brotli rows decode +> with plain Brotli; all 24 require `brotli -D`.** A dictionary stream and a plain one are not +> interchangeable, so anything reading or writing `Content` has to go through the dictionary. +> +> No Python Brotli binding exposes a custom dictionary (the `brotli` package has no such parameter; +> `brotlicffi` dropped `BrotliDecoderSetCustomDictionary` in 1.2), so all three writers shell out to +> the **`brotli` CLI** — which therefore has to be installed wherever the pipeline runs. Both +> workflows install it. +> +> | Writer | Dictionary handling | +> |---|---| +> | `ProcessKotlinWebsiteJSON/populate_db.py` | `DictionaryCompressor` + `load_or_create_dictionary`; trains one only if the table is absent/empty, and **never** retrains. | +> | `ProcessKotlinWebsiteJSON/insert_optimized_media.py` | Reads the existing dictionary via `load_dictionary`; never creates one. | +> | `scripts/sync_kotlin_stdlib_docs/sync_kdoc_json_to_db.py` | `DictionaryBrotli` + `load_compression_dictionary`; falls back to plain Brotli only for pre-2.0.0 databases. | +> +> Retraining an existing dictionary would orphan every row already compressed against the old one, +> which is why all three only ever read what is already stored. +> `migrate_content_to_dictionary_brotli.py` recompresses any remaining plain-Brotli rows. +> +> Note also that `image/webp` (id 26) and `video/quicktime` (id 28) now exist in `ContentTypes`, so +> the workflows' `INSERT OR IGNORE ... image/webp` step is a no-op against current copies. + +The schema below is what `~/documentation.db` (Alex's current production copy) actually contains, +as of 2026-08-05 — predating the 2.0.0 bump described above, which additionally adds the +`CompressionDictionary` and `DocumentationDatabaseVersion` tables: + +```sql +CREATE TABLE Languages (id INTEGER PRIMARY KEY AUTOINCREMENT, value TEXT NOT NULL UNIQUE); +CREATE TABLE ContentTypes (id INTEGER PRIMARY KEY AUTOINCREMENT, value TEXT NOT NULL UNIQUE, compression TEXT NOT NULL); +CREATE TABLE TooltipCategories (id INTEGER PRIMARY KEY, category TEXT NOT NULL); +CREATE TABLE TooltipButtonNumbers (id INTEGER UNIQUE); -- manually assigned display order +CREATE TABLE Content ( + id INTEGER PRIMARY KEY AUTOINCREMENT, path TEXT NOT NULL, languageID INTEGER NOT NULL, + content BLOB NOT NULL, contentTypeID INTEGER NOT NULL, templateId INTEGER NOT NULL DEFAULT 0, + FOREIGN KEY (languageID) REFERENCES Languages(id), FOREIGN KEY (contentTypeID) REFERENCES ContentTypes(id), + UNIQUE('path') +); +CREATE TABLE Tooltips ( + id INTEGER PRIMARY KEY AUTOINCREMENT, categoryId INTEGER NOT NULL, tag TEXT NOT NULL, + summary TEXT NOT NULL, detail TEXT NOT NULL, UNIQUE (categoryId, tag), + FOREIGN KEY(categoryId) REFERENCES TooltipCategories(id) +); +CREATE TABLE TooltipButtons ( + tooltipId INTEGER, buttonNumberId INTEGER, description TEXT, uri TEXT, + FOREIGN KEY(tooltipId) REFERENCES Tooltips(id), FOREIGN KEY(buttonNumberId) REFERENCES TooltipButtonNumbers(id) +); +CREATE TABLE LastChange (documentationSet TEXT, changeTime TIMESTAMP DEFAULT CURRENT_TIMESTAMP, who TEXT); +CREATE TABLE Templates (id INTEGER PRIMARY KEY AUTOINCREMENT, name TEXT NOT NULL, content BLOB NOT NULL, UNIQUE('name')); +CREATE TABLE BookCategories (id INTEGER PRIMARY KEY AUTOINCREMENT, category STRING, description STRING DEFAULT '', UNIQUE('category')); +CREATE TABLE Bookshelf (contentID INTEGER NOT NULL, title STRING DEFAULT '', description STRING DEFAULT '', + bookCategoryID INTEGER, FOREIGN KEY (bookCategoryID) REFERENCES BookCategories(id), UNIQUE(title, bookCategoryId)); +-- Triggers keep Bookshelf in sync when a .pdf row is added to/removed from Content. +CREATE TABLE PUCC_Students (...), PUCC_Classes (...), PUCC_Sections (...), PUCC_Professors (...), + PUCC_StudentAssignments (...), PUCC_ProfessorAssignments (...) +-- Unrelated to documentation tooling (confirmed by Alex) — ignore, leave as-is, do not +-- document or maintain further in this repo. +``` + +**`Templates`, `BookCategories`, and `Bookshelf` are not documentation cruft — `WebServer.kt` +actively depends on them.** Its `/pr/bs` endpoint builds a JSON "bookshelf" payload straight from +`Content` + `Bookshelf` + `BookCategories`, looks up a template named `'bookshelf'` in `Templates`, +and renders it with the Pebble template engine. More generally, any `Content` row with a non-zero +`templateId` gets its stored (decompressed) content run through the matching row in `Templates` as +a Pebble template before being served. This is a real, current feature of the shipped server, not +a placeholder. + +**Nothing that currently builds or writes to the database in this repository knows about any of +that — and that's expected.** `Templates`/`Bookshelf`/`BookCategories` are populated by a separate +plugin system, not by anything in this repo: App Dev For All supports plugins that write into the +documentation database, including the bookshelf feature specifically — +[appdevforall/bookshelf-plugin](https://github.com/appdevforall/bookshelf-plugin). So the absence +of any `Templates`/`Bookshelf`/`BookCategories` handling here is not a gap to fill; it's out of +scope for this repo. (A repo-wide search for `Templates`, `Bookshelf`, `BookCategories`, or `PUCC` +turns up zero matches outside `WebServer.kt` itself, which is consistent with that division of +responsibility. `templateId` itself is a different story - `populate_db.py` and +`insert_optimized_media.py` both read/write it directly, since it's a plain column on `Content` +they populate; it's only the `Templates` table and the plugin system that reference it that stay +out of scope.) Concretely, relative to the schema above: + +| Piece | What it thinks the schema is | Consequence | +| --- | --- | --- | +| `scripts/DocumentationDatabase.py` (used by `scripts/ingest.py`, and hence by `.github/workflows/publish-doc-db.yaml`) | `Content` / `Languages` / `ContentTypes` only, plus an optional `ide_tooltip_table`. Its constructor explicitly **raises `ValueError`** if it opens a DB containing any table outside that whitelist. | **This will refuse to open the current production `documentation.db` at all** — it will list `Tooltips`, `TooltipCategories`, `TooltipButtons`, `TooltipButtonNumbers`, `LastChange`, `Templates`, `BookCategories`, `Bookshelf`, and every `PUCC_*` table as "unexpected." This is the single biggest blocker to reusing this script as-is. | +| `docdb-studio/SCHEMA.md` / `AGENTS.md` (states the schema is "locked," no migrations) | `Content` (no `templateId`, no `UNIQUE(path)`), `Tooltips`, `TooltipButtons`, `TooltipCategories`, `TooltipButtonNumbers`, `LastChange` (with a *different* shape: `documentationSet`/`changeTime`/`who` — this part does match current), plus a legacy `ide_tooltip_table`. Missing `templateId`, `Templates`, `BookCategories`, `Bookshelf`, `PUCC_*`. | Closest of the three documented schemas to reality, but still out of date. `docdb_studio.py`'s own "never change the schema" policy is itself now stale, since the live schema has already changed underneath it. | +| `check-tools/README.md`'s embedded schema (and by extension the mental model behind `check-tools/db_health_checker.py`) | `Content` (no `templateId`, no `UNIQUE(path)`), `Tooltips`, `TooltipButtons`, `TooltipCategories`, `TooltipButtonNumbers`, and a *third* variant of `LastChange` (`now`/`who`). No `Templates`/`Bookshelf`/`BookCategories`/`PUCC_*`. | The health checker's required-table check still passes (it only checks that its known tables exist, not that no others do). Since `Templates`/`Bookshelf`/`BookCategories` are out of scope for this repo (see above), this is not being treated as something to fix right now. | + +There also appear to be **two unrelated tooltip storage formats** in this repo's history, and it's +worth being deliberate about which one is current: + +- The **normalized** format (`Tooltips` + `TooltipCategories` + `TooltipButtons` + + `TooltipButtonNumbers`) — this is what's in the live schema above, what `docdb-studio` edits, + what `check-tools/db_health_checker.py` validates, and what `scripts/TooltipManager.py` + dumps/rebuilds via CSV. +- A **legacy flat** format, a single `ide_tooltip_table(tooltipCategory, tooltipTag, + tooltipSummary, tooltipDetail, tooltipButtons)` table (button data packed as a JSON string in + one column) — written by `scripts/tooltips.py` (`TooltipDatabase`, driven by + `scripts/import_tooltips.py` from `SourceDocs/Tooltips/tooltips.xlsx`) and by + `scripts/load_android_data.py` (fed by pickle files that `scripts/android_tooltips.py` / + `scripts/java_tooltips.py` scrape from Android/Java HTML doc trees). **`ide_tooltip_table` does + not exist in the current production schema at all.** + +**`ide_tooltip_table` is officially dead (confirmed by Alex).** That means the entire chain that +targets it — `scripts/tooltips.py`, `scripts/import_tooltips.py`, `scripts/android_tooltips.py`, +`scripts/java_tooltips.py`, `scripts/android_html_page.py`, and `scripts/load_android_data.py` — is +**deprecated legacy code**. It's left in the repo for reference/history, but none of it should be +extended or relied on, and none of it writes to a table the shipped app or `docdb-studio` actually +uses. Any future Android/Java tooltip work should target the normalized `Tooltips` / +`TooltipCategories` / `TooltipButtons` / `TooltipButtonNumbers` tables instead (the same ones +`docdb-studio` and `scripts/TooltipManager.py` already use for Kotlin tooltips). + +## Repository tour + +- **`docdb-studio/`** — a Flet (Flutter-for-Python) desktop GUI for browsing/editing `Tooltips` / + `TooltipCategories` / `TooltipButtons` and importing `Content`. Has its own `CLAUDE.md`, + `AGENTS.md`, `SCHEMA.md`, and a real pytest suite. Actively maintained (most recent commits in + the repo touch this tool), but per the table above, its documented schema is behind the live one. + That's an accepted state, not an active problem: schema evolution happens outside + `docdb-studio` (and outside this repo, e.g. via plugins — see below), and `docdb-studio` is + expected to catch up after the fact rather than lead. Its `AGENTS.md`/`SCHEMA.md` "never migrate + the schema" language should be read as "don't migrate it from in here," not as a claim that the + schema never changes. +- **`check-tools/`** — `db_health_checker.py` (schema/integrity/referential checks against the + *old* normalized schema) plus `download_database.py`, a working Google Drive downloader + authenticated via GCP Workload Identity Federation (no long-lived keys). Wired into + `.github/workflows/docdb-regression-test.yaml`, which runs it daily against the production DB on + Drive. +- **`scripts/`** — the original CLI toolbox. Live/current: `DocumentationDatabase.py` (Content + ingestion — see whitelist issue above), `ingest.py` (thin CLI over it, used by + `publish-doc-db.yaml`), `TooltipManager.py` (CSV ⇄ normalized-Tooltips round-trip), + `create_empty_database.py`, `list_database_documents.py`. **Deprecated/dead** (target the + removed `ide_tooltip_table` — see above, kept for reference only): `tooltips.py`, + `import_tooltips.py`, `android_tooltips.py`, `java_tooltips.py`, `android_html_page.py`, + `load_android_data.py`. +- **`scripts/myServer.py`** — a minimal Python `http.server` reference implementation that predates + `WebServer.kt`. It queries a differently-cased `Documentation.db`, doesn't implement Brotli + decompression (there's a literal `TODO: Replace this function with Brotli decompression`), and + knows nothing about compression-aware content types, templates, or fragmentation. **This is not + what ships in the app** — treat it as historical/reference only, not as documentation of current + server behavior. `WebServer.kt` is the real thing. +- **`Dokka-plugin-kdoc2json/`** — the Dokka `JsonRenderer`/`ModelMapper`/`LinkPostProcessor` plugin, + its test suite, and the `kotlin-stdlib-docs` build scripts, merged to `main` via `fix/ADFA-4514` + (`4c6b8aef`). Consumed by `scripts/kotlin/build-stdlib-json-docs.sh` and + `scripts/sync_kotlin_stdlib_docs/sync_kdoc_json_to_db.py` (ADFA-4739) to generate and load + kotlin-stdlib/-reflect/-test JSON docs. +- **`ProcessDocs/`** — HTML-processing pipelines that predate the "build docs as JSON" goal: + `ProcessKotlinDocs/` (turns Kotlin's HTML doc export into a self-contained HTML set + table of + contents, used by `.github/workflows/automate-kotlin.yaml`), `ProcessAndroidDevSite/`, `AndroidDocs/` + (holds `android-tooltips.pkl`, the pickle consumed by the now-deprecated `load_android_data.py`), + `ProcessPDFs/`. +- **`SourceDocs/`** — raw inputs: `KotlinDocs/html`, `JavaDocs/html` + `java_keywords.html`, + `Tooltips/tooltips.xlsx`, `KotlinDocs/kotlin-spec.pdf`. +- **`DocumentationAnalysis/`, `DocAnalysis/`, `png_optimization/`, `androidxtooltips/`** — Jupyter + notebooks and one-off scripts for doc-set size analysis, image/PNG compression experiments, and a + one-time AndroidX tooltip import (ADFA-1419). Not part of the critical build path. +- **`.github/workflows/`** — three workflows: `automate-kotlin.yaml` (tag-triggered, builds the + Kotlin HTML doc bundle as a GitHub release asset), `publish-doc-db.yaml` (tag-triggered, runs the + `scripts/ingest.py` pipeline and releases the resulting `.sqlite`), `docdb-regression-test.yaml` + (daily cron, downloads the production DB from Google Drive via WIF and runs + `check-tools/main.py` against it). None of these have any Slack integration yet. + +## Decisions log + +Settled with Alex on 2026-08-05, folded into the sections above; recorded here so the reasoning +isn't lost: + +- `ide_tooltip_table` and everything that targets it are dead. Treat as deprecated, not as a gap. +- `Templates`/`Bookshelf`/`BookCategories` are populated by App Dev For All's plugin system + (e.g. [bookshelf-plugin](https://github.com/appdevforall/bookshelf-plugin)), not by this repo. + Not a gap to fill here. +- `PUCC_*` tables are unrelated to documentation tooling. Ignore; leave as-is. +- `docdb-studio`'s schema is expected to lag the live schema and catch up after the fact; that's + fine, no urgent update needed. +- `check-tools/db_health_checker.py` is not being extended with `Templates`/`Bookshelf` checks + right now — deliberately out of scope for the moment. + +The one piece of this document that still describes an *active* problem rather than a settled +scope boundary is `scripts/DocumentationDatabase.py`'s hard failure on unrecognized tables (see the +table above) — that will need to be addressed before `scripts/ingest.py` / +`publish-doc-db.yaml` can run against a current-schema database. diff --git a/Dokka-plugin-kdoc2json/scripts/kotlin/build-stdlib-json-docs.sh b/Dokka-plugin-kdoc2json/scripts/kotlin/build-stdlib-json-docs.sh new file mode 100755 index 000000000..8d160825d --- /dev/null +++ b/Dokka-plugin-kdoc2json/scripts/kotlin/build-stdlib-json-docs.sh @@ -0,0 +1,95 @@ +#!/usr/bin/env bash +# Builds the kotlin-stdlib/kotlin-test/kotlin-reflect API docs as JSON via the +# kdoc-to-json Dokka plugin, against a full kotlin/ (https://github.com/JetBrains/kotlin) +# repo checkout - freshly compiling and publishing the plugin from source +# first, so every run picks up whatever's currently in +# Dokka-plugin-kdoc2json/kdoc-to-json/src, not a jar left over from an +# earlier run. +# +# Only generates the JSON output (dokkaGenerateModuleJson), not the default +# HTML - JSON/latest/all-libs is the only thing this project's pipeline +# (sync_kdoc_json_to_db.py) consumes. Use build-kotlin-stdlib.sh directly, +# against libraries/tools/kotlin-stdlib-docs, if you also want the HTML +# comparison output that test_kotlin_stdlib.sh checks against. +# +# The target kotlin-stdlib-docs project's build.gradle.kts is swapped out +# for this directory's own (JSON-plugin-enabled) copy for the duration of +# the build, then restored automatically on exit - the kotlin checkout is +# left exactly as it was found, whether the build succeeds or fails. +# +# Only the final output path is written to stdout; every other message goes +# to stderr, so this composes as: +# STDLIB_ALL_LIBS="$(build-stdlib-json-docs.sh )" +set -euo pipefail + +log() { echo "$@" >&2; } + +if [ $# -lt 1 ]; then + log "Usage: $0 [output-dir]" + exit 1 +fi + +KOTLIN_ROOT="$(cd "$1" && pwd)" +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PLUGIN_DIR="$(cd "$SCRIPT_DIR/../../kdoc-to-json" && pwd)" +STDLIB_DOCS_DIR="$KOTLIN_ROOT/libraries/tools/kotlin-stdlib-docs" +OUTPUT_ROOT="$(mkdir -p "${2:-$SCRIPT_DIR/build-output}" && cd "${2:-$SCRIPT_DIR/build-output}" && pwd)" +JSON_OUTPUT_DIR="$OUTPUT_ROOT/json" + +if [ ! -f "$KOTLIN_ROOT/gradle.properties" ]; then + log "error: '$KOTLIN_ROOT' doesn't look like a kotlin repo checkout (missing gradle.properties)." + exit 1 +fi +if [ ! -f "$STDLIB_DOCS_DIR/settings.gradle.kts" ] || [ ! -x "$STDLIB_DOCS_DIR/gradlew" ]; then + log "error: '$STDLIB_DOCS_DIR' doesn't look like a kotlin-stdlib-docs project (missing settings.gradle.kts or gradlew)." + exit 1 +fi +if [ ! -x "$PLUGIN_DIR/gradlew" ]; then + log "error: kdoc-to-json plugin project not found at '$PLUGIN_DIR' (missing gradlew)." + exit 1 +fi + +# The JSON-plugin-enabled build.gradle.kts we're about to install reads +# dokka_version as a plain Gradle project property (-Pdokka_version=...) +# rather than through this repo's own version catalog, so it has to be +# supplied explicitly - pulled from the same catalog entry the rest of the +# kotlin repo's Dokka usage is pinned to, so it never drifts out of sync. +DOKKA_VERSION="$(grep -m1 '^dokka[[:space:]]*=' "$KOTLIN_ROOT/gradle/libs.versions.toml" | sed -E 's/^dokka[[:space:]]*=[[:space:]]*"([^"]*)".*/\1/')" +if [ -z "$DOKKA_VERSION" ]; then + log "error: couldn't find a 'dokka = \"...\"' entry in $KOTLIN_ROOT/gradle/libs.versions.toml" + exit 1 +fi + +log "==> [1/2] Building and publishing a fresh copy of the kdoc-to-json plugin..." +# Sent to stderr (fd 2), not left on stdout - a caller doing +# STDLIB_ALL_LIBS="$(build-stdlib-json-docs.sh ...)" must only capture the +# final path this script echoes, not gradlew's own build console output. +( cd "$PLUGIN_DIR" && ./gradlew clean publishToMavenLocal ) >&2 + +log "==> Installing kdoc-to-json-enabled build.gradle.kts into $STDLIB_DOCS_DIR" +ORIGINAL_BUILD_GRADLE="$(mktemp)" +cp "$STDLIB_DOCS_DIR/build.gradle.kts" "$ORIGINAL_BUILD_GRADLE" +restore_build_gradle() { + cp "$ORIGINAL_BUILD_GRADLE" "$STDLIB_DOCS_DIR/build.gradle.kts" + rm -f "$ORIGINAL_BUILD_GRADLE" +} +trap restore_build_gradle EXIT +cp "$SCRIPT_DIR/build.gradle.kts" "$STDLIB_DOCS_DIR/build.gradle.kts" + +log "==> [2/2] Generating JSON documentation via kdoc-to-json (dokka $DOKKA_VERSION)..." +# --refresh-dependencies forces Gradle to re-resolve the just-published +# SNAPSHOT jar from mavenLocal() rather than serving a same-GAV copy it +# cached from an earlier run of this same script. +( cd "$STDLIB_DOCS_DIR" && ./gradlew dokkaGenerateModuleJson \ + "-PdocsBuildDir=$JSON_OUTPUT_DIR" \ + "-Pdokka_version=$DOKKA_VERSION" \ + --refresh-dependencies ) >&2 + +ALL_LIBS_DIR="$JSON_OUTPUT_DIR/latest/all-libs" +if [ ! -d "$ALL_LIBS_DIR" ]; then + log "error: expected output at '$ALL_LIBS_DIR' but it wasn't created." + exit 1 +fi + +log "==> Done." +echo "$ALL_LIBS_DIR" diff --git a/Dokka-plugin-kdoc2json/scripts/kotlin/build.gradle.kts b/Dokka-plugin-kdoc2json/scripts/kotlin/build.gradle.kts index db4c72983..48f68c3f2 100644 --- a/Dokka-plugin-kdoc2json/scripts/kotlin/build.gradle.kts +++ b/Dokka-plugin-kdoc2json/scripts/kotlin/build.gradle.kts @@ -46,7 +46,14 @@ allprojects { // 3. Maven Central (Keep this for standard standard stable libraries like Gson/Coroutines) mavenCentral() - // ALL REMOTE JETBRAINS SNAPSHOT SERVERS HAVE BEEN REMOVED! + // 4. Dokka's own dev-snapshot server - required by plugins:dokka-samples-transformer-plugin + // and plugins:dokka-version-filter-plugin (both included by kotlin-stdlib-docs' + // settings.gradle.kts and pulled onto the build graph by its dokka-convention plugin), + // which pin to a Dokka dev build rather than a Maven Central release. Same property + + // default kotlin-stdlib-docs' own settings.gradle.kts uses, so this only ever points + // wherever that project already expects it to. + maven(url = providers.gradleProperty("dokka_repository") + .getOrElse("https://redirector.kotlinlang.org/maven/dokka-dev")) } // --- ADDED THIS EXCLUSION BLOCK --- diff --git a/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/README.md b/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/README.md new file mode 100644 index 000000000..91ca97129 --- /dev/null +++ b/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/README.md @@ -0,0 +1,171 @@ +# Process Kotlin Website JSON + +Converts a `kotlin-web-site/docs` checkout (JetBrains Writerside-flavored +Markdown) into the JSON block schema this project's templating engine +renders, then (optionally) builds the sidebar nav and loads everything +straight into a `documentation.db`-schema SQLite database. + +## Scripts + +| Script | Purpose | +|---|---| +| [`md_to_json.py`](md_to_json.py) | Converts every `topics/**/*.md` page into one JSON file. Writes `theme.json` and copies `images/` into the output directory. See "Page JSON schema" below. | +| [`build_nav.py`](build_nav.py) | Builds `nav.json`/`nav.html` sidebar navigation from `kr.tree`, resolving each `` against `md_to_json.py`'s output. | +| [`find_missing_assets.py`](find_missing_assets.py) | QA pass: reports cross-page links, images, and `` targets in the source tree that don't resolve to anything. Reuses `md_to_json.py`'s own resolution logic, so it flags exactly what would end up broken on the rendered site. | +| [`populate_db.py`](populate_db.py) | The database path: converts the docs tree the same way `md_to_json.py` does, builds nav the same way `build_nav.py` does, and inserts pages + nav + images + CSS/JS directly into `documentation.db` (replacing everything under `k/html/` and `assets/`). Supports pruning whole `kr.tree` subtrees via `--blacklisted-element-titles`. | +| [`optimize_media.py`](optimize_media.py) | Standalone media optimizer: downscales/recompresses a directory of images (pngquant, Pillow, Scour/cairosvg for SVG) into a mirrored output directory. | +| [`insert_optimized_media.py`](insert_optimized_media.py) | Runs `optimize_media.py`'s pipeline over a directory of raw media, then replaces the corresponding `k/html/images/*` rows in an existing database, rewriting any page that referenced a renamed file and deleting anything left unreferenced. | + +## Requirements + +- Python 3.10+ and [`uv`](https://docs.astral.sh/uv/getting-started/installation/) — every command below is run as `uv run --with-requirements /requirements.txt + + + + +{# + Recursive block renderer. Macros only see the variables passed to them, so + every block that can nest other blocks (blockquote, note/tip/warning, list + items, table cells, tabs) passes its children back through renderBlock(). + Macros defined in a template are directly visible to themselves and to each + other within that same template, so no self-import is needed for recursion. +#} +{% macro renderBlock(b) %} +{% if b.type == "heading" %} +{{ b.html|raw }} + +{% elseif b.type == "paragraph" %} +

{{ b.html|raw }}

+ +{% elseif b.type == "code" %} +
{{ b.code }}
+ +{% elseif b.type == "blockquote" %} +
+{% if b.attrs.title %}

{{ b.attrs.title }}

{% endif %} +{% for child in b.blocks %}{{ renderBlock(child) }} +{% endfor %}
+ +{% elseif b.type == "note" or b.type == "tip" or b.type == "warning" %} +
+{% if b.attrs.title %}

{{ b.attrs.title }}

{% endif %} +{% for child in b.blocks %}{{ renderBlock(child) }} +{% endfor %}
+ +{% elseif b.type == "list" %} +{% if b.ordered %}
    {% else %}
      {% endif %} +{% for item in b.items %}
    • {% for child in item.blocks %}{{ renderBlock(child) }}{% endfor %}
      • +{% endfor %}{% if b.ordered %}
{% else %}{% endif %} + +{% elseif b.type == "table" %} + +{% if b.headers is not empty %} +{% for h in b.headers %}{% endfor %} +{% endif %} + +{% for row in b.rows %}{% for cell in row %}{% endfor %} +{% endfor %} +
{{ h|raw }}
{{ cell|raw }}
+ +{% elseif b.type == "image" %} +{{ b.alt|default('') }} + +{% elseif b.type == "hr" %} +
+ +{% elseif b.type == "tabs" and b.tabs is not empty %} +{# + Tab switching + the group-key syncing (e.g. picking "Groovy" in one + Kotlin/Groovy/Maven tabs block switches every other tabs block sharing the + same data-group on the page, matching Writerside's data-sync-tabs + behavior) is implemented in assets/tabs.js. md_to_json.py's + _finalize_container always gives every "tabs" block a non-empty "tabs" + list (synthesizing one from code-block languages, or dropping the wrapper + entirely, when the source had no children) - the "b.tabs is not + empty" guard here is just a defensive backstop against any other producer + of this JSON schema making the same mistake, not something this pipeline + itself still needs. +#} +
+
+ {% for tab in b.tabs %}{% set tabKey = tab.attrs["group-key"]|default(tab.title)|default(loop.index) %} + {% endfor %}
+ {% for tab in b.tabs %}{% set tabKey = tab.attrs["group-key"]|default(tab.title)|default(loop.index) %}
+ {% for child in tab.blocks %}{{ renderBlock(child) }} + {% endfor %}
+ {% endfor %} +
+ +{% elseif b.type == "html" %} +{{ b.html|raw }} + +{% elseif b.type == "tab" %} +{# A lone not wrapped in (e.g. seen in eap.json's HTML-table + compatibility layout); render its children rather than dropping them. #} +{% for child in b.blocks %}{{ renderBlock(child) }} +{% endfor %} + +{% else %} +{% if b.html %}{{ b.html|raw }}{% endif %} + +{% endif %} +{% endmacro %} diff --git a/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/test_migrate_content_to_dictionary_brotli.py b/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/test_migrate_content_to_dictionary_brotli.py new file mode 100644 index 000000000..d3b2da30d --- /dev/null +++ b/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/test_migrate_content_to_dictionary_brotli.py @@ -0,0 +1,342 @@ +#!/usr/bin/env python3 +"""Tests for migrate_content_to_dictionary_brotli.py (ADFA-5153). + +Run directly: python3 test_migrate_content_to_dictionary_brotli.py +""" +import random +import sqlite3 +import tempfile +import unittest +from pathlib import Path + +import brotli + +from migrate_content_to_dictionary_brotli import collect_training_samples, migrate, read_item +from populate_db import CHUNK_SIZE, DictionaryCompressor, load_dictionary + +SCHEMA_SQL = """ +CREATE TABLE Languages (id INTEGER PRIMARY KEY AUTOINCREMENT, value TEXT NOT NULL UNIQUE); +CREATE TABLE ContentTypes (id INTEGER PRIMARY KEY AUTOINCREMENT, value TEXT NOT NULL UNIQUE, compression TEXT NOT NULL); +CREATE TABLE Content ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + path TEXT NOT NULL, + languageID INTEGER NOT NULL, + content BLOB NOT NULL, + contentTypeID INTEGER NOT NULL, + templateId INTEGER, + UNIQUE(path) +); +""" + +WORDS = [ + "kotlin", "class", "fun", "val", "var", "override", "interface", "object", "companion", + "sidebar", "nav", "template", "docs-sidebar", "toc-element", "page.peb", "Content-Type", +] + + +def make_text(word_count: int, seed: int) -> bytes: + rng = random.Random(seed) + return (" ".join(rng.choice(WORDS) for _ in range(word_count))).encode("utf-8") + + +def insert_plain_chunked(conn, path, language_id, content_type_id, template_id, plain_bytes): + """Mimics populate_db.py's insert_chunked_content, but with plain + (no-dictionary) Brotli - i.e. exactly what every pre-ADFA-5153 pipeline + actually wrote.""" + compressed = brotli.compress(plain_bytes) + conn.execute( + "INSERT INTO Content (path, languageID, content, contentTypeID, templateId) VALUES (?, ?, ?, ?, ?)", + (path, language_id, compressed[:CHUNK_SIZE], content_type_id, template_id), + ) + offset = CHUNK_SIZE + n = 1 + while offset < len(compressed): + conn.execute( + "INSERT INTO Content (path, languageID, content, contentTypeID, templateId) VALUES (?, ?, ?, ?, ?)", + (f"{path}-{n}", language_id, compressed[offset:offset + CHUNK_SIZE], content_type_id, template_id), + ) + offset += CHUNK_SIZE + n += 1 + return compressed + + +def reassemble(conn, path, first_content): + if len(first_content) < CHUNK_SIZE: + return first_content + parts = [first_content] + n = 1 + while True: + row = conn.execute("SELECT content FROM Content WHERE path = ?", (f"{path}-{n}",)).fetchone() + if row is None: + break + parts.append(row[0]) + if len(row[0]) < CHUNK_SIZE: + break + n += 1 + return b"".join(parts) + + +class MigrateContentToDictionaryBrotliTest(unittest.TestCase): + def setUp(self): + # A real file, not :memory: - the tests below reopen the database to + # check what was actually committed, and the trigger fixture needs a + # schema an in-memory connection would not outlive. + fd, path = tempfile.mkstemp(suffix=".db") + Path(path).unlink(missing_ok=True) + self.db_path = Path(path) + self.conn = sqlite3.connect(self.db_path) + self.conn.executescript(SCHEMA_SQL) + self.conn.execute("INSERT INTO Languages (value) VALUES ('en-US')") + self.conn.execute("INSERT INTO ContentTypes (value, compression) VALUES ('text/html', 'brotli')") + self.conn.execute("INSERT INTO ContentTypes (value, compression) VALUES ('image/png', 'none')") + self.language_id = 1 + self.html_type_id = 1 + self.png_type_id = 2 + self.conn.commit() + + def tearDown(self): + self.conn.close() + self.db_path.unlink(missing_ok=True) + + def test_migrates_plain_brotli_rows_preserving_content(self): + originals = {} + for i in range(20): + plain = make_text(200, seed=i) + insert_plain_chunked(self.conn, f"k/html/page{i}.html", self.language_id, self.html_type_id, 5, plain) + originals[f"k/html/page{i}.html"] = plain + self.conn.execute( + "INSERT INTO Content (path, languageID, content, contentTypeID, templateId) VALUES (?, ?, ?, ?, ?)", + ("assets/logo.png", self.language_id, b"\x89PNG-not-really-compressed", self.png_type_id, 0), + ) + self.conn.commit() + + stats = migrate(self.conn, sample_size=20, dict_size=16384) + self.assertEqual(stats["scanned"], 20) + self.assertEqual(stats["migrated"], 20) + self.assertEqual(stats["already"], 0) + + dictionary_data = load_dictionary(self.conn) + with DictionaryCompressor(dictionary_data) as compressor: + for path, plain in originals.items(): + row = self.conn.execute("SELECT content, templateId FROM Content WHERE path = ?", (path,)).fetchone() + first_content, template_id = row + full = reassemble(self.conn, path, first_content) + self.assertEqual(compressor.decompress(full), plain) + self.assertEqual(template_id, 5) + + # untouched: not a 'brotli' content type + png_row = self.conn.execute("SELECT content FROM Content WHERE path = 'assets/logo.png'").fetchone() + self.assertEqual(png_row[0], b"\x89PNG-not-really-compressed") + + def test_preserves_chunked_rows_across_the_1mb_boundary(self): + # A handful of small filler rows so the dictionary trainer has + # more than one sample to work with (zstd's trainer refuses "too + # few samples" on just one row) - a realistic database always has + # many rows, this test's chunked row just happens to be one of them. + for i in range(20): + insert_plain_chunked(self.conn, f"k/html/filler{i}.html", self.language_id, self.html_type_id, 0, + make_text(150, seed=i)) + + # High-entropy bytes to push the compressed form over CHUNK_SIZE (text + # from make_text's small vocabulary compresses far too well to cross + # that boundary at any realistic size), plus a text tail so the encoder + # actually references the shared dictionary. Noise alone references + # nothing, and migrate() then correctly reports the row as having + # nothing to gain rather than migrating it. + plain = random.Random(777).randbytes(int(CHUNK_SIZE * 1.2)) + make_text(4000, seed=777) + insert_plain_chunked(self.conn, "k/html/big.html", self.language_id, self.html_type_id, 5, plain) + self.conn.commit() + + # Confirm the fixture actually produced a chunked row before relying on it + fragment_exists = self.conn.execute( + "SELECT 1 FROM Content WHERE path = 'k/html/big.html-1'" + ).fetchone() + self.assertIsNotNone(fragment_exists, "test fixture did not produce a chunked row; adjust its size") + + stats = migrate(self.conn, sample_size=21, dict_size=16384) + self.assertEqual(stats["migrated"], 21) + + dictionary_data = load_dictionary(self.conn) + first_content = self.conn.execute( + "SELECT content FROM Content WHERE path = 'k/html/big.html'" + ).fetchone()[0] + with DictionaryCompressor(dictionary_data) as compressor: + full = reassemble(self.conn, "k/html/big.html", first_content) + self.assertEqual(compressor.decompress(full), plain) + + def test_idempotent_second_run_is_a_no_op(self): + originals = {} + for i in range(15): + plain = make_text(150, seed=100 + i) + insert_plain_chunked(self.conn, f"k/html/p{i}.html", self.language_id, self.html_type_id, 0, plain) + originals[f"k/html/p{i}.html"] = plain + self.conn.commit() + + first_stats = migrate(self.conn, sample_size=15, dict_size=16384) + self.assertEqual(first_stats["migrated"], 15) + dictionary_after_first_run = load_dictionary(self.conn) + + snapshot = { + path: self.conn.execute("SELECT content FROM Content WHERE path = ?", (path,)).fetchone()[0] + for path in originals + } + + second_stats = migrate(self.conn, sample_size=15, dict_size=16384) + self.assertEqual(second_stats["migrated"], 0) + self.assertEqual(second_stats["already"], 15) + + # dictionary must not have been retrained + self.assertEqual(load_dictionary(self.conn), dictionary_after_first_run) + # and no row's bytes changed on the no-op second pass + for path, before in snapshot.items(): + after = self.conn.execute("SELECT content FROM Content WHERE path = ?", (path,)).fetchone()[0] + self.assertEqual(before, after) + + def test_chain_numbered_from_minus_two_is_migrated_not_miscounted(self): + """ADFA-5171 chains must not read as 'already dictionary-compressed'. + + Probing "-1" returns a truncated stream for a chain numbered from + -2; the decode then fails, and counting that as already-migrated leaves + the row plain-Brotli while the run reports a clean finish.""" + for i in range(20): + insert_plain_chunked(self.conn, f"k/html/filler{i}.html", self.language_id, self.html_type_id, 0, + make_text(150, seed=i)) + plain = random.Random(4242).randbytes(int(CHUNK_SIZE * 1.4)) + make_text(4000, seed=4242) + compressed = brotli.compress(plain) + self.assertGreater(len(compressed), CHUNK_SIZE, "fixture must be chunked; adjust its size") + self.conn.execute( + "INSERT INTO Content (path, languageID, content, contentTypeID, templateId) VALUES (?, ?, ?, ?, ?)", + ("k/html/misnumbered.html", self.language_id, compressed[:CHUNK_SIZE], self.html_type_id, 0), + ) + # the continuation numbered from -2, with no -1 at all + offset, number = CHUNK_SIZE, 2 + while offset < len(compressed): + self.conn.execute( + "INSERT INTO Content (path, languageID, content, contentTypeID, templateId) VALUES (?, ?, ?, ?, ?)", + (f"k/html/misnumbered.html-{number}", self.language_id, + compressed[offset:offset + CHUNK_SIZE], self.html_type_id, 0), + ) + offset += CHUNK_SIZE + number += 1 + self.conn.commit() + + stats = migrate(self.conn, sample_size=21, dict_size=16384) + self.assertEqual(stats["errors"], 0) + self.assertEqual(stats["already"], 0, "a -2 chain was miscounted as already migrated") + self.assertEqual(stats["migrated"], 21) + + # Content survives, and the rewritten chain is -1-based, which is what + # WebServer.kt's reassembly loop actually probes. + with DictionaryCompressor(load_dictionary(self.conn)) as compressor: + self.assertEqual(compressor.decompress(read_item(self.conn, "k/html/misnumbered.html")), plain) + self.assertIsNotNone( + self.conn.execute("SELECT 1 FROM Content WHERE path = 'k/html/misnumbered.html-1'").fetchone()) + + def test_undecodable_row_is_an_error_not_a_success(self): + for i in range(20): + insert_plain_chunked(self.conn, f"k/html/ok{i}.html", self.language_id, self.html_type_id, 0, + make_text(150, seed=i)) + self.conn.execute( + "INSERT INTO Content (path, languageID, content, contentTypeID, templateId) VALUES (?, ?, ?, ?, ?)", + ("k/html/corrupt.html", self.language_id, b"not brotli at all", self.html_type_id, 0), + ) + self.conn.commit() + + stats = migrate(self.conn, sample_size=20, dict_size=16384) + self.assertEqual(stats["errors"], 1) + self.assertEqual(stats["already"], 0) + self.assertEqual([path for path, _detail in stats["problems"]], ["k/html/corrupt.html"]) + # left exactly as it was, not half-written + self.assertEqual( + self.conn.execute("SELECT content FROM Content WHERE path = 'k/html/corrupt.html'").fetchone()[0], + b"not brotli at all") + + def test_bookshelf_entry_survives_migration(self): + """Content's AddBook/DeleteBook triggers make a delete+insert cycle on a + '%.pdf' path silently replace the curated Bookshelf row.""" + self.conn.executescript(""" + CREATE TABLE BookCategories (id INTEGER PRIMARY KEY AUTOINCREMENT, category TEXT); + CREATE TABLE Bookshelf ( + contentID INTEGER NOT NULL, title STRING DEFAULT '', description STRING DEFAULT '', + bookCategoryID INTEGER, + FOREIGN KEY (bookCategoryID) REFERENCES BookCategories(id), UNIQUE(title, bookCategoryId)); + CREATE TRIGGER DeleteBook AFTER DELETE ON Content WHEN OLD.path LIKE '%.pdf' + BEGIN DELETE FROM Bookshelf WHERE contentID = OLD.id; END; + CREATE TRIGGER AddBook AFTER INSERT ON Content WHEN NEW.path LIKE '%.pdf' + BEGIN INSERT INTO Bookshelf (contentID, title) VALUES (NEW.id, CURRENT_TIMESTAMP || NEW.id); END; + """) + self.conn.execute("INSERT INTO ContentTypes (value, compression) VALUES ('application/pdf', 'brotli')") + pdf_type_id = self.conn.execute( + "SELECT id FROM ContentTypes WHERE value = 'application/pdf'").fetchone()[0] + self.conn.execute("INSERT INTO BookCategories (category) VALUES ('Programming')") + + for i in range(20): + insert_plain_chunked(self.conn, f"k/html/f{i}.html", self.language_id, self.html_type_id, 0, + make_text(150, seed=i)) + insert_plain_chunked(self.conn, "bookshelfplugin/Notes.pdf", self.language_id, pdf_type_id, 0, + make_text(400, seed=9)) + pdf_id = self.conn.execute( + "SELECT id FROM Content WHERE path = 'bookshelfplugin/Notes.pdf'").fetchone()[0] + self.conn.execute("DELETE FROM Bookshelf") # drop what AddBook just generated + self.conn.execute( + "INSERT INTO Bookshelf (contentID, bookCategoryID, title, description) VALUES (?, 1, ?, ?)", + (pdf_id, "Notes for Professionals", "A curated description")) + self.conn.commit() + + stats = migrate(self.conn, sample_size=21, dict_size=16384) + self.assertEqual(stats["migrated"], 21) + + shelf = self.conn.execute( + "SELECT contentID, bookCategoryID, title, description FROM Bookshelf").fetchall() + self.assertEqual(shelf, [(pdf_id, 1, "Notes for Professionals", "A curated description")], + "the curated Bookshelf entry was replaced") + + def test_second_run_leaves_rows_whose_dictionary_was_never_referenced(self): + """A payload the encoder never needed the dictionary for decodes both + ways, so 'a plain decode succeeded' does not mean 'not yet migrated'.""" + for i in range(20): + insert_plain_chunked(self.conn, f"k/html/t{i}.html", self.language_id, self.html_type_id, 0, + make_text(150, seed=i)) + # 24 bytes of high-entropy noise: nothing in any dictionary can help it. + self.conn.execute( + "INSERT INTO Content (path, languageID, content, contentTypeID, templateId) VALUES (?, ?, ?, ?, ?)", + ("k/html/tiny.bin", self.language_id, brotli.compress(random.Random(5).randbytes(24)), + self.html_type_id, 0), + ) + self.conn.commit() + + migrate(self.conn, sample_size=20, dict_size=16384) + snapshot = self.conn.execute("SELECT path, content FROM Content").fetchall() + second = migrate(self.conn, sample_size=20, dict_size=16384) + self.assertEqual(second["migrated"], 0, "a second run re-migrated rows it should have left alone") + self.assertEqual(second["errors"], 0) + self.assertEqual(self.conn.execute("SELECT path, content FROM Content").fetchall(), snapshot) + + def test_training_sample_is_stratified_and_reproducible(self): + for doc_set, count in (("a", 40), ("j", 30), ("k", 10)): + for i in range(count): + insert_plain_chunked(self.conn, f"{doc_set}/page{i}.html", self.language_id, + self.html_type_id, 0, make_text(200, seed=hash((doc_set, i)) % 10_000)) + self.conn.commit() + base_rows = self.conn.execute( + "SELECT C.path, C.languageID, C.contentTypeID, C.templateId, LENGTH(C.content) " + "FROM Content C, ContentTypes CT WHERE C.contentTypeID = CT.id AND CT.compression = 'brotli' " + "ORDER BY C.path").fetchall() + + first = collect_training_samples(self.conn, base_rows, sample_size=30, byte_budget=1 << 20, seed=7) + again = collect_training_samples(self.conn, base_rows, sample_size=30, byte_budget=1 << 20, seed=7) + self.assertEqual(first, again, "same seed must produce the same training set") + + # Every doc set should be represented -- the whole point of stratifying. + # Sample paths back out by matching the plaintext we know we inserted. + sampled_sets = set() + for sample in first: + row = self.conn.execute( + "SELECT path FROM Content WHERE content = ?", (brotli.compress(sample),)).fetchone() + if row: + sampled_sets.add(row[0].split("/", 1)[0]) + self.assertGreaterEqual(len(sampled_sets), 2, f"sample covered only {sampled_sets}") + + +if __name__ == "__main__": + unittest.main() diff --git a/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/test_populate_db_dictionary.py b/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/test_populate_db_dictionary.py new file mode 100644 index 000000000..de128250d --- /dev/null +++ b/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/test_populate_db_dictionary.py @@ -0,0 +1,132 @@ +#!/usr/bin/env python3 +"""Tests for populate_db.py's shared-dictionary Brotli compression (ADFA-5153). + +Run directly: python3 test_populate_db_dictionary.py +""" +import random +import sqlite3 +import unittest + +from populate_db import DictionaryCompressor, load_dictionary, load_or_create_dictionary, train_dictionary + +WORDS = [ + "kotlin", "class", "fun", "val", "var", "override", "interface", "object", "companion", + "sidebar", "nav", "template", "docs-sidebar", "toc-element", "page.peb", "Content-Type", +] + + +def make_samples(count: int, seed: int = 1) -> list: + rng = random.Random(seed) + return [ + (" ".join(rng.choice(WORDS) for _ in range(150))).encode("utf-8") + for _ in range(count) + ] + + +class TrainDictionaryTest(unittest.TestCase): + def test_produces_nonempty_dictionary(self): + dictionary_data = train_dictionary(make_samples(120)) + self.assertGreater(len(dictionary_data), 0) + + +class DictionaryCompressorTest(unittest.TestCase): + def setUp(self): + self.dictionary_data = train_dictionary(make_samples(120)) + + def test_round_trip(self): + payload = make_samples(1)[0] + with DictionaryCompressor(self.dictionary_data) as compressor: + compressed = compressor.compress(payload) + self.assertNotEqual(compressed, payload) + self.assertEqual(compressor.decompress(compressed), payload) + + def test_compresses_smaller_than_plain_brotli_for_repetitive_corpus(self): + # The whole point of a shared dictionary: content similar to the + # training samples should compress smaller with the dictionary than + # without one. + import brotli + payload = make_samples(1)[0] + with DictionaryCompressor(self.dictionary_data) as compressor: + with_dict = compressor.compress(payload) + without_dict = brotli.compress(payload) + self.assertLess(len(with_dict), len(without_dict)) + + def test_wrong_dictionary_never_returns_the_original_bytes(self): + # A mismatched dictionary is not guaranteed to fail loudly. Measured on + # the real corpus (perturbing one 16 KiB region of the real dictionary, + # decoding real rows): 50% raised, 38% decoded with no error into + # *different* bytes, 12% decoded identically because the perturbed + # region was never referenced. + # + # So asserting that the decode does not raise would be asserting a coin + # flip, brittle across brotli versions and payloads. The invariant that + # actually holds is the one that matters: a wrong dictionary never + # yields the original bytes *and* reports success. That is why + # load_or_create_dictionary must never retrain over a stored dictionary + # - no runtime check can catch the mismatch afterwards. + other_dictionary_data = train_dictionary(make_samples(120, seed=99)) + payload = make_samples(1)[0] + with DictionaryCompressor(self.dictionary_data) as compressor: + compressed = compressor.compress(payload) + with DictionaryCompressor(other_dictionary_data) as wrong_compressor: + try: + result = wrong_compressor.decompress(compressed) + except RuntimeError: + return # failed loudly, which is the other acceptable outcome + self.assertNotEqual(result, payload, "a wrong dictionary must not appear to succeed") + + def test_no_dictionary_stream_fails_to_decode_with_dictionary_attached(self): + import brotli + payload = make_samples(1)[0] + plain_compressed = brotli.compress(payload) + with DictionaryCompressor(self.dictionary_data) as compressor: + with self.assertRaises(RuntimeError): + compressor.decompress(plain_compressed) + + +class LoadOrCreateDictionaryTest(unittest.TestCase): + def setUp(self): + self.conn = sqlite3.connect(":memory:") + + def tearDown(self): + self.conn.close() + + def test_first_call_trains_and_stores(self): + dictionary_data = load_or_create_dictionary(self.conn, make_samples(120)) + self.assertGreater(len(dictionary_data), 0) + row = self.conn.execute("SELECT data FROM CompressionDictionary WHERE id = 1").fetchone() + self.assertEqual(row[0], dictionary_data) + + def test_second_call_reuses_stored_dictionary_without_retraining(self): + first = load_or_create_dictionary(self.conn, make_samples(120, seed=1)) + second = load_or_create_dictionary(self.conn, make_samples(120, seed=2)) + self.assertEqual(first, second) + + def test_content_survives_a_reused_dictionary_across_separate_connections(self): + # Mirrors the real cross-repo split: populate_db.py trains/stores the + # dictionary once; a later run (or a different process entirely, + # like WebServer.kt) must be able to decode against the same bytes + # loaded back from the database. + dictionary_data = load_or_create_dictionary(self.conn, make_samples(120)) + payload = make_samples(1)[0] + with DictionaryCompressor(dictionary_data) as compressor: + compressed = compressor.compress(payload) + + reloaded = load_dictionary(self.conn) + self.assertEqual(reloaded, dictionary_data) + with DictionaryCompressor(reloaded) as compressor: + self.assertEqual(compressor.decompress(compressed), payload) + + +class LoadDictionaryTest(unittest.TestCase): + def test_raises_when_missing(self): + conn = sqlite3.connect(":memory:") + try: + with self.assertRaises(RuntimeError): + load_dictionary(conn) + finally: + conn.close() + + +if __name__ == "__main__": + unittest.main() diff --git a/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/test_remint_dictionary.py b/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/test_remint_dictionary.py new file mode 100644 index 000000000..143991fb4 --- /dev/null +++ b/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/test_remint_dictionary.py @@ -0,0 +1,118 @@ +#!/usr/bin/env python3 +"""Tests for remint_dictionary.py / verify_remint_dictionary.py (ADFA-5153). + +Run directly: python3 test_remint_dictionary.py +""" +import random +import shutil +import sqlite3 +import tempfile +import unittest +from pathlib import Path + +import brotli + +from migrate_content_to_dictionary_brotli import migrate +from populate_db import CHUNK_SIZE, DictionaryCompressor, load_dictionary +from remint_dictionary import remint +from verify_remint_dictionary import verify +from test_migrate_content_to_dictionary_brotli import SCHEMA_SQL, insert_plain_chunked, make_text + + +class RemintDictionaryTest(unittest.TestCase): + def setUp(self): + fd, path = tempfile.mkstemp(suffix=".db") + Path(path).unlink(missing_ok=True) + self.db_path = Path(path) + self.conn = sqlite3.connect(self.db_path) + self.conn.executescript(SCHEMA_SQL) + self.conn.execute("INSERT INTO Languages (value) VALUES ('en-US')") + self.conn.execute("INSERT INTO ContentTypes (value, compression) VALUES ('text/html', 'brotli')") + self.html_type_id = 1 + self.originals = {} + # Spread across doc sets, so the stratified sampler has something to stratify. + for doc_set in ("a", "j", "k"): + for i in range(12): + path_ = f"{doc_set}/page{i}.html" + plain = make_text(220, seed=hash((doc_set, i)) % 9973) + insert_plain_chunked(self.conn, path_, 1, self.html_type_id, 0, plain) + self.originals[path_] = plain + self.conn.commit() + migrate(self.conn, sample_size=30, dict_size=16384) + self.conn.commit() + self.first_dictionary = load_dictionary(self.conn) + + def tearDown(self): + self.conn.close() + self.db_path.unlink(missing_ok=True) + + def snapshot(self) -> Path: + """A copy of the database as it stands, to verify a re-mint against.""" + fd, path = tempfile.mkstemp(suffix=".db") + Path(path).unlink(missing_ok=True) + copy = Path(path) + shutil.copy2(self.db_path, copy) + self.addCleanup(copy.unlink, True) + return copy + + def plaintext(self, path: str) -> bytes: + with DictionaryCompressor(load_dictionary(self.conn)) as compressor: + row = self.conn.execute("SELECT content FROM Content WHERE path = ?", (path,)).fetchone() + return compressor.decompress(row[0]) + + def test_replaces_the_dictionary_and_preserves_every_payload(self): + before = self.snapshot() + # A different seed is what makes this a real re-mint: the sampler is + # deterministic, so re-training on the same corpus with the same seed + # reproduces the stored dictionary byte for byte (see the sibling test). + stats = remint(self.conn, sample_size=30, dict_size=16384, training_bytes=1 << 20, seed=999) + self.assertEqual(stats["items"], len(self.originals)) + + self.assertNotEqual(load_dictionary(self.conn), self.first_dictionary, + "the dictionary was not actually re-minted") + for path, plain in self.originals.items(): + self.assertEqual(self.plaintext(path), plain, f"{path} lost its content") + self.assertEqual(verify(before, self.db_path), []) + + def test_same_seed_reproduces_the_stored_dictionary(self): + """The sampler is seeded so a dictionary's training set can be + reproduced later -- which also means re-minting with the same seed and + corpus is a no-op, and the verifier says so rather than pretending + something changed.""" + before = self.snapshot() + remint(self.conn, sample_size=30, dict_size=16384, training_bytes=1 << 20) + self.assertEqual(load_dictionary(self.conn), self.first_dictionary) + self.assertEqual(verify(before, self.db_path), []) + + def test_undecodable_row_aborts_with_nothing_written(self): + self.conn.execute( + "INSERT INTO Content (path, languageID, content, contentTypeID, templateId) VALUES (?, 1, ?, ?, 0)", + ("a/broken.html", b"not brotli at all", self.html_type_id)) + self.conn.commit() + snapshot = {path: content for path, content in + self.conn.execute("SELECT path, content FROM Content")} + + with self.assertRaises(RuntimeError): + remint(self.conn, sample_size=30, dict_size=16384, training_bytes=1 << 20) + + self.assertEqual(load_dictionary(self.conn), self.first_dictionary, "dictionary was replaced anyway") + self.assertEqual({path: content for path, content in + self.conn.execute("SELECT path, content FROM Content")}, snapshot, + "rows were rewritten despite the abort") + + def test_verifier_catches_a_corrupted_remint(self): + """Guards against a vacuous verifier: hand it a database whose content + does not match and it must object.""" + before = self.snapshot() + remint(self.conn, sample_size=30, dict_size=16384, training_bytes=1 << 20, seed=4242) + with DictionaryCompressor(load_dictionary(self.conn)) as compressor: + wrong = compressor.compress(b"replaced with something else entirely") + self.conn.execute("UPDATE Content SET content = ? WHERE path = 'j/page3.html'", (wrong,)) + self.conn.commit() + + problems = verify(before, self.db_path) + self.assertEqual([path for path, _why in problems], ["j/page3.html"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/test_renumber_misnumbered_fragments.py b/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/test_renumber_misnumbered_fragments.py new file mode 100644 index 000000000..dba7f948c --- /dev/null +++ b/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/test_renumber_misnumbered_fragments.py @@ -0,0 +1,195 @@ +#!/usr/bin/env python3 +"""Tests for renumber_misnumbered_fragments.py (ADFA-5171). + +Run directly: python3 test_renumber_misnumbered_fragments.py +""" +import sqlite3 +import tempfile +import unittest +from pathlib import Path + +from populate_db import CHUNK_SIZE +from renumber_misnumbered_fragments import repair + +SCHEMA_SQL = """ +CREATE TABLE Languages (id INTEGER PRIMARY KEY AUTOINCREMENT, value TEXT NOT NULL UNIQUE); +CREATE TABLE ContentTypes (id INTEGER PRIMARY KEY AUTOINCREMENT, value TEXT NOT NULL UNIQUE, compression TEXT NOT NULL); +CREATE TABLE Content ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + path TEXT NOT NULL, + languageID INTEGER NOT NULL, + content BLOB NOT NULL, + contentTypeID INTEGER NOT NULL, + templateId INTEGER, + UNIQUE(path) +); +""" + + +def chunk_bytes(n: int, fill: bytes) -> bytes: + return (fill * (n // len(fill) + 1))[:n] + + +class RenumberMisnumberedFragmentsTest(unittest.TestCase): + def setUp(self): + fd, path = tempfile.mkstemp(suffix=".db") + Path(path).unlink(missing_ok=True) + self.db_path = Path(path) + self.conn = sqlite3.connect(self.db_path) + self.conn.executescript(SCHEMA_SQL) + self.conn.execute("INSERT INTO Languages (value) VALUES ('en-US')") + self.conn.execute("INSERT INTO ContentTypes (value, compression) VALUES ('image/gif', 'none')") + self.conn.commit() + + def tearDown(self): + self.conn.close() + self.db_path.unlink(missing_ok=True) + + def insert(self, path: str, content: bytes): + self.conn.execute( + "INSERT INTO Content (path, languageID, content, contentTypeID) VALUES (?, 1, ?, 1)", + (path, content), + ) + + def all_paths(self) -> set: + return {row[0] for row in self.conn.execute("SELECT path FROM Content")} + + def content_at(self, path: str) -> bytes: + return self.conn.execute("SELECT content FROM Content WHERE path = ?", (path,)).fetchone()[0] + + def test_renumbers_chain_starting_at_minus_2(self): + base = "a/devsite/media/size-range.gif" + self.insert(base, chunk_bytes(CHUNK_SIZE, b"A")) + self.insert(f"{base}-2", chunk_bytes(CHUNK_SIZE, b"B")) + self.insert(f"{base}-3", chunk_bytes(CHUNK_SIZE, b"C")) + self.insert(f"{base}-4", chunk_bytes(CHUNK_SIZE, b"D")) + self.insert(f"{base}-5", b"E" * 100) + self.conn.commit() + + stats = repair(self.conn) + self.conn.commit() + + self.assertEqual(stats["chains_renumbered"], 1) + self.assertEqual(stats["fragments_moved"], 4) + self.assertEqual(stats["chains_gapped"], 0) + self.assertEqual( + self.all_paths(), + {base, f"{base}-1", f"{base}-2", f"{base}-3", f"{base}-4"}, + ) + self.assertEqual(self.content_at(f"{base}-1"), chunk_bytes(CHUNK_SIZE, b"B")) + self.assertEqual(self.content_at(f"{base}-2"), chunk_bytes(CHUNK_SIZE, b"C")) + self.assertEqual(self.content_at(f"{base}-3"), chunk_bytes(CHUNK_SIZE, b"D")) + self.assertEqual(self.content_at(f"{base}-4"), b"E" * 100) + + def test_renumbers_zero_based_chain(self): + """A chain numbered from -0 shifts *up*, where renaming in ascending + order would land on a slot still occupied and trip UNIQUE(path) - + rolling back every other repair in the same pass. It is as broken as a + -2 chain: WebServer.kt probes "-1", finds it, and serves the chain with + "-0" silently dropped.""" + base = "a/devsite/media/zero-based.gif" + self.insert(base, chunk_bytes(CHUNK_SIZE, b"A")) + self.insert(f"{base}-0", chunk_bytes(CHUNK_SIZE, b"B")) + self.insert(f"{base}-1", chunk_bytes(CHUNK_SIZE, b"C")) + self.insert(f"{base}-2", b"D" * 100) + self.conn.commit() + + stats = repair(self.conn) + self.conn.commit() + + self.assertEqual(stats["chains_renumbered"], 1) + self.assertEqual(stats["chains_gapped"], 0) + self.assertEqual(self.all_paths(), {base, f"{base}-1", f"{base}-2", f"{base}-3"}) + # order preserved: -0 -> -1, -1 -> -2, -2 -> -3 + self.assertEqual(self.content_at(f"{base}-1"), chunk_bytes(CHUNK_SIZE, b"B")) + self.assertEqual(self.content_at(f"{base}-2"), chunk_bytes(CHUNK_SIZE, b"C")) + self.assertEqual(self.content_at(f"{base}-3"), b"D" * 100) + + def test_zero_based_chain_does_not_block_other_repairs(self): + """One chain tripping UNIQUE(path) used to roll back the whole run.""" + zero_based = "a/devsite/media/zero.gif" + self.insert(zero_based, chunk_bytes(CHUNK_SIZE, b"A")) + self.insert(f"{zero_based}-0", b"B" * 100) + two_based = "a/devsite/media/two.gif" + self.insert(two_based, chunk_bytes(CHUNK_SIZE, b"C")) + self.insert(f"{two_based}-2", b"D" * 100) + self.conn.commit() + + stats = repair(self.conn) + self.conn.commit() + + self.assertEqual(stats["chains_renumbered"], 2) + self.assertEqual(self.all_paths(), + {zero_based, f"{zero_based}-1", two_based, f"{two_based}-1"}) + + def test_single_orphaned_continuation(self): + base = "j/html/api/index-all.html" + self.insert(base, chunk_bytes(CHUNK_SIZE, b"A")) + self.insert(f"{base}-2", b"tail" * 10) + self.conn.commit() + + stats = repair(self.conn) + self.conn.commit() + + self.assertEqual(stats["chains_renumbered"], 1) + self.assertEqual(stats["fragments_moved"], 1) + self.assertEqual(self.all_paths(), {base, f"{base}-1"}) + self.assertEqual(self.content_at(f"{base}-1"), b"tail" * 10) + + def test_correctly_numbered_chain_untouched(self): + base = "k/html/already-fine.html" + self.insert(base, chunk_bytes(CHUNK_SIZE, b"A")) + self.insert(f"{base}-1", chunk_bytes(CHUNK_SIZE, b"B")) + self.insert(f"{base}-2", b"tail") + self.conn.commit() + + stats = repair(self.conn) + self.conn.commit() + + self.assertEqual(stats["chains_renumbered"], 0) + self.assertEqual(stats["fragments_moved"], 0) + self.assertEqual(self.all_paths(), {base, f"{base}-1", f"{base}-2"}) + + def test_idempotent_second_run(self): + base = "a/devsite/media/size-range.gif" + self.insert(base, chunk_bytes(CHUNK_SIZE, b"A")) + self.insert(f"{base}-2", b"tail") + self.conn.commit() + + repair(self.conn) + self.conn.commit() + stats = repair(self.conn) + self.conn.commit() + + self.assertEqual(stats["chains_renumbered"], 0) + self.assertEqual(stats["fragments_moved"], 0) + + def test_exact_size_file_with_no_continuation_left_alone(self): + path = "k/html/exactly-one-mb.bin" + self.insert(path, chunk_bytes(CHUNK_SIZE, b"A")) + self.conn.commit() + + stats = repair(self.conn) + self.conn.commit() + + self.assertEqual(stats["chains_renumbered"], 0) + self.assertEqual(stats["chains_gapped"], 0) + self.assertEqual(self.all_paths(), {path}) + + def test_chain_with_real_gap_reported_and_left_untouched(self): + base = "k/html/actually-missing-a-chunk.html" + self.insert(base, chunk_bytes(CHUNK_SIZE, b"A")) + self.insert(f"{base}-2", chunk_bytes(CHUNK_SIZE, b"B")) + self.insert(f"{base}-4", b"tail") # -3 is genuinely missing + self.conn.commit() + + stats = repair(self.conn) + self.conn.commit() + + self.assertEqual(stats["chains_renumbered"], 0) + self.assertEqual(stats["chains_gapped"], 1) + self.assertEqual(self.all_paths(), {base, f"{base}-2", f"{base}-4"}) + + +if __name__ == "__main__": + unittest.main() diff --git a/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/tests/conftest.py b/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/tests/conftest.py new file mode 100644 index 000000000..ece8e0c12 --- /dev/null +++ b/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/tests/conftest.py @@ -0,0 +1,4 @@ +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) diff --git a/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/tests/test_find_missing_assets.py b/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/tests/test_find_missing_assets.py new file mode 100644 index 000000000..33c79c321 --- /dev/null +++ b/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/tests/test_find_missing_assets.py @@ -0,0 +1,73 @@ +"""Regression test for find_missing_assets.py's exit-code behavior (PR #24 +review). Runs the script as a subprocess against the real md_to_json.py +(merged from ADFA-5039); a non-UTF-8 .md file gives convert_file a genuine +reason to raise, matching the pattern md_to_json.py's own test suite uses +for its equivalent main()-exit-code tests. +""" +import subprocess +import sys +from pathlib import Path + +import find_missing_assets as fma + + +def _write_minimal_docs_root(tmp_path, *, with_failure=False): + docs_root = tmp_path / "docs" + (docs_root / "topics").mkdir(parents=True) + (docs_root / "topics" / "good.md").write_text("# Good\n\nHello.\n", encoding="utf-8") + if with_failure: + # Not valid UTF-8 - convert_file's read_text(encoding="utf-8") raises. + (docs_root / "topics" / "bad.md").write_bytes(b"\xff\xfe not utf-8") + return docs_root + + +def _run(*args): + script = Path(__file__).resolve().parent.parent / "find_missing_assets.py" + return subprocess.run([sys.executable, str(script), *map(str, args)], capture_output=True, text=True) + + +def test_exits_zero_and_reports_zero_failures_when_nothing_fails(tmp_path): + docs_root = _write_minimal_docs_root(tmp_path) + report = tmp_path / "report.md" + result = _run(docs_root, report) + assert result.returncode == 0 + assert "0 file(s) failed to scan" in report.read_text(encoding="utf-8") + + +def test_exits_nonzero_when_a_file_fails_to_scan(tmp_path): + """A per-file scan failure used to be printed to stderr and otherwise + ignored - the report still claimed a clean summary and the process + still exited 0, so a totally broken corpus was indistinguishable from a + clean one (this is the pre-flight gate run before populate_db.py). The + bad file is scanned by two independent passes (the main conversion loop + and find_include_warnings' own scan), so it counts twice.""" + docs_root = _write_minimal_docs_root(tmp_path, with_failure=True) + report = tmp_path / "report.md" + result = _run(docs_root, report) + assert result.returncode == 1 + text = report.read_text(encoding="utf-8") + assert "2 file(s) failed to scan" in text + assert "incomplete" in text.lower() + + +def test_allow_failures_exits_zero_despite_failure(tmp_path): + docs_root = _write_minimal_docs_root(tmp_path, with_failure=True) + report = tmp_path / "report.md" + result = _run(docs_root, report, "--allow-failures") + assert result.returncode == 0 + + +def test_find_include_warnings_reports_failure_instead_of_raising(tmp_path): + """find_include_warnings had its own unguarded read_text(encoding="utf-8") + outside the main loop's try/except - a non-UTF-8 file crashed the whole + process with an uncaught traceback, bypassing --allow-failures entirely + rather than being counted as a scan failure like every other file-read + in this script.""" + topics_dir = tmp_path / "topics" + topics_dir.mkdir() + (topics_dir / "good.md").write_text("no includes here\n", encoding="utf-8") + (topics_dir / "bad.md").write_bytes(b"\xff\xfe not utf-8") + + warnings, failed = fma.find_include_warnings(topics_dir) + assert warnings == [] + assert failed == 1 diff --git a/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/tests/test_insert_optimized_media.py b/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/tests/test_insert_optimized_media.py new file mode 100644 index 000000000..71a8e657d --- /dev/null +++ b/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/tests/test_insert_optimized_media.py @@ -0,0 +1,150 @@ +"""Regression tests for insert_optimized_media.py's destructive paths. + +Every test here covers a way this script could delete a row it shouldn't - +the LIKE-wildcard over-match in delete_content, and the "no pages to check +against" case that would otherwise wipe the whole image corpus. +""" +import shutil +import sqlite3 + +import pytest + +from insert_optimized_media import ( + IMAGES_URL_PREFIX, + collect_referenced_media, + delete_content, + delete_unreferenced_media, +) +from optimize_media import Logger +from populate_db import DictionaryCompressor + +SCHEMA = """ +CREATE TABLE Languages (id INTEGER PRIMARY KEY AUTOINCREMENT, value TEXT NOT NULL UNIQUE); +CREATE TABLE ContentTypes (id INTEGER PRIMARY KEY AUTOINCREMENT, value TEXT NOT NULL UNIQUE, + compression TEXT NOT NULL); +CREATE TABLE Content ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + path TEXT NOT NULL, + languageID INTEGER NOT NULL, + content BLOB NOT NULL, + contentTypeID INTEGER NOT NULL, + templateId INTEGER NOT NULL DEFAULT 0, + UNIQUE(path) +); +""" + +PAGE_TYPE_ID = 12 +# Content is dictionary-compressed from schema 2.0.0 on (ADFA-5153), and +# insert_optimized_media reads pages back through that dictionary, so these +# fixtures have to speak it too. +DICTIONARY = bytes(range(256)) * 64 +needs_brotli_cli = pytest.mark.skipif(shutil.which("brotli") is None, reason="brotli CLI not installed") + +pytestmark = needs_brotli_cli + + +@pytest.fixture +def compressor(): + instance = DictionaryCompressor(DICTIONARY) + yield instance + instance.close() + + +@pytest.fixture +def conn(): + connection = sqlite3.connect(":memory:") + connection.executescript(SCHEMA) + connection.execute("INSERT INTO Languages (id, value) VALUES (1, 'en-US')") + connection.execute("INSERT INTO ContentTypes (id, value, compression) VALUES (?, 'text/html', 'brotli')", + (PAGE_TYPE_ID,)) + yield connection + connection.close() + + +def add_row(conn, path, blob=b"x", template_id=0): + conn.execute( + "INSERT INTO Content (path, languageID, content, contentTypeID, templateId) VALUES (?, 1, ?, ?, ?)", + (path, blob, PAGE_TYPE_ID, template_id), + ) + + +def paths(conn): + return {row[0] for row in conn.execute("SELECT path FROM Content")} + + +def page_blob(compressor, *image_names): + """A stored page blob referencing each image the way md_to_json bakes it + in: an HTML src="..." attribute inside JSON, so the filename is followed + by an escaped quote.""" + srcs = "".join(f'' for name in image_names) + return compressor.compress(f'{{"blocks":[{{"html":"{srcs}"}}]}}'.encode("utf-8")) + + +class TestDeleteContent: + def test_removes_the_row_and_its_chunk_fragments(self, conn): + add_row(conn, "k/html/images/big.png") + add_row(conn, "k/html/images/big.png-1") + add_row(conn, "k/html/images/big.png-2") + add_row(conn, "k/html/images/other.png") + + delete_content(conn, "k/html/images/big.png") + + assert paths(conn) == {"k/html/images/other.png"} + + def test_underscore_is_not_treated_as_a_wildcard(self, conn): + # "_" in a LIKE pattern matches any single character, so an unescaped + # "k/html/_nav.html-%" would also match "k/html/Xnav.html-1" and take + # an unrelated page's chunk fragment with it. + add_row(conn, "k/html/_nav.html") + add_row(conn, "k/html/_nav.html-1") + add_row(conn, "k/html/Xnav.html") + add_row(conn, "k/html/Xnav.html-1") + + delete_content(conn, "k/html/_nav.html") + + assert paths(conn) == {"k/html/Xnav.html", "k/html/Xnav.html-1"} + + def test_percent_is_not_treated_as_a_wildcard(self, conn): + add_row(conn, "k/html/images/100%.png") + add_row(conn, "k/html/images/100%.png-1") + add_row(conn, "k/html/images/100-other.png-1") + + delete_content(conn, "k/html/images/100%.png") + + assert paths(conn) == {"k/html/images/100-other.png-1"} + + +class TestDeleteUnreferencedMedia: + def test_removes_only_images_no_page_references(self, conn, compressor): + add_row(conn, "k/html/page.html", page_blob(compressor, "kept.png"), template_id=2) + add_row(conn, "k/html/images/kept.png") + add_row(conn, "k/html/images/orphan.png") + add_row(conn, "k/html/images/orphan.png-1") + + removed = delete_unreferenced_media(conn, PAGE_TYPE_ID, Logger(None), compressor) + + assert removed == 1 + assert paths(conn) == {"k/html/page.html", "k/html/images/kept.png"} + + def test_refuses_to_run_when_no_page_references_any_image(self, conn, compressor): + # populate_db.py hasn't written its pages yet (or was skipped): the + # reference scan comes back empty and every stored image looks like + # garbage. Deleting the whole corpus is never what was meant. + add_row(conn, "k/html/images/a.png") + add_row(conn, "k/html/images/b.png") + + with pytest.raises(RuntimeError, match="no page references any image"): + delete_unreferenced_media(conn, PAGE_TYPE_ID, Logger(None), compressor) + + assert paths(conn) == {"k/html/images/a.png", "k/html/images/b.png"} + + def test_empty_database_is_not_an_error(self, conn, compressor): + assert delete_unreferenced_media(conn, PAGE_TYPE_ID, Logger(None), compressor) == 0 + + def test_untemplated_rows_are_not_scanned_for_references(self, conn, compressor): + # templateId 0 marks a raw asset, not a page; only page/nav rows carry + # the JSON that image references live in. + add_row(conn, "k/html/page.html", page_blob(compressor, "kept.png"), template_id=2) + add_row(conn, "k/html/images/kept.png") + + assert collect_referenced_media(conn, PAGE_TYPE_ID, compressor) == {"kept.png"} diff --git a/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/tests/test_md_to_json.py b/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/tests/test_md_to_json.py new file mode 100644 index 000000000..9eec163fd --- /dev/null +++ b/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/tests/test_md_to_json.py @@ -0,0 +1,954 @@ +"""Regression tests for md_to_json.py, one per bug found in PR #23 review. + +Each test's docstring names the finding it guards; several reuse the +reviewers' own repro snippets verbatim so a future regression reproduces the +exact case that was originally reported. +""" +import json +import subprocess +import sys +from pathlib import Path + +import pytest + +import md_to_json as m + + +class FakeToken: + """Minimal stand-in for markdown_it.token.Token - just the attributes + fold_image_attrs/extract_trailing_attrs actually touch.""" + + def __init__(self, type_, content="", children=None): + self.type = type_ + self.content = content + self.children = children + self.attrs = {} + + +def make_converter(**kwargs): + return m.Converter(m.make_markdown_it(), {}, **kwargs) + + +# --- TAG_RE: must not match as tag "tab" ----------------------- + +def test_tag_re_does_not_match_table(): + """TAG_RE previously had no word boundary after the tag-name + alternation, so
/
matched as tag "tab" with attrs "le", + silently eating every raw HTML table in the corpus.""" + assert m.TAG_RE.match("") is None + assert m.TAG_RE.match("
") is None + assert m.TAG_RE.match("") is None + + +def test_tag_re_still_matches_real_container_tags(): + assert m.TAG_RE.match('') is not None + assert m.TAG_RE.match('') is not None + assert m.TAG_RE.match("") is not None + + +# --- TAG_RE / html_block: self-closing tags must not stay open ---------- + +def test_tag_re_captures_self_closing_marker_separately(): + """The attrs group used to be greedy, so it swallowed a self-closing + tag's trailing "/" before the optional "/?" at the end ever got a + chance to match it - "" came out with an empty closing + group, i.e. indistinguishable from a plain opener.""" + closing, tag, attrs, self_closing = m.TAG_RE.match('').groups() + assert (closing, tag, self_closing) == ("", "tab", "/") + assert attrs.strip() == 'title="A"' + + closing, tag, attrs, self_closing = m.TAG_RE.match('').groups() + assert self_closing == "" + + +def test_html_block_self_closing_tag_emits_open_and_close_markers(): + """convert_node's html_block dispatch must turn a self-closing tag into + an immediate open/close pair rather than a bare opener - otherwise the + container never closes and silently nests the rest of the page.""" + node = m.Node(FakeToken("html_block", content='')) + conv = make_converter() + result = conv.convert_node(node) + assert [(b["type"], b["closing"], b["tag"]) for b in result] == [ + ("tag_marker", False, "tab"), + ("tag_marker", True, "tab"), + ] + + +def test_self_closing_tag_does_not_swallow_trailing_content(): + """End-to-end repro: a self-closing immediately followed by a + paragraph must not leave that paragraph nested inside the tab.""" + node = m.Node(FakeToken("html_block", content='')) + conv = make_converter() + conv.current_source = "test.md" + markers = conv.convert_node(node) + blocks = markers + [{"type": "paragraph", "html": "After."}] + result = conv.group_containers(blocks) + assert [b["html"] for b in result if b.get("type") == "paragraph"] == ["After."] + assert conv.warnings == [] + + +# --- convert_node: indented code_block must render as code, not html ---- + +def test_indented_code_block_renders_as_code_not_html(): + """convert_node only handled markdown-it's "fence" (```-delimited) + token; an indented (4-space) code block produces a different token + type, "code_block", which fell through to the generic fallback and + came out as an unescaped "html" block instead - raw "<"/"&" in the + code would be interpreted as markup rather than shown as text, and the + block lost its code typing entirely.""" + md = m.make_markdown_it() + conv = m.Converter(md, {}) + tokens = md.parse(' List x = listOf("a", "b");\n') + tree = m.build_tree(tokens) + block = conv.convert_node(tree[0]) + assert block["type"] == "code" + assert block["lang"] is None + assert "List" in block["code"] + + +# --- convert_node: html_block preserves blank lines in a raw run ------- + +def test_html_block_preserves_blank_lines_within_a_raw_run(): + """CommonMark's html_block type 1 (
/