diff --git a/.github/workflows/build-kotlin-docs.yaml b/.github/workflows/build-kotlin-docs.yaml new file mode 100644 index 000000000..c63415568 --- /dev/null +++ b/.github/workflows/build-kotlin-docs.yaml @@ -0,0 +1,381 @@ +name: Build Kotlin Docs + +# CI counterpart of ProcessDocs/ProcessKotlinDocs/run_e2e_pipeline_test.sh - +# same five steps (find_missing_assets -> populate_db -> insert_optimized_media +# -> build-stdlib-json-docs -> sync_kdoc_json_to_db), same ADFA-4737 blacklist, +# but sourcing its inputs from fresh git checkouts instead of a developer's +# local machine, and reading/writing the real database on Google Drive +# (GOOGLE_DRIVE_FILE_ID) instead of a local SOURCE_DB copy. +# +# KNOWN LIMITATION: populate_db.py requires Writerside's own image export +# ("webHelpImages.zip"), which JetBrains only produces via IntelliJ IDEA's +# Writerside plugin build/export action - there is no headless/CLI way to +# generate it (see ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/README.md, +# "Inputs you need before starting"). So this workflow downloads it from +# Google Drive rather than generating it itself; someone has to run that IDE +# export, upload the zip to Drive, and supply its file ID (see +# images_zip_file_id / GOOGLE_DRIVE_IMAGES_ZIP_FILE_ID below) before +# triggering a run that touches the website docs. Use skip_website_docs to +# bypass this entirely and only refresh the kotlin-stdlib/-reflect/-test +# JSON content. +# +# Required secrets (already configured - see docdb-regression-test.yaml for +# their other use in this repo): +# GCP_WIF_PROVIDER - Workload Identity Federation provider name +# GCP_WIF_SERVICE_ACCOUNT - Service account email for WIF (needs read +# access to the images-zip file below, and +# write access - not just view - to the +# database file, since this workflow +# overwrites it) +# GOOGLE_DRIVE_FILE_ID - File ID of the production documentation.db +# (stored on Drive as a zip) +# +# Optional secret (falls back to the images_zip_file_id input if unset; see +# also the hard-coded TEST_*_FILE_ID overrides below for one-off testing): +# GOOGLE_DRIVE_IMAGES_ZIP_FILE_ID - File ID of Writerside's webHelpImages.zip export +# +# Optional secret (Slack notifications are skipped with a warning if unset): +# SLACK_WEBHOOK_URL - Incoming Webhook URL for the "Notify Slack" steps +# below ("Grabbing baton" on start, "...Dropping +# baton" on finish - org shorthand for lock +# acquire/release, since this workflow mutates a +# single shared Drive file). + +permissions: + contents: read + id-token: write + +# This workflow overwrites a single shared Drive file - never let two runs +# race to upload against each other. +concurrency: + group: build-kotlin-docs + cancel-in-progress: false + +on: + workflow_dispatch: + inputs: + kotlin_web_site_ref: + description: >- + Branch/tag/commit of JetBrains/kotlin-web-site to check out for the + "docs" tree (topics/, images/, kr.tree, v.list). Leave empty to use + the repo's default branch. + required: false + default: '' + kotlin_ref: + description: >- + Branch/tag/commit of JetBrains/kotlin to check out for the + kotlin-stdlib-docs build. Leave empty to use the repo's default + branch. Pin this to a real release tag for a reproducible build. + required: false + default: '' + images_zip_file_id: + description: >- + Google Drive file ID for Writerside's webHelpImages.zip export + matching kotlin_web_site_ref (see KNOWN LIMITATION above). Falls + back to the GOOGLE_DRIVE_IMAGES_ZIP_FILE_ID secret if left empty. + Ignored if skip_website_docs is true. + required: false + default: '' + skip_website_docs: + description: 'Skip the kotlin-web-site steps and only refresh kotlin-stdlib/-reflect/-test JSON content.' + required: false + default: false + type: boolean + dry_run: + description: >- + If true, build and verify everything but do NOT upload the result + back to Google Drive - the production database is left untouched. + Set to false only once you trust a given ref/URL combination (see + this workflow's testing notes). + required: false + default: true + type: boolean + +jobs: + build-kotlin-docs: + runs-on: ubuntu-latest + timeout-minutes: 180 + env: + KOTLIN_WEB_SITE_REF: ${{ inputs.kotlin_web_site_ref }} + KOTLIN_REF: ${{ inputs.kotlin_ref }} + DB_FILE_ID_SECRET: ${{ secrets.GOOGLE_DRIVE_FILE_ID }} + IMAGES_ZIP_FILE_ID_INPUT: ${{ inputs.images_zip_file_id }} + IMAGES_ZIP_FILE_ID_SECRET: ${{ secrets.GOOGLE_DRIVE_IMAGES_ZIP_FILE_ID }} + # --- Hard-coded overrides for one-off manual testing ----------------- + # Fill in either of these with a literal Google Drive file ID to + # bypass the secret/input resolution above for a quick, repeatable + # test run (e.g. against scratch copies of the database/images zip on + # Drive). Leave both empty ('') for normal operation. + TEST_DB_FILE_ID: '' + TEST_IMAGES_ZIP_FILE_ID: '' + steps: + - name: Checkout OfflineDocumentationTools + uses: actions/checkout@v4 + + - name: Resolve Google Drive file IDs + run: | + DB_FILE_ID="${TEST_DB_FILE_ID:-$DB_FILE_ID_SECRET}" + IMG_FILE_ID="${TEST_IMAGES_ZIP_FILE_ID:-${IMAGES_ZIP_FILE_ID_INPUT:-$IMAGES_ZIP_FILE_ID_SECRET}}" + if [ -z "$DB_FILE_ID" ]; then + echo "Error: no database file ID resolved - set the GOOGLE_DRIVE_FILE_ID secret, or TEST_DB_FILE_ID above for a test run" >&2 + exit 1 + fi + echo "Resolved DB_FILE_ID: ${DB_FILE_ID:+(set)}" + echo "Resolved IMG_FILE_ID: ${IMG_FILE_ID:+(set)}" + echo "DB_FILE_ID=$DB_FILE_ID" >> "$GITHUB_ENV" + echo "IMG_FILE_ID=$IMG_FILE_ID" >> "$GITHUB_ENV" + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Set up JDK (for the kdoc-to-json / kotlin-stdlib-docs Gradle builds) + uses: actions/setup-java@v4 + with: + distribution: temurin + # kdoc-to-json's own Gradle wrapper is pinned to Gradle 9.1.0, which + # needs JDK 17+. Bump this if the kotlin checkout's own wrapper + # (invoked by build-stdlib-json-docs.sh against kotlin-stdlib-docs) + # turns out to need something newer - verify on first real run. + java-version: '17' + + - name: Install system dependencies + run: | + sudo apt-get update -y + # brotli: the CLI, not the Python package. populate_db.py's + # DictionaryCompressor and sync_kdoc_json_to_db.py shell out to it because + # no Python binding exposes a custom dictionary (ADFA-5153). + sudo apt-get install -y pngquant unzip zip sqlite3 brotli + + - name: Install Python dependencies + run: | + pip install -r requirements.txt + # markdown-it-py/scour/cairosvg: ProcessKotlinWebsiteJSON's own + # requirements (see its README), not in the root requirements.txt. + # google-api-python-client & friends: Drive download/upload, same + # libraries check-tools/download_database.py already depends on. + pip install markdown-it-py scour cairosvg \ + google-api-python-client google-auth-httplib2 google-auth-oauthlib + + - name: Authenticate to Google Cloud using Workload Identity Federation + uses: google-github-actions/auth@v2 + with: + workload_identity_provider: ${{ secrets.GCP_WIF_PROVIDER }} + service_account: ${{ secrets.GCP_WIF_SERVICE_ACCOUNT }} + access_token_scopes: | + https://www.googleapis.com/auth/drive.file + + - name: Download current documentation.db from Google Drive + run: | + python3 check-tools/download_database.py "$DB_FILE_ID" documentation.zip + unzip -o documentation.zip + if [ ! -f documentation.db ]; then + found="$(find . -maxdepth 2 -name documentation.db | head -n1)" + [ -n "$found" ] && mv "$found" documentation.db + fi + test -f documentation.db + sqlite3 documentation.db "SELECT 1;" > /dev/null + rm -f documentation.zip + echo "DB_SIZE=$(stat -c%s documentation.db 2>/dev/null || stat -f%z documentation.db)" >> "$GITHUB_ENV" + + - name: 'Notify Slack: build started' + env: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} + run: | + if [ -z "$SLACK_WEBHOOK_URL" ]; then + echo "SLACK_WEBHOOK_URL not set - skipping Slack notification" >&2 + else + curl -sS -X POST -H 'Content-type: application/json' \ + --data '{"text": "Grabbing baton"}' \ + "$SLACK_WEBHOOK_URL" || echo "warning: Slack notification failed" >&2 + fi + + - name: Clone kotlin-web-site + if: ${{ !inputs.skip_website_docs }} + run: | + ARGS=(--depth 1) + [ -n "$KOTLIN_WEB_SITE_REF" ] && ARGS+=(--branch "$KOTLIN_WEB_SITE_REF") + git clone "${ARGS[@]}" https://github.com/JetBrains/kotlin-web-site.git kotlin-web-site + + - name: Download Writerside image export from Google Drive + if: ${{ !inputs.skip_website_docs }} + run: | + if [ -z "$IMG_FILE_ID" ]; then + echo "Error: no images-zip file ID resolved - set images_zip_file_id, the GOOGLE_DRIVE_IMAGES_ZIP_FILE_ID secret, or TEST_IMAGES_ZIP_FILE_ID above (see KNOWN LIMITATION in this workflow's header comment). Required unless skip_website_docs is true." >&2 + exit 1 + fi + # download_database.py is a generic Drive-file-by-ID downloader + # despite its name - reused here rather than duplicating the + # WIF/Drive-API download logic for a second file type. + python3 check-tools/download_database.py "$IMG_FILE_ID" webHelpImages.zip + + - name: 'Step 1/5: find_missing_assets.py (source QA report)' + if: ${{ !inputs.skip_website_docs }} + run: | + python3 ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/find_missing_assets.py \ + kotlin-web-site/docs missing-assets-report.md + + - name: Upload missing-assets report + if: ${{ !inputs.skip_website_docs }} + uses: actions/upload-artifact@v4 + with: + name: missing-assets-report + path: missing-assets-report.md + + - name: 'Step 2/5: populate_db.py (convert docs, prune blacklist, insert into db)' + if: ${{ !inputs.skip_website_docs }} + run: | + # Same three blacklist entries as run_e2e_pipeline_test.sh + # (ADFA-4737) - re-derive these from kotlin-web-site/docs/kr.tree + # if its nav structure has changed since this was written. + python3 ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/populate_db.py \ + kotlin-web-site/docs \ + ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/config.json \ + webHelpImages.zip \ + documentation.db \ + --blacklisted-element-titles \ + 'Development\/Web development' \ + 'Interoperability\/Swift/Objective-C and C interop' \ + 'Interoperability\/JavaScript interop' + + - name: 'Step 3/5: insert_optimized_media.py (re-optimize + reinsert images)' + if: ${{ !inputs.skip_website_docs }} + run: | + # --webp requires an "image/webp" ContentTypes row, which this + # database doesn't ship with by default (idempotent). + sqlite3 documentation.db \ + "INSERT OR IGNORE INTO ContentTypes (value, compression) VALUES ('image/webp', 'brotli');" + mkdir -p media + unzip -q webHelpImages.zip -d media + python3 ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/insert_optimized_media.py \ + media documentation.db \ + --jpeg-quality 85 --webp --webp-quality 90 --verbose + + - name: Clone kotlin (for kotlin-stdlib-docs) + run: | + ARGS=(--depth 1) + [ -n "$KOTLIN_REF" ] && ARGS+=(--branch "$KOTLIN_REF") + git clone "${ARGS[@]}" https://github.com/JetBrains/kotlin.git kotlin-repo + + - name: 'Step 4/5: build-stdlib-json-docs.sh (fresh plugin build -> kotlin-stdlib/-reflect/-test JSON)' + id: stdlib_docs + run: | + OUTPUT="$(Dokka-plugin-kdoc2json/scripts/kotlin/build-stdlib-json-docs.sh kotlin-repo stdlib-json-build)" + echo "Generated JSON docs at $OUTPUT" + echo "all_libs_dir=$OUTPUT" >> "$GITHUB_OUTPUT" + + - name: 'Step 5/5: sync_kdoc_json_to_db.py (overwrite kotlin-stdlib/-reflect/-test content)' + run: | + python3 scripts/sync_kotlin_stdlib_docs/sync_kdoc_json_to_db.py \ + "${{ steps.stdlib_docs.outputs.all_libs_dir }}" --db documentation.db + + - name: Summary + run: | + python3 - documentation.db <<'PYEOF' + import sqlite3 + import sys + + conn = sqlite3.connect(sys.argv[1]) + + def count(where, params=()): + return conn.execute(f"SELECT count(*) FROM Content WHERE {where}", params).fetchone()[0] + + print(f"Database: {sys.argv[1]}") + print(f" k/html/* rows: {count('path LIKE ?', ('k/html/%',))}") + print(f" k/html/images/* rows: {count('path LIKE ?', ('k/html/images/%',))}") + print(f" k/html/images/*.webp rows: {count('path LIKE ?', ('k/html/images/%.webp%',))}") + print(f" k/kotlin-stdlib/* rows: {count('path LIKE ? OR path = ?', ('k/kotlin-stdlib/%', 'k/kotlin-stdlib'))}") + print(f" k/kotlin-reflect/* rows: {count('path LIKE ? OR path = ?', ('k/kotlin-reflect/%', 'k/kotlin-reflect'))}") + print(f" k/kotlin-test/* rows: {count('path LIKE ? OR path = ?', ('k/kotlin-test/%', 'k/kotlin-test'))}") + conn.close() + PYEOF + + - name: Blacklist pruning verification + if: ${{ !inputs.skip_website_docs }} + run: | + python3 - ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON kotlin-web-site/docs documentation.db \ + 'Development\/Web development' \ + 'Interoperability\/Swift/Objective-C and C interop' \ + 'Interoperability\/JavaScript interop' <<'PYEOF' + import sqlite3 + import sys + import xml.etree.ElementTree as ET + from pathlib import Path + + process_dir, docs_root, db_path, *blacklist_raw = sys.argv[1:] + sys.path.insert(0, process_dir) + import populate_db # noqa: E402 + + root = ET.parse(Path(docs_root) / "kr.tree").getroot() + blacklisted_paths = {populate_db.parse_blacklist_path(raw) for raw in blacklist_raw} + blacklisted_stems, unmatched_paths = populate_db.prune_blacklisted_elements(root, blacklisted_paths) + + conn = sqlite3.connect(db_path) + leftover = [] + for stem in sorted(blacklisted_stems): + path = f"k/html/{stem}.html" + if conn.execute("SELECT 1 FROM Content WHERE path = ?", (path,)).fetchone(): + leftover.append(path) + conn.close() + + print(f"Blacklisted toc-element path(s) checked: {len(blacklisted_paths)}") + for path in sorted(blacklisted_paths): + status = "unmatched (no such element in kr.tree)" if path in unmatched_paths else "matched" + print(f" {' > '.join(path)}: {status}") + print(f"Topic page(s) expected removed: {len(blacklisted_stems)}") + + if unmatched_paths: + print(f"FAIL: {len(unmatched_paths)} blacklist path(s) never matched a .") + sys.exit(1) + if leftover: + print(f"FAIL: {len(leftover)} blacklisted page(s) still present in the database:") + for path in leftover: + print(f" {path}") + sys.exit(1) + + print(f"PASS: all {len(blacklisted_stems)} blacklisted topic page(s) confirmed absent from {db_path}.") + PYEOF + + - name: Upload built database as workflow artifact + uses: actions/upload-artifact@v4 + with: + name: documentation-db-${{ github.run_number }} + path: documentation.db + retention-days: 14 + + - name: Zip updated database for upload + if: ${{ !inputs.dry_run }} + run: zip -j documentation.zip documentation.db + + - name: Upload updated database to Google Drive + if: ${{ !inputs.dry_run }} + run: | + python3 - <<'PYEOF' + import os + from google.auth import default + from googleapiclient.discovery import build + from googleapiclient.http import MediaFileUpload + + file_id = os.environ["DB_FILE_ID"] + credentials, _ = default() + service = build("drive", "v3", credentials=credentials) + media = MediaFileUpload("documentation.zip", mimetype="application/zip", resumable=True) + updated = service.files().update( + fileId=file_id, media_body=media, fields="id, modifiedTime, md5Checksum" + ).execute() + print(f"Uploaded new revision of {file_id}: {updated}") + PYEOF + + - name: 'Notify Slack: build complete' + if: ${{ !inputs.dry_run }} + env: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} + run: | + if [ -z "$SLACK_WEBHOOK_URL" ]; then + echo "SLACK_WEBHOOK_URL not set - skipping Slack notification" >&2 + else + curl -sS -X POST -H 'Content-type: application/json' \ + --data '{"text": "Updated Kotlin documentation. Dropping baton"}' \ + "$SLACK_WEBHOOK_URL" || echo "warning: Slack notification failed" >&2 + fi diff --git a/.github/workflows/docdb-regression-test.yaml b/.github/workflows/docdb-regression-test.yaml index fa5346e04..d9222dfa8 100644 --- a/.github/workflows/docdb-regression-test.yaml +++ b/.github/workflows/docdb-regression-test.yaml @@ -98,7 +98,9 @@ jobs: echo "Extracting database from zip file..." # Install unzip if not available - sudo apt-get update -qq && sudo apt-get install -y unzip + # brotli: docdb_studio reads dictionary-compressed Content rows through the + # CLI (ADFA-5153); the downloaded production database is one of those. + sudo apt-get update -qq && sudo apt-get install -y unzip brotli # Extract the zip file if ! unzip -o documentation.zip; then diff --git a/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/README.md b/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/README.md new file mode 100644 index 000000000..47008e279 --- /dev/null +++ b/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/README.md @@ -0,0 +1,153 @@ +# Process Kotlin Website JSON + +Scripts for converting a `kotlin-web-site/docs` checkout (JetBrains +Writerside-flavored Markdown) into the JSON block schema this project's +templating engine renders, and for loading that JSON, its navigation tree, +and its media straight into a `documentation.db`-schema SQLite database. + +## Scripts + +| Script | Purpose | +|---|---| +| [`md_to_json.py`](md_to_json.py) | Converts every `topics/**/*.md` page into one JSON file (see schema below). Writes `theme.json` and copies `images/` into the output directory. | +| [`build_nav.py`](build_nav.py) | Builds `nav.json`/`nav.html` sidebar navigation from `kr.tree`, resolving each `` against `md_to_json.py`'s output. | +| [`find_missing_assets.py`](find_missing_assets.py) | QA pass: reports cross-page links, images, and `` targets in the source tree that don't resolve to anything. Reuses `md_to_json.py`'s own resolution logic, so it flags exactly what would end up broken on the rendered site. | +| [`populate_db.py`](populate_db.py) | The database path: converts the docs tree the same way `md_to_json.py` does, builds nav the same way `build_nav.py` does, and inserts pages + nav + images + CSS/JS directly into `documentation.db` (replacing everything under `k/html/` and `assets/`). Supports pruning whole `kr.tree` subtrees via `--blacklisted-element-titles`. | +| [`migrate_content_to_dictionary_brotli.py`](migrate_content_to_dictionary_brotli.py) | One-off, resumable: recompresses every `brotli` Content row against the database's shared `CompressionDictionary`, training one first if there is none (ADFA-5153). Covers the rows `populate_db.py` never touches. | +| [`renumber_misnumbered_fragments.py`](renumber_misnumbered_fragments.py) | One-off repair: chunked rows whose continuations start at `-2` (or `-0`) instead of `-1`, which `WebServer.kt` reassembles truncated (ADFA-5171). Moves paths only, never content. | +| [`remint_dictionary.py`](remint_dictionary.py) | One-off, **destructive**: trains a *new* shared dictionary and recompresses every row against it, in one transaction. The only safe way to change a dictionary, since the stored one is otherwise permanent for that database's content. Pair with `verify_remint_dictionary.py` before putting the result in place. | +| [`verify_remint_dictionary.py`](verify_remint_dictionary.py) | Read-only gate for the above: decodes every row out of both databases and requires the plaintexts to match, exiting non-zero otherwise. A mismatched dictionary decodes into wrong bytes without erroring, so this is what makes re-minting safe. | +| [`optimize_media.py`](optimize_media.py) | Standalone media optimizer: downscales/recompresses a directory of images (pngquant, Pillow, Scour/cairosvg for SVG) into a mirrored output directory. | +| [`insert_optimized_media.py`](insert_optimized_media.py) | Runs `optimize_media.py`'s pipeline over a directory of raw media, then replaces the corresponding `k/html/images/*` rows in an existing database, rewriting any page that referenced a renamed file and deleting anything left unreferenced. | + +## Requirements + +- Python 3.10+ +- `pip install markdown-it-py Pillow scour brotli` +- `cairosvg` (only needed if an optimized SVG exceeds `--svg-rasterize-threshold`): `pip install cairosvg` +- `pngquant` on `PATH` (e.g. `apt install pngquant`) — required by `optimize_media.py`/`insert_optimized_media.py`, and by `populate_db.py` for the images it inserts directly from the Writerside export. +- `brotli` on `PATH` (e.g. `apt install brotli`) — the **command-line tool**, which is a different artifact from the `brotli` Python package listed above. `populate_db.py`, `insert_optimized_media.py`, `migrate_content_to_dictionary_brotli.py` and `remint_dictionary.py` compress against the shared dictionary in `CompressionDictionary` (ADFA-5153), and no Python binding exposes a custom dictionary, so they shell out to this binary. Without it they fail at startup. + +`populate_db.py` also expects, relative to its own location, and already +included in this directory: + +- `templates/page.peb`, `templates/nav.peb` — Pebble templates upserted into the `Templates` table. +- `assets/docs.css`, `assets/tabs.js`, `assets/sidebar.js` — static assets inserted at `assets/`. + +## Inputs you need before starting + +- A checkout of `kotlin-web-site/docs` (the `` argument below) — contains `topics/`, `images/`, `v.list`, and `kr.tree`. +- A config JSON with theming colors, e.g.: + ```json + {"broken-ext-link-color": "#cc0000", "menu-no-link-color": "#999999"} + ``` +- Writerside's own image export zip (e.g. `webHelpImages.zip`, found next to `kr.tree`) if you're using `populate_db.py`. + +## Workflow: generate JSON + nav for a static/templated preview + +Use this to produce standalone JSON pages and nav data (not the database) +for local inspection or a different renderer. + +```bash +# 1. Convert every topic .md into JSON, one file per page +python3 md_to_json.py config.json + +# 2. Build the sidebar nav from kr.tree against that JSON output +python3 build_nav.py + +# 3. (optional) Check for broken links/images/includes in the source tree +python3 find_missing_assets.py missing-assets-report.md +``` + +`` ends up containing: +- `topics/**/*.json` — one page per source `.md` file (schema below) +- `theme.json` — the two theming colors, carried from `config.json` +- `images/` — copied straight from `/images/` +- `nav.json` / `nav.html` — sidebar tree and a pre-rendered static copy + +### Page JSON schema + +```json +{ + "id": "enum-classes", + "sourceFile": "topics/enum-classes.md", + "title": "Enum classes", + "blocks": [ { "type": "heading", "level": 2, "id": "...", "html": "..." }, "..." ] +} +``` + +Block types: `heading`, `paragraph`, `code`, `blockquote`, `list`, `table`, +`image`, `hr`, `tabs`, `note`/`tip`/`warning`, `html` (raw passthrough). See +the module docstring in [`md_to_json.py`](md_to_json.py) for full shapes and +known limitations (nested tabs, `` resolution, variable +substitution). + +## Workflow: generate + insert directly into the documentation database + +This is the path that actually populates `documentation.db`. It performs +the same conversion as `md_to_json.py`/`build_nav.py` internally — you don't +run those scripts first. + +```bash +python3 populate_db.py config.json [db-path] +``` + +- `db-path` defaults to `documentation.db` in the current directory, and must already exist with the expected schema (`Languages`, `ContentTypes`, `Templates` tables populated). +- A timestamped backup (`.backup-`) is written before any changes, via SQLite's `VACUUM INTO`. +- Everything under `k/html/` and `assets/` is deleted and re-inserted in a single transaction (rolled back on error), then the database is `VACUUM`ed. + +### Pruning documentation you don't want (ADFA-4737) + +To leave a whole `kr.tree` subtree out of the database entirely — nav +entry, converted pages, and all — pass `--blacklisted-element-titles` with +the full `toc-title` path from a top-level element down to the one you want +to drop. Levels are joined with `\/` (backslash-slash), not a bare `/`, +since a bare `/` commonly appears inside a real title. The example below is +illustrative only — open `/kr.tree` and copy the actual +`toc-title` chain for whatever section you're dropping (e.g. Kotlin/Wasm): + +```bash +python3 populate_db.py config.json documentation.db \ + --blacklisted-element-titles \ + "\/" +``` + +Any other page's in-content link to a pruned topic renders as a styled +"broken" link (via `broken-ext-link-color`) rather than a dead link with no +indication anything changed. Run with `--blacklisted-element-titles` first +against a scratch copy of the database and check the warnings on stderr for +any path that didn't match — that usually means the toc-title or ancestor +chain was copied wrong. + +## Workflow: optimizing and inserting media + +Two options, depending on whether the database already has pages loaded: + +**Standalone optimization only** (no database involved): + +```bash +python3 optimize_media.py [--max-width 500] [--webp] [...] +``` + +**Optimize and update an existing database's images in place:** + +```bash +python3 insert_optimized_media.py [work-dir] [options] +``` + +This re-runs `optimize_media.py`'s pipeline, backs up the database first, +replaces each `k/html/images/` row with the optimized bytes, rewrites +any page/nav reference to a file that got renamed during optimization (e.g. +`--webp` conversion or SVG rasterization), and deletes any image no page +references anymore. Both scripts share the same tuning flags +(`--max-width`, `--jpeg-quality`, `--webp`, `--webp-quality`, +`--pngquant-speed`, `--svg-precision`, `--svg-rasterize-threshold`, +`--verbose`, `--log-file`), settable via `--config ` instead of the +command line — see either script's module docstring for the full option +reference. + +## Recommended order for a full refresh + +1. `find_missing_assets.py` against the new `` — fix anything broken in the source before converting it. +2. `populate_db.py`, with `--blacklisted-element-titles` for anything you don't want documented (e.g. Kotlin/Wasm per ADFA-4737). +3. `insert_optimized_media.py` against the raw media directory, if you want optimized (resized/compressed) images rather than Writerside's own export as-is. diff --git a/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/insert_optimized_media.py b/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/insert_optimized_media.py new file mode 100644 index 000000000..9da4fc8ed --- /dev/null +++ b/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/insert_optimized_media.py @@ -0,0 +1,479 @@ +#!/usr/bin/env python3 +""" +insert_optimized_media.py + +Runs optimize_media.py's image optimizer over a directory of raw media, +then updates an existing documentation.db-schema database (as +populate_db.py produces) with the optimized results, fixing up every page +that referenced a file under its old name. + +What this does, inside a single transaction (rolled back on any error): + 1. Backs up first, same as populate_db.py (VACUUM INTO a + timestamped sibling file). + 2. Optimizes every file under into a staging directory + (--work-dir, or a temporary one removed afterwards) via + optimize_media.py's own pipeline - see its own module docstring for + what "optimized" means (resize, pngquant, Scour, optional WEBP + conversion / SVG rasterization). Aborts before touching the database + if any file fails to optimize. + 3. Replaces every "k/html/images/" Content row with the optimized + bytes, deleting the old row (and any leftover chunked fragments) first + - Content.path is UNIQUE, so a stale row has to go before its + replacement can be inserted. Images are addressed by bare filename + only, matching populate_db.py's own flat "k/html/images/*" convention: + subdirectories are flattened to their basename, and a + basename collision across two different subdirectories is a warning + (keeping the first, sorted, skipping the rest), not an error. + 4. Wherever optimization renamed a file (webp conversion, or an oversized + SVG rasterized to PNG/WEBP), rewrites every "/k/html/images/" + reference still pointing at the old name, in every k/html/*.html page + and the nav row, to the new name - so a page doesn't end up linking to + a filename that no longer exists. + 5. Deletes every remaining "k/html/images/" row (base row and any + chunked fragments) that, after the rename rewriting above, no + k/html/*.html page or the nav row references even once - not just ones + touched by this run's rename_map, but every currently-stored image, + so media that fell out of use in an earlier run (e.g. a topic's .md + was deleted, or an reference was removed by hand) gets cleaned + up too, not just this run's renames. + 6. VACUUMs the database afterwards (outside the transaction - SQLite + refuses to VACUUM inside one), same as populate_db.py. + +Usage: + python3 insert_optimized_media.py [work_dir] [options] + python3 insert_optimized_media.py --config myjob.config + + are optimize_media.py's own tuning flags (--max-width, +--jpeg-quality, --webp, --webp-quality, --pngquant-speed, --svg-precision, +--svg-rasterize-threshold, --verbose, --log-file, --config) - see +optimize_media.py's own docstring for what each one does. media-dir/db-path/ +work-dir can also be set via --config (as "input-dir"/"db-path"/ +"output-dir"), the same as optimize_media.py's own options. + +Note: --webp requires this database's ContentTypes table to already have an +"image/webp" row (checked up front, before any optimization work starts) - +this project's documentation.db doesn't ship with one. +""" +import argparse +import re +import shutil +import sqlite3 +import sys +import tempfile +from pathlib import Path + +from optimize_media import ( + BUILTIN_DEFAULTS, Logger, OPTION_SPECS, add_optimize_arguments, find_pngquant, optimize_directory, + resolve_config, +) +from populate_db import ( + CHUNK_SIZE, DictionaryCompressor, EXTENSION_TO_CONTENT_TYPE, IMAGES_DB_PATH_PREFIX, IMAGES_URL_PREFIX, + LANGUAGE, PAGE_CONTENT_TYPE, backup_database, fragment_chain, get_content_type, get_id, + insert_chunked_content, load_dictionary, +) + +WEBP_CONTENT_TYPE = "image/webp" +# populate_db.py's own EXTENSION_TO_CONTENT_TYPE has no ".webp" entry - its +# image source (a Writerside export) never produces one, but +# optimize_media.py's --webp does, so it's added here rather than touching +# that shared dict. +IMAGE_EXTENSION_TO_CONTENT_TYPE = {**EXTENSION_TO_CONTENT_TYPE, ".webp": WEBP_CONTENT_TYPE} + +# This script's own options, layered on top of optimize_media.py's (db-path +# has no equivalent there) - passed to resolve_config/load_config_file so +# --config can set any of them, the same mechanism optimize_media.py uses +# for its own options. +OWN_OPTION_SPECS = {**OPTION_SPECS, "db-path": ("db_path", Path)} + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("input_dir", type=Path, nargs="?", default=None, metavar="media_dir", + help="Directory of raw media to optimize, recursively (or set input-dir in --config)") + parser.add_argument("db_path", type=Path, nargs="?", default=None, + help="SQLite database to update, e.g. documentation.db (or set db-path in --config)") + parser.add_argument("output_dir", type=Path, nargs="?", default=None, metavar="work_dir", + help="Staging directory for optimized files; default: a temporary directory removed " + "afterwards (or set output-dir in --config)") + add_optimize_arguments(parser) + return parser + + +def delete_content(conn, path: str) -> None: + """Deletes a Content row and any chunked continuation fragments for it + (see insert_chunked_content/CHUNK_SIZE) - safe to call even if nothing + exists yet at that path. Content.path is UNIQUE, so this has to run + before any re-insert at the same path. + + Deletes by exact path rather than by a LIKE pattern. `_` is a single- + character wildcard in LIKE and the `-%` suffix does not restrict the tail to + digits, so "DELETE ... WHERE path LIKE '-%'" also removes rows that + merely resemble a continuation - and those are never re-inserted, so the + loss is permanent. populate_db.fragment_chain does the over-matching query + once and re-checks every candidate's suffix, which is what makes the result + exact.""" + conn.execute("DELETE FROM Content WHERE path = ?", (path,)) + for _number, fragment_path in fragment_chain(conn, path): + conn.execute("DELETE FROM Content WHERE path = ?", (fragment_path,)) + + +def insert_optimized_file(conn, data: bytes, name: str, db_path: str, language_id: int, content_type_cache: dict, + chunked_log: list, compressor: DictionaryCompressor) -> bool: + """Inserts one already-optimized file's bytes as-is. Unlike + populate_db.py's own insert_file, this does not run pngquant itself - + optimize_media.py already did, and running it again here would just + re-quantize an already-quantized image for no benefit. Returns False + (skipping the file, with a warning) for an extension with no known + content type.""" + content_type_value = IMAGE_EXTENSION_TO_CONTENT_TYPE.get(Path(name).suffix.lower()) + if content_type_value is None: + print(f"warning: no known content type for {name!r}; skipping", file=sys.stderr) + return False + if content_type_value not in content_type_cache: + content_type_cache[content_type_value] = get_content_type(conn, content_type_value) + content_type_id, compress = content_type_cache[content_type_value] + + if compress: + data = compressor.compress(data) + delete_content(conn, db_path) + insert_chunked_content(conn, db_path, language_id, content_type_id, 0, data, chunked_log) + return True + + +def build_rename_map(manifest: dict, logger: Logger) -> dict: + """Flattens optimize_directory's {relative_src: relative_dst} manifest + to {old_basename: new_basename}, matching k/html/images/*'s bare-filename + addressing. Warns (keeping the first) if two different renames collide + on the same old basename - e.g. two identically-named files in + different subdirectories of media_dir.""" + rename_map = {} + for old_rel, new_rel in sorted(manifest.items()): + old_name = Path(old_rel).name + new_name = Path(new_rel).name + if old_name == new_name: + continue + if old_name in rename_map and rename_map[old_name] != new_name: + logger.error( + f"warning: {old_rel!r} and an earlier file both renamed from {old_name!r}, to different names " + f"({rename_map[old_name]!r} vs {new_name!r}); keeping the first" + ) + continue + rename_map[old_name] = new_name + return rename_map + + +def reassemble_content(conn, path: str, first_content: bytes) -> bytes: + """Reassembles a possibly-chunked row's full bytes - mirrors + WebServer.kt's own reassembly protocol (see CHUNK_SIZE's docstring in + populate_db.py): a row is fragmented purely when its content is exactly + CHUNK_SIZE bytes, in which case "-1", "-2", ... are + concatenated until a missing or shorter-than-CHUNK_SIZE row is hit.""" + if len(first_content) < CHUNK_SIZE: + return first_content + parts = [first_content] + n = 1 + while True: + row = conn.execute("SELECT content FROM Content WHERE path = ?", (f"{path}-{n}",)).fetchone() + if row is None: + break + parts.append(row[0]) + if len(row[0]) < CHUNK_SIZE: + break + n += 1 + return b"".join(parts) + + +def rewrite_pages(conn, rename_map: dict, language_id: int, page_content_type_id: int, logger: Logger, + chunked_log: list, compressor: DictionaryCompressor) -> int: + """Rewrites every k/html/*.html page (and the nav row) that references a + renamed image, replacing "/k/html/images/" with + "/k/html/images/" wherever it appears. Operates directly on + each row's decompressed JSON text rather than parsing it: every image + reference is a literal IMAGES_URL_PREFIX+filename substring, baked in at + conversion time by md_to_json.py's Converter (resolve_image_src), so a + plain text substitution finds it correctly regardless of which block + type it ends up nested inside - no need to understand that nested block + schema here. The match is anchored on the escaped quote (\\") that + always immediately follows a rewritten src="..." attribute in the + stored JSON (see resolve_image_src/rewrite_urls - image references are + only ever embedded as HTML attributes, never as a bare JSON field on + their own), so a renamed file's name can't accidentally match as a + prefix of some other, unrelated, longer filename. Returns the number of + rows changed. + + ".html" is the exact literal suffix populate_db.py gives every base + page/nav row; fragment continuation rows are named "-" (the + "-N" appended after the ".html" already in path), so the path filter + below naturally excludes them without needing to detect chunking up + front. + + Substitutes in a single pass over each row's original text (one regex + covering every old_name, dispatched through `replacements` by exact + match) rather than N sequential str.replace calls on a mutating buffer. + Sequential replaces would risk a chain rename: if one rename's new_name + equals another rename's old_name (e.g. foo.png -> foo.webp and, + unrelated, foo.webp -> foo-2.webp), a later replace could re-match text + an earlier replace just wrote, sending an original foo.png reference to + foo-2.webp instead of foo.webp. Scanning the untouched original text + once makes that impossible.""" + if not rename_map: + return 0 + + rows = conn.execute( + "SELECT path, content, templateId FROM Content WHERE path LIKE 'k/html/%.html' AND contentTypeID = ? " + "AND templateId != 0", + (page_content_type_id,), + ).fetchall() + + replacements = { + f'{IMAGES_URL_PREFIX}{old_name}\\"': f'{IMAGES_URL_PREFIX}{new_name}\\"' + for old_name, new_name in rename_map.items() + } + old_ref_pattern = re.compile("|".join(re.escape(old_ref) for old_ref in replacements)) + + changed = 0 + for path, first_content, template_id in rows: + full = reassemble_content(conn, path, first_content) + text = compressor.decompress(full).decode("utf-8") + hits = len(old_ref_pattern.findall(text)) + if not hits: + continue + new_text = old_ref_pattern.sub(lambda m: replacements[m.group(0)], text) + blob = compressor.compress(new_text.encode("utf-8")) + delete_content(conn, path) + insert_chunked_content(conn, path, language_id, page_content_type_id, template_id, blob, chunked_log) + changed += 1 + logger.info(f"[URL FIX] {path}: updated {hits} image reference(s)") + return changed + + +# Matches a rewritten image src's filename, anchored the same way +# rewrite_pages' own known-rename substitutions are: resolve_image_src/ +# rewrite_urls only ever embed an image reference as an HTML src="..." +# attribute, which - JSON-encoded - always has the escaped quote (\") right +# after it, so this can't accidentally swallow past the end of the filename. +IMAGE_REF_RE = re.compile(re.escape(IMAGES_URL_PREFIX) + r'([^\\"]+)\\"') + + +def collect_referenced_media(conn, page_content_type_id: int, compressor: DictionaryCompressor) -> set: + """Bare filenames (e.g. "mascot.png") referenced by at least one + src="/k/html/images/" anywhere across current k/html/*.html page + content and the nav row - the same row selection/reassembly + rewrite_pages uses, just extracting every image reference found instead + of only substituting the ones in a known rename_map.""" + rows = conn.execute( + "SELECT path, content FROM Content WHERE path LIKE 'k/html/%.html' AND contentTypeID = ? AND templateId != 0", + (page_content_type_id,), + ).fetchall() + referenced = set() + for path, first_content in rows: + full = reassemble_content(conn, path, first_content) + text = compressor.decompress(full).decode("utf-8") + referenced.update(IMAGE_REF_RE.findall(text)) + return referenced + + +def list_stored_media(conn) -> dict: + """Bare filename -> Content.path (e.g. "mascot.png" -> "k/html/images/ + mascot.png") for every image currently stored under IMAGES_DB_PATH_PREFIX, + collapsing chunked continuation fragments ("-1", "-2", ...) + back into their base row, since deleting the base via delete_content + already takes its fragments with it (see CHUNK_SIZE's docstring in + populate_db.py for that fragmentation convention). A path is treated as + a fragment when stripping a trailing "-" yields another path + that's also present - the same convention this whole pipeline already + relies on elsewhere, ambiguous only for a base filename that itself + looks like "-", which no real optimized + media filename does.""" + paths = {row[0] for row in conn.execute( + "SELECT path FROM Content WHERE path LIKE ?", (f"{IMAGES_DB_PATH_PREFIX}%",) + )} + + def is_fragment(path: str) -> bool: + prefix, sep, suffix = path.rpartition("-") + return sep == "-" and suffix.isdigit() and prefix in paths + + return {path[len(IMAGES_DB_PATH_PREFIX):]: path for path in paths if not is_fragment(path)} + + +def delete_unreferenced_media(conn, page_content_type_id: int, logger: Logger, + compressor: DictionaryCompressor) -> int: + """Deletes every currently-stored k/html/images/ row (base row and + any chunked fragments) that no page or the nav row references even once. + Must run after insertion and rename-rewriting, so it sees the final, + up-to-date state of both stored media and in-content references - a file + renamed this run is only "unreferenced" under its stale old name, which + rewrite_pages will have already fixed up by the time this runs. Returns + the number of images removed.""" + stored = list_stored_media(conn) + referenced = collect_referenced_media(conn, page_content_type_id, compressor) + removed = 0 + for name, path in sorted(stored.items()): + if name in referenced: + continue + delete_content(conn, path) + removed += 1 + logger.info(f"[UNUSED] removed {path} (not referenced by any page)") + return removed + + +def main() -> None: + parser = build_parser() + args = parser.parse_args() + + try: + cfg = resolve_config(args, OWN_OPTION_SPECS, BUILTIN_DEFAULTS) + except RuntimeError as exc: + parser.error(str(exc)) + return + + if cfg["input_dir"] is None or cfg["db_path"] is None: + parser.error("media_dir and db_path must be given either as positional arguments or in --config") + + log_file_handle = open(cfg["log_file"], "w", encoding="utf-8") if cfg["log_file"] else None + logger = Logger(log_file_handle) + work_dir_is_temp = cfg["output_dir"] is None + work_dir = cfg["output_dir"] or Path(tempfile.mkdtemp(prefix="insert_optimized_media_")) + + try: + if not cfg["input_dir"].is_dir(): + logger.error(f"error: {cfg['input_dir']} is not a directory") + sys.exit(1) + if not cfg["db_path"].is_file(): + logger.error(f"error: {cfg['db_path']} does not exist") + sys.exit(1) + + if cfg["verbose"]: + logger.info("Config parameters:") + for key, (dest, _converter) in OWN_OPTION_SPECS.items(): + logger.info(f" {key} = {cfg.get(dest)}") + logger.info(f" work-dir = {work_dir}{' (temporary)' if work_dir_is_temp else ''}") + if args.config: + logger.info(f" (loaded from {args.config})") + + try: + pngquant_path = find_pngquant() + except RuntimeError as exc: + logger.error(f"error: {exc}") + sys.exit(1) + + # Fail fast on a schema this database doesn't support - before + # spending time optimizing every file - rather than discovering it + # partway through the (rolled-back, but still wasted) DB transaction. + preflight_conn = sqlite3.connect(cfg["db_path"]) + try: + get_id(preflight_conn, "Languages", LANGUAGE) + get_id(preflight_conn, "ContentTypes", PAGE_CONTENT_TYPE) + if cfg["webp"]: + get_content_type(preflight_conn, WEBP_CONTENT_TYPE) + except RuntimeError as exc: + logger.error(f"error: {exc}") + sys.exit(1) + finally: + preflight_conn.close() + + work_dir.mkdir(parents=True, exist_ok=True) + stats = {"raster": 0, "svg": 0, "svg_rasterized": 0, "copied": 0, "errors": 0, "original_bytes": 0, + "optimized_bytes": 0} + logger.info(f"Optimizing media from {cfg['input_dir']} into {work_dir}...") + manifest = optimize_directory(cfg["input_dir"], work_dir, cfg=cfg, pngquant_path=pngquant_path, + logger=logger, stats=stats) + if stats["errors"]: + logger.error( + f"error: {stats['errors']} file(s) failed to optimize; aborting before touching the database" + ) + sys.exit(1) + rename_map = build_rename_map(manifest, logger) + + logger.info(f"Backing up {cfg['db_path']}...") + backup_path = backup_database(cfg["db_path"]) + logger.info(f"Backup written to {backup_path}") + + conn = sqlite3.connect(cfg["db_path"]) + try: + conn.execute("BEGIN") + language_id = get_id(conn, "Languages", LANGUAGE) + page_content_type_id = get_id(conn, "ContentTypes", PAGE_CONTENT_TYPE) + # This script only ever runs against a database populate_db.py + # already populated (see module docstring), so its + # CompressionDictionary must already exist - never train a new + # one here, since that would orphan every row already + # compressed against the existing one (see DictionaryCompressor). + compressor = DictionaryCompressor(load_dictionary(conn)) + + content_type_cache = {} + chunked_log = [] + inserted = 0 + seen_names = {} + try: + for out_path in sorted(work_dir.rglob("*")): + if out_path.is_dir(): + continue + name = out_path.name + if name in seen_names: + logger.error( + f"warning: {out_path} has the same filename as {seen_names[name]}; keeping the first, " + "skipping this one" + ) + continue + seen_names[name] = out_path + db_path = f"{IMAGES_DB_PATH_PREFIX}{name}" + if insert_optimized_file(conn, out_path.read_bytes(), name, db_path, language_id, + content_type_cache, chunked_log, compressor): + inserted += 1 + if cfg["verbose"]: + logger.info(f"[OK] {out_path} -> {db_path}") + + # A renamed file's old basename no longer appears anywhere under + # work_dir (that's what makes it a rename), so the loop above + # never visits its old db_path to replace it - it'd otherwise + # linger forever as an orphaned, no-longer-referenced row. + removed = 0 + for old_name in rename_map: + old_db_path = f"{IMAGES_DB_PATH_PREFIX}{old_name}" + delete_content(conn, old_db_path) + removed += 1 + if cfg["verbose"]: + logger.info( + f"[REMOVED] {old_db_path} (renamed to {IMAGES_DB_PATH_PREFIX}{rename_map[old_name]})" + ) + + changed_pages = rewrite_pages(conn, rename_map, language_id, page_content_type_id, logger, + chunked_log, compressor) + + unreferenced_removed = delete_unreferenced_media(conn, page_content_type_id, logger, compressor) + finally: + compressor.close() + + conn.commit() + except Exception: + conn.rollback() + raise + finally: + conn.close() + + logger.info("Vacuuming database to reclaim freed space...") + vacuum_conn = sqlite3.connect(cfg["db_path"]) + try: + vacuum_conn.execute("VACUUM") + finally: + vacuum_conn.close() + + logger.info( + f"Done: inserted/updated {inserted} image(s) in {cfg['db_path']}, {removed} stale renamed-away row(s) " + f"removed, {changed_pages} page(s)/nav row(s) updated to match {len(rename_map)} renamed file(s), " + f"{unreferenced_removed} unreferenced image(s) deleted." + ) + if chunked_log: + logger.info(f"Chunked {len(chunked_log)} file(s) over {CHUNK_SIZE:,} bytes:") + for path, total_size, chunk_count in chunked_log: + logger.info(f" {path}: {total_size:,} bytes -> {chunk_count} chunks") + finally: + if log_file_handle is not None: + log_file_handle.close() + if work_dir_is_temp: + shutil.rmtree(work_dir, ignore_errors=True) + + +if __name__ == "__main__": + main() diff --git a/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/migrate_content_to_dictionary_brotli.py b/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/migrate_content_to_dictionary_brotli.py new file mode 100644 index 000000000..129929ff7 --- /dev/null +++ b/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/migrate_content_to_dictionary_brotli.py @@ -0,0 +1,409 @@ +#!/usr/bin/env python3 +""" +migrate_content_to_dictionary_brotli.py + +One-time, resumable, whole-database migration: recompresses every Content row +whose ContentTypes.compression is 'brotli' against this database's shared +CompressionDictionary (see ADFA-5153), replacing plain (no dictionary) Brotli +blobs with dictionary-compressed ones in place. + +Why this exists: populate_db.py and insert_optimized_media.py only ever touch +their own subset of Content ("k/html/%", "assets/%"). Every other Content row +in documentation.db - reference docs, tooltip-linked pages, whatever else - was +compressed with plain Brotli by whichever pipeline wrote it. Dictionary +compression pays off best when it covers the whole corpus, so this script is +what converts the rows outside populate_db.py's reach. + +Note what it does NOT establish: an invariant that every 'brotli' row in a +shipped database uses the dictionary. That is unachievable by construction - a +plugin installed on-device contributes plain-Brotli rows at any time (see +PluginDocumentationManager/BrotliCompressor in the app). WebServer.kt therefore +tries a dictionary-attached decode and falls back to a plain one, and that +fallback is load-bearing rather than defensive. Any other reader of this +database needs the same fallback. + +Classification, per row, is by *decoding* rather than by assumption, because +"plain decode failed" alone means very little: + + * decodes plainly AND with the dictionary, to identical bytes -> the encoder + never referenced the dictionary (small or already-compressed payloads, + ~0.5% of the real corpus). Nothing to gain; left untouched, so re-runs do + not churn it. This is the case that makes a naive "plain decode succeeded, + so it needs migrating" test re-migrate the same rows on every run. + * decodes plainly only -> not yet migrated. Recompress. + * decodes with the dictionary only -> already migrated. + * decodes neither way -> reported as an ERROR, never counted as success. A + truncated ADFA-5171 chain lands here, and silently counting it as + "already migrated" is exactly how such a row stays plain-Brotli while the + run reports a clean finish. + +Chunked rows are reassembled via populate_db.fragment_chain, which finds a +chain by LIKE plus parsed suffix rather than by probing "-1" - an +ADFA-5171 chain numbered from -2 would otherwise reassemble truncated. Run +renumber_misnumbered_fragments.py first if the database still has those; this +script reports them rather than repairing them. + +Writes are in place: UPDATE on the base row, then the continuation rows are +reconciled by exact path. The base row is never DELETEd and re-INSERTed, +because Content carries AddBook/DeleteBook triggers on paths matching +'%.pdf' - a delete/insert cycle drops the curated Bookshelf entry (title, +description, bookCategoryID) and replaces it with 'CURRENT_TIMESTAMP || id' +under a fresh Content.id. 15 brotli-typed .pdf rows and all 7 Bookshelf rows +are in scope on the real database. + +Dictionary training (only when CompressionDictionary does not exist yet) draws +a sample stratified across doc sets in proportion to their stored bytes, and is +bounded by a plaintext byte budget rather than a row count. Both halves matter, +measured on the real corpus with only the sampling varied: + + first 300 rows by path (all under "a/") 36.2% smaller than plain + 300 rows stratified across doc sets 33.2% <- worse + stratified, 32 MiB plaintext budget 48.3% <- best + first-by-path, same 32 MiB budget 36.4% <- volume alone: nil + +Stratifying at a fixed row count draws quotas from smaller doc sets, so total +material falls and the trainer cannot even fill a 256 KiB dictionary. The +spread is the win; the byte budget is what makes the spread affordable. + +Safety: backs up the database first (VACUUM INTO, same as populate_db.py), +commits in batches (a single transaction spanning the whole run holds a write +lock that the readers below cannot work around under rollback-journal mode), +verifies every recompressed row round-trips before writing it, and VACUUMs +afterward on a separate connection (SQLite refuses VACUUM inside a +transaction). Interrupting it is safe: finished batches stand, and re-running +resumes. + +Performance: all database access happens on the calling thread; the worker pool +only ever receives bytes. Each recompress spawns a `brotli` subprocess, so the +work parallelizes well, and keeping SQLite single-threaded avoids the +"database is locked" failure that worker-owned read connections hit under +journal_mode=delete (documentation.db's actual mode) once the write +transaction's page cache spills. + +Usage: + python3 migrate_content_to_dictionary_brotli.py [--sample-size N] + [--dict-size BYTES] [--max-workers N] [--training-bytes BYTES] [--sample-seed N] +""" +import argparse +import random +import sqlite3 +import sys +import threading +from collections import defaultdict +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path + +import brotli + +from populate_db import ( + CHUNK_SIZE, DEFAULT_DICT_SIZE, DictionaryCompressor, backup_database, fragment_chain, + load_or_create_dictionary, +) + +DEFAULT_SAMPLE_SIZE = 300 +# ~128x the 256 KiB dictionary. zstd's cover trainers want roughly two orders of +# magnitude more material than the dictionary they produce; below that they +# return a dictionary smaller than the cap, which measurably compresses worse. +DEFAULT_TRAINING_BYTES = 32 * 1024 * 1024 +DEFAULT_SAMPLE_SEED = 0x5153 +# Rows per write transaction. Small enough that the write lock is never held +# across a long stretch of compression work, large enough that commit overhead +# stays negligible against a q11 recompress. +BATCH_ROWS = 200 + +_thread_local = threading.local() + + +def read_item(conn, path: str) -> bytes: + """The full stored bytes of one logical item: its base row plus every + continuation row in its chain, in suffix order. Suffix-agnostic (see + populate_db.fragment_chain), so an ADFA-5171 chain numbered from -2 + reassembles correctly rather than truncating.""" + row = conn.execute("SELECT content FROM Content WHERE path = ?", (path,)).fetchone() + if row is None: + return b"" + parts = [row[0]] + if len(row[0]) < CHUNK_SIZE: + return parts[0] + for _n, fragment_path in fragment_chain(conn, path): + fragment = conn.execute("SELECT content FROM Content WHERE path = ?", (fragment_path,)).fetchone() + if fragment is not None: + parts.append(fragment[0]) + return b"".join(parts) + + +def write_item(conn, path: str, language_id: int, content_type_id: int, template_id: int, data: bytes) -> None: + """Replaces one item's stored bytes in place: UPDATE on the base row, then + the continuation rows reconciled by exact path (updated, inserted, or + deleted as the new chunk count requires). + + Deliberately never DELETEs the base row: Content's AddBook/DeleteBook + triggers fire on '%.pdf' paths and a delete/insert cycle silently replaces + the curated Bookshelf entry with a timestamp title under a new + Content.id. Continuation paths end in "-", so they never match those + triggers and are safe to delete. Nothing here goes through LIKE, so no + unrelated row can be caught by a `_` wildcard in a path.""" + conn.execute("UPDATE Content SET content = ? WHERE path = ?", (data[:CHUNK_SIZE], path)) + + wanted = {} + for number, offset in enumerate(range(CHUNK_SIZE, len(data), CHUNK_SIZE), start=1): + wanted[f"{path}-{number}"] = data[offset:offset + CHUNK_SIZE] + + existing = {fragment_path for _n, fragment_path in fragment_chain(conn, path)} + for fragment_path, blob in wanted.items(): + if fragment_path in existing: + conn.execute("UPDATE Content SET content = ? WHERE path = ?", (blob, fragment_path)) + else: + conn.execute( + "INSERT INTO Content (path, languageID, content, contentTypeID, templateId) " + "VALUES (?, ?, ?, ?, ?)", + (fragment_path, language_id, blob, content_type_id, template_id), + ) + for surplus in sorted(existing - set(wanted)): + conn.execute("DELETE FROM Content WHERE path = ?", (surplus,)) + + +def _thread_compressor(dictionary_data: bytes) -> DictionaryCompressor: + """One DictionaryCompressor per worker thread, reused across every row that + thread processes - creating one per row would re-write the same dictionary + bytes to a fresh temp file on every call for no benefit.""" + compressor = getattr(_thread_local, "compressor", None) + if compressor is None: + compressor = DictionaryCompressor(dictionary_data) + _thread_local.compressor = compressor + return compressor + + +def classify(compressor: DictionaryCompressor, stored: bytes) -> tuple: + """Returns (state, plaintext) for one item's stored bytes, by decoding it + both ways. See the module docstring for why each state means what it does. + state is one of "both", "plain", "dictionary", "undecodable".""" + try: + plain = brotli.decompress(stored) + except brotli.error: + plain = None + try: + via_dictionary = compressor.decompress(stored) + except RuntimeError: + via_dictionary = None + + if plain is not None and via_dictionary is not None and plain == via_dictionary: + return "both", plain + if plain is not None: + return "plain", plain + if via_dictionary is not None: + return "dictionary", via_dictionary + return "undecodable", None + + +def recompress_item(dictionary_data: bytes, path: str, stored: bytes) -> dict: + """Runs on a worker thread: pure bytes in, pure bytes out, no database + access. Returns a result dict the caller writes back (or reports).""" + compressor = _thread_compressor(dictionary_data) + state, plain = classify(compressor, stored) + if state == "undecodable": + return {"path": path, "state": "error", + "detail": "decodes neither plainly nor with the dictionary; " + "run renumber_misnumbered_fragments.py if its chain is numbered from -2"} + if state in ("dictionary", "both"): + return {"path": path, "state": "already"} + + recompressed = compressor.compress(plain) + # A migration that does not round-trip is worse than no migration: the row + # would only fail later, on-device, at read time. + try: + if compressor.decompress(recompressed) != plain: + return {"path": path, "state": "error", "detail": "recompressed bytes do not decode back to the original"} + except RuntimeError as exc: + return {"path": path, "state": "error", "detail": f"recompressed bytes failed to decode: {exc}"} + + return {"path": path, "state": "migrated", "data": recompressed, + "before": len(stored), "after": len(recompressed)} + + +def doc_set(path: str) -> str: + return path.split("/", 1)[0] if "/" in path else "(root)" + + +def collect_training_samples(conn, base_rows: list, sample_size: int, + byte_budget: int = DEFAULT_TRAINING_BYTES, + seed: int = DEFAULT_SAMPLE_SEED, decode=None) -> list: + """Plaintext samples for training a new dictionary, stratified across doc + sets in proportion to their stored bytes and bounded by `byte_budget` of + plaintext. See the module docstring for the measurements behind both + choices. Deterministic for a given seed, because a dictionary is never + retrained once stored - being able to reproduce the training set later is + the only way to explain the bytes you are then stuck with. + + `decode` reads one item's stored bytes back to plaintext, defaulting to plain + Brotli because this only ever runs before CompressionDictionary exists. A + re-mint (see remint_dictionary.py) passes one that decodes against the + outgoing dictionary instead.""" + decode = decode or brotli.decompress + by_set = defaultdict(list) + for row in base_rows: + by_set[doc_set(row[0])].append(row) + + weight = {name: sum(row[4] for row in rows) for name, rows in by_set.items()} + total_weight = sum(weight.values()) or 1 + rng = random.Random(seed) + + drawn = [] + for name, rows in by_set.items(): + shuffled = list(rows) + rng.shuffle(shuffled) + # Oversample per set: the byte budget below is the real limit, and a + # short set should not strand budget that another set could use. + quota = max(1, round(sample_size * weight[name] / total_weight)) + drawn.append((name, shuffled, quota)) + + ordered = [] + for name, shuffled, quota in drawn: + ordered.extend(shuffled[:quota * 3]) + rng.shuffle(ordered) + + samples = [] + used = 0 + for row in ordered: + if used >= byte_budget or len(samples) >= sample_size * 3: + break + try: + plain = decode(read_item(conn, row[0])) + except brotli.error as exc: + print(f"warning: could not decompress {row[0]!r} for training sample: {exc}", file=sys.stderr) + continue + samples.append(plain) + used += len(plain) + + histogram = defaultdict(int) + for row in ordered[:len(samples)]: + histogram[doc_set(row[0])] += 1 + print(f"Training dictionary on {len(samples)} rows, {used / 1048576:.1f} MiB of plaintext " + f"across {len(histogram)} doc set(s): {dict(sorted(histogram.items(), key=lambda kv: -kv[1]))}", + file=sys.stderr) + return samples + + +def load_base_rows(conn) -> list: + """Every brotli-typed base row as (path, language_id, content_type_id, + template_id, stored_len). Blobs are deliberately not selected here - the + real table is ~130 MB of compressed content, and each row's bytes are read + only when its turn comes.""" + rows = conn.execute( + "SELECT C.path, C.languageID, C.contentTypeID, C.templateId, LENGTH(C.content) " + "FROM Content C, ContentTypes CT " + "WHERE C.contentTypeID = CT.id AND CT.compression = 'brotli' " + "ORDER BY C.path" + ).fetchall() + all_paths = {row[0] for row in conn.execute("SELECT path FROM Content")} + base_rows = [] + for row in rows: + prefix, sep, suffix = row[0].rpartition("-") + if sep == "-" and suffix.isdigit() and prefix in all_paths: + continue # a continuation row; handled with its base + base_rows.append(row) + return base_rows + + +def migrate(conn, sample_size: int, dict_size: int, max_workers: int | None = None, + training_bytes: int = DEFAULT_TRAINING_BYTES, sample_seed: int = DEFAULT_SAMPLE_SEED) -> dict: + base_rows = load_base_rows(conn) + + has_dictionary = conn.execute( + "SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'CompressionDictionary'" + ).fetchone() is not None + training_samples = [] if has_dictionary else collect_training_samples( + conn, base_rows, sample_size, training_bytes, sample_seed + ) + dictionary_data = load_or_create_dictionary(conn, training_samples, dict_size) + conn.commit() + + stats = {"scanned": len(base_rows), "migrated": 0, "already": 0, "errors": 0, + "bytes_before": 0, "bytes_after": 0} + problems = [] + + with ThreadPoolExecutor(max_workers=max_workers) as executor: + for start in range(0, len(base_rows), BATCH_ROWS): + batch = base_rows[start:start + BATCH_ROWS] + payloads = [(row, read_item(conn, row[0])) for row in batch] + results = executor.map( + lambda item: recompress_item(dictionary_data, item[0][0], item[1]), payloads + ) + for row, result in zip(batch, results): + if result["state"] == "already": + stats["already"] += 1 + continue + if result["state"] == "error": + stats["errors"] += 1 + problems.append((result["path"], result["detail"])) + continue + stats["migrated"] += 1 + stats["bytes_before"] += result["before"] + stats["bytes_after"] += result["after"] + write_item(conn, row[0], row[1], row[2], row[3], result["data"]) + conn.commit() + + stats["problems"] = problems + return stats + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("db_path", type=Path, help="SQLite database to migrate, e.g. documentation.db") + parser.add_argument("--sample-size", type=int, default=DEFAULT_SAMPLE_SIZE, + help=f"Rows to aim for when training a dictionary, if none exists yet " + f"(default: {DEFAULT_SAMPLE_SIZE}); --training-bytes is the real limit") + parser.add_argument("--training-bytes", type=int, default=DEFAULT_TRAINING_BYTES, + help=f"Plaintext byte budget for dictionary training " + f"(default: {DEFAULT_TRAINING_BYTES:,})") + parser.add_argument("--sample-seed", type=int, default=DEFAULT_SAMPLE_SEED, + help="Seed for the stratified training sample, so a dictionary's training set " + "stays reproducible (default: %(default)s)") + parser.add_argument("--dict-size", type=int, default=DEFAULT_DICT_SIZE, + help=f"Dictionary size in bytes if training a new one (default: {DEFAULT_DICT_SIZE})") + parser.add_argument("--max-workers", type=int, default=None, + help="Worker threads for the recompress phase (default: ThreadPoolExecutor's own " + "min(32, cpu_count+4)); database access stays on the calling thread") + args = parser.parse_args() + + if not args.db_path.is_file(): + print(f"error: {args.db_path} does not exist", file=sys.stderr) + sys.exit(1) + + print(f"Backing up {args.db_path}...", file=sys.stderr) + backup_path = backup_database(args.db_path) + print(f"Backup written to {backup_path}", file=sys.stderr) + + conn = sqlite3.connect(args.db_path) + try: + stats = migrate(conn, args.sample_size, args.dict_size, args.max_workers, + args.training_bytes, args.sample_seed) + conn.commit() + finally: + conn.close() + + print("Vacuuming database to reclaim freed space...", file=sys.stderr) + vacuum_conn = sqlite3.connect(args.db_path) + try: + vacuum_conn.execute("VACUUM") + finally: + vacuum_conn.close() + + print( + f"Scanned {stats['scanned']} brotli row(s): migrated {stats['migrated']}, " + f"already dictionary-compressed {stats['already']}, errors {stats['errors']}." + ) + if stats["migrated"]: + before, after = stats["bytes_before"], stats["bytes_after"] + pct = (1 - after / before) * 100 if before else 0.0 + print(f"Migrated bytes: {before:,} -> {after:,} ({pct:.1f}% smaller)") + for path, detail in stats["problems"][:20]: + print(f"error: {path}: {detail}", file=sys.stderr) + if len(stats["problems"]) > 20: + print(f"error: ... and {len(stats['problems']) - 20} more", file=sys.stderr) + if stats["errors"]: + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/populate_db.py b/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/populate_db.py new file mode 100644 index 000000000..ce4b28e39 --- /dev/null +++ b/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/populate_db.py @@ -0,0 +1,820 @@ +#!/usr/bin/env python3 +""" +Populates a documentation.db-schema SQLite database with the same content +templates/page.peb and this project's md_to_json.py conversion pipeline +produce for the static site, replacing what's currently at k/html/*. + +Usage: + python3 populate_db.py [db-path] + [--tree-file kr.tree] [--topics-subdir topics] + [--blacklisted-element-titles "Ancestor\\/.../Element Title" ...] + +--blacklisted-element-titles names element(s) +to drop from kr.tree entirely before anything else below reads it: the +element and its whole subtree get no nav entry, none of their .md sub-topics +get converted or inserted, and any *other*, non-blacklisted page's in-content +link to one of those .md files renders broken/styled (same as any other +unresolved link - see broken-ext-link-color) rather than pointing somewhere +that no longer exists. + +Each value is the *full* toc-title path from a top-level down +to the one being blacklisted, since toc-title alone is not unique across +kr.tree (e.g. plenty of "Overview"s). Levels are joined by the two-character +sequence "\\/" (backslash then slash) rather than a bare "/", because a bare +"/" routinely appears *within* a single real toc-title (e.g. "Swift/ +Objective-C and C interop") and this way that overwhelmingly common case +needs no escaping at all - only the rare level separator does. So to +blacklist the "Swift/Objective-C and C interop" element nested under the +top-level "Interoperability" element, pass +"Interoperability\\/Swift/Objective-C and C interop": split on "\\/" that's +["Interoperability", "Swift/Objective-C and C interop"], matching kr.tree's +actual nesting - the inner "/" is left untouched since it wasn't preceded by +a backslash. + + defaults to "documentation.db". A safety backup (via SQLite's +"VACUUM INTO", which is safe even against a live/WAL-mode database) is +written next to it before any changes: ".backup-". + + is Writerside's own official image output for this doc set +(e.g. "webHelpImages.zip", found next to kr.tree) - a flat archive with no +subdirectories, one entry per image, already exactly as Writerside itself +would serve them. Rather than re-deriving image content/sizing ourselves +from the raw source tree (which is a plain, uncompressed truecolor export - +several times larger than what a real Writerside build actually ships, +since it applies its own image optimization we have no easy way to +replicate faithfully), this script just copies that zip's entries in +directly, so k/html/images/ ends up byte-for-byte what Writerside +itself produces. + +What this does, inside a single transaction (rolled back on any error): + 1. Deletes every Content row with path LIKE 'k/html/%' or 'assets/%' - the + former includes the existing *.html doc pages AND everything else + parked there (images, the old Writerside JS bundle under + k/html/frontend/, none of which this script replaces); the latter is + wherever a previous run of this script put images/CSS/JS, all of + which get freshly re-inserted below. + 2. Upserts templates/page.peb and templates/nav.peb into Templates. + page.peb's