diff --git a/.github/workflows/publish-micropython-lib.yml b/.github/workflows/publish-micropython-lib.yml new file mode 100644 index 0000000..7c554fe --- /dev/null +++ b/.github/workflows/publish-micropython-lib.yml @@ -0,0 +1,157 @@ +# Publish pure-Python usdl2 to micropython-lib, MIP index, and TestPyPI (usdl2-py). +# Native usdl2 wheels are published by publish-testpypi.yml on the same tag. +# +# Release trigger: push tag vX.Y.Z (see scripts/publish_release_tag.sh). +# +# Secrets: +# MICROPYTHON_LIB_DEPLOY_TOKEN — PAT with contents:write on PyDevices/micropython-lib +# TESTPYPI_API_TOKEN — TestPyPI API token + +name: Publish micropython-lib + +on: + push: + tags: + - 'v*.*.*' + workflow_dispatch: + inputs: + version: + description: Semver X.Y.Z + type: string + required: false + sync_sources: + description: Sync usdl2 into micropython-lib + type: boolean + default: true + upload_testpypi: + description: Upload usdl2-py wheels to TestPyPI + type: boolean + default: false + publish_mip_index: + description: Rebuild mip/PyDevices and push gh-pages + type: boolean + default: true + commit_message: + description: micropython-lib commit message + type: string + required: false + +permissions: + contents: read + +# Sibling repos also push to micropython-lib; scripts rebase-retry on conflict. +concurrency: + group: publish-micropython-lib-${{ github.repository }} + cancel-in-progress: false + +jobs: + publish: + runs-on: ubuntu-latest + if: github.repository == 'PyDevices/usdl2' + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Resolve release version + id: version + env: + INPUT_VERSION: ${{ inputs.version }} + run: | + set -euo pipefail + normalize() { + local v="${1#v}" + v="$(echo "$v" | tr -d '[:space:]')" + if [[ ! "$v" =~ ^[0-9]+\.[0-9]+\.[0-9]+(-[a-zA-Z0-9.]+)?$ ]]; then + echo "Invalid semver: $1" >&2 + exit 1 + fi + echo "$v" + } + if [[ "${{ github.event_name }}" == "push" ]]; then + VERSION="$(normalize "${GITHUB_REF_NAME}")" + elif [[ -n "${INPUT_VERSION:-}" ]]; then + VERSION="$(normalize "$INPUT_VERSION")" + else + TAG="$(git describe --tags --exact-match 2>/dev/null || true)" + if [[ -z "$TAG" ]]; then + echo "Set version input or run from a vX.Y.Z tag." >&2 + exit 1 + fi + VERSION="$(normalize "$TAG")" + fi + echo "version=$VERSION" >> "$GITHUB_OUTPUT" + echo "Publishing usdl2-py / MIP release $VERSION" + + - name: Verify publish secrets + env: + DEPLOY_TOKEN: ${{ secrets.MICROPYTHON_LIB_DEPLOY_TOKEN }} + TESTPYPI_TOKEN: ${{ secrets.TESTPYPI_API_TOKEN }} + run: | + set -euo pipefail + missing=() + if [[ -z "${DEPLOY_TOKEN}" ]]; then + missing+=("MICROPYTHON_LIB_DEPLOY_TOKEN") + fi + if [[ "${{ github.event_name }}" == "push" || "${{ inputs.upload_testpypi }}" == "true" ]]; then + if [[ -z "${TESTPYPI_TOKEN}" ]]; then + missing+=("TESTPYPI_API_TOKEN") + fi + fi + if [[ ${#missing[@]} -gt 0 ]]; then + echo "::error::GitHub did not supply secrets to PyDevices/usdl2: ${missing[*]}" + echo "Org Settings → Secrets → Actions → each secret → Repository access → include usdl2." + exit 1 + fi + + - uses: actions/checkout@v4 + with: + repository: PyDevices/micropython-lib + ref: PyDevices + path: micropython-lib + token: ${{ secrets.MICROPYTHON_LIB_DEPLOY_TOKEN }} + + - uses: actions/setup-python@v5 + with: + python-version: '3.13' + + - name: Install publish tools + if: github.event_name == 'push' || inputs.upload_testpypi == true + run: pip install hatch twine + + - name: Configure micropython-lib git + run: | + git -C micropython-lib config user.name 'github-actions[bot]' + git -C micropython-lib config user.email 'github-actions[bot]@users.noreply.github.com' + git -C micropython-lib remote set-url origin \ + "https://x-access-token:${{ secrets.MICROPYTHON_LIB_DEPLOY_TOKEN }}@github.com/PyDevices/micropython-lib.git" + + - name: Sync usdl2 into micropython-lib + if: github.event_name == 'push' || inputs.sync_sources == true + env: + MICROPYTHON_LIB_DIR: ${{ github.workspace }}/micropython-lib + USDL2_VERSION: ${{ steps.version.outputs.version }} + TESTPYPI_API_TOKEN: ${{ secrets.TESTPYPI_API_TOKEN }} + run: | + VERSION="${{ steps.version.outputs.version }}" + MSG="${{ inputs.commit_message }}" + if [[ -z "$MSG" ]]; then + MSG="usdl2: Release v${VERSION} (${GITHUB_SHA::7})." + fi + EXTRA=(--commit-message "$MSG" --push) + if [[ "${{ github.event_name }}" != "push" && "${{ inputs.upload_testpypi }}" != "true" ]]; then + EXTRA=(--skip-pypi --commit-message "$MSG" --push) + fi + chmod +x scripts/publish_micropython_lib.sh + ./scripts/publish_micropython_lib.sh "${EXTRA[@]}" + + - name: Publish MIP index to gh-pages + if: github.event_name == 'push' || inputs.publish_mip_index == true + env: + MICROPYTHON_LIB_DIR: ${{ github.workspace }}/micropython-lib + USDL2_DIR: ${{ github.workspace }} + MICROPYTHON_DIR: /tmp/micropython + MIP_INDEX_OUTPUT: /tmp/mip-index + run: | + chmod +x scripts/publish_mip_ghpages.sh + ./scripts/publish_mip_ghpages.sh diff --git a/.gitignore b/.gitignore index a207eac..c0462de 100644 --- a/.gitignore +++ b/.gitignore @@ -19,3 +19,4 @@ default.profraw # Cursor IDE throwaways (do not ignore the whole .cursor/ tree) .cursor/debug* .cursor/plans/ +wheels/ diff --git a/AGENTS.md b/AGENTS.md index 86c7478..2ad8c53 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,10 +1,16 @@ # AGENTS.md — usdl2 -Native **SDL2 subset for Python** (`import usdl2`). Pure C for MicroPython, -CircuitPython, and CPython. Android APK packaging is in -`pydisplay_android` (TestPyPI wheel + p4a recipes), not this repo. When this -module is not linked, pydisplay falls back to `src/add_ons/usdl2.py` (same -public contract). +Native and pure-Python **SDL2 subset for Python** (`import usdl2`). C for +MicroPython, CircuitPython, and CPython; ctypes/ffi fallback at `lib/usdl2.py`. +Android APK packaging is in `pydisplay_android` (TestPyPI wheel + p4a recipes), +not this repo. + +| Product | Role | +|---------|------| +| **usdl2** (native) | C extension / usermod — TestPyPI `usdl2` | +| **usdl2-py** | Pure Python `lib/usdl2.py` — TestPyPI `usdl2-py`, MIP `usdl2` | + +One tag `vX.Y.Z` publishes both (see `PUBLISHING.md`). ## Hard rule: SDL2 symbols only @@ -27,7 +33,7 @@ or put the helper in the consumer package — never grow usdl2’s public surfac When adding a binding, check the SDL2 docs/headers first. If the name is not `SDL_*` (or an established SDL macro like `SDL_DEFINE_PIXELFORMAT`), it does -not belong here. +not belong here. Keep `lib/usdl2.py` in lockstep with the C public surface. ## Layout @@ -35,7 +41,13 @@ not belong here. - `src/usdl2_mp.c` — MicroPython + CircuitPython - `src/usdl2_cpy.c` — CPython extension - `include/` — shared headers / `usdl2_module_globals.inc` / qstrs -- No `.c` / `.h` / `.inc` at repo root; no ctypes `python/` package +- `lib/usdl2.py` — pure-Python module (`src/` is C only) +- No `.c` / `.h` / `.inc` at repo root + +## Publish + +1. **Native wheels** — `publish-testpypi.yml` (cibuildwheel) +2. **usdl2-py** + MIP — `publish-micropython-lib.yml` (micropython-lib + TestPyPI + gh-pages) ## Smoke diff --git a/PUBLISHING.md b/PUBLISHING.md index ba23e15..fee58cb 100644 --- a/PUBLISHING.md +++ b/PUBLISHING.md @@ -1,8 +1,29 @@ # Publishing and releases -How changes in this repo become versioned **`usdl2`** wheels on [TestPyPI](https://test.pypi.org/project/usdl2/), and how to install them. +One annotated tag `vX.Y.Z` publishes **both** products at that version: -The CPython package ships a native extension (`src/usdl2_cpy.c`) linked against libSDL2. CI builds platform wheels with [cibuildwheel](https://cibuildwheel.pypa.io/): +| Product | Channel | Workflow | +|---------|---------|----------| +| **usdl2** (native) | TestPyPI (platform wheels) | `publish-testpypi.yml` | +| **usdl2-py** | TestPyPI (pure Python) + micropython-lib / MIP | `publish-micropython-lib.yml` | + +## Pipeline + +```text +usdl2 (commit on main) + ./scripts/publish_release_tag.sh --push # next patch after highest v* + │ + ├─► publish-testpypi.yml + │ cibuildwheel → Linux + Windows + Android → usdl2 + │ + └─► publish-micropython-lib.yml + sync → micropython/usdl2/ + hatch + twine → usdl2-py + rebuild mip/PyDevices → gh-pages +``` + +The CPython native package ships an extension (`src/usdl2_cpy.c`) linked against +libSDL2. CI builds platform wheels with [cibuildwheel](https://cibuildwheel.pypa.io/): | Platform | Wheel tag | |----------|-----------| @@ -13,36 +34,28 @@ The CPython package ships a native extension (`src/usdl2_cpy.c`) linked against Android wheels link against SDL2 prepared in CI (`scripts/ci_prepare_sdl2_android.sh`) but do **not** vendor `libSDL2.so` — the p4a SDL2 bootstrap provides it inside the APK. -MicroPython / CircuitPython consume `src/usdl2_mp.c` via `micropython.mk` / `circuitpython.mk` (not the PyPI wheel). - -APK packaging and p4a recipes live in [pydisplay_android](https://github.com/PyDevices/pydisplay_android). - -When the native module is unavailable, pydisplay uses `src/add_ons/usdl2.py` as a pure-Python fallback — that file lives in pydisplay, not this repo. - -## Pipeline overview - -```text -usdl2 (your machine) - commit → push main - │ - ▼ - ./scripts/publish_release_tag.sh --push (or manual git tag vX.Y.Z) - │ - ▼ -usdl2: Publish TestPyPI - cibuildwheel → Linux + Windows + Android wheels → twine upload -``` +MicroPython / CircuitPython consume `src/usdl2_mp.c` via `micropython.mk` / `circuitpython.mk` (not the PyPI wheel). APK packaging lives in [pydisplay_android](https://github.com/PyDevices/pydisplay_android). ## Version numbers -Format: **`X.Y.Z`** (semver). Later releases use the highest existing tag + 1 patch (`v0.0.7` → `0.0.8`, …). +Continue the native **usdl2** line (`v0.0.11` → `v0.0.12`, …). Preview: ```bash ./scripts/next_release_version.sh --verbose +./scripts/publish_release_tag.sh --dry-run ``` TestPyPI rejects re-uploading the same version — each release needs a new tag. +## Secrets + +| Secret | Purpose | +|--------|---------| +| `TESTPYPI_API_TOKEN` | TestPyPI upload (native + usdl2-py) | +| `MICROPYTHON_LIB_DEPLOY_TOKEN` | PAT with `contents:write` on PyDevices/micropython-lib | + +Grant both secrets to the **usdl2** repository (org secret repository access). + ## Release (local clone) ```bash @@ -50,6 +63,20 @@ git push origin main ./scripts/publish_release_tag.sh --push ``` +## Install + +```bash +# native +pip install -i https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple/ usdl2 + +# pure Python +pip install -i https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple/ usdl2-py +``` + +```python +mip.install("usdl2", index="https://PyDevices.github.io/micropython-lib/mip/PyDevices") +``` + ## Local wheel builds (cibuildwheel) ```bash @@ -69,9 +96,3 @@ echo "0.0.0.dev" > VERSION pipx run cibuildwheel --platform android ls wheelhouse/*android*.whl ``` - -## Install from TestPyPI - -```bash -pip install -i https://test.pypi.org/simple/ usdl2 -``` diff --git a/README.md b/README.md index e871dfd..462957d 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,19 @@ # usdl2 -Native **SDL2 subset** for Python (`import usdl2`) — CPython wheels (Linux, Windows, Android) plus MicroPython / CircuitPython user C modules. Public names are **SDL2 symbols only**. +Native and pure-Python **SDL2 subset** for Python (`import usdl2`) — CPython +wheels (Linux, Windows, Android), MicroPython / CircuitPython user C modules, +plus a ctypes/ffi fallback. Public names are **SDL2 symbols only**. -When this module is not linked or installed, [pydisplay](https://github.com/PyDevices/pydisplay) falls back to [`add_ons/usdl2.py`](https://github.com/PyDevices/pydisplay/blob/main/src/add_ons/usdl2.py). +| Product | Pip / MIP | Role | +|---------|-----------|------| +| **usdl2** | TestPyPI `usdl2` | Native C extension (prefer on desktop/Android when available) | +| **usdl2-py** | TestPyPI `usdl2-py`, MIP `usdl2` | Pure-Python package (same public API) | + +One release tag `vX.Y.Z` publishes both products at that version. See [PUBLISHING.md](PUBLISHING.md). ## Install -### CPython (TestPyPI) +### Native (TestPyPI) ```bash pip install \ @@ -17,6 +24,22 @@ pip install \ Requires a system or bundled **SDL2** shared library at runtime (`libSDL2.so` / `SDL2.dll`). Android APKs use wheels tagged `android_21_*` with the APK’s p4a SDL2 bootstrap — see [pydisplay_android](https://github.com/PyDevices/pydisplay_android). +### Pure Python (TestPyPI) + +```bash +pip install \ + -i https://test.pypi.org/simple/ \ + --extra-index-url https://pypi.org/simple/ \ + usdl2-py +``` + +### MicroPython (MIP) + +```python +import mip +mip.install("usdl2", index="https://PyDevices.github.io/micropython-lib/mip/PyDevices") +``` + ### Quick check ```bash @@ -53,6 +76,7 @@ usdl2/ src/usdl2_mp.c # MicroPython + CircuitPython src/usdl2_cpy.c # CPython Extension include/usdl2.h, usdl2_module_globals.inc, … + lib/usdl2.py # pure-Python fallback (usdl2-py) test_usdl2.py ``` diff --git a/lib/usdl2.py b/lib/usdl2.py new file mode 100644 index 0000000..e87c180 --- /dev/null +++ b/lib/usdl2.py @@ -0,0 +1,926 @@ +# SPDX-FileCopyrightText: 2024 Brad Barnett +# +# SPDX-License-Identifier: MIT +""" +Pure-Python ``usdl2`` fallback: an **SDL2 subset for Python**. + +Used when the native C ``usdl2`` module is unavailable. Must stay in lockstep +with the C sources in this repo (see ``include/usdl2_module_globals.inc`` and +``src/usdl2_cpy.c`` / ``src/usdl2_mp.c``). + +Hard rule — **SDL2 symbols only**: + Export only names that exist in SDL2 (``SDL_*`` functions/constants/macros + and binding constructors for SDL types: ``SDL_Rect``, ``SDL_Point``, + ``SDL_Event``, ``SDL_TimerCallback``, ``SDL_DEFINE_PIXELFORMAT``, …). + Do **not** invent helpers such as ``process_exit``, ``pump_scheduler``, a + bare ``Event`` type, or other non-SDL module attributes. Past agents added + those for timing/shutdown; put such logic in the *consumer* (e.g. + ``sdldisplay``, ``multimer``) instead. MP cooperative timer delivery may + ride inside real SDL entry points (e.g. ``SDL_PumpEvents``) as a private + implementation detail — never as a new public name. + +Public surface matches native usdl2: same ``SDL_*`` API, 56-byte ``SDL_Event`` +with the same subviews, ``SDL_Rect``/``SDL_Point`` as packed bytes, timer API +via ``SDL_TimerCallback`` / ``SDL_AddTimer`` / ``SDL_RemoveTimer``. Opaque +handles are plain ints (falsy when 0/NULL). + +Lives at ``lib/usdl2.py`` in this repo. Published as TestPyPI ``usdl2-py`` / +MIP ``usdl2`` (same release tag as the native ``usdl2`` wheels). Uses +**ctypes** on CPython (unix and win32); **ffi** on MicroPython unix. +""" + +import struct +import sys + +try: + from micropython import const +except ImportError: + + def const(x): + return x + + +############################################################################### +# SDL2 Constants # +############################################################################### + +# SDL_WindowPos values +SDL_WINDOWPOS_UNDEFINED = const(0x1FFF0000) +SDL_WINDOWPOS_CENTERED = const(0x2FFF0000) + +# SDL_Window flags +SDL_WINDOW_FULLSCREEN = const(0x00000001) +SDL_WINDOW_OPENGL = const(0x00000002) +SDL_WINDOW_SHOWN = const(0x00000004) +SDL_WINDOW_HIDDEN = const(0x00000008) +SDL_WINDOW_BORDERLESS = const(0x00000010) +SDL_WINDOW_RESIZABLE = const(0x00000020) +SDL_WINDOW_MINIMIZED = const(0x00000040) +SDL_WINDOW_MAXIMIZED = const(0x00000080) +SDL_WINDOW_INPUT_GRABBED = const(0x00000100) +SDL_WINDOW_INPUT_FOCUS = const(0x00000200) +SDL_WINDOW_MOUSE_FOCUS = const(0x00000400) +SDL_WINDOW_FULLSCREEN_DESKTOP = const(0x00001001) +SDL_WINDOW_ALLOW_HIGHDPI = const(0x00002000) +SDL_WINDOW_MOUSE_CAPTURE = const(0x00004000) +SDL_WINDOW_ALWAYS_ON_TOP = const(0x00008000) +SDL_WINDOW_SKIP_TASKBAR = const(0x00010000) +SDL_WINDOW_UTILITY = const(0x00020000) +SDL_WINDOW_TOOLTIP = const(0x00040000) +SDL_WINDOW_POPUP_MENU = const(0x00080000) +SDL_WINDOW_VULKAN = const(0x10000000) + +# SDL_Renderer flags +SDL_RENDERER_SOFTWARE = const(0x00000001) +SDL_RENDERER_ACCELERATED = const(0x00000002) +SDL_RENDERER_PRESENTVSYNC = const(0x00000004) +SDL_RENDERER_TARGETTEXTURE = const(0x00000008) + +# SDL_Init flags +SDL_INIT_TIMER = const(0x00000001) +SDL_INIT_AUDIO = const(0x00000010) +SDL_INIT_VIDEO = const(0x00000020) +SDL_INIT_JOYSTICK = const(0x00000200) +SDL_INIT_HAPTIC = const(0x00001000) +SDL_INIT_GAMECONTROLLER = const(0x00002000) +SDL_INIT_EVENTS = const(0x00004000) +SDL_INIT_EVERYTHING = const(0x0000000F) +SDL_INIT_NOPARACHUTE = const(0x00100000) + +# SDL_Texture values +SDL_TEXTUREACCESS_STATIC = const(0) +SDL_TEXTUREACCESS_STREAMING = const(1) +SDL_TEXTUREACCESS_TARGET = const(2) + +# SDL_BlendMode values +SDL_BLENDMODE_NONE = const(1) +SDL_BLENDMODE_BLEND = const(2) +SDL_BLENDMODE_ADD = const(3) +SDL_BLENDMODE_MOD = const(4) +SDL_BLENDMODE_MUL = const(5) + +# SDL_Event types (not complete) +SDL_QUIT = const(0x100) # User clicked the window close button +SDL_KEYDOWN = const(0x300) # Key pressed +SDL_KEYUP = const(0x301) # Key released +SDL_MOUSEMOTION = const(0x400) # Mouse moved +SDL_MOUSEBUTTONDOWN = const(0x401) # Mouse button pressed +SDL_MOUSEBUTTONUP = const(0x402) # Mouse button released +SDL_MOUSEWHEEL = const(0x403) # Mouse wheel motion +SDL_FINGERDOWN = const(0x700) # Finger touched +SDL_FINGERUP = const(0x701) # Finger lifted +SDL_FINGERMOTION = const(0x702) # Finger moved +SDL_JOYAXISMOTION = const(0x600) # Joystick axis motion +SDL_JOYBALLMOTION = const(0x601) # Joystick trackball motion +SDL_JOYHATMOTION = const(0x602) # Joystick hat position change +SDL_JOYBUTTONDOWN = const(0x603) # Joystick button pressed +SDL_JOYBUTTONUP = const(0x604) # Joystick button released +SDL_JOYDEVICEADDED = const(0x605) # A joystick was connected +SDL_JOYDEVICEREMOVED = const(0x606) # A joystick was disconnected +SDL_POLLSENTINEL = const(0x7F00) # Signals the end of an event poll cycle + +# SDL_MouseMotionEvent button masks +SDL_BUTTON_LMASK = const(1 << 0) # Left mouse button +SDL_BUTTON_MMASK = const(1 << 1) # Middle mouse button +SDL_BUTTON_RMASK = const(1 << 2) # Right mouse button + +# SDL_JoyHatEvent position masks +SDL_HAT_CENTERED = const(0x00) +SDL_HAT_UP = const(0x01) +SDL_HAT_RIGHT = const(0x02) +SDL_HAT_DOWN = const(0x04) +SDL_HAT_LEFT = const(0x08) + + +############################################################################### +# SDL2 Pixel Formats # +############################################################################### + + +def SDL_DEFINE_PIXELFORMAT(type, order, layout, bits, bytes): + """ + Define a pixel format. + """ + return ( + (1 << 28) + | ((type) << 24) + | ((order) << 20) + | ((layout) << 16) + | ((bits) << 8) + | ((bytes) << 0) + ) + + +# SDL_PIXELTYPE values +SDL_PIXELTYPE_UNKNOWN = const(0) +SDL_PIXELTYPE_INDEX1 = const(1) +SDL_PIXELTYPE_INDEX4 = const(2) +SDL_PIXELTYPE_INDEX8 = const(3) +SDL_PIXELTYPE_PACKED8 = const(4) +SDL_PIXELTYPE_PACKED16 = const(5) +SDL_PIXELTYPE_PACKED32 = const(6) +SDL_PIXELTYPE_ARRAYU8 = const(7) +SDL_PIXELTYPE_ARRAYU16 = const(8) +SDL_PIXELTYPE_ARRAYU32 = const(9) +SDL_PIXELTYPE_ARRAYF16 = const(10) +SDL_PIXELTYPE_ARRAYF32 = const(11) + +# SDL_PACKEDORDER values +SDL_PACKEDORDER_NONE = const(0) +SDL_PACKEDORDER_XRGB = const(1) +SDL_PACKEDORDER_RGBX = const(2) +SDL_PACKEDORDER_ARGB = const(3) +SDL_PACKEDORDER_RGBA = const(4) +SDL_PACKEDORDER_XBGR = const(5) +SDL_PACKEDORDER_BGRX = const(6) +SDL_PACKEDORDER_ABGR = const(7) +SDL_PACKEDORDER_BGRA = const(8) + +# SDL_ARRAYORDER values +SDL_ARRAYORDER_NONE = const(0) +SDL_ARRAYORDER_RGB = const(1) +SDL_ARRAYORDER_RGBA = const(2) +SDL_ARRAYORDER_ARGB = const(3) +SDL_ARRAYORDER_BGR = const(4) +SDL_ARRAYORDER_BGRA = const(5) +SDL_ARRAYORDER_ABGR = const(6) + +# SDL_PACKEDLAYOUT values +SDL_PACKEDLAYOUT_NONE = const(0) +SDL_PACKEDLAYOUT_332 = const(1) +SDL_PACKEDLAYOUT_4444 = const(2) +SDL_PACKEDLAYOUT_1555 = const(3) +SDL_PACKEDLAYOUT_5551 = const(4) +SDL_PACKEDLAYOUT_565 = const(5) +SDL_PACKEDLAYOUT_8888 = const(6) +SDL_PACKEDLAYOUT_2101010 = const(7) +SDL_PACKEDLAYOUT_1010102 = const(8) + +# SDL_BITMAPORDER values +SDL_BITMAPORDER_NONE = const(0) +SDL_BITMAPORDER_4321 = const(1) +SDL_BITMAPORDER_1234 = const(2) + +# SDL_PIXELFORMAT values +SDL_PIXELFORMAT_UNKNOWN = const(0) +SDL_PIXELFORMAT_INDEX1LSB = SDL_DEFINE_PIXELFORMAT( + SDL_PIXELTYPE_INDEX1, SDL_BITMAPORDER_4321, 0, 1, 0 +) +SDL_PIXELFORMAT_INDEX1MSB = SDL_DEFINE_PIXELFORMAT( + SDL_PIXELTYPE_INDEX1, SDL_BITMAPORDER_1234, 0, 1, 0 +) +SDL_PIXELFORMAT_INDEX4LSB = SDL_DEFINE_PIXELFORMAT( + SDL_PIXELTYPE_INDEX4, SDL_BITMAPORDER_4321, 0, 4, 0 +) +SDL_PIXELFORMAT_INDEX4MSB = SDL_DEFINE_PIXELFORMAT( + SDL_PIXELTYPE_INDEX4, SDL_BITMAPORDER_1234, 0, 4, 0 +) +SDL_PIXELFORMAT_INDEX8 = SDL_DEFINE_PIXELFORMAT(SDL_PIXELTYPE_INDEX8, 0, 0, 8, 1) +SDL_PIXELFORMAT_RGB332 = SDL_DEFINE_PIXELFORMAT( + SDL_PIXELTYPE_PACKED8, SDL_PACKEDORDER_XRGB, SDL_PACKEDLAYOUT_332, 8, 1 +) +SDL_PIXELFORMAT_XRGB4444 = SDL_DEFINE_PIXELFORMAT( + SDL_PIXELTYPE_PACKED16, SDL_PACKEDORDER_XRGB, SDL_PACKEDLAYOUT_4444, 12, 2 +) +SDL_PIXELFORMAT_RGB444 = SDL_PIXELFORMAT_XRGB4444 +SDL_PIXELFORMAT_XBGR4444 = SDL_DEFINE_PIXELFORMAT( + SDL_PIXELTYPE_PACKED16, SDL_PACKEDORDER_XBGR, SDL_PACKEDLAYOUT_4444, 12, 2 +) +SDL_PIXELFORMAT_BGR444 = SDL_PIXELFORMAT_XBGR4444 +SDL_PIXELFORMAT_XRGB1555 = SDL_DEFINE_PIXELFORMAT( + SDL_PIXELTYPE_PACKED16, SDL_PACKEDORDER_XRGB, SDL_PACKEDLAYOUT_1555, 15, 2 +) +SDL_PIXELFORMAT_RGB555 = SDL_PIXELFORMAT_XRGB1555 +SDL_PIXELFORMAT_XBGR1555 = SDL_DEFINE_PIXELFORMAT( + SDL_PIXELTYPE_PACKED16, SDL_PACKEDORDER_XBGR, SDL_PACKEDLAYOUT_1555, 15, 2 +) +SDL_PIXELFORMAT_BGR555 = SDL_PIXELFORMAT_XBGR1555 +SDL_PIXELFORMAT_ARGB4444 = SDL_DEFINE_PIXELFORMAT( + SDL_PIXELTYPE_PACKED16, SDL_PACKEDORDER_ARGB, SDL_PACKEDLAYOUT_4444, 16, 2 +) +SDL_PIXELFORMAT_RGBA4444 = SDL_DEFINE_PIXELFORMAT( + SDL_PIXELTYPE_PACKED16, SDL_PACKEDORDER_RGBA, SDL_PACKEDLAYOUT_4444, 16, 2 +) +SDL_PIXELFORMAT_ABGR4444 = SDL_DEFINE_PIXELFORMAT( + SDL_PIXELTYPE_PACKED16, SDL_PACKEDORDER_ABGR, SDL_PACKEDLAYOUT_4444, 16, 2 +) +SDL_PIXELFORMAT_BGRA4444 = SDL_DEFINE_PIXELFORMAT( + SDL_PIXELTYPE_PACKED16, SDL_PACKEDORDER_BGRA, SDL_PACKEDLAYOUT_4444, 16, 2 +) +SDL_PIXELFORMAT_ARGB1555 = SDL_DEFINE_PIXELFORMAT( + SDL_PIXELTYPE_PACKED16, SDL_PACKEDORDER_ARGB, SDL_PACKEDLAYOUT_1555, 16, 2 +) +SDL_PIXELFORMAT_RGBA5551 = SDL_DEFINE_PIXELFORMAT( + SDL_PIXELTYPE_PACKED16, SDL_PACKEDORDER_RGBA, SDL_PACKEDLAYOUT_5551, 16, 2 +) +SDL_PIXELFORMAT_ABGR1555 = SDL_DEFINE_PIXELFORMAT( + SDL_PIXELTYPE_PACKED16, SDL_PACKEDORDER_ABGR, SDL_PACKEDLAYOUT_1555, 16, 2 +) +SDL_PIXELFORMAT_BGRA5551 = SDL_DEFINE_PIXELFORMAT( + SDL_PIXELTYPE_PACKED16, SDL_PACKEDORDER_BGRA, SDL_PACKEDLAYOUT_5551, 16, 2 +) +SDL_PIXELFORMAT_RGB565 = SDL_DEFINE_PIXELFORMAT( + SDL_PIXELTYPE_PACKED16, SDL_PACKEDORDER_XRGB, SDL_PACKEDLAYOUT_565, 16, 2 +) +SDL_PIXELFORMAT_BGR565 = SDL_DEFINE_PIXELFORMAT( + SDL_PIXELTYPE_PACKED16, SDL_PACKEDORDER_XBGR, SDL_PACKEDLAYOUT_565, 16, 2 +) +SDL_PIXELFORMAT_RGB24 = SDL_DEFINE_PIXELFORMAT(SDL_PIXELTYPE_ARRAYU8, SDL_ARRAYORDER_RGB, 0, 24, 3) +SDL_PIXELFORMAT_BGR24 = SDL_DEFINE_PIXELFORMAT(SDL_PIXELTYPE_ARRAYU8, SDL_ARRAYORDER_BGR, 0, 24, 3) +SDL_PIXELFORMAT_XRGB8888 = SDL_DEFINE_PIXELFORMAT( + SDL_PIXELTYPE_PACKED32, SDL_PACKEDORDER_XRGB, SDL_PACKEDLAYOUT_8888, 24, 4 +) +SDL_PIXELFORMAT_RGB888 = SDL_PIXELFORMAT_XRGB8888 +SDL_PIXELFORMAT_RGBX8888 = SDL_DEFINE_PIXELFORMAT( + SDL_PIXELTYPE_PACKED32, SDL_PACKEDORDER_RGBX, SDL_PACKEDLAYOUT_8888, 24, 4 +) +SDL_PIXELFORMAT_XBGR8888 = SDL_DEFINE_PIXELFORMAT( + SDL_PIXELTYPE_PACKED32, SDL_PACKEDORDER_XBGR, SDL_PACKEDLAYOUT_8888, 24, 4 +) +SDL_PIXELFORMAT_BGR888 = SDL_PIXELFORMAT_XBGR8888 +SDL_PIXELFORMAT_BGRX8888 = SDL_DEFINE_PIXELFORMAT( + SDL_PIXELTYPE_PACKED32, SDL_PACKEDORDER_BGRX, SDL_PACKEDLAYOUT_8888, 24, 4 +) +SDL_PIXELFORMAT_ARGB8888 = SDL_DEFINE_PIXELFORMAT( + SDL_PIXELTYPE_PACKED32, SDL_PACKEDORDER_ARGB, SDL_PACKEDLAYOUT_8888, 32, 4 +) +SDL_PIXELFORMAT_RGBA8888 = SDL_DEFINE_PIXELFORMAT( + SDL_PIXELTYPE_PACKED32, SDL_PACKEDORDER_RGBA, SDL_PACKEDLAYOUT_8888, 32, 4 +) +SDL_PIXELFORMAT_ABGR8888 = SDL_DEFINE_PIXELFORMAT( + SDL_PIXELTYPE_PACKED32, SDL_PACKEDORDER_ABGR, SDL_PACKEDLAYOUT_8888, 32, 4 +) +SDL_PIXELFORMAT_BGRA8888 = SDL_DEFINE_PIXELFORMAT( + SDL_PIXELTYPE_PACKED32, SDL_PACKEDORDER_BGRA, SDL_PACKEDLAYOUT_8888, 32, 4 +) +SDL_PIXELFORMAT_ARGB2101010 = SDL_DEFINE_PIXELFORMAT( + SDL_PIXELTYPE_PACKED32, SDL_PACKEDORDER_ARGB, SDL_PACKEDLAYOUT_2101010, 32, 4 +) + + +############################################################################### +# SDL_Rect / SDL_Point # +############################################################################### + +# Packed as bytes (not a ctypes.Structure / uctypes struct) so the same value +# is usable, unmodified, on every backend -- matches py_SDL_Rect()/py_SDL_Point() +# in usdl2_cpy.c, which pack a bytes object directly. + + +def SDL_Rect(x=0, y=0, w=0, h=0): + return struct.pack(" a new zero-filled event. + * ``SDL_Event()`` -> a new event, copied from the buffer. + * ``SDL_Event()`` -> the same instance (identity). + """ + + def __new__(cls, event=None): + if isinstance(event, cls): + return event + self = super().__new__(cls) + if event is None: + self._data = bytearray(_EVENT_SIZE) + else: + buf = memoryview(event) + if len(buf) < _EVENT_SIZE: + raise ValueError("event buffer too small") + self._data = bytearray(buf[:_EVENT_SIZE]) + return self + + def __len__(self): + return _EVENT_SIZE + + @property + def type(self): + return _u32(self._data, 0) + + @type.setter + def type(self, value): + struct.pack_into(" "SDL_PollEvent" / "SDL_GetError".""" + if name.startswith("_lib_") or name.startswith("_raw_"): + return name[5:] + return name + + +def _bind_ffi(lib, specs): + for name, ret, args in specs: + globals()[name] = lib.func(ret, _libsym(name), args) + + +def _bind_ctypes(lib, specs): + for name, restype, argtypes in specs: + fn = getattr(lib, _libsym(name)) + fn.restype = restype + fn.argtypes = list(argtypes) + globals()[name] = fn + + +if _use_ffi: + _libSDL2 = ffi.open("libSDL2-2.0.so.0") + _bind_ffi(_libSDL2, _FFI_FUNCS) + _raw_SDL_GetError = globals()["_raw_SDL_GetError"] + _raw_SDL_GetKeyName = globals()["_raw_SDL_GetKeyName"] + + def _wrap_buf(buf): + return buf + + # modffi's "s" return type yields None for a NULL C string; SDL never + # actually returns NULL for either of these, but fall back to "" to + # match usdl2_cpy.c's PyUnicode_FromString(err ? err : "") exactly. + def SDL_GetError(): + err = _raw_SDL_GetError() + return err if err is not None else "" + + def SDL_GetKeyName(sym): + name = _raw_SDL_GetKeyName(sym) + return name if name is not None else "" + +else: + import ctypes + + if sys.platform == "win32": + _libSDL2 = ctypes.CDLL("SDL2.dll") + else: + _libSDL2 = ctypes.CDLL("libSDL2-2.0.so.0") + + _c = ctypes + _v = _c.c_void_p + _i = _c.c_int + _u = _c.c_uint + _d = _c.c_double + + # Rect/Point/Event/DisplayMode arguments are all c_void_p: SDL_Rect()/ + # SDL_Point() return plain bytes (accepted directly by ctypes for a + # c_void_p argument), and _wrap_buf() below adapts writable bytearrays + # (SDL_Event, out-params) the same way usdl2_cpy.c's PyObject_GetBuffer() + # accepts any bytes-like object. + _CTYPES_FUNCS = ( + ("SDL_Init", _i, (_u,)), + ("SDL_InitSubSystem", _i, (_u,)), + ("SDL_Quit", None, ()), + ("_raw_SDL_GetError", _c.c_char_p, ()), + ("_raw_SDL_CreateWindow", _v, (_c.c_char_p, _i, _i, _i, _i, _u)), + ("SDL_DestroyWindow", None, (_v,)), + ("SDL_SetWindowSize", None, (_v, _i, _i)), + ("SDL_SetWindowResizable", None, (_v, _i)), + ("SDL_SetWindowMinimumSize", None, (_v, _i, _i)), + ("SDL_SetWindowMaximumSize", None, (_v, _i, _i)), + ("SDL_CreateRenderer", _v, (_v, _i, _u)), + ("SDL_DestroyRenderer", None, (_v,)), + ("SDL_SetRenderDrawColor", _i, (_v, _u, _u, _u, _u)), + ("SDL_SetRenderTarget", _i, (_v, _v)), + ("SDL_RenderClear", _i, (_v,)), + ("SDL_RenderCopy", _i, (_v, _v, _v, _v)), + ("SDL_RenderCopyEx", _i, (_v, _v, _v, _v, _d, _v, _i)), + ("SDL_RenderPresent", None, (_v,)), + ("SDL_RenderFillRect", _i, (_v, _v)), + ("SDL_RenderSetLogicalSize", _i, (_v, _i, _i)), + ("SDL_CreateTexture", _v, (_v, _u, _i, _i, _i)), + ("SDL_DestroyTexture", None, (_v,)), + ("SDL_SetTextureBlendMode", _i, (_v, _i)), + ("SDL_NumJoysticks", _i, ()), + ("SDL_JoystickOpen", _v, (_i,)), + ("SDL_JoystickClose", None, (_v,)), + ("SDL_JoystickInstanceID", _i, (_v,)), + ("_raw_SDL_GetKeyName", _c.c_char_p, (_i,)), + ("_lib_SDL_PumpEvents", None, ()), + ("_lib_SDL_PollEvent", _i, (_v,)), + ("_lib_SDL_UpdateTexture", _i, (_v, _v, _v, _i)), + ("_lib_SDL_GetDisplayUsableBounds", _i, (_i, _v)), + ("_lib_SDL_GetDesktopDisplayMode", _i, (_i, _v)), + # SDL_AddTimer/SDL_RemoveTimer are bound separately below (need the + # timer trampoline's CFUNCTYPE to exist first). + ) + _bind_ctypes(_libSDL2, _CTYPES_FUNCS) + _raw_SDL_GetError = globals()["_raw_SDL_GetError"] + _raw_SDL_GetKeyName = globals()["_raw_SDL_GetKeyName"] + _raw_SDL_CreateWindow = globals()["_raw_SDL_CreateWindow"] + + def _wrap_buf(buf): + """Adapt a bytes-like object for a ctypes c_void_p argument. + + Immutable ``bytes`` (e.g. from SDL_Rect()) already convert directly; + anything else supporting the buffer protocol (bytearray, memoryview, + array.array, ...) is wrapped with from_buffer() so the callee can + read/write through the same memory in place, matching usdl2_cpy.c's + PyObject_GetBuffer() flexibility. + """ + if buf is None or isinstance(buf, (bytes, int, ctypes.Array)): + return buf + return (ctypes.c_char * len(buf)).from_buffer(buf) + + # SDL_GetError()/SDL_GetKeyName() use c_char_p (not modelled as "P" in the + # ffi table above) so ctypes decodes the returned C string for us; wrap + # them to fall back to "" for NULL, matching usdl2_cpy.c. + def SDL_GetError(): + err = _raw_SDL_GetError() + return err.decode("utf-8") if err else "" + + def SDL_GetKeyName(sym): + name = _raw_SDL_GetKeyName(sym) + return name.decode("utf-8") if name else "" + + def SDL_CreateWindow(title, x, y, w, h, flags): + if isinstance(title, str): + title = title.encode("utf-8") + return _raw_SDL_CreateWindow(title, x, y, w, h, flags) + + +# _bind_ffi / _bind_ctypes assign these via globals()[name]; materialize so +# static analysis and the buffer wrappers below see real module bindings. +_lib_SDL_PumpEvents = globals()["_lib_SDL_PumpEvents"] +_lib_SDL_PollEvent = globals()["_lib_SDL_PollEvent"] +_lib_SDL_UpdateTexture = globals()["_lib_SDL_UpdateTexture"] +_lib_SDL_GetDisplayUsableBounds = globals()["_lib_SDL_GetDisplayUsableBounds"] +_lib_SDL_GetDesktopDisplayMode = globals()["_lib_SDL_GetDesktopDisplayMode"] + + +############################################################################### +# Timer API # +############################################################################### + + +class _TimerCallback: + """Opaque token returned by SDL_TimerCallback(); only usable via SDL_AddTimer().""" + + __slots__ = ("callback",) + + def __init__(self, callback): + if not callable(callback): + raise TypeError("callback must be callable") + self.callback = callback + + +def SDL_TimerCallback(callback): + return _TimerCallback(callback) + + +if _use_ffi: + # Real SDL timers fire on an SDL-owned pthread that MicroPython's runtime + # never registered (mp_thread_init() was never called for it); invoking + # any Python callback from that thread segfaults unconditionally -- + # verified experimentally, even a trivial ffi.callback(..., lock=True) + # trampoline crashes the moment SDL's timer thread calls it. So on + # MicroPython, timers are cooperative/software instead of real SDL ones: + # SDL_AddTimer() just records a deadline, and SDL_PumpEvents()/ + # SDL_PollEvent() -- already polled regularly by pydisplay's event loop, + # on a safe/registered thread -- fire any due callbacks in-line. + import time + + _sw_timers = {} + _sw_timer_next_id = [1] + + def _sw_timers_poll(): + if not _sw_timers: + return + now = time.ticks_ms() + for timer_id, entry in list(_sw_timers.items()): + deadline, interval, callback, user_param = entry + if time.ticks_diff(now, deadline) < 0: + continue + if timer_id not in _sw_timers: + continue + _sw_timers[timer_id] = ( + time.ticks_add(deadline, interval), + interval, + callback, + user_param, + ) + try: + callback(interval, user_param) + except Exception: + pass + + def SDL_AddTimer(interval, tcb, user_param): + if not isinstance(tcb, _TimerCallback): + raise TypeError("callback must be from SDL_TimerCallback()") + timer_id = _sw_timer_next_id[0] + _sw_timer_next_id[0] += 1 + _sw_timers[timer_id] = ( + time.ticks_add(time.ticks_ms(), interval), + interval, + tcb.callback, + user_param, + ) + return timer_id + + def SDL_RemoveTimer(timer): + return 1 if _sw_timers.pop(timer, None) is not None else 0 + +else: + _TIMER_MAX = const(8) + _timer_slots = [None] * _TIMER_MAX # (callback, user_param, ret_interval) or None + _timer_id_to_slot = {} + + def _sw_timers_poll(): + pass # ctypes timers run for real via SDL's own thread; nothing to poll. + + def _timer_trampoline(interval, slot): + """ + Runs on an SDL-owned thread, but ctypes callbacks always re-acquire the + GIL for us (PyGILState_Ensure()/Release() around every callback + invocation), so this is safe unlike the MicroPython ffi case above. + Looks the entry up through the slot table (not a raw pointer) so a + concurrent SDL_RemoveTimer() cannot use freed state; mirrors + timer_trampoline() in usdl2_cpy.c. The next interval is always the one + the timer was created with (not the callback's return value). + """ + entry = _timer_slots[slot] if 0 <= slot < _TIMER_MAX else None + if entry is None: + return 0 + callback, user_param, ret_interval = entry + try: + callback(interval, user_param) + except Exception: + pass + return ret_interval + + _sdl_timer_functype = ctypes.CFUNCTYPE(ctypes.c_uint32, ctypes.c_uint32, ctypes.c_size_t) + _sdl_timer_cfunc = _sdl_timer_functype(_timer_trampoline) + _lib_SDL_AddTimer = _libSDL2.SDL_AddTimer + _lib_SDL_AddTimer.restype = ctypes.c_int + _lib_SDL_AddTimer.argtypes = (ctypes.c_uint32, _sdl_timer_functype, ctypes.c_size_t) + _lib_SDL_RemoveTimer = _libSDL2.SDL_RemoveTimer + _lib_SDL_RemoveTimer.restype = ctypes.c_int + _lib_SDL_RemoveTimer.argtypes = (ctypes.c_int,) + + def SDL_AddTimer(interval, tcb, user_param): + if not isinstance(tcb, _TimerCallback): + raise TypeError("callback must be from SDL_TimerCallback()") + slot = -1 + for i in range(_TIMER_MAX): + if _timer_slots[i] is None: + slot = i + break + if slot < 0: + raise RuntimeError("too many SDL timers") + _timer_slots[slot] = (tcb.callback, user_param, interval) + timer_id = _lib_SDL_AddTimer(interval, _sdl_timer_cfunc, slot) + if not timer_id: + _timer_slots[slot] = None + return 0 + _timer_id_to_slot[timer_id] = slot + return timer_id + + def SDL_RemoveTimer(timer): + slot = _timer_id_to_slot.pop(timer, None) + if slot is not None: + _timer_slots[slot] = None + return 1 if _lib_SDL_RemoveTimer(timer) else 0 + + +############################################################################### +# Buffer-taking wrappers # +############################################################################### + +# Defined once, backend-independent: _wrap_buf() and the private _lib_SDL_* +# names above are already backend-specific; the logic here mirrors +# usdl2_cpy.c's Python-facing signatures exactly. SDL_PumpEvents()/ +# SDL_PollEvent() also drive the MicroPython software-timer poll above (a +# no-op on the ctypes backend, where real SDL timers do the work). + + +def SDL_PumpEvents(): + _lib_SDL_PumpEvents() + _sw_timers_poll() + + +def SDL_PollEvent(event): + _sw_timers_poll() + data = event._data if isinstance(event, SDL_Event) else event + return bool(_lib_SDL_PollEvent(_wrap_buf(data))) + + +def SDL_UpdateTexture(texture, rect, pixels, pitch): + return _lib_SDL_UpdateTexture(texture, _wrap_buf(rect), _wrap_buf(pixels), pitch) + + +def SDL_GetDisplayUsableBounds(display_index, rect=None): + return _lib_SDL_GetDisplayUsableBounds(display_index, _wrap_buf(rect)) + + +def SDL_GetDesktopDisplayMode(display_index, mode=None): + return _lib_SDL_GetDesktopDisplayMode(display_index, _wrap_buf(mode)) diff --git a/scripts/build.py b/scripts/build.py new file mode 100755 index 0000000..1a489e4 --- /dev/null +++ b/scripts/build.py @@ -0,0 +1,504 @@ +#!/usr/bin/env python3 +# +# This file is part of the MicroPython project, http://micropython.org/ +# +# The MIT License (MIT) +# +# Copyright (c) 2022 Jim Mussared +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. + +# pydisplay — upstream micropython-lib ``tools/build.py`` (MIP index compiler). +# Renamed from ``scripts/publish_mip_index.py``; kept as ``build.py`` so +# ``publish_make_pyproject.py`` can ``from build import ensure_path_exists, error_color``. +# +# Used in this repo: +# - ``scripts/publish_mip_ghpages.sh`` — build ``mip/PyDevices`` and push gh-pages +# - ``.github/workflows/publish-micropython-lib.yml`` — calls publish_mip_ghpages.sh +# - ``scripts/publish_make_pyproject.py`` — shared ``ensure_path_exists`` / ``error_color`` +# +# Direct use (micropython-lib checkout required as ``--lib-dir``): +# ./scripts/build.py --lib-dir ../micropython-lib \\ +# --micropython /tmp/micropython \\ +# --mpy-cross /tmp/micropython/mpy-cross/build/mpy-cross \\ +# --output /tmp/mip-out + +# This script compiles all packages in this repository (excluding unix-ffi) +# into a directory suitable for serving to "mip" via a static web server. + +# Usage: +# ./scripts/build.py --output /tmp/micropython-lib/v2 --lib-dir ../micropython-lib + +# The output directory (--output) will have the following layout +# / +# index.json +# file/ +# 1d/ +# 1dddc25d +# c3/ +# c31d7eb7 +# c3a3934b +# e3/ +# e39dbf64 +# ... +# package/ +# 6/ <-- mpy version +# aioble/ +# latest.json +# 0.1.json +# ... +# hmac/ +# latest.json +# 3.4.2-3.json +# ... +# pyjwt/ +# latest.json +# 0.1.json +# ... +# 7/ <-- other mpy versions +# ... +# py/ <-- "source" distribution +# ... +# ... + +# index.json is: +# { +# "v": 2, <-- file format version +# "updated": , +# "packages": { +# { +# "name": "aioble", +# "version": "0.1", <-- Latest version of this package (always present, may be empty). +# "author": "", <-- Optional author (always present, may be empty). +# "description": "...", <-- Optional description (always present, may be empty). +# "license": "MIT", <-- SPDX short identifier (required). +# "versions": { +# "6": ["0.1", "0.2"], +# "7": ["0.2", "0.3", "0.4"], +# ... <-- Other bytecode versions +# "py": ["0.1", "0.2", "0.3", "0.4"] +# }, +# // The following entries were added in file format version 2. +# path: "micropython/bluetooth/aioble", +# }, +# ... +# } +# } + +# Each file in the "file" directory is the file contents (usually .mpy), named +# by the prefix of the sha256 hash of the contents. Files are never removed, and +# collisions are detected and will fail the compile, and the prefix length should +# be increased. +# As of September 2022, there are no collisions with a hash prefix length of 4, +# so the default of 8 should be sufficient for a while. Increasing the length +# doesn't invalidate old packages. + +# Each package json (either latest.json or {version}.json) is: +# { +# "v": 1, <-- file format version +# "hashes": [ +# ["aioble/server.mpy", "e39dbf64"], +# ... +# ], +# "urls": [ <-- not used by micropython-lib packages +# ["target/path.py", "http://url/foo/bar/path.py"], +# ... +# ], +# "deps": [ <-- not used by micropython-lib packages +# ["name", "version"], +# ... +# ] +# "version": "0.1" +# } + +# mip (or other tools) should request /package/{mpy_version}/{package_name}/{version}.json. + +import glob +import hashlib +import json +import os +import shutil +import sys +import tempfile +import time + +_JSON_VERSION_INDEX = 2 +_JSON_VERSION_PACKAGE = 1 + + +_COLOR_ERROR_ON = "\033[1;31m" +_COLOR_ERROR_OFF = "\033[0m" + + +# Create all directories in the path (such that the file can be created). +def ensure_path_exists(file_path): + path = os.path.dirname(file_path) + if not os.path.isdir(path): + os.makedirs(path) + + +# Returns the sha256 of the specified file object. +def _get_file_hash(f): + hs256 = hashlib.sha256() + hs256.update(f.read()) + return hs256.hexdigest() + + +# Returns true if the two files contain identical contents. +def _identical_files(path_a, path_b): + with open(path_a, "rb") as fa, open(path_b, "rb") as fb: + return fa.read() == fb.read() + + +# Helper to write the object as json to the specified path, creating any +# directories as required. +def _write_json(obj, path, minify=False): + ensure_path_exists(path) + with open(path, "w") as f: + json.dump( + obj, f, indent=(None if minify else 2), separators=((",", ":") if minify else None) + ) + f.write("\n") + + +# Write the package json to package/{"py" or mpy_version}/{package}/{version}.json. +def _write_package_json( + package_json, out_package_dir, mpy_version, package_name, version, replace +): + path = os.path.join(out_package_dir, mpy_version, package_name, version + ".json") + if replace or not os.path.exists(path): + _write_json(package_json, path, minify=True) + + +# Format s with bold red. +def error_color(s): + return _COLOR_ERROR_ON + s + _COLOR_ERROR_OFF + + +# Copy src to "file"/{short_hash[0:2]}/{short_hash}. +def _write_hashed_file(package_name, src, target_path, out_file_dir, hash_prefix_len): + # Generate the full sha256 and the hash prefix to use as the output path. + file_hash = _get_file_hash(src) + short_file_hash = file_hash[:hash_prefix_len] + # Group files into subdirectories using the first two bytes of the hash prefix. + output_file = os.path.join(short_file_hash[:2], short_file_hash) + output_file_path = os.path.join(out_file_dir, output_file) + + if os.path.exists(output_file_path): + # If the file exists (e.g. from a previous run of this script), then ensure + # that it's actually the same file. + if not _identical_files(src.name, output_file_path): + print( + error_color("Hash collision processing:"), + package_name, + file=sys.stderr, + ) + print(" File: ", target_path, file=sys.stderr) + print(" Short hash: ", short_file_hash, file=sys.stderr) + print(" Full hash: ", file_hash, file=sys.stderr) + with open(output_file_path, "rb") as f: + print(" Target hash: ", _get_file_hash(f), file=sys.stderr) + print("Try increasing --hash-prefix (currently {})".format(hash_prefix_len)) + sys.exit(1) + else: + # Create new file. + ensure_path_exists(output_file_path) + shutil.copyfile(src.name, output_file_path) + + return short_file_hash + + +# Convert the tagged .py file into a .mpy file and copy to the "file" output +# directory with it's hashed name. Updates the package_json with the file +# hash. +def _compile_as_mpy( + package_name, + package_json, + tagged_path, + target_path, + opt, + mpy_cross, + mpy_cross_path, + out_file_dir, + hash_prefix_len, +): + with tempfile.NamedTemporaryFile(mode="rb", suffix=".mpy", delete=True) as mpy_tempfile: + try: + mpy_cross.compile( + tagged_path, + dest=mpy_tempfile.name, + src_path=target_path, + opt=opt, + mpy_cross=mpy_cross_path, + ) + except mpy_cross.CrossCompileError as e: + print( + error_color("Error:"), + "Unable to compile", + target_path, + "in package", + package_name, + file=sys.stderr, + ) + print(e) + sys.exit(1) + + short_mpy_hash = _write_hashed_file( + package_name, mpy_tempfile, target_path, out_file_dir, hash_prefix_len + ) + + # Add the file to the package json. + target_path_mpy = target_path[:-2] + "mpy" + package_json["hashes"].append((target_path_mpy, short_mpy_hash)) + + +# Copy the tagged .py file to the "file" output directory with it's hashed +# name. Updates the package_json with the file hash. +def _copy_as_py( + package_name, package_json, tagged_path, target_path, out_file_dir, hash_prefix_len +): + with open(tagged_path, "rb") as tagged_file: + short_py_hash = _write_hashed_file( + package_name, tagged_file, target_path, out_file_dir, hash_prefix_len + ) + # Add the file to the package json. + package_json["hashes"].append((target_path, short_py_hash)) + + +# Update to the latest metadata, and add any new versions to the package in +# the index json. +def _update_index_package_metadata(index_package_json, metadata, mpy_version, package_path): + index_package_json["version"] = metadata.version or "" + index_package_json["author"] = "" # TODO: Make manifestfile.py capture this. + index_package_json["description"] = metadata.description or "" + index_package_json["license"] = metadata.license or "MIT" + if "versions" not in index_package_json: + index_package_json["versions"] = {} + if metadata.version: + for v in ("py", mpy_version): + if v not in index_package_json["versions"]: + index_package_json["versions"][v] = [] + if metadata.version not in index_package_json["versions"][v]: + print(" New version {}={}".format(v, metadata.version)) + index_package_json["versions"][v].append(metadata.version) + + # The following entries were added in file format version 2. + index_package_json["path"] = package_path + + +def build(output_path, hash_prefix_len, mpy_cross_path, lib_dir): + import manifestfile + import mpy_cross + + out_file_dir = os.path.join(output_path, "file") + out_package_dir = os.path.join(output_path, "package") + + path_vars = { + "MPY_LIB_DIR": lib_dir, + } + + index_json_path = os.path.join(output_path, "index.json") + + try: + with open(index_json_path) as f: + print("Updating existing index.json") + index_json = json.load(f) + except FileNotFoundError: + print("Creating new index.json") + index_json = {"packages": []} + + index_json["v"] = _JSON_VERSION_INDEX + index_json["updated"] = int(time.time()) + + # For now, don't process unix-ffi. In the future this can be extended to + # allow a way to request unix-ffi packages via mip. + lib_dirs = ["micropython", "python-stdlib", "python-ecosys"] + + mpy_version, _mpy_sub_version = mpy_cross.mpy_version(mpy_cross=mpy_cross_path) + mpy_version = str(mpy_version) + print("Generating bytecode version", mpy_version) + + prev_cwd = os.getcwd() + os.chdir(lib_dir) + try: + for lib_dir_name in lib_dirs: + for manifest_path in glob.glob( + os.path.join(lib_dir_name, "**", "manifest.py"), recursive=True + ): + package_path = os.path.dirname(manifest_path) + print("{}".format(package_path)) + # .../foo/manifest.py -> foo + package_name = os.path.basename(os.path.dirname(manifest_path)) + + # Compile the manifest. + manifest = manifestfile.ManifestFile(manifestfile.MODE_COMPILE, path_vars) + manifest.execute(manifest_path) + + # Append this package to the index. + if not manifest.metadata().version: + print(error_color("Warning:"), package_name, "doesn't have a version.") + + # Try to find this package in the previous index.json. + for p in index_json["packages"]: + if p["name"] == package_name: + index_package_json = p + break + else: + print(" First-time package") + index_package_json = { + "name": package_name, + } + index_json["packages"].append(index_package_json) + + _update_index_package_metadata( + index_package_json, manifest.metadata(), mpy_version, package_path + ) + + # This is the package json that mip/mpremote downloads. + mpy_package_json = { + "v": _JSON_VERSION_PACKAGE, + "hashes": [], + "version": manifest.metadata().version or "", + } + py_package_json = { + "v": _JSON_VERSION_PACKAGE, + "hashes": [], + "version": manifest.metadata().version or "", + } + + for result in manifest.files(): + # This isn't allowed in micropython-lib anyway. + if result.file_type != manifestfile.FILE_TYPE_LOCAL: + print( + error_color("Error:"), "Non-local file not supported.", file=sys.stderr + ) + sys.exit(1) + + if not result.target_path.endswith(".py"): + print( + error_color("Error:"), + "Target path isn't a .py file:", + result.target_path, + file=sys.stderr, + ) + sys.exit(1) + + # Tag each file with the package metadata and compile to .mpy + # (and copy the .py directly). + with manifestfile.tagged_py_file( + result.full_path, result.metadata + ) as tagged_path: + _compile_as_mpy( + package_name, + mpy_package_json, + tagged_path, + result.target_path, + result.opt, + mpy_cross, + mpy_cross_path, + out_file_dir, + hash_prefix_len, + ) + _copy_as_py( + package_name, + py_package_json, + tagged_path, + result.target_path, + out_file_dir, + hash_prefix_len, + ) + + # Create/replace {package}/latest.json. + _write_package_json( + mpy_package_json, + out_package_dir, + mpy_version, + package_name, + "latest", + replace=True, + ) + _write_package_json( + py_package_json, out_package_dir, "py", package_name, "latest", replace=True + ) + + # Write {package}/{version}.json, but only if it doesn't already + # exist. A package version is "locked" the first time it's seen + # by this script. + if manifest.metadata().version: + _write_package_json( + mpy_package_json, + out_package_dir, + mpy_version, + package_name, + manifest.metadata().version, + replace=False, + ) + _write_package_json( + py_package_json, + out_package_dir, + "py", + package_name, + manifest.metadata().version, + replace=False, + ) + + finally: + os.chdir(prev_cwd) + + # Write updated package index json, sorted by package name. + index_json["packages"].sort(key=lambda p: p["name"]) + _write_json(index_json, index_json_path, minify=False) + + +def main(): + import argparse + + cmd_parser = argparse.ArgumentParser(description="Compile micropython-lib for serving to mip.") + cmd_parser.add_argument("--output", required=True, help="output directory") + cmd_parser.add_argument("--hash-prefix", default=8, type=int, help="hash prefix length") + cmd_parser.add_argument("--mpy-cross", default=None, help="optional path to mpy-cross binary") + cmd_parser.add_argument("--micropython", default=None, help="path to micropython repo") + cmd_parser.add_argument( + "--lib-dir", + default=None, + help="micropython-lib repo root (scan cwd); default: pydisplay repo root", + ) + args = cmd_parser.parse_args() + + _scripts_dir = os.path.dirname(os.path.abspath(__file__)) + sys.path.insert(0, _scripts_dir) + + if args.micropython: + sys.path.append(os.path.join(args.micropython, "tools")) # for manifestfile + sys.path.append(os.path.join(args.micropython, "mpy-cross")) # for mpy_cross + + lib_dir = os.path.abspath( + args.lib_dir or os.path.join(os.path.dirname(__file__), ".."), + ) + + build( + args.output, + hash_prefix_len=max(4, args.hash_prefix), + mpy_cross_path=args.mpy_cross, + lib_dir=lib_dir, + ) + + +if __name__ == "__main__": + main() diff --git a/scripts/manifestfile.py b/scripts/manifestfile.py new file mode 100755 index 0000000..ece8305 --- /dev/null +++ b/scripts/manifestfile.py @@ -0,0 +1,668 @@ +#!/usr/bin/env python3 +# +# This file is part of the MicroPython project, http://micropython.org/ +# +# The MIT License (MIT) +# +# Copyright (c) 2022 Jim Mussared +# Copyright (c) 2019 Damien P. George +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. + +from __future__ import print_function + +import os +import sys +import tempfile +from collections import namedtuple + +__all__ = ["ManifestFile", "ManifestFileError"] + +# Allow freeze*() etc. +MODE_FREEZE = 1 +# Only allow include/require/module/package. +MODE_COMPILE = 2 +# Same as compile, but handles require(..., pypi="name") as a requirements.txt entry. +MODE_PYPROJECT = 3 + +# In compile mode, .py -> KIND_COMPILE_AS_MPY +# In freeze mode, .py -> KIND_FREEZE_AS_MPY, .mpy->KIND_FREEZE_MPY +KIND_AUTO = 1 +# Freeze-mode only, .py -> KIND_FREEZE_AS_MPY, .mpy->KIND_FREEZE_MPY +KIND_FREEZE_AUTO = 2 + +# Freeze-mode only, The .py file will be frozen as text. +KIND_FREEZE_AS_STR = 3 +# Freeze-mode only, The .py file will be compiled and frozen as bytecode. +KIND_FREEZE_AS_MPY = 4 +# Freeze-mode only, The .mpy file will be frozen directly. +KIND_FREEZE_MPY = 5 +# Compile mode only, the .py file should be compiled to .mpy. +KIND_COMPILE_AS_MPY = 6 + +# File on the local filesystem. +FILE_TYPE_LOCAL = 1 +# URL to file. (TODO) +FILE_TYPE_HTTP = 2 + +# Default list of libraries in micropython-lib to search for library packages. +BASE_LIBRARY_NAMES = ("micropython", "python-stdlib", "python-ecosys") + + +class ManifestFileError(Exception): + pass + + +class ManifestIgnoreException(Exception): + pass + + +class ManifestUsePyPIException(Exception): + def __init__(self, pypi_name): + self.pypi_name = pypi_name + + +# The set of files that this manifest references. +ManifestOutput = namedtuple( # noqa: PYI024 + "ManifestOutput", + [ + "file_type", # FILE_TYPE_*. + "full_path", # The input file full path. + "target_path", # The target path on the device. + "timestamp", # Last modified date of the input file. + "kind", # KIND_*. + "metadata", # Metadata for the containing package. + "opt", # Optimisation level (or None). + ], +) + + +# Represents the metadata for a package. +class ManifestPackageMetadata: + def __init__(self, is_require=False): + self._is_require = is_require + self._initialised = False + + self.version = None + self.description = None + self.license = None + self.author = None + + # Annotate a package as being from the python standard library. + self.stdlib = False + + # Allows a python-ecosys package to be annotated with the + # corresponding name in PyPI. e.g. micropython-lib/requests is based + # on pypi/requests. + self.pypi = None + # For a micropython package, this is the name that we will publish it + # to PyPI as. e.g. micropython-lib/senml publishes as + # pypi/micropython-senml. + self.pypi_publish = None + + def update( + self, + mode, + description=None, + version=None, + license=None, + author=None, + stdlib=False, + pypi=None, + pypi_publish=None, + ): + if self._initialised: + raise ManifestFileError("Duplicate call to metadata().") + + # In MODE_PYPROJECT, if this manifest is being evaluated as a result + # of a require(), then figure out if it should be replaced by a PyPI + # dependency instead. + if mode == MODE_PYPROJECT and self._is_require: + if stdlib: + # No dependency required at all for CPython. + raise ManifestIgnoreException + if pypi_publish or pypi: + # In the case where a package is both based on a PyPI package and + # provides one, preference depending on the published one. + # (This should be pretty rare). + raise ManifestUsePyPIException(pypi_publish or pypi) + + self.description = description + self.version = version + self.license = license + self.author = author + self.pypi = pypi + self.pypi_publish = pypi_publish + self._initialised = True + + def check_initialised(self, mode): + # Ensure that metadata() is the first thing a manifest.py does. + # This is to ensure that we early-exit if it should be replaced by a pypi dependency. + if mode in (MODE_COMPILE, MODE_PYPROJECT) and not self._initialised: + raise ManifestFileError("metadata() must be the first command in a manifest file.") + + def __str__(self): + return "version={} description={} license={} author={} pypi={} pypi_publish={}".format( + self.version, self.description, self.license, self.author, self.pypi, self.pypi_publish + ) + + +# Turns a dict of options into a object with attributes used to turn the +# kwargs passed to include() and require into the "options" global in the +# included manifest. +# options = IncludeOptions(foo="bar", blah="stuff") +# options.foo # "bar" +# options.blah # "stuff" +class IncludeOptions: + def __init__(self, **kwargs): + self._kwargs = kwargs + self._defaults = {} + + def defaults(self, **kwargs): + self._defaults = kwargs + + def __getattr__(self, name): + return self._kwargs.get(name, self._defaults.get(name, None)) + + +class ManifestFile: + def __init__(self, mode, path_vars=None): + # See MODE_* constants above. + self._mode = mode + # Path substitution variables. + self._path_vars = path_vars or {} + # List of files (as ManifestFileResult) references by this manifest. + self._manifest_files = [] + # List of PyPI dependencies (when mode=MODE_PYPROJECT). + self._pypi_dependencies = [] + # Don't allow including the same file twice. + self._visited = set() + # Stack of metadata for each level. + self._metadata = [ManifestPackageMetadata()] + # Registered external libraries. + self._libraries = {} + # List of directories to search for packages. + self._library_dirs = [] + # Add default micropython-lib libraries if $(MPY_LIB_DIR) has been specified. + if self._path_vars["MPY_LIB_DIR"]: + for lib in BASE_LIBRARY_NAMES: + self.add_library(lib, os.path.join("$(MPY_LIB_DIR)", lib)) + + def _resolve_path(self, path): + # Convert path to an absolute path, applying variable substitutions. + for name, value in self._path_vars.items(): + if value is not None: + path = path.replace("$({})".format(name), value) + return os.path.abspath(path) + + def _manifest_globals(self, kwargs): + # This is the "API" available to a manifest file. + g = { + "metadata": self.metadata, + "include": self.include, + "require": self.require, + "add_library": self.add_library, + "package": self.package, + "module": self.module, + "options": IncludeOptions(**kwargs), + } + + # Extra legacy functions only for freeze mode. + if self._mode == MODE_FREEZE: + g.update( + { + "freeze": self.freeze, + "freeze_as_str": self.freeze_as_str, + "freeze_as_mpy": self.freeze_as_mpy, + "freeze_mpy": self.freeze_mpy, + } + ) + + return g + + def files(self): + return self._manifest_files + + def pypi_dependencies(self): + # In MODE_PYPROJECT, this will return a list suitable for requirements.txt. + return self._pypi_dependencies + + def execute(self, manifest_file): + if manifest_file.endswith(".py"): + # Execute file from filesystem. + self.include(manifest_file) + else: + # Execute manifest code snippet. + try: + exec(manifest_file, self._manifest_globals({})) + except Exception as er: + raise ManifestFileError("Error in manifest: {}".format(er)) from er + + def _add_file(self, full_path, target_path, kind=KIND_AUTO, opt=None): + # Check file exists and get timestamp. + try: + stat = os.stat(full_path) + timestamp = stat.st_mtime + except OSError as e: + raise ManifestFileError("Cannot stat {}".format(full_path)) from e + + # Map the AUTO kinds to their actual kind based on mode and extension. + _, ext = os.path.splitext(full_path) + if self._mode == MODE_FREEZE: + if kind in ( + KIND_AUTO, + KIND_FREEZE_AUTO, + ): + if ext.lower() == ".py": + kind = KIND_FREEZE_AS_MPY + elif ext.lower() == ".mpy": + kind = KIND_FREEZE_MPY + else: + if kind != KIND_AUTO: + raise ManifestFileError("Not in freeze mode") + if ext.lower() != ".py": + raise ManifestFileError("Expected .py file") + kind = KIND_COMPILE_AS_MPY + + self._manifest_files.append( + ManifestOutput( + FILE_TYPE_LOCAL, full_path, target_path, timestamp, kind, self._metadata[-1], opt + ) + ) + + def _search(self, base_path, package_path, files, exts, kind, opt=None, strict=False): + base_path = self._resolve_path(base_path) + + if files is not None: + # Use explicit list of files (relative to package_path). + for file in files: + if package_path: + file = os.path.join(package_path, file) + self._add_file(os.path.join(base_path, file), file, kind=kind, opt=opt) + else: + if base_path: + prev_cwd = os.getcwd() + os.chdir(self._resolve_path(base_path)) + + # Find all candidate files. + for dirpath, _, filenames in os.walk(package_path or ".", followlinks=True): + for file in filenames: + file = os.path.relpath(os.path.join(dirpath, file), ".") + _, ext = os.path.splitext(file) + if ext.lower() in exts: + self._add_file( + os.path.join(base_path, file), + file, + kind=kind, + opt=opt, + ) + elif strict: + raise ManifestFileError("Unexpected file type") + + if base_path: + os.chdir(prev_cwd) + + def metadata(self, **kwargs): + """ + From within a manifest file, use this to set the metadata for the + package described by current manifest. + + After executing a manifest file (via execute()), call this + to obtain the metadata for the top-level manifest file. + + See ManifestPackageMetadata.update() for valid kwargs. + """ + if kwargs: + self._metadata[-1].update(self._mode, **kwargs) + return self._metadata[-1] + + def include(self, manifest_path, is_require=False, **kwargs): + """ + Include another manifest. + + The manifest argument can be a string (filename) or an iterable of + strings. + + Relative paths are resolved with respect to the current manifest file. + + If the path is to a directory, then it implicitly includes the + manifest.py file inside that directory. + + Optional kwargs can be provided which will be available to the + included script via the `options` variable. + + e.g. include("path.py", extra_features=True) + + in path.py: + options.defaults(standard_features=True) + + # freeze minimal modules. + if options.standard_features: + # freeze standard modules. + if options.extra_features: + # freeze extra modules. + """ + if is_require: + self._metadata[-1].check_initialised(self._mode) + + if not isinstance(manifest_path, str): + for m in manifest_path: + self.include(m, **kwargs) + else: + manifest_path = self._resolve_path(manifest_path) + # Including a directory grabs the manifest.py inside it. + if os.path.isdir(manifest_path): + manifest_path = os.path.join(manifest_path, "manifest.py") + if manifest_path in self._visited: + return + self._visited.add(manifest_path) + if is_require: + # This include is the result of require("name"), so push a new + # package metadata onto the stack. + self._metadata.append(ManifestPackageMetadata(is_require=True)) + try: + with open(manifest_path) as f: + # Make paths relative to this manifest file while processing it. + # Applies to includes and input files. + prev_cwd = os.getcwd() + os.chdir(os.path.dirname(manifest_path)) + try: + exec(f.read(), self._manifest_globals(kwargs)) + finally: + os.chdir(prev_cwd) + except ManifestIgnoreException: + # e.g. MODE_PYPROJECT and this was a stdlib dependency. No-op. + pass + except ManifestUsePyPIException as e: + # e.g. MODE_PYPROJECT and this was a package from + # python-ecosys. Add PyPI dependency instead. + self._pypi_dependencies.append(e.pypi_name) + except Exception as e: + raise ManifestFileError( + "Error in manifest file: {}: {}".format(manifest_path, e) + ) from e + if is_require: + self._metadata.pop() + + def _require_from_path(self, library_path, name, version, extra_kwargs): + for root, _dirnames, filenames in os.walk(library_path): # type: ignore[type-arg] + if os.path.basename(root) == name and "manifest.py" in filenames: + self.include(root, is_require=True, **extra_kwargs) + return True + return False + + def require(self, name, version=None, pypi=None, library=None, **kwargs): + """ + Require a package by name from micropython-lib. + + Optionally specify pipy="package-name" to indicate that this should + use the named package from PyPI when building for CPython. + + Optionally specify library="name" to reference a package from a + library that has been previously registered with add_library(). Otherwise + the list of library paths will be used. + """ + self._metadata[-1].check_initialised(self._mode) + + if self._mode == MODE_PYPROJECT and pypi: + # In PYPROJECT mode, allow overriding the PyPI dependency name + # explicitly. Otherwise if the dependent package has metadata + # (pypi_publish) or metadata(pypi) we will use that. + self._pypi_dependencies.append(pypi) + return + + if library is not None: + # Find package in external library. + if library not in self._libraries: + raise ValueError("Unknown library '{}' for require('{}').".format(library, name)) + library_path = self._libraries[library] + # Search for {library_path}/**/{name}/manifest.py. + if self._require_from_path(library_path, name, version, kwargs): + return + raise ValueError( + "Package '{}' not found in external library '{}' ({}).".format( + name, library, library_path + ) + ) + + for lib_dir in self._library_dirs: + # Search for {lib_dir}/**/{name}/manifest.py. + if self._require_from_path(lib_dir, name, version, kwargs): + return + + if pypi: + # PyPI-only dependency (e.g. pygame-ce). Not bundled in MIP / micropython-lib. + return + + raise ValueError("Package '{}' not found in any known library.".format(name)) + + def add_library(self, library, library_path, prepend=False): + """ + Register the path to an external named library. + + The path will be automatically searched when using require(). By default the + added library is added to the end of the list of libraries to search. Pass + `prepend=True` to add it to the start of the list. + + Additionally, the added library can be explicitly requested by using + `require("name", library="library")`. + """ + library_path = self._resolve_path(library_path) + self._libraries[library] = library_path + self._library_dirs.insert(0 if prepend else len(self._library_dirs), library_path) + + def package(self, package_path, files=None, base_path=".", opt=None): + """ + Define a package, optionally restricting to a set of files. + + Simple case, a package in the current directory: + package("foo") + will include all .py files in foo, and will be stored as foo/bar/baz.py. + + If the package isn't in the current directory, use base_path: + package("foo", base_path="src") + + To restrict to certain files in the package use files (note: paths should be relative to the package): + package("foo", files=["bar/baz.py"]) + """ + self._metadata[-1].check_initialised(self._mode) + + # Include "base_path/package_path/**/*.py" --> "package_path/**/*.py" + self._search(base_path, package_path, files, exts=(".py",), kind=KIND_AUTO, opt=opt) + + def module(self, module_path, base_path=".", opt=None): + """ + Include a single Python file as a module. + + If the file is in the current directory: + module("foo.py") + + Otherwise use base_path to locate the file: + module("foo.py", "src/drivers") + """ + self._metadata[-1].check_initialised(self._mode) + + # Include "base_path/module_path" --> "module_path" + base_path = self._resolve_path(base_path) + _, ext = os.path.splitext(module_path) + if ext.lower() != ".py": + raise ManifestFileError("module must be .py file") + # TODO: version None + self._add_file(os.path.join(base_path, module_path), module_path, opt=opt) + + def _freeze_internal(self, path, script, exts, kind, opt): + if script is None: + self._search(path, None, None, exts=exts, kind=kind, opt=opt) + elif isinstance(script, str) and os.path.isdir(os.path.join(path, script)): + self._search(path, script, None, exts=exts, kind=kind, opt=opt) + elif not isinstance(script, str): + self._search(path, None, script, exts=exts, kind=kind, opt=opt) + else: + self._search(path, None, (script,), exts=exts, kind=kind, opt=opt) + + def freeze(self, path, script=None, opt=None): + """ + Freeze the input, automatically determining its type. A .py script + will be compiled to a .mpy first then frozen, and a .mpy file will be + frozen directly. + + `path` must be a directory, which is the base directory to _search for + files from. When importing the resulting frozen modules, the name of + the module will start after `path`, ie `path` is excluded from the + module name. + + If `path` is relative, it is resolved to the current manifest.py. + Use $(MPY_DIR), $(MPY_LIB_DIR), $(PORT_DIR), $(BOARD_DIR) if you need + to access specific paths. + + If `script` is None all files in `path` will be frozen. + + If `script` is an iterable then freeze() is called on all items of the + iterable (with the same `path` and `opt` passed through). + + If `script` is a string then it specifies the file or directory to + freeze, and can include extra directories before the file or last + directory. The file or directory will be _searched for in `path`. If + `script` is a directory then all files in that directory will be frozen. + + `opt` is the optimisation level to pass to mpy-cross when compiling .py + to .mpy. + """ + self._freeze_internal( + path, + script, + exts=( + ".py", + ".mpy", + ), + kind=KIND_FREEZE_AUTO, + opt=opt, + ) + + def freeze_as_str(self, path): + """ + Freeze the given `path` and all .py scripts within it as a string, + which will be compiled upon import. + """ + self._search(path, None, None, exts=(".py",), kind=KIND_FREEZE_AS_STR) + + def freeze_as_mpy(self, path, script=None, opt=None): + """ + Freeze the input (see above) by first compiling the .py scripts to + .mpy files, then freezing the resulting .mpy files. + """ + self._freeze_internal(path, script, exts=(".py",), kind=KIND_FREEZE_AS_MPY, opt=opt) + + def freeze_mpy(self, path, script=None, opt=None): + """ + Freeze the input (see above), which must be .mpy files that are + frozen directly. + """ + self._freeze_internal(path, script, exts=(".mpy",), kind=KIND_FREEZE_MPY, opt=opt) + + +# Generate a temporary file with a line appended to the end that adds __version__. +class tagged_py_file: + def __init__(self, path, metadata): + self._path = path + self._metadata = metadata + self._dest_path = None + + def __enter__(self): + dest_fd, self._dest_path = tempfile.mkstemp(suffix=".py", text=True) + try: + with os.fdopen(dest_fd, "w") as dest, open(self._path, "r") as src: + contents = src.read() + dest.write(contents) + + # Don't overwrite a version definition if the file already has one in it. + if self._metadata.version and "__version__ =" not in contents: + dest.write("\n\n__version__ = {}\n".format(repr(self._metadata.version))) + except Exception: + os.unlink(self._dest_path) + raise + return self._dest_path + + def __exit__(self, exc_type, exc, tb): + os.unlink(self._dest_path) + return False + + +def main(): + import argparse + + cmd_parser = argparse.ArgumentParser(description="List the files referenced by a manifest.") + cmd_parser.add_argument("--freeze", action="store_true", help="freeze mode") + cmd_parser.add_argument("--compile", action="store_true", help="compile mode") + cmd_parser.add_argument("--pyproject", action="store_true", help="pyproject mode") + cmd_parser.add_argument( + "--lib", + default=os.path.join(os.path.dirname(__file__), "../lib/micropython-lib"), + help="path to micropython-lib repo", + ) + cmd_parser.add_argument( + "--unix-ffi", action="store_true", help="prepend unix-ffi to the library path" + ) + cmd_parser.add_argument("--port", default=None, help="path to port dir") + cmd_parser.add_argument("--board", default=None, help="path to board dir") + cmd_parser.add_argument( + "--top", + default=os.path.join(os.path.dirname(__file__), ".."), + help="path to micropython repo", + ) + cmd_parser.add_argument("files", nargs="+", help="input manifest.py") + args = cmd_parser.parse_args() + + path_vars = { + "MPY_DIR": os.path.abspath(args.top) if args.top else None, + "BOARD_DIR": os.path.abspath(args.board) if args.board else None, + "PORT_DIR": os.path.abspath(args.port) if args.port else None, + "MPY_LIB_DIR": os.path.abspath(args.lib) if args.lib else None, + } + + mode = None + if args.freeze: + mode = MODE_FREEZE + elif args.compile: + mode = MODE_COMPILE + elif args.pyproject: + mode = MODE_PYPROJECT + else: + print("Error: No mode specified.", file=sys.stderr) + sys.exit(1) + + m = ManifestFile(mode, path_vars) + if args.unix_ffi: + m.add_library("unix-ffi", os.path.join("$(MPY_LIB_DIR)", "unix-ffi"), prepend=True) + for manifest_file in args.files: + try: + m.execute(manifest_file) + except ManifestFileError as er: + print(er, file=sys.stderr) + sys.exit(1) + print(m.metadata()) + for f in m.files(): + print(f) + if mode == MODE_PYPROJECT: + for r in m.pypi_dependencies(): + print("pypi-require:", r) + + +if __name__ == "__main__": + main() diff --git a/scripts/publish_make_pyproject.py b/scripts/publish_make_pyproject.py new file mode 100755 index 0000000..a00a1ba --- /dev/null +++ b/scripts/publish_make_pyproject.py @@ -0,0 +1,258 @@ +#!/usr/bin/env python3 +# +# This file is part of the MicroPython project, http://micropython.org/ +# +# The MIT License (MIT) +# +# Copyright (c) 2023 Jim Mussared +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. + +# This script makes a CPython-compatible package from a micropython-lib package +# with a pyproject.toml that can be built (via hatch) and deployed to PyPI. +# Requires that the project sets the pypi_publish= kwarg in its metadata(). + +# Usage: +# ./scripts/publish_make_pyproject.py --output /tmp/foo micropython/foo +# python -m build /tmp/foo +# python -m twine upload /tmp/foo/dist/*.whl + +import os +import re +import shutil +import sys +from email.utils import parseaddr + +_scripts_dir = os.path.dirname(os.path.abspath(__file__)) +sys.path.insert(0, _scripts_dir) + +from build import ensure_path_exists, error_color # noqa: E402 + +DEFAULT_AUTHOR = "micropython-lib " +DEFAULT_LICENSE = "MIT" +HOME_PAGE = "https://github.com/PyDevices/usdl2" +DOCS_PAGE = "https://pydevices.github.io/usdl2" +ISSUES_PAGE = "https://github.com/PyDevices/usdl2/issues" +TESTPYPI = "https://test.pypi.org/simple/" + + +def _resolve_mpy_lib_dir(): + """Micropython-lib root for require() resolution when building PyPI packages.""" + env_dir = os.environ.get("MICROPYTHON_LIB_DIR") + if env_dir: + return os.path.abspath(env_dir) + repo_root = os.path.abspath(os.path.join(os.path.dirname(__file__), "..")) + sibling = os.path.join(os.path.dirname(repo_root), "micropython-lib") + if os.path.isdir(os.path.join(sibling, "micropython")): + return sibling + return repo_root + + +def quoted_escape(s): + return s.replace('"', '\\"') + + +def build(manifest_path, output_path): + import manifestfile + + if not manifest_path.endswith(".py"): + # Allow specifying either the directory or the manifest file explicitly. + manifest_path = os.path.join(manifest_path, "manifest.py") + + print("Generating pyproject for {} in {}...".format(manifest_path, output_path)) + + toml_path = os.path.join(output_path, "pyproject.toml") + ensure_path_exists(toml_path) + + path_vars = { + "MPY_LIB_DIR": _resolve_mpy_lib_dir(), + } + + # .../foo/manifest.py -> foo + package_name = os.path.basename(os.path.dirname(manifest_path)) + + # Compile the manifest. + manifest = manifestfile.ManifestFile(manifestfile.MODE_PYPROJECT, path_vars) + manifest.add_library("testpypi", TESTPYPI, prepend=False) + manifest.execute(manifest_path) + + # If a package doesn't have a pypi name, then assume it isn't intended to + # be publishable. + if not manifest.metadata().pypi_publish: + print(error_color("Error:"), package_name, "doesn't have a pypi_publish name.") + sys.exit(1) + + # These should be in all packages eventually. + if not manifest.metadata().version: + print(error_color("Error:"), package_name, "doesn't have a version.") + sys.exit(1) + if not manifest.metadata().description: + print(error_color("Error:"), package_name, "doesn't have a description.") + sys.exit(1) + + # This is the root path of all .py files that are copied. We ensure that + # they all match. + top_level_package = None + root_modules = [] + + for result in manifest.files(): + # This isn't allowed in micropython-lib anyway. + if result.file_type != manifestfile.FILE_TYPE_LOCAL: + print(error_color("Error:"), "Non-local file not supported.", file=sys.stderr) + sys.exit(1) + + # "foo/bar/baz.py" --> "foo" + # "foo/nested/bar/baz.py" --> "foo" (not "foo/nested") + # "baz.py" --> "" + normalized = result.target_path.replace("\\", "/") + result_package = normalized.split("/", 1)[0] if "/" in normalized else "" + + if not result_package: + root_modules.append(normalized) + with manifestfile.tagged_py_file(result.full_path, result.metadata) as tagged_path: + dest_path = os.path.join(output_path, result.target_path) + ensure_path_exists(dest_path) + shutil.copyfile(tagged_path, dest_path) + continue + if top_level_package and result_package != top_level_package: + # This likely suggests that something needs to use require(..., pypi="..."). + print( + error_color("Error:"), + "More than one top-level package: {}, {}.".format( + result_package, top_level_package + ), + file=sys.stderr, + ) + sys.exit(1) + top_level_package = result_package + + # Tag each file with the package metadata and copy the .py directly. + with manifestfile.tagged_py_file(result.full_path, result.metadata) as tagged_path: + dest_path = os.path.join(output_path, result.target_path) + ensure_path_exists(dest_path) + shutil.copyfile(tagged_path, dest_path) + + if not top_level_package and not root_modules: + print(error_color("Error:"), "No package or module files in manifest.", file=sys.stderr) + sys.exit(1) + + # Copy README.md if it exists + readme_path = os.path.join(os.path.dirname(manifest_path), "README.md") + readme_toml = "" + if os.path.exists(readme_path): + shutil.copyfile(readme_path, os.path.join(output_path, "README.md")) + readme_toml = 'readme = "README.md"' + + # Apply default author and license, otherwise use the package metadata. + license_toml = 'license = {{ text = "{}" }}'.format( + quoted_escape(manifest.metadata().license or DEFAULT_LICENSE) + ) + author_name, author_email = parseaddr(manifest.metadata().author or DEFAULT_AUTHOR) + author_toml = 'authors = [ {{ name = "{}", email = "{}"}} ]'.format( + quoted_escape(author_name), quoted_escape(author_email) + ) + + # Write pyproject.toml. + with open(toml_path, "w") as toml_file: + print("# Generated by makepyproject.py", file=toml_file) + + print( + """ +[build-system] +requires = [ + "hatchling" +] +build-backend = "hatchling.build" +""", + file=toml_file, + ) + + print( + """ +[project] +name = "{}" +description = "{}" +{} +{} +version = "{}" +dependencies = [{}] +urls = {{ Homepage = "{}", Documentation = "{}", Issues = "{}" }} +{} +""".format( + quoted_escape(manifest.metadata().pypi_publish), + quoted_escape(manifest.metadata().description), + author_toml, + license_toml, + quoted_escape(manifest.metadata().version), + ", ".join('"{}"'.format(quoted_escape(r)) for r in manifest.pypi_dependencies()), + quoted_escape(HOME_PAGE), + quoted_escape(DOCS_PAGE), + quoted_escape(ISSUES_PAGE), + readme_toml, + ), + file=toml_file, + ) + + if top_level_package: + print( + """ +[tool.hatch.build.targets.wheel] +packages = ["{}"] +""".format(top_level_package), + file=toml_file, + ) + if root_modules: + print("[tool.hatch.build.targets.wheel.force-include]", file=toml_file) + for module_path in root_modules: + print( + '"{}" = "{}"'.format(module_path, module_path), + file=toml_file, + ) + else: + only_include = ", ".join('"{}"'.format(m) for m in root_modules) + print( + """ +[tool.hatch.build.targets.wheel] +only-include = [{}] +""".format(only_include), + file=toml_file, + ) + + print("Done.") + + +def main(): + import argparse + + cmd_parser = argparse.ArgumentParser( + description="Generate a project that can be pushed to PyPI." + ) + cmd_parser.add_argument("--output", required=True, help="output directory") + cmd_parser.add_argument("--micropython", default=None, help="path to micropython repo") + cmd_parser.add_argument("manifest", help="input package path") + args = cmd_parser.parse_args() + + if args.micropython: + sys.path.append(os.path.join(args.micropython, "tools")) # for manifestfile + + build(args.manifest, args.output) + + +if __name__ == "__main__": + main() diff --git a/scripts/publish_micropython_lib.sh b/scripts/publish_micropython_lib.sh new file mode 100755 index 0000000..3dc1e51 --- /dev/null +++ b/scripts/publish_micropython_lib.sh @@ -0,0 +1,187 @@ +#!/usr/bin/env bash +# Sync usdl2 into PyDevices/micropython-lib, build TestPyPI wheels, push MIP index. +# +# CI: MICROPYTHON_LIB_DIR=../micropython-lib ./scripts/publish_micropython_lib.sh --push +# MIP: mip.install("usdl2", index="https://PyDevices.github.io/micropython-lib/mip/PyDevices") + +set -euo pipefail + +SKIP_PYPI=0 +DO_PUSH=0 +COMMIT_MESSAGE="" +INTERACTIVE_COMMIT=0 +CLI_VERSION="" + +usage() { + cat <<'EOF' +Usage: ./scripts/publish_micropython_lib.sh [OPTION] + +Copy lib/usdl2.py into micropython-lib, optionally upload TestPyPI wheels, +then commit (and optionally push) on the PyDevices branch. + +Options: + --skip-pypi Sync manifests only; skip hatch/twine TestPyPI uploads. + --version X.Y.Z Release version (overrides tag / USDL2_VERSION). + --commit-message MSG Commit micropython-lib changes (non-interactive). + --push Push micropython-lib after commit. + --help, -h Show this message. + +Environment: + MICROPYTHON_LIB_DIR micropython-lib checkout (default: ../micropython-lib) + USDL2_VERSION Release version (overrides git tag on current commit) + TESTPYPI_API_TOKEN TestPyPI token for twine (when not using --skip-pypi) +EOF +} + +while [[ $# -gt 0 ]]; do + case "$1" in + --skip-pypi) SKIP_PYPI=1; shift ;; + --version) CLI_VERSION=$2; shift 2 ;; + --commit-message) COMMIT_MESSAGE=$2; shift 2 ;; + --push) DO_PUSH=1; shift ;; + --help | -h) usage; exit 0 ;; + *) echo "Unknown option: $1" >&2; usage >&2; exit 1 ;; + esac +done + +if [[ -z "$COMMIT_MESSAGE" ]] && [[ -t 0 ]]; then + INTERACTIVE_COMMIT=1 +fi + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +SOURCE_REPO="$(cd "$SCRIPT_DIR/.." && pwd)" + +normalize_version() { + local v="${1#v}" + v="$(echo "$v" | tr -d '[:space:]')" + if [[ ! "$v" =~ ^[0-9]+\.[0-9]+\.[0-9]+(-[a-zA-Z0-9.]+)?$ ]]; then + echo "Error: invalid semver: $1 (expected X.Y.Z)" >&2 + return 1 + fi + echo "$v" +} + +resolve_version() { + if [[ -n "$CLI_VERSION" ]]; then + normalize_version "$CLI_VERSION" + return + fi + if [[ -n "${USDL2_VERSION:-}" ]]; then + normalize_version "$USDL2_VERSION" + return + fi + local tag + tag="$(git -C "$SOURCE_REPO" describe --tags --exact-match 2>/dev/null || true)" + if [[ -n "$tag" ]]; then + normalize_version "$tag" + return + fi + echo "Error: no release version. Tag HEAD (vX.Y.Z), pass --version, or set USDL2_VERSION." >&2 + return 1 +} + +VERSION="$(resolve_version)" || exit 1 +echo "Release version: $VERSION" + +DESCRIPTION_PREFIX="usdl2" +AUTHOR="Brad Barnett " +LICENSE="MIT" +BASENAME=usdl2 +PYPI_NAME=usdl2-py +DEST_REPO="${MICROPYTHON_LIB_DIR:-$SOURCE_REPO/../micropython-lib}" +DEST_REPO="$(cd "$DEST_REPO" 2>/dev/null && pwd || echo "$DEST_REPO")" +export MICROPYTHON_LIB_DIR="$DEST_REPO" +SOURCE_MODULE=$SOURCE_REPO/lib/usdl2.py +DEST_DIR=$DEST_REPO/micropython/$BASENAME +PYPI_DIR=$SOURCE_REPO/wheels +README_FULL_PATH=$SOURCE_REPO/README.md + +build_and_upload_pypi() { + if [[ "$SKIP_PYPI" -eq 1 ]]; then + return 0 + fi + rm -rf dist + hatch build + if [[ -n "${TESTPYPI_API_TOKEN:-}" ]]; then + TWINE_USERNAME=__token__ TWINE_PASSWORD="$TESTPYPI_API_TOKEN" \ + twine upload --repository testpypi --verbose dist/* + else + twine upload --repository testpypi --verbose dist/* + fi +} + +# Concurrent tag publishes from sibling repos share micropython-lib PyDevices. +push_micropython_lib() { + local repo="$1" + local branch + branch="$(git -C "$repo" rev-parse --abbrev-ref HEAD)" + local max_attempts=8 + local attempt=1 + while true; do + if git -C "$repo" push origin "HEAD:${branch}"; then + return 0 + fi + if (( attempt >= max_attempts )); then + echo "Error: push to micropython-lib ${branch} failed after ${max_attempts} attempts" >&2 + return 1 + fi + echo "Push rejected (likely concurrent publish); rebase onto origin/${branch} and retry (${attempt}/${max_attempts})..." + git -C "$repo" fetch origin "${branch}" + if ! git -C "$repo" rebase "origin/${branch}"; then + git -C "$repo" rebase --abort 2>/dev/null || true + git -C "$repo" fetch --deepen=100 origin "${branch}" || git -C "$repo" fetch --unshallow origin || true + git -C "$repo" rebase "origin/${branch}" + fi + attempt=$((attempt + 1)) + sleep "$attempt" + done +} + +echo +echo "Processing $BASENAME" +mkdir -p "$DEST_DIR" +# Single-module layout (like micropython-lib upysh): micropython/usdl2/usdl2.py +rm -rf "$DEST_DIR/$BASENAME" "$DEST_DIR/__pycache__" +cp "$SOURCE_MODULE" "$DEST_DIR/usdl2.py" + +cat < "$DEST_DIR/manifest.py" +metadata( + description="Pure-Python SDL2 subset for MicroPython/CircuitPython/CPython; import as usdl2", + version="$VERSION", + author="$AUTHOR", + license="$LICENSE", + pypi_publish="$PYPI_NAME", +) +module("usdl2.py") +EOF + +cp "$README_FULL_PATH" "$DEST_DIR/README.md" + +if [[ "$SKIP_PYPI" -eq 0 ]]; then + ./scripts/publish_make_pyproject.py --output "$PYPI_DIR/$BASENAME" "$DEST_DIR/manifest.py" + pushd "$PYPI_DIR/$BASENAME" + build_and_upload_pypi + popd +fi + +find "$DEST_DIR" \( \ + -type d \( -name __pycache__ -o -name .mypy_cache -o -name .ruff_cache \) \ + -o -type f \( -name '*.pyc' -o -name '*.pyo' \) \ +\) -print0 2>/dev/null | xargs -0 rm -rf 2>/dev/null || true + +if [[ "$INTERACTIVE_COMMIT" -eq 1 ]] || [[ -n "$COMMIT_MESSAGE" ]]; then + if [[ "$INTERACTIVE_COMMIT" -eq 1 ]] && [[ -z "$COMMIT_MESSAGE" ]]; then + read -r -p "Enter micropython-lib commit message: " COMMIT_MESSAGE + fi + if [[ -n "$COMMIT_MESSAGE" ]]; then + if [[ -z "$(git -C "$DEST_REPO" status --porcelain)" ]]; then + echo "No changes to commit in $DEST_REPO" + else + git -C "$DEST_REPO" add . + git -C "$DEST_REPO" commit -s -m "$COMMIT_MESSAGE" + if [[ "$DO_PUSH" -eq 1 ]]; then + push_micropython_lib "$DEST_REPO" + fi + fi + fi +fi diff --git a/scripts/publish_mip_ghpages.sh b/scripts/publish_mip_ghpages.sh new file mode 100755 index 0000000..6a6d707 --- /dev/null +++ b/scripts/publish_mip_ghpages.sh @@ -0,0 +1,83 @@ +#!/usr/bin/env bash +# Build mip/PyDevices from micropython-lib and push to gh-pages. +# +# Requires: +# MICROPYTHON_LIB_DIR checkout of PyDevices/micropython-lib (PyDevices branch) +# +# Optional: +# MICROPYTHON_DIR micropython source for mpy-cross (default: /tmp/micropython) +# MIP_INDEX_OUTPUT build output dir (default: /tmp/mip-index) +# MIP_INDEX_SUBDIR gh-pages subdir under mip/ (default: PyDevices) +# GITHUB_SHA used in commit message (Actions sets this) +# +# Push credentials: configure git remote on MICROPYTHON_LIB_DIR before calling +# (see .github/workflows/publish-micropython-lib.yml). + +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +USDL2_DIR="${USDL2_DIR:-$ROOT}" +LIB_DIR="${MICROPYTHON_LIB_DIR:?set MICROPYTHON_LIB_DIR}" +MPY_DIR="${MICROPYTHON_DIR:-/tmp/micropython}" +INDEX_OUT="${MIP_INDEX_OUTPUT:-/tmp/mip-index}" +MIP_SUBDIR="${MIP_INDEX_SUBDIR:-PyDevices}" +MPY_CROSS="$MPY_DIR/mpy-cross/build/mpy-cross" +PAGES_PATH="${MIP_GHPAGES_WORKTREE:-/tmp/micropython-lib-gh-pages}" + +if [[ ! -x "$MPY_CROSS" ]]; then + echo "Building mpy-cross in $MPY_DIR" + if [[ ! -d "$MPY_DIR/.git" ]]; then + git clone --depth=1 https://github.com/micropython/micropython.git "$MPY_DIR" + fi + make -C "$MPY_DIR/mpy-cross" -j"$(nproc)" CFLAGS_EXTRA=-O0 +fi + +rm -rf "$INDEX_OUT" +mkdir -p "$INDEX_OUT" + +echo "Compiling MIP index from $LIB_DIR -> $INDEX_OUT" +python3 "$USDL2_DIR/scripts/build.py" \ + --lib-dir "$LIB_DIR" \ + --micropython "$MPY_DIR" \ + --mpy-cross "$MPY_CROSS" \ + --output "$INDEX_OUT" + +cd "$LIB_DIR" +git config user.name 'github-actions[bot]' +git config user.email 'github-actions[bot]@users.noreply.github.com' + +NEW_BRANCH=0 +if git fetch --depth=1 origin gh-pages:gh-pages; then + if git worktree list | grep -q "$PAGES_PATH"; then + git worktree remove --force "$PAGES_PATH" || true + fi + git worktree add "$PAGES_PATH" gh-pages +else + echo "Creating gh-pages branch..." + git worktree add --force "$PAGES_PATH" HEAD + cd "$PAGES_PATH" + git switch --orphan gh-pages + NEW_BRANCH=1 + cd "$LIB_DIR" +fi + +DEST_PATH="$PAGES_PATH/mip/$MIP_SUBDIR" +rm -rf "$DEST_PATH" +mkdir -p "$DEST_PATH" +cp -r "$INDEX_OUT/." "$DEST_PATH/" + +cd "$PAGES_PATH" +git add . +SHA="${GITHUB_SHA:-local}" +git diff --staged --quiet && { + echo "No MIP index changes to publish" + exit 0 +} +git commit -m "usdl2: Update mip/$MIP_SUBDIR from PyDevices/usdl2 ${SHA}." + +if [[ "$NEW_BRANCH" -eq 0 ]]; then + git pull --rebase origin gh-pages +fi +git push origin gh-pages + +echo "Published https://PyDevices.github.io/micropython-lib/mip/$MIP_SUBDIR"