From cf24902255854a981a00c00d7c6d707d08e21f0e Mon Sep 17 00:00:00 2001 From: Brad Barnett <127794626+bdbarnett@users.noreply.github.com> Date: Thu, 22 Oct 2026 01:53:29 -0500 Subject: [PATCH 1/2] A dev container per port, built and proven in CI; workbench on a webassembly build --- .devcontainer/Dockerfile | 15 ++++ .devcontainer/esp32/devcontainer.json | 22 +++++ .devcontainer/esp32/prefetch.sh | 8 ++ .devcontainer/on-create.sh | 7 ++ .devcontainer/rp2/devcontainer.json | 15 ++++ .devcontainer/unix/devcontainer.json | 15 ++++ .devcontainer/webassembly/devcontainer.json | 24 ++++++ .devcontainer/windows/devcontainer.json | 15 ++++ .github/workflows/devcontainers.yml | 94 +++++++++++++++++++++ README.md | 6 ++ ROADMAP.md | 4 + docs/newcomers.md | 40 ++++++++- scripts/serve_workbench.sh | 76 +++++++++++++++++ 13 files changed, 340 insertions(+), 1 deletion(-) create mode 100644 .devcontainer/Dockerfile create mode 100644 .devcontainer/esp32/devcontainer.json create mode 100755 .devcontainer/esp32/prefetch.sh create mode 100755 .devcontainer/on-create.sh create mode 100644 .devcontainer/rp2/devcontainer.json create mode 100644 .devcontainer/unix/devcontainer.json create mode 100644 .devcontainer/webassembly/devcontainer.json create mode 100644 .devcontainer/windows/devcontainer.json create mode 100644 .github/workflows/devcontainers.yml create mode 100755 scripts/serve_workbench.sh diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile new file mode 100644 index 0000000..7c7b046 --- /dev/null +++ b/.devcontainer/Dockerfile @@ -0,0 +1,15 @@ +# The image every dev container here builds from: Ubuntu, and the Debian +# packages for one port, which install.sh --deps installs from the same list it +# prints for a machine of your own. Nothing else is baked in: build_mp.py +# fetches the port's toolchain (ESP-IDF, emsdk, the pico-sdk, SDL2 for MinGW) +# on its first build, into deps/, which each config keeps in a volume. +FROM mcr.microsoft.com/devcontainers/base:ubuntu-24.04 + +ARG PORT +ARG INSTALL_SH=https://raw.githubusercontent.com/PyDevices/PyDevices.github.io/install-deps/install.sh + +# Downloaded first, not piped: a failed download must fail the image. +RUN test -n "$PORT" \ + && curl -fsSL --retry 3 -o /tmp/install.sh "$INSTALL_SH" \ + && sh /tmp/install.sh --deps "$PORT" \ + && rm -rf /tmp/install.sh /var/lib/apt/lists/* diff --git a/.devcontainer/esp32/devcontainer.json b/.devcontainer/esp32/devcontainer.json new file mode 100644 index 0000000..119f2e4 --- /dev/null +++ b/.devcontainer/esp32/devcontainer.json @@ -0,0 +1,22 @@ +{ + "name": "esp32: ESP32-S3, ESP32-P4 and the other ESP chips", + "build": { + "dockerfile": "../Dockerfile", + "context": "..", + "args": { + "PORT": "esp32" + } + }, + "mounts": [ + "source=pydevices-deps-${devcontainerId},target=${containerWorkspaceFolder}/deps,type=volume" + ], + "containerEnv": { + "IDF_TOOLS_PATH": "${containerWorkspaceFolder}/deps/espressif-tools" + }, + "hostRequirements": { + "cpus": 4, + "storage": "32gb" + }, + "onCreateCommand": "sh .devcontainer/on-create.sh && sh .devcontainer/esp32/prefetch.sh", + "postCreateCommand": "echo 'To build: ./build_mp.py --port esp32 --board ESP32_GENERIC_S3 --variant SPIRAM_OCT --flash 8MB --modules all'" +} diff --git a/.devcontainer/esp32/prefetch.sh b/.devcontainer/esp32/prefetch.sh new file mode 100755 index 0000000..b1b11c0 --- /dev/null +++ b/.devcontainer/esp32/prefetch.sh @@ -0,0 +1,8 @@ +#!/bin/sh +# The esp32 container fetches, when it's created, what build_mp.py would fetch +# on its first esp32 build: MicroPython, the modules, and ESP-IDF with its +# compilers (about 4 GB, into the deps/ volume). In a Codespaces prebuild that +# happens once, ahead of time, so your first build starts compiling. +set -eu +cd "$(dirname "$0")/../.." +python3 -c 'import build_mp as b; b.ensure_micropython(None); b.ensure_modules(None); b.ensure_dep("esp-idf", None)' diff --git a/.devcontainer/on-create.sh b/.devcontainer/on-create.sh new file mode 100755 index 0000000..e563b63 --- /dev/null +++ b/.devcontainer/on-create.sh @@ -0,0 +1,7 @@ +#!/bin/sh +# Every config's first step when its container is created. deps/ is a Docker +# volume, so a rebuilt container keeps the toolchains it fetched; Docker makes +# a new volume root's, and build_mp.py writes to it as you. +set -eu +cd "$(dirname "$0")/.." +sudo chown "$(id -u):$(id -g)" deps diff --git a/.devcontainer/rp2/devcontainer.json b/.devcontainer/rp2/devcontainer.json new file mode 100644 index 0000000..09d0d56 --- /dev/null +++ b/.devcontainer/rp2/devcontainer.json @@ -0,0 +1,15 @@ +{ + "name": "rp2: Raspberry Pi Pico and Pico 2", + "build": { + "dockerfile": "../Dockerfile", + "context": "..", + "args": { + "PORT": "rp2" + } + }, + "mounts": [ + "source=pydevices-deps-${devcontainerId},target=${containerWorkspaceFolder}/deps,type=volume" + ], + "onCreateCommand": "sh .devcontainer/on-create.sh", + "postCreateCommand": "echo 'To build: ./build_mp.py --port rp2 --board RPI_PICO2 --modules all'" +} diff --git a/.devcontainer/unix/devcontainer.json b/.devcontainer/unix/devcontainer.json new file mode 100644 index 0000000..a835ef4 --- /dev/null +++ b/.devcontainer/unix/devcontainer.json @@ -0,0 +1,15 @@ +{ + "name": "unix: MicroPython for Linux, with a desktop display", + "build": { + "dockerfile": "../Dockerfile", + "context": "..", + "args": { + "PORT": "unix" + } + }, + "mounts": [ + "source=pydevices-deps-${devcontainerId},target=${containerWorkspaceFolder}/deps,type=volume" + ], + "onCreateCommand": "sh .devcontainer/on-create.sh", + "postCreateCommand": "echo 'To build: ./build_mp.py --port unix --variant pydevices --modules all'" +} diff --git a/.devcontainer/webassembly/devcontainer.json b/.devcontainer/webassembly/devcontainer.json new file mode 100644 index 0000000..50af4fb --- /dev/null +++ b/.devcontainer/webassembly/devcontainer.json @@ -0,0 +1,24 @@ +{ + "name": "webassembly: MicroPython in the browser, opened in workbench", + "build": { + "dockerfile": "../Dockerfile", + "context": "..", + "args": { + "PORT": "webassembly" + } + }, + "mounts": [ + "source=pydevices-deps-${devcontainerId},target=${containerWorkspaceFolder}/deps,type=volume" + ], + "onCreateCommand": "sh .devcontainer/on-create.sh", + "forwardPorts": [ + 8000 + ], + "portsAttributes": { + "8000": { + "label": "workbench", + "onAutoForward": "openBrowser" + } + }, + "postCreateCommand": "echo 'To build: ./build_mp.py --port webassembly --variant pydevices --modules all, then scripts/serve_workbench.sh'" +} diff --git a/.devcontainer/windows/devcontainer.json b/.devcontainer/windows/devcontainer.json new file mode 100644 index 0000000..1a90146 --- /dev/null +++ b/.devcontainer/windows/devcontainer.json @@ -0,0 +1,15 @@ +{ + "name": "windows: micropython.exe, cross-built from Linux", + "build": { + "dockerfile": "../Dockerfile", + "context": "..", + "args": { + "PORT": "windows" + } + }, + "mounts": [ + "source=pydevices-deps-${devcontainerId},target=${containerWorkspaceFolder}/deps,type=volume" + ], + "onCreateCommand": "sh .devcontainer/on-create.sh", + "postCreateCommand": "echo 'To build: ./build_mp.py --port windows --variant pydevices --modules all'" +} diff --git a/.github/workflows/devcontainers.yml b/.github/workflows/devcontainers.yml new file mode 100644 index 0000000..bb6024f --- /dev/null +++ b/.github/workflows/devcontainers.yml @@ -0,0 +1,94 @@ +# Each dev container in .devcontainer/ builds its port from nothing, as a +# codespace does: the image is made from the Dockerfile (install.sh --deps for +# that port, on a clean Ubuntu), the container is created with its deps/ volume +# empty, and build_mp.py fetches MicroPython, the modules and the port's +# toolchain before it builds. One job per config, one build each. +# +# It runs when a config changes, and weekly, which also catches a change to +# install.sh's package list or to a pin that stops a port building from a +# clean machine. +name: Dev containers + +on: + push: + branches: [main] + paths: + - '.devcontainer/**' + - 'scripts/serve_workbench.sh' + - '.github/workflows/devcontainers.yml' + pull_request: + paths: + - '.devcontainer/**' + - 'scripts/serve_workbench.sh' + - '.github/workflows/devcontainers.yml' + schedule: + - cron: '41 4 * * 3' + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: devcontainers-${{ github.event_name == 'pull_request' && github.ref || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + build: + name: ${{ matrix.port }} + runs-on: ubuntu-latest + timeout-minutes: ${{ matrix.port == 'esp32' && 150 || 75 }} + strategy: + fail-fast: false + matrix: + include: + - port: unix + run: | + ./build_mp.py --port unix --variant pydevices --modules all + builds/unix/pydevices/micropython -c "import sys; print(sys.implementation); import displaydev; print('displaydev imports')" + - port: webassembly + run: | + ./build_mp.py --port webassembly --variant pydevices --modules all + b=builds/webassembly/pydevices + printf 'import sys\nprint(sys.implementation)\nimport displaydev\nprint("displaydev imports")\n' > /tmp/check.py + node $b/micropython.mjs /tmp/check.py + # workbench, with this build in it, on the port the config forwards + WORKBENCH_PORT=8000 scripts/serve_workbench.sh < /dev/null > /tmp/serve.log 2>&1 & + srv=$! + for i in $(seq 60); do curl -fs -o /dev/null http://127.0.0.1:8000/ && break; sleep 2; done + cat /tmp/serve.log + curl -fsS http://127.0.0.1:8000/index.html | grep -qi '&2; exit 1; } + +repo=$(cd "$(dirname "$0")/.." && pwd) +build=${1:-$repo/builds/webassembly/pydevices} +port=${WORKBENCH_PORT:-8000} +bind=${WORKBENCH_BIND:-127.0.0.1} +cache=$repo/deps/workbench +site=$repo/builds/workbench + +[ -f "$build/micropython.mjs" ] && [ -f "$build/micropython.wasm" ] || + fail "no webassembly build in $build; build one with ./build_mp.py --port webassembly --variant pydevices --modules all" + +# The newest release's tag, or the newest one already downloaded when GitHub +# can't be reached. +tag=$(curl -fsSL --retry 2 https://api.github.com/repos/PyDevices/workbench/releases/latest 2>/dev/null | + python3 -c 'import json, sys; print(json.load(sys.stdin)["tag_name"])' 2>/dev/null) || tag= +if [ -z "$tag" ]; then + zip=$(ls -t "$cache"/workbench-v*.zip 2>/dev/null | head -n 1) || + zip= + [ -n "$zip" ] || fail "can't reach GitHub for workbench's newest release, and none is downloaded yet" + say "GitHub can't be reached; using $(basename "$zip")" +else + zip=$cache/workbench-$tag.zip + if [ ! -f "$zip" ]; then + mkdir -p "$cache" + say "Downloading workbench $tag" + curl -fsSL --retry 3 -o "$zip.part" \ + "https://github.com/PyDevices/workbench/releases/download/$tag/workbench-$tag.zip" || + fail "could not download workbench-$tag.zip" + mv "$zip.part" "$zip" + fi +fi + +rm -rf "$site" +mkdir -p "$site" +python3 -m zipfile -e "$zip" "$site" +[ -f "$site/index.html" ] && [ -d "$site/assets/pydevices" ] || + fail "$(basename "$zip") has no index.html and assets/pydevices/; is it a workbench release?" + +# Your build in place of the release's. The release's provenance describes its +# own runtime, so it goes; your build's record goes beside the runtime instead. +cp "$build/micropython.mjs" "$build/micropython.wasm" "$site/assets/pydevices/" +rm -f "$site/assets/pydevices/micropython.wasm.provenance" +[ ! -f "$build/pydevices-build.json" ] || cp "$build/pydevices-build.json" "$site/assets/pydevices/" + +if [ -n "${CODESPACE_NAME:-}" ] && [ -n "${GITHUB_CODESPACES_PORT_FORWARDING_DOMAIN:-}" ]; then + url="https://$CODESPACE_NAME-$port.$GITHUB_CODESPACES_PORT_FORWARDING_DOMAIN/" +else + url="http://localhost:$port/" +fi +say "workbench $(basename "$zip" .zip | sed 's/^workbench-//') with $build" +say "Serving it at $url (Ctrl+C stops it)" +exec python3 -m http.server "$port" --bind "$bind" --directory "$site" From 0b352878625823d76844f85b4d8fd23758edbf2e Mon Sep 17 00:00:00 2001 From: Brad Barnett <127794626+bdbarnett@users.noreply.github.com> Date: Thu, 22 Oct 2026 02:31:38 -0500 Subject: [PATCH 2/2] Dev containers take install.sh from pydevices.github.io; esp32 lists libusb-1.0-0 --- .devcontainer/Dockerfile | 2 +- docs/newcomers.md | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 7c7b046..e01730b 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -6,7 +6,7 @@ FROM mcr.microsoft.com/devcontainers/base:ubuntu-24.04 ARG PORT -ARG INSTALL_SH=https://raw.githubusercontent.com/PyDevices/PyDevices.github.io/install-deps/install.sh +ARG INSTALL_SH=https://pydevices.github.io/install.sh # Downloaded first, not piped: a failed download must fail the image. RUN test -n "$PORT" \ diff --git a/docs/newcomers.md b/docs/newcomers.md index 6446b51..d1acfee 100644 --- a/docs/newcomers.md +++ b/docs/newcomers.md @@ -80,7 +80,7 @@ fetch are your distribution's packages. On Debian or Ubuntu (WSL included): |---|---| | every port | `git make cmake python3 gcc g++` | | unix | `pkg-config libsdl2-dev libffi-dev` (the desktop display) | -| esp32 | `ninja-build python3-venv` | +| esp32 | `ninja-build python3-venv libusb-1.0-0` (ESP-IDF's installer checks the openocd it fetches, which needs libusb) | | rp2 | `gcc-arm-none-eabi libnewlib-arm-none-eabi` | | windows, cross-built from Linux | `gcc-mingw-w64-x86-64 binutils-mingw-w64-x86-64 libtool libltdl-dev autoconf automake` (the binutils package has `windres`) | | webassembly | `nodejs` | @@ -89,9 +89,9 @@ All of it in one line: ```bash sudo apt-get update && sudo apt-get install -y git make cmake ninja-build \ - python3 python3-venv python3-pip gcc g++ pkg-config libffi-dev libsdl2-dev \ - gcc-mingw-w64-x86-64 binutils-mingw-w64-x86-64 libtool libltdl-dev \ - autoconf automake \ + python3 python3-venv python3-pip libusb-1.0-0 gcc g++ pkg-config libffi-dev \ + libsdl2-dev gcc-mingw-w64-x86-64 binutils-mingw-w64-x86-64 libtool \ + libltdl-dev autoconf automake \ gcc-arm-none-eabi libnewlib-arm-none-eabi nodejs ```