diff --git a/.github/workflows/electron-release.yml b/.github/workflows/electron-release.yml new file mode 100644 index 0000000..680c47b --- /dev/null +++ b/.github/workflows/electron-release.yml @@ -0,0 +1,584 @@ +name: Electron release (reusable) + +# The release pipeline for a de-shell app, generalised from SpyDE's release.yml +# so Ground Crew, Autopilot and SpyDE stop carrying three drifting copies. +# An app calls it from its own tag-triggered workflow: +# +# # .github/workflows/release.yml in the app's repo +# on: +# push: { tags: ["v*"] } +# jobs: +# release: +# permissions: { contents: write } +# uses: directelectron/de-shell/.github/workflows/electron-release.yml@v0.5.0 +# with: +# app-name: Ground Crew +# wheelhouse-extras: gpu +# include-mingit: true +# azure-endpoint: https://eus.codesigning.azure.net +# azure-account: de-signing +# azure-profile: de-public-trust +# azure-publisher-name: Direct Electron +# secrets: inherit +# +# IT RUNS IN THE APP'S REPOSITORY: `actions/checkout` below checks out the +# CALLER, and publishes to the caller's releases (GITHUB_TOKEN cannot publish +# cross-repo anyway). This repo only supplies the pipeline, and the build +# scripts it calls come from the de-shell wheel the app's lock installs +# (electron/shell/packaging/…), so the pipeline and the scripts cannot be at +# different versions than the app expects. +# +# The app must provide: electron/package.json with a `deShell` block (appId, +# pythonDist, pythonVersion …; see packaging/bundle-python.mjs), a committed +# uv.lock, an electron-builder.yml that extends +# shell/packaging/electron-builder.base.yml, and a build script reachable as +# `electron-vite build`. +# +# Job order is load-bearing, as in the SpyDE original: the release is CREATED +# once, sequentially, BEFORE the matrix -- electron-builder's GitHub publisher +# finds-or-creates a release by tag, and parallel matrix legs hitting that +# together is a known race (electron-builder#6676, duplicate drafts). + +on: + workflow_call: + inputs: + app-name: + description: Display name, used in the release title and the wheelhouse zip name. + type: string + required: true + electron-dir: + description: The app's Electron project directory, relative to the repo root. + type: string + default: electron + python-dir: + description: The uv project (pyproject.toml, uv.lock), relative to the repo root. + type: string + default: "." + npm-dir: + description: Where package-lock.json lives (`npm ci` runs here), relative to the repo root. + type: string + default: "." + builder-config: + description: electron-builder config file, relative to electron-dir. + type: string + default: electron-builder.yml + platforms: + description: JSON array of runner labels to build on. + type: string + default: '["windows-latest","macos-latest","ubuntu-latest"]' + wheelhouse-extras: + description: >- + Comma-separated extras to include in the offline Windows wheelhouse + (e.g. "gpu"). Empty builds the wheelhouse with no extras. + type: string + default: "" + wheelhouse: + description: Build the offline wheelhouse and attach its zip to the release (Windows leg only). + type: boolean + default: true + wheelhouse-platform: + description: pip platform tag of the wheelhouse. + type: string + default: win_amd64 + include-mingit: + description: Vendor MinGit into the Windows installer (for git+https dependencies). + type: boolean + default: false + mingit-version: + type: string + default: "2.51.0" + mingit-tag: + type: string + default: "v2.51.0.windows.1" + uv-version: + description: >- + Pinned, never "latest": resolving latest hits the GitHub API and took out every leg during an API outage. + type: string + default: "0.10.11" + node-version: + type: string + default: "24" + sentry-dsn-env: + description: Name of the env var the app reads its Sentry DSN from (e.g. SPYDE_SENTRY_DSN). Empty = none. + type: string + default: "" + azure-endpoint: + description: Azure Trusted Signing endpoint. Signing happens only if this AND the Azure secrets are set. + type: string + default: "" + azure-account: + description: Azure Trusted Signing code-signing account name. + type: string + default: "" + azure-profile: + description: Azure Trusted Signing certificate profile name. + type: string + default: "" + azure-publisher-name: + description: Publisher (certificate subject) name for azureSignOptions. + type: string + default: "" + secrets: + SENTRY_DSN: + required: false + MAC_CERT_P12_BASE64: + required: false + MAC_CERT_PASSWORD: + required: false + APPLE_API_KEY_P8: + required: false + APPLE_API_KEY_ID: + required: false + APPLE_API_ISSUER_ID: + required: false + APPLE_TEAM_ID: + required: false + AZURE_TENANT_ID: + required: false + AZURE_CLIENT_ID: + required: false + AZURE_CLIENT_SECRET: + required: false + +jobs: + # ── Derive the update channel from the tag shape (stable vs beta) ─────────── + # Plain vX.Y.Z -> stable. Any prerelease suffix (-rc.N / -beta.N / -alpha.N) + # -> beta. electron-updater understands GitHub's own "prerelease" flag + # natively (allowPrerelease gates on it), so this one regex is what the whole + # channel feature hinges on downstream. + channel: + name: Derive channel from tag + runs-on: ubuntu-latest + outputs: + channel: ${{ steps.detect.outputs.channel }} + prerelease: ${{ steps.detect.outputs.prerelease }} + version: ${{ steps.detect.outputs.version }} + steps: + - name: Detect channel + version from tag + id: detect + run: | + tag="${GITHUB_REF_NAME}" + version="${tag#v}" + if [[ "$tag" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + echo "channel=stable" >> "$GITHUB_OUTPUT" + echo "prerelease=false" >> "$GITHUB_OUTPUT" + else + echo "channel=beta" >> "$GITHUB_OUTPUT" + echo "prerelease=true" >> "$GITHUB_OUTPUT" + fi + echo "version=${version}" >> "$GITHUB_OUTPUT" + + # ── Create the (draft) release BEFORE the matrix builds ───────────────────── + # CRITICAL -- the release MUST be a DRAFT when the builds publish into it. + # electron-builder's GitHub publisher only uploads into an EXISTING release + # when that release is a draft; if it finds a NON-draft one it skips every + # asset ("existing type not compatible with publishing type") and, because + # that skip is non-fatal, the run goes GREEN with an EMPTY release. That + # shipped an asset-less release once. So we do NOT set `prerelease:` here and + # we FORCE the release back to draft if it already exists; the prerelease flag + # is applied in `finalize`, after the assets attach. + prepare-release: + name: Create draft release + runs-on: ubuntu-latest + needs: channel + permissions: + contents: write + steps: + - name: Create draft GitHub Release + uses: softprops/action-gh-release@v2 + with: + name: ${{ inputs.app-name }} ${{ github.ref_name }} + draft: true + body: | + ## ${{ inputs.app-name }} ${{ github.ref_name }} + + Desktop app (Electron) with a uv-managed Python backend. The + installer is small: the Python environment is set up on **first + launch**, which needs a network connection and may take a few + minutes -- or, on a machine with no network, from the offline + wheelhouse zip below. Later launches are instant. + + ${{ needs.channel.outputs.channel == 'beta' && '**Beta channel** — only offered to users who opted into beta updates.' || '' }} + + ### Downloads + | Platform | File | + |----------|------| + | Windows | `* Setup *.exe` (NSIS installer) | + | macOS | `*.dmg` | + | Linux | `*.AppImage` | + | Offline (Windows) | `*-wheels-win_amd64.zip` -- every dependency plus Python, for installs with no network | + generate_release_notes: true + + # softprops/action-gh-release only CREATES a draft; if a release for this + # tag already exists it just updates it and leaves its draft flag alone. + # Force draft, and strip assets a prior half-run left attached, so the + # "release has assets" check reflects THIS run only. + - name: Force the release to draft (and clear stale assets) + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + TAG: ${{ github.ref_name }} + REPO: ${{ github.repository }} + run: | + gh release edit "$TAG" --repo "$REPO" --draft=true + assets=$(gh release view "$TAG" --repo "$REPO" \ + --json assets --jq '.assets[].name' 2>/dev/null || true) + for a in $assets; do + echo "Removing stale asset: $a" + gh release delete-asset "$TAG" "$a" --repo "$REPO" --yes || true + done + + # ── Build the Electron app + staged Python sidecar on each platform ───────── + build: + name: Build (${{ matrix.os }}) + runs-on: ${{ matrix.os }} + needs: [channel, prepare-release] + permissions: + contents: write # electron-builder publishes assets to the release + # The `secrets` context is NOT available in step-level `if:` -- only `env` + # is. So surface "is signing configured?" as job-level env flags the steps + # can gate on (a fork / secret-less run then produces an unsigned build). + env: + MAC_SIGNING_ENABLED: ${{ secrets.MAC_CERT_P12_BASE64 != '' && 'true' || 'false' }} + AZURE_SIGNING_ENABLED: ${{ secrets.AZURE_CLIENT_SECRET != '' && secrets.AZURE_CLIENT_ID != '' && secrets.AZURE_TENANT_ID != '' && inputs.azure-endpoint != '' && 'true' || 'false' }} + # windows-latest's default shell is PowerShell, which has killed releases + # before (bash version guards that do not parse; npm.cmd mangling + # -c.publish.repo=x into a config FILE path). Every step is POSIX or a + # command verified under Git Bash, so default the whole job. + defaults: + run: + shell: bash + strategy: + fail-fast: true # abort all legs if one fails -- keeps releases clean + matrix: + os: ${{ fromJSON(inputs.platforms) }} + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + # Full history + tags so setuptools_scm derives the real version from + # the pushed tag (a shallow checkout has none -> 0.0.postN.devN). + with: + fetch-depth: 0 + + # electron/package.json's "version" is what electron-builder and + # electron-updater treat as the app's semver; a drift from the tag would + # ship an installer the updater cannot reason about. Fail fast. + - name: Verify electron package.json version matches the pushed tag + run: | + pkg_version=$(node -p "require('./${{ inputs.electron-dir }}/package.json').version") + tag_version="${{ needs.channel.outputs.version }}" + if [ "$pkg_version" != "$tag_version" ]; then + echo "::error::${{ inputs.electron-dir }}/package.json version ($pkg_version) does not match tag v$tag_version" + exit 1 + fi + + - name: Set up Node + uses: actions/setup-node@v4 + with: + node-version: ${{ inputs.node-version }} + + # bundle-python.mjs ships the `uv` on PATH when electron/vendor/uv/ is + # absent, so installing uv here is what populates the shipped binary. + - name: Set up uv + uses: astral-sh/setup-uv@v6 + with: + version: ${{ inputs.uv-version }} + enable-cache: true + + # The installer bundles pyproject.toml + uv.lock and resolves the venv on + # the USER's first launch, so an out-of-sync lock would ship a broken + # installer that only fails for them. + - name: Verify uv.lock is in sync with pyproject.toml + working-directory: ${{ inputs.python-dir }} + run: uv lock --check + + # AppImage tooling needs FUSE on the runner. + - name: Install Linux packaging deps + if: runner.os == 'Linux' + run: | + sudo apt-get update -q + sudo apt-get install -y --no-install-recommends fuse libfuse2 + + # `uv sync` resolves git+https dependencies by shelling out to `git`, and + # a clean Windows user machine often has NO git. bundle-python.mjs stages + # vendor/git/win32 into the installer; this populates it with MinGit. + - name: Vendor MinGit (Windows) + if: runner.os == 'Windows' && inputs.include-mingit + env: + MINGIT_VER: ${{ inputs.mingit-version }} + MINGIT_TAG: ${{ inputs.mingit-tag }} + run: | + dest="${{ inputs.electron-dir }}/vendor/git/win32" + mkdir -p "$dest" + url="https://github.com/git-for-windows/git/releases/download/${MINGIT_TAG}/MinGit-${MINGIT_VER}-64-bit.zip" + echo "Downloading $url" + curl -fsSL -o mingit.zip "$url" + unzip -q mingit.zip -d "$dest" + test -f "$dest/cmd/git.exe" || test -f "$dest/bin/git.exe" + "$dest/cmd/git.exe" --version || "$dest/bin/git.exe" --version + + - name: Install npm dependencies + working-directory: ${{ inputs.npm-dir }} + run: npm ci + + # The shell's TypeScript ships INSIDE the de-shell wheel; electron/shell + # is a link to it. `uv run` syncs the locked environment first (the same + # install the app's tests use), then the stdlib CLI makes the link. + # Heavy for an app with a big science stack, but it is the one honest way + # to get the version the lock pins -- the link has to point at THAT. + - name: Link the shell (python -m de_shell.js link) + working-directory: ${{ inputs.python-dir }} + run: uv run --frozen --no-dev python -m de_shell.js link "${GITHUB_WORKSPACE}/${{ inputs.electron-dir }}" + + # ── macOS code signing (only when secrets are present) ─────────────────── + # Import the Developer ID Application cert into a throwaway keychain so + # electron-builder can sign + (with notarize:true) notarize. A fork / + # secret-less run skips it and electron-builder produces an unsigned build. + - name: Import Apple signing certificate (macOS only) + if: runner.os == 'macOS' && env.MAC_SIGNING_ENABLED == 'true' + env: + MAC_CERT_P12_BASE64: ${{ secrets.MAC_CERT_P12_BASE64 }} + MAC_CERT_PASSWORD: ${{ secrets.MAC_CERT_PASSWORD }} + run: | + set -euo pipefail + KEYCHAIN=build.keychain + KEYCHAIN_PW=$(openssl rand -base64 24) + echo "$MAC_CERT_P12_BASE64" | base64 --decode > cert.p12 + security create-keychain -p "$KEYCHAIN_PW" "$KEYCHAIN" + security set-keychain-settings -lut 21600 "$KEYCHAIN" + security unlock-keychain -p "$KEYCHAIN_PW" "$KEYCHAIN" + security import cert.p12 -k "$KEYCHAIN" -P "$MAC_CERT_PASSWORD" \ + -T /usr/bin/codesign -T /usr/bin/security + # The Developer ID G2 intermediate, in case the .p12 did not bundle it. + curl -fsSO https://www.apple.com/certificateauthority/DeveloperIDG2CA.cer + security import DeveloperIDG2CA.cer -k "$KEYCHAIN" || true + security set-key-partition-list -S apple-tool:,apple:,codesign: \ + -s -k "$KEYCHAIN_PW" "$KEYCHAIN" + security list-keychains -d user -s "$KEYCHAIN" login.keychain + rm -f cert.p12 DeveloperIDG2CA.cer + security find-identity -v -p codesigning "$KEYCHAIN" + + # electron-builder's notarize step wants APPLE_API_KEY as a PATH to the .p8. + - name: Stage notarization API key (macOS only) + if: runner.os == 'macOS' && env.MAC_SIGNING_ENABLED == 'true' + env: + APPLE_API_KEY_P8: ${{ secrets.APPLE_API_KEY_P8 }} + run: | + printf '%s' "$APPLE_API_KEY_P8" > "$RUNNER_TEMP/apple_api_key.p8" + echo "APPLE_API_KEY=$RUNNER_TEMP/apple_api_key.p8" >> "$GITHUB_ENV" + + # Where Help -> Report a Problem sends reports. A Sentry DSN is a + # write-only public key, so baking it into the build is how Sentry is + # meant to be used; it lives in secrets so it can be rotated without a + # release, and so a fork builds with reporting simply off (reports are + # then written to the user's data directory). The variable NAME is the + # app's (`sentry-dsn-env`), exported here for the bundler that reads it. + - name: Export the Sentry DSN + if: inputs.sentry-dsn-env != '' + env: + SENTRY_DSN_VALUE: ${{ secrets.SENTRY_DSN }} + SENTRY_DSN_NAME: ${{ inputs.sentry-dsn-env }} + run: | + if [ -n "$SENTRY_DSN_VALUE" ]; then + echo "::add-mask::$SENTRY_DSN_VALUE" + echo "$SENTRY_DSN_NAME=$SENTRY_DSN_VALUE" >> "$GITHUB_ENV" + fi + + - name: Build renderer + main (electron-vite) + working-directory: ${{ inputs.electron-dir }} + run: npx electron-vite build + + # Stage the Python sidecar payload (see packaging/bundle-python.mjs). All + # configuration comes from electron/package.json's `deShell` block; the + # only flag is MinGit: required where we vendored it, off elsewhere. + - name: Stage the Python payload + working-directory: ${{ inputs.electron-dir }} + run: | + flags=() + if [ "${{ inputs.include-mingit }}" = "true" ]; then + [ "$RUNNER_OS" = "Windows" ] && flags+=(--mingit) + else + flags+=(--no-mingit) + fi + node shell/packaging/bundle-python.mjs "${flags[@]}" + + # electron-builder --publish always: finds the draft `prepare-release` + # created (by tag) and uploads this platform's installer PLUS the + # update-feed metadata (latest.yml / latest-mac.yml / latest-linux.yml) + # electron-updater reads. -c.publish.owner/.repo pin the publish target to + # THE REPO THIS WORKFLOW RUNS IN, so a static owner in the app's config + # can neither 404 the build nor survive a repo transfer stale. + # Windows signing is opt-in via -c.win.azureSignOptions.* (Azure Trusted + # Signing; the AZURE_* env below is what its credential chain reads). + # electron-builder's "skipped publishing" is a NON-FATAL warning -- it + # exits 0 while attaching ZERO assets -- so tee the output and hard-fail + # on it: the tripwire that catches an empty release at build time. + - name: Build + publish desktop app + working-directory: ${{ inputs.electron-dir }} + run: | + set -o pipefail + args=(--config "${{ inputs.builder-config }}" --publish always + "-c.publish.owner=${{ github.repository_owner }}" + "-c.publish.repo=${{ github.event.repository.name }}") + if [ "$RUNNER_OS" = "Windows" ] && [ "$AZURE_SIGNING_ENABLED" = "true" ]; then + args+=("-c.win.azureSignOptions.endpoint=${{ inputs.azure-endpoint }}" + "-c.win.azureSignOptions.codeSigningAccountName=${{ inputs.azure-account }}" + "-c.win.azureSignOptions.certificateProfileName=${{ inputs.azure-profile }}") + if [ -n "${{ inputs.azure-publisher-name }}" ]; then + args+=("-c.win.azureSignOptions.publisherName=${{ inputs.azure-publisher-name }}") + fi + fi + npx electron-builder "${args[@]}" 2>&1 | tee eb-publish.log + if grep -qiE "skipped publishing|GitHub release not created" eb-publish.log; then + echo "::error::electron-builder SKIPPED publishing assets — the release is not a writable draft. See the 'skipped publishing' / 'release not created' lines above." + exit 1 + fi + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + APPLE_API_KEY_ID: ${{ secrets.APPLE_API_KEY_ID }} + APPLE_API_ISSUER: ${{ secrets.APPLE_API_ISSUER_ID }} + APPLE_TEAM_ID: ${{ secrets.APPLE_TEAM_ID }} + AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }} + AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }} + AZURE_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }} + + # The offline wheelhouse: every locked wheel for one platform plus the + # Python interpreter, zipped and attached to the SAME release so a lab + # machine with no network can install from a file. Built on the Windows + # leg only (that is the platform that needs it; the tool can target any + # platform from any host). The zip name is --wheels-.zip. + - name: Build + attach the offline wheelhouse + if: runner.os == 'Windows' && inputs.wheelhouse + working-directory: ${{ inputs.electron-dir }} + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + TAG: ${{ github.ref_name }} + REPO: ${{ github.repository }} + EXTRAS: ${{ inputs.wheelhouse-extras }} + APP_NAME: ${{ inputs.app-name }} + PLATFORM: ${{ inputs.wheelhouse-platform }} + run: | + flags=() + IFS=',' read -ra xs <<< "$EXTRAS" + for x in "${xs[@]}"; do + x="${x// /}" + [ -n "$x" ] && flags+=(--extra "$x") + done + node shell/packaging/build-wheelhouse.mjs \ + --platform "$PLATFORM" --app-name "$APP_NAME" --zip "${flags[@]}" + zip_file=$(ls ./*-wheels-"$PLATFORM".zip) + gh release upload "$TAG" "$zip_file" --repo "$REPO" --clobber + + - name: Upload build artifacts (for inspection) + uses: actions/upload-artifact@v4 + with: + name: installer-${{ runner.os }} + path: | + ${{ inputs.electron-dir }}/dist/*.exe + ${{ inputs.electron-dir }}/dist/*.dmg + ${{ inputs.electron-dir }}/dist/*.zip + ${{ inputs.electron-dir }}/dist/*.AppImage + if-no-files-found: error + + # ── Un-draft the release now that every leg has published ─────────────────── + finalize: + name: Publish (un-draft) the release + runs-on: ubuntu-latest + needs: [channel, build] # only runs when ALL build legs pass + permissions: + contents: write + steps: + # Last line of defence: every leg went green, but confirm assets ACTUALLY + # landed. A release with no installers is worse than a failed run (it + # looks shippable), so refuse to un-draft an empty one. Each platform + # contributes at least an installer, its blockmap or feed -- 3 apiece is + # the usual; require 2 per platform as the floor. + - name: Verify the release has assets before publishing + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + TAG: ${{ github.ref_name }} + REPO: ${{ github.repository }} + PLATFORMS: ${{ inputs.platforms }} + run: | + count=$(gh release view "$TAG" --repo "$REPO" --json assets --jq '.assets | length') + legs=$(printf '%s' "$PLATFORMS" | grep -o ',' | wc -l) + legs=$((legs + 1)) + echo "Release $TAG has $count asset(s); expected at least $((legs * 2))." + gh release view "$TAG" --repo "$REPO" --json assets --jq '.assets[].name' + if [ "$count" -lt $((legs * 2)) ]; then + echo "::error::Release $TAG has only $count assets — the installers did not attach. Refusing to publish an empty release." + exit 1 + fi + + # AUTO-UPDATE FEED GATE. electron-updater reads latest*.yml to decide what + # to download; on macOS it CANNOT apply a .dmg, only a .zip. A dmg-only + # mac feed builds and publishes fine yet every mac auto-update then fails. + # Validate each built platform's feed references an INSTALLABLE payload + # AND that the referenced file is an attached asset. + - name: Validate the auto-update feeds are installable + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + TAG: ${{ github.ref_name }} + REPO: ${{ github.repository }} + PLATFORMS: ${{ inputs.platforms }} + run: | + set -euo pipefail + mapfile -t ASSETS < <(gh release view "$TAG" --repo "$REPO" --json assets --jq '.assets[].name') + has_asset() { for a in "${ASSETS[@]}"; do [ "$a" = "$1" ] && return 0; done; return 1; } + + # feed file -> the extension(s) electron-updater can install from it, + # for the platforms this run built. + declare -A FEED_EXT=() + case "$PLATFORMS" in *macos*) FEED_EXT[latest-mac.yml]='\.zip$' ;; esac + case "$PLATFORMS" in *windows*) FEED_EXT[latest.yml]='\.(exe|nupkg)$' ;; esac + case "$PLATFORMS" in *ubuntu*) FEED_EXT[latest-linux.yml]='\.(AppImage|deb|rpm)$' ;; esac + + # This runs while the release is STILL A DRAFT, whose assets are not + # reachable at the public releases/download// URL (404), so go + # through the API with `gh release download`. + tmp=$(mktemp -d) + fail=0 + for feed in "${!FEED_EXT[@]}"; do + if ! has_asset "$feed"; then + echo "::error::update feed $feed is not attached to the release — electron-updater can't detect updates on that platform." + fail=1; continue + fi + if ! gh release download "$TAG" --repo "$REPO" --pattern "$feed" --dir "$tmp" --clobber; then + echo "::error::could not download update feed $feed from the (draft) release." + fail=1; continue + fi + body=$(cat "$tmp/$feed") + mapfile -t urls < <(printf '%s\n' "$body" | sed -nE 's/^[[:space:]]*-?[[:space:]]*url:[[:space:]]*//p') + if [ "${#urls[@]}" -eq 0 ]; then + echo "::error::$feed lists no files — malformed update feed."; fail=1; continue + fi + want="${FEED_EXT[$feed]}" + ok=0 + for u in "${urls[@]}"; do + u="${u%$'\r'}" + if printf '%s' "$u" | grep -qiE "$want"; then + if has_asset "$u"; then ok=1; echo " $feed -> $u ok"; else + echo "::error::$feed references '$u' but it is NOT an attached asset."; fail=1 + fi + fi + done + if [ "$ok" -eq 0 ]; then + echo "::error::$feed has no installable payload (need $want). Files: ${urls[*]}. On macOS a dmg-only feed CANNOT auto-update — add the zip target." + fail=1 + fi + done + [ "$fail" -eq 0 ] || { echo "::error::auto-update feed validation failed — refusing to publish a release users can't update from."; exit 1; } + echo "All auto-update feeds reference an installable, attached payload." + + # Un-draft AND stamp the channel: the prerelease flag is applied HERE (not + # at draft-creation) so the release stays a plain draft during the upload + # phase -- see the prepare-release note. + - name: Publish the release (un-draft + set channel) + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + TAG: ${{ github.ref_name }} + REPO: ${{ github.repository }} + PRERELEASE: ${{ needs.channel.outputs.prerelease }} + run: | + gh release edit "$TAG" --repo "$REPO" \ + --draft=false --prerelease="$PRERELEASE" diff --git a/README.md b/README.md index 411e27d..46ed543 100644 --- a/README.md +++ b/README.md @@ -63,13 +63,23 @@ it into the project at a fixed path, then alias and `paths` through the link: python -m de_shell.js # prints …/de_shell/js ``` -Autopilot's `electron/scripts/shell-link.mjs` is the reference: it makes -`electron/shell` a junction (a symlink off Windows) to that folder, runs from -npm's postinstall and from the vite config on every build, and re-points a -stale link rather than trusting it. With it in place: +The shell makes the link itself: + +``` +uv run python -m de_shell.js link electron # electron/shell -> …/de_shell/js +``` + +That makes `electron/shell` a junction (a symlink off Windows) to that +folder. It re-points a stale link rather than trusting it, and refuses to +replace a real directory; `--soft` turns the refusal into a warning, which is +what npm's postinstall wants. The same logic ships as +`shell/packaging/shell-link.mjs` for the vite config to call on every build, +once the first link exists. The copies the apps carried as +`electron/scripts/shell-link.mjs` can go. With the link in place: ```ts // electron.vite.config.ts +import { ensureShellLink } from './shell/packaging/shell-link.mjs' const shell = ensureShellLink(__dirname) resolve: { alias: { '@de/shell-main': resolve(shell, 'main', 'index.ts') }, dedupe: ['react', 'react-dom'] } @@ -92,6 +102,92 @@ All three apps are wired this way: Autopilot, SpyDE and Ground Crew ([directelectron/de_ground_crew#183](https://github.com/directelectron/de_ground_crew/pull/183)), pinned to the `0.2` line. +## Shipping an app + +An app ships as a per-user NSIS installer (an AppImage on Linux, a dmg on +macOS). The installer is about 150 MB because the Python environment is **not** +inside it. On first launch the shell builds the environment into `userData`, +and an update re-syncs it only when the lock changed. Everything the pipeline +needs lives in `shell/packaging/`. + +**The app provides:** + +* **A `deShell` block in `electron/package.json`.** This is read by + `bundle-python.mjs` and `build-wheelhouse.mjs`: + + ```json + "deShell": { "appId": "groundcrew", "pythonDist": "de-ground-crew", + "pythonVersion": "3.13", "wheelhouseExtras": ["gpu"] } + ``` + +* **`electron/electron-builder.yml`.** It extends the shell's base and adds + only what is the app's own: + + ```yaml + extends: shell/packaging/electron-builder.base.yml + appId: com.directelectron.groundcrew + productName: Ground Crew + publish: { provider: github, owner: directelectron, repo: de_ground_crew } + ``` + +* **`.github/workflows/release.yml`.** It calls the shared pipeline on a tag. + The pipeline runs in the app's repo and publishes to the app's releases: + + ```yaml + on: { push: { tags: ["v*"] } } + jobs: + release: + permissions: { contents: write } + uses: directelectron/de-shell/.github/workflows/electron-release.yml@v0.5.0 + with: { app-name: Ground Crew, wheelhouse-extras: gpu } + secrets: inherit + ``` + + The workflow links the shell with `uv run --frozen --no-dev`, so it syncs the + app's locked environment on the runner. It also attaches a + `--wheels-win_amd64.zip` offline wheelhouse to the release, and + signs Windows builds with Azure Trusted Signing when the `AZURE_*` secrets + and the `azure-*` inputs are set. + +* **Main-process wiring:** + + ```ts + configureShell({ appId: 'groundcrew', appName: 'Ground Crew', + pythonModule: 'de_groundcrew', pythonDist: 'de-ground-crew', + gpuExtras: ['gpu'] }) + registerShellDialogs(); registerUpdaterIpc(); registerProblemReportIpc() + const py = app.isPackaged + ? await runEnvSetup(win, { isPackaged: true, resourcesPath: process.resourcesPath, + projectRoot, userData: app.getPath('userData') }) + : await resolvePythonEnv({ isPackaged: false, resourcesPath: '', projectRoot, userData: '' }) + ``` + + The renderer shows `EnvSetupOverlay` while `state.envSetup` is set, and + `UpdateCard`/`UpdateDialog` and `ReportProblemDialog` where the app wants + them. + +**Extras and the GPU.** `pythonExtras` are always installed. `gpuExtras` are +installed when `nvidia-smi` finds a device. `_GPU=0|1` overrides the +probe either way. + +**Offline sites.** Unzip the release's wheelhouse into the install +folder (next to the app's `.exe`) or into `userData/wheelhouse`, or let the +user choose it with "Install from a folder…" when the online install fails. The first launch then installs with no network: + +* the interpreter comes from the wheelhouse's python-build-standalone mirror; +* the packages are the exported lock, installed with `uv pip install --offline + --no-index --require-hashes`. + +A frozen `uv sync --offline --find-links` cannot do this, because uv looks for +registry-locked wheels only in its cache. A GPU machine given a wheelhouse +built without the GPU extra installs CPU-only and remembers that. It does not +fail. + +**No residue.** The env, the uv cache and uv's managed interpreters all live +under `userData`. The uninstaller offers to remove them. The answer defaults to +No, and is always No for silent runs and updates, so an update never +re-downloads the environment. + ## Developing it ```bash diff --git a/de_shell/js/__init__.py b/de_shell/js/__init__.py index 68a0429..1da7328 100644 --- a/de_shell/js/__init__.py +++ b/de_shell/js/__init__.py @@ -1,6 +1,6 @@ """The TypeScript half of the shell, shipped inside the wheel. -Four folders, one per Electron target, each a self-contained tree of +Four folders, one per Electron target (plus ``packaging/``, below), each a self-contained tree of TypeScript that the consuming app's bundler compiles: * ``main/`` — the Electron main process: the window and menus, the Python @@ -18,6 +18,12 @@ (``python -m de_shell.js``) and links it into its Electron project — see the README — which also makes an editable install live-editable. +``packaging/`` is not compiled into the app: it is the build-time half — the +Node scripts and config an app runs from its ``electron/`` directory to ship an +installer (``bundle-python.mjs``, ``build-wheelhouse.mjs``, ``shell-link.mjs``, +``installer.nsh``, ``electron-builder.base.yml``). ``python -m de_shell.js link +`` creates the ``shell`` link without needing any of them. + The peer dependencies (react, electron, electron-updater, @playwright/test) are the app's to declare; every app already does. """ @@ -25,7 +31,7 @@ from pathlib import Path -TARGETS = ("main", "preload", "renderer", "testing") +TARGETS = ("main", "preload", "renderer", "testing", "packaging") def path(target: str | None = None) -> Path: diff --git a/de_shell/js/__main__.py b/de_shell/js/__main__.py index 898aa3d..25492bb 100644 --- a/de_shell/js/__main__.py +++ b/de_shell/js/__main__.py @@ -1,4 +1,115 @@ -"""``python -m de_shell.js`` prints where the TypeScript is, for build scripts.""" +"""``python -m de_shell.js`` — where the TypeScript is, and how to link it. + + python -m de_shell.js print …/de_shell/js (build scripts rely on this) + python -m de_shell.js link [--soft] + make /shell point at it + +``link`` is the Python twin of ``packaging/shell-link.mjs``: a junction on +Windows (no admin or developer mode needed, unlike a symlink), a symlink +elsewhere. CI and ``postinstall`` use it so an app does not need to carry its +own copy of the script. Semantics, identical to the mjs: + +* an up-to-date link is left alone; +* a stale link (a venv that moved, an editable install that was re-pointed) is + re-pointed, never trusted; +* a REAL directory at the link path is refused, not deleted — someone put it + there, and a build script has no business removing it; +* ``--soft`` turns every failure into a warning and exit 0, for an ``npm + install`` that runs before the first ``uv sync``. +""" +from __future__ import annotations + +import os +import stat +import subprocess +import sys +from pathlib import Path + from . import path -print(path()) +_REPARSE_POINT = getattr(stat, "FILE_ATTRIBUTE_REPARSE_POINT", 0x400) + + +class LinkError(RuntimeError): + pass + + +def _is_link(p: Path) -> bool: + """A symlink OR a Windows junction (``Path.is_symlink`` misses junctions + before 3.12, and a junction is exactly what we create there).""" + try: + st = os.lstat(p) + except OSError: + return False + if stat.S_ISLNK(st.st_mode): + return True + return bool(getattr(st, "st_file_attributes", 0) & _REPARSE_POINT) + + +def _unlink(p: Path) -> None: + # rmdir removes a junction / directory symlink without touching its target + # on Windows; unlink is the POSIX spelling. Try both. + try: + os.unlink(p) + except OSError: + os.rmdir(p) + + +def _make_link(link: Path, target: Path) -> None: + if os.name != "nt": + os.symlink(target, link, target_is_directory=True) + return + try: + import _winapi + _winapi.CreateJunction(str(target), str(link)) + except (ImportError, AttributeError, OSError): + r = subprocess.run(["cmd", "/c", "mklink", "/J", str(link), str(target)], + capture_output=True, text=True) + if r.returncode != 0: + raise LinkError(f"could not create a junction at {link}: {(r.stderr or r.stdout).strip()}") + + +def link(electron_dir: str | os.PathLike, *, quiet: bool = False) -> Path: + """Make ``/shell`` point at this install's ``de_shell/js``.""" + electron_dir = Path(electron_dir) + if not electron_dir.is_dir(): + raise LinkError(f"{electron_dir} is not a directory") + target = path().resolve() + lnk = electron_dir / "shell" + if _is_link(lnk): + if lnk.resolve() == target: + return lnk + _unlink(lnk) + elif lnk.exists(): + raise LinkError(f"{lnk} exists and is not a link; refusing to replace it. Remove it by hand.") + _make_link(lnk, target) + if not quiet: + print(f"[shell-link] {lnk} -> {target}") + return lnk + + +def main(argv: list[str] | None = None) -> int: + argv = sys.argv[1:] if argv is None else argv + if not argv: + print(path()) + return 0 + if argv[0] != "link": + print(f"usage: python -m de_shell.js [link [--soft]]", file=sys.stderr) + return 2 + soft = "--soft" in argv + rest = [a for a in argv[1:] if a != "--soft"] + if len(rest) != 1: + print("usage: python -m de_shell.js link [--soft]", file=sys.stderr) + return 2 + try: + link(rest[0]) + except (LinkError, OSError) as err: + if not soft: + print(f"[shell-link] {err}", file=sys.stderr) + return 1 + print(f"[shell-link] not linked yet: {err}", file=sys.stderr) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/de_shell/js/main/config.ts b/de_shell/js/main/config.ts index 4d47af7..d25cf96 100644 --- a/de_shell/js/main/config.ts +++ b/de_shell/js/main/config.ts @@ -24,6 +24,16 @@ export interface ShellConfig { * (e.g. module `spyde` / dist `spyde`). Used for the pre-built wheel prefix * and the SETUPTOOLS_SCM_PRETEND_VERSION_FOR_ env var. */ pythonDist?: string + /** Optional-dependency groups (`[project.optional-dependencies]`) the packaged + * first-run install ALWAYS adds, each as `uv sync --extra `. Changing + * this list changes the lock stamp, so existing installs re-sync once. */ + pythonExtras?: string[] + /** Extras added ONLY when an NVIDIA GPU is detected (`nvidia-smi -L`), e.g. + * Ground Crew's CuPy stack behind `gpu`. The probe is skipped entirely when + * this is empty. Overridable per run with `_GPU=0|1` or + * `EnsureOptions.gpu`. A machine that gains a GPU later re-syncs, because the + * effective extras are part of the lock stamp. */ + gpuExtras?: string[] } let _config: ShellConfig | null = null diff --git a/de_shell/js/main/envSetup.ts b/de_shell/js/main/envSetup.ts new file mode 100644 index 0000000..958ebdd --- /dev/null +++ b/de_shell/js/main/envSetup.ts @@ -0,0 +1,79 @@ +/** + * envSetup.ts — `runEnvSetup`: what a packaged app calls INSTEAD of + * `resolvePythonEnv` directly. + * + * It does the same resolve, but narrates it to the renderer as `env_setup` + * shell messages (so the first run shows a real progress overlay rather than a + * blank window) and survives failure by asking the user what to do — retry, + * install from a local wheelhouse folder, or quit. The state machine itself is + * envSetupMachine.ts; this file is only the Electron glue: the IPC answer + * channel, the native folder dialog, and app.quit. + * + * Messages ride the window's normal message pipe (`sendToRenderer`), so they + * are buffered until the renderer listens — a progress line sent before React + * mounted is not lost. + */ +import { app, dialog, ipcMain } from 'electron' + +import { channel, shellConfig } from './config' +import { runEnvSetupMachine } from './envSetupMachine' +import type { EnvSetupAnswer } from './envSetupMachine' +import { resolvePythonEnv } from './pythonEnv' +import type { EnsureOptions, ResolvedPython } from './pythonEnv' +import type { ShellWindow } from './window' + +export type { EnvSetupAnswer, EnvSetupWireMessage } from './envSetupMachine' +export { EnvSetupQuitError } from './envSetupMachine' + +const ANSWERS: readonly EnvSetupAnswer[] = ['retry', 'use-wheelhouse', 'quit'] + +/** The answer the machine is currently waiting on, if any. An answer arriving + * while nothing waits (a double click, a stale renderer) is dropped. */ +let waiter: ((a: EnvSetupAnswer) => void) | null = null +let listening = false + +function listenForAnswers(): void { + if (listening) return + listening = true + ipcMain.on(channel('env-setup-answer'), (_e, answer: unknown) => { + if (!ANSWERS.includes(answer as EnvSetupAnswer)) return + const w = waiter + waiter = null + w?.(answer as EnvSetupAnswer) + }) +} + +/** + * Resolve the Python environment with progress and a recovery path. + * + * `opts.wheelhouse`, if given, is used for the first attempt; later attempts use + * whatever folder the user picks. Rejects only with EnvSetupQuitError. + */ +export function runEnvSetup( + win: Pick, + opts: EnsureOptions, +): Promise { + listenForAnswers() + return runEnvSetupMachine({ + initialWheelhouse: opts.wheelhouse ?? null, + run: (wheelhouse, onProgress) => resolvePythonEnv({ + ...opts, + wheelhouse, + // Keep the app's own sink working alongside ours. + onProgress: (line: string) => { opts.onProgress?.(line); onProgress(line) }, + }), + emit: (msg) => win.sendToRenderer({ ...msg }), + waitForAnswer: () => new Promise((resolve) => { waiter = resolve }), + pickFolder: async () => { + const parent = win.get() + const options = { + title: `Choose the folder of packages to install ${shellConfig().appName} from`, + properties: ['openDirectory' as const], + } + const r = parent ? await dialog.showOpenDialog(parent, options) + : await dialog.showOpenDialog(options) + return r.canceled || !r.filePaths.length ? null : r.filePaths[0] + }, + quit: () => app.quit(), + }) +} diff --git a/de_shell/js/main/envSetupMachine.test.ts b/de_shell/js/main/envSetupMachine.test.ts new file mode 100644 index 0000000..c3c4ea1 --- /dev/null +++ b/de_shell/js/main/envSetupMachine.test.ts @@ -0,0 +1,154 @@ +/** + * envSetupMachine.test.ts — the first-run setup conversation, with a scripted + * resolver and renderer standing in for uv and the overlay. + */ +import { test, describe } from 'node:test' +import assert from 'node:assert/strict' + +import { + createProgressBatcher, runEnvSetupMachine, EnvSetupQuitError, +} from './envSetupMachine.ts' +import type { EnvSetupAnswer, EnvSetupWireMessage } from './envSetupMachine.ts' + +/** A manual clock + timer so batching is deterministic. */ +function fakeClock() { + let t = 1_000 + let pending: { fn: () => void; at: number } | null = null + return { + now: () => t, + setTimer: (fn: () => void, ms: number) => { pending = { fn, at: t + ms }; return pending }, + clearTimer: () => { pending = null }, + advance(ms: number) { + t += ms + if (pending && pending.at <= t) { const p = pending; pending = null; p.fn() } + }, + } +} + +describe('progress batching', () => { + test('the first chunk goes out at once, later ones coalesce until the interval', () => { + const clock = fakeClock() + const out: EnvSetupWireMessage[] = [] + const b = createProgressBatcher({ emit: m => out.push(m), intervalMs: 100, ...clock }) + b.push('Resolved 10 packages\n') + assert.equal(out.length, 1) + b.push('Downloading torch (825.0 MiB)\n') + b.push('torch ====> 100.0 MiB/825.0 MiB\r') + assert.equal(out.length, 1, 'throttled') + clock.advance(100) + assert.equal(out.length, 2) + assert.equal(out[1].line, 'Downloading torch (825.0 MiB)\ntorch ====> 100.0 MiB/825.0 MiB') + // The newest classified event wins, carrying the measured percent. + assert.equal(out[1].event?.percent, 12) + assert.equal(out[1].state, 'progress') + }) + + test('flush drains the pending batch and blank chunks emit nothing', () => { + const clock = fakeClock() + const out: EnvSetupWireMessage[] = [] + const b = createProgressBatcher({ emit: m => out.push(m), ...clock }) + b.push('Resolved 1 package\n') + b.push('\r\n \n') + b.push('Installed 1 package\n') + b.flush() + assert.equal(out.length, 2) + b.flush() + assert.equal(out.length, 2, 'nothing left to flush') + }) + + test('a burst is capped so one IPC cannot balloon', () => { + const clock = fakeClock() + const out: EnvSetupWireMessage[] = [] + const b = createProgressBatcher({ emit: m => out.push(m), intervalMs: 100, ...clock }) + b.push('first\n') + b.push(Array.from({ length: 500 }, (_, i) => `l${i}`).join('\n')) + b.flush() + assert.equal(out[1].line!.split('\n').length, 40) + assert.ok(out[1].line!.endsWith('l499'), 'keeps the newest lines') + }) +}) + +/** Build deps whose run() outcomes and renderer answers are scripted. */ +function scripted(outcomes: Array<'ok' | Error>, answers: EnvSetupAnswer[], folders: Array = []) { + const emitted: EnvSetupWireMessage[] = [] + const runs: Array = [] + let quit = 0 + return { + emitted, runs, get quit() { return quit }, + deps: { + intervalMs: 0, + run: async (wh: string | null, onProgress: (c: string) => void) => { + runs.push(wh) + onProgress('Resolved 3 packages\n') + const o = outcomes.shift()! + if (o !== 'ok') throw o + return { cmd: ['python'], cwd: '/x' } + }, + emit: (m: EnvSetupWireMessage) => emitted.push(m), + waitForAnswer: async () => answers.shift()!, + pickFolder: async () => folders.shift() ?? null, + quit: () => { quit++ }, + }, + } +} + +const states = (s: { emitted: EnvSetupWireMessage[] }) => s.emitted.map(m => m.state) + +describe('the setup state machine', () => { + test('success: start, progress, done, and resolves with the result', async () => { + const s = scripted(['ok'], []) + const r = await runEnvSetupMachine(s.deps) + assert.deepEqual(r, { cmd: ['python'], cwd: '/x' }) + assert.deepEqual(states(s), ['start', 'progress', 'done']) + }) + + test('failure does not reject: it reports the error and retry runs again', async () => { + const s = scripted([new Error('network down'), 'ok'], ['retry']) + await runEnvSetupMachine(s.deps) + assert.deepEqual(states(s), ['start', 'progress', 'error', 'start', 'progress', 'done']) + const err = s.emitted.find(m => m.state === 'error')! + assert.equal(err.message, 'network down') + assert.deepEqual(s.runs, [null, null]) + }) + + test('the last uv lines reach the renderer before the error', async () => { + const s = scripted([new Error('boom')], ['quit']) + await assert.rejects(runEnvSetupMachine(s.deps), EnvSetupQuitError) + const i = s.emitted.findIndex(m => m.state === 'error') + assert.equal(s.emitted[i - 1].state, 'progress') + }) + + test('use-wheelhouse re-runs with the picked folder', async () => { + const s = scripted([new Error('offline'), 'ok'], ['use-wheelhouse'], ['D:/wheels']) + await runEnvSetupMachine(s.deps) + assert.deepEqual(s.runs, [null, 'D:/wheels']) + const restart = s.emitted.filter(m => m.state === 'start')[1] + assert.equal(restart.wheelhouse, 'D:/wheels') + }) + + test('a cancelled folder dialog keeps waiting rather than re-running', async () => { + const s = scripted([new Error('offline'), 'ok'], ['use-wheelhouse', 'retry'], [null]) + await runEnvSetupMachine(s.deps) + assert.deepEqual(s.runs, [null, null]) + assert.equal(s.emitted.filter(m => m.state === 'error').length, 1) + }) + + test('a chosen wheelhouse sticks for later retries', async () => { + const s = scripted( + [new Error('a'), new Error('b'), 'ok'], ['use-wheelhouse', 'retry'], ['D:/w']) + await runEnvSetupMachine(s.deps) + assert.deepEqual(s.runs, [null, 'D:/w', 'D:/w']) + }) + + test('quit calls quit() once and rejects with EnvSetupQuitError', async () => { + const s = scripted([new Error('x')], ['quit']) + await assert.rejects(runEnvSetupMachine(s.deps), EnvSetupQuitError) + assert.equal(s.quit, 1) + }) + + test('an app-supplied initial wheelhouse is used on the first run', async () => { + const s = scripted(['ok'], []) + await runEnvSetupMachine({ ...s.deps, initialWheelhouse: 'E:/offline' }) + assert.deepEqual(s.runs, ['E:/offline']) + }) +}) diff --git a/de_shell/js/main/envSetupMachine.ts b/de_shell/js/main/envSetupMachine.ts new file mode 100644 index 0000000..8eda693 --- /dev/null +++ b/de_shell/js/main/envSetupMachine.ts @@ -0,0 +1,163 @@ +/** + * envSetupMachine.ts — the first-run environment setup state machine, with no + * Electron in it. + * + * A packaged first run downloads ~1.8 GB. Two things go wrong with a bare + * `await resolvePythonEnv()`: the window sits blank for minutes, and a failure + * (offline lab PC, proxy, full disk) leaves the app dead with no recourse. + * This turns the run into a conversation with the renderer: + * + * start -> progress* -> done (resolves) + * start -> progress* -> error -> retry (runs again) + * -> use-wheelhouse (folder picked, runs again offline) + * -> quit (rejects with EnvSetupQuitError) + * + * Failure deliberately does NOT reject: the user is the only one who can fix a + * dead network, so the promise stays pending until they answer. Everything + * Electron-shaped (the IPC listener, the directory dialog, app.quit) is injected + * by envSetup.ts, so this file is unit-testable under node --test. + */ +// Extension spelled out so node:test can load this module without a bundler. +import { parseUvLine } from './envProgress.ts' +import type { EnvProgressEvent } from './envProgress.ts' + +export type EnvSetupAnswer = 'retry' | 'use-wheelhouse' | 'quit' + +/** What the renderer receives (`type: 'env_setup'`). Flat and JSON-safe. */ +export interface EnvSetupWireMessage { + type: 'env_setup' + state: 'start' | 'progress' | 'done' | 'error' + /** Latest classified uv event in this batch (progress only). */ + event?: EnvProgressEvent + /** The raw lines of this batch, newline-joined (progress only). */ + line?: string + /** Human-readable failure (error only). */ + message?: string + /** The wheelhouse folder being installed from, when there is one. */ + wheelhouse?: string +} + +/** Thrown when the user answers `quit` after a failure. The app is exiting; + * callers should swallow it rather than report it as a crash. */ +export class EnvSetupQuitError extends Error { + constructor() { + super('environment setup abandoned: the user chose to quit') + this.name = 'EnvSetupQuitError' + } +} + +export interface BatcherDeps { + emit: (msg: EnvSetupWireMessage) => void + /** Minimum gap between progress messages. uv repaints a bar many times a + * second; one IPC per repaint would flood the renderer for no visible gain. */ + intervalMs?: number + now?: () => number + setTimer?: (fn: () => void, ms: number) => unknown + clearTimer?: (t: unknown) => void +} + +/** Cap on raw lines carried by one message, so a burst cannot make a huge IPC. */ +const MAX_LINES_PER_BATCH = 40 + +/** + * Coalesce uv output chunks into at most one `progress` message per interval. + * The newest classified event wins (it is the current state); every raw line is + * kept (up to the cap) so the log tail still moves. + */ +export function createProgressBatcher(deps: BatcherDeps) { + const interval = deps.intervalMs ?? 100 + const now = deps.now ?? Date.now + const setTimer = deps.setTimer ?? ((fn: () => void, ms: number) => setTimeout(fn, ms)) + const clearTimer = deps.clearTimer + ?? ((t: unknown) => clearTimeout(t as ReturnType)) + + let lines: string[] = [] + let event: EnvProgressEvent | undefined + let timer: unknown = null + let lastEmit = -Infinity + + const flush = () => { + if (timer !== null) { clearTimer(timer); timer = null } + if (lines.length === 0 && !event) return + const msg: EnvSetupWireMessage = { + type: 'env_setup', state: 'progress', line: lines.join('\n'), + } + if (event) msg.event = event + lines = [] + event = undefined + lastEmit = now() + deps.emit(msg) + } + + const push = (chunk: string) => { + // uv repaints with \r, so split on both; blank pieces carry nothing. + for (const raw of chunk.split(/[\r\n]+/)) { + const text = raw.trimEnd() + if (!text.trim()) continue + lines.push(text) + const ev = parseUvLine(text) + if (ev) event = ev + } + if (lines.length > MAX_LINES_PER_BATCH) lines = lines.slice(-MAX_LINES_PER_BATCH) + if (lines.length === 0 && !event) return + const wait = interval - (now() - lastEmit) + if (wait <= 0) flush() + else if (timer === null) timer = setTimer(flush, wait) + } + + return { push, flush } +} + +export interface MachineDeps { + /** Run the real resolve. `wheelhouse` is null until the user picks a folder. */ + run: (wheelhouse: string | null, onProgress: (chunk: string) => void) => Promise + emit: (msg: EnvSetupWireMessage) => void + /** Resolve with the renderer's next answer. Only called while in `error`. */ + waitForAnswer: () => Promise + /** Native folder picker; null when cancelled. */ + pickFolder: () => Promise + quit: () => void + /** Wheelhouse to start with (an app that already knows the machine is offline). */ + initialWheelhouse?: string | null + intervalMs?: number +} + +export async function runEnvSetupMachine(deps: MachineDeps): Promise { + let wheelhouse = deps.initialWheelhouse ?? null + for (;;) { + const batcher = createProgressBatcher({ emit: deps.emit, intervalMs: deps.intervalMs }) + deps.emit({ type: 'env_setup', state: 'start', ...(wheelhouse ? { wheelhouse } : {}) }) + try { + const result = await deps.run(wheelhouse, batcher.push) + batcher.flush() + deps.emit({ type: 'env_setup', state: 'done' }) + return result + } catch (err) { + // Flush first so the last uv lines (usually the actual reason) reach the + // log tail BEFORE the error replaces the progress view. + batcher.flush() + deps.emit({ + type: 'env_setup', state: 'error', + message: (err as Error)?.message ?? String(err), + ...(wheelhouse ? { wheelhouse } : {}), + }) + } + + // Wait for a usable answer. A cancelled folder dialog is not an answer: the + // error stays on screen and we keep listening. + for (;;) { + const answer = await deps.waitForAnswer() + if (answer === 'quit') { + deps.quit() + throw new EnvSetupQuitError() + } + if (answer === 'retry') break + if (answer === 'use-wheelhouse') { + const dir = await deps.pickFolder() + if (!dir) continue + wheelhouse = dir + break + } + } + } +} diff --git a/de_shell/js/main/index.ts b/de_shell/js/main/index.ts index 8d59c30..9c370ad 100644 --- a/de_shell/js/main/index.ts +++ b/de_shell/js/main/index.ts @@ -42,9 +42,12 @@ export type { BackendHandlers } from './backendProcess' export { resolvePythonEnv, managedEnvPaths, venvPython, readLockedTorchVersion, - installTorchPerMachine, + installTorchPerMachine, hasNvidiaGpu, readWheelhouse, findWheelhouse, + wheelhouseCandidates, lockStamp, buildSyncArgs, offlinePlan, +} from './pythonEnv' +export type { + ResolvedPython, EnsureOptions, WheelhouseManifest, CommandRunner, OfflinePlan, } from './pythonEnv' -export type { ResolvedPython, EnsureOptions } from './pythonEnv' export { parseUvLine } from './envProgress' export type { EnvPhase, EnvProgressEvent } from './envProgress' @@ -59,3 +62,8 @@ export type { UpdateChannel, UpdateStatus } from './updater' export { isPrereleaseVersion, defaultChannelForVersion, truncateMessage, } from './updaterErrors' + +export { registerUpdaterIpc, registerProblemReportIpc } from './shellIpc' + +export { runEnvSetup, EnvSetupQuitError } from './envSetup' +export type { EnvSetupAnswer, EnvSetupWireMessage } from './envSetup' diff --git a/de_shell/js/main/pythonEnv.test.ts b/de_shell/js/main/pythonEnv.test.ts index fd2ecef..ca41c7b 100644 --- a/de_shell/js/main/pythonEnv.test.ts +++ b/de_shell/js/main/pythonEnv.test.ts @@ -25,7 +25,11 @@ import { dirname, join } from 'path' import { tmpdir } from 'os' import { createHash } from 'crypto' import { configureShell } from './config.ts' -import { findUv, resolvePythonEnv, venvPython } from './pythonEnv.ts' +import { + findUv, resolvePythonEnv, hasNvidiaGpu, readWheelhouse, findWheelhouse, + wheelhouseCandidates, lockStamp, buildSyncArgs, offlinePlan, effectiveExtras, + gpuEnvOverride, platformMatches, fileUrl, checkWheelhouseExtras, venvPython, +} from './pythonEnv.ts' configureShell({ appId: 'testapp', @@ -145,10 +149,152 @@ test('dev-mode resolvePythonEnv leaves a bare uv when none is found (the spawn t } }) +// ── extras, GPU probe, stamp ───────────────────────────────────────────────── + +test('lockStamp is the bare hash with no extras (existing installs do not re-sync)', () => { + assert.equal(lockStamp('abc', []), 'abc') +}) + +test('lockStamp appends sorted extras, order-insensitively', () => { + assert.equal(lockStamp('abc', ['gpu', 'cuda']), 'abc+extras=cuda,gpu') + assert.equal(lockStamp('abc', ['cuda', 'gpu']), lockStamp('abc', ['gpu', 'cuda'])) +}) + +test('effectiveExtras adds gpuExtras only with a GPU, de-duplicated', () => { + assert.deepEqual(effectiveExtras(['a'], ['gpu'], false), ['a']) + assert.deepEqual(effectiveExtras(['a'], ['gpu'], true), ['a', 'gpu']) + assert.deepEqual(effectiveExtras(['gpu'], ['gpu'], true), ['gpu']) + assert.deepEqual(effectiveExtras(undefined, undefined, true), []) +}) + +test('buildSyncArgs passes --extra for each extra, torch skip last', () => { + assert.deepEqual( + buildSyncArgs(['--no-install-project'], ['gpu', 'x'], ['torch']), + ['sync', '--frozen', '--no-dev', '--no-install-project', + '--extra', 'gpu', '--extra', 'x', '--no-install-package', 'torch']) + assert.deepEqual(buildSyncArgs([], []), ['sync', '--frozen', '--no-dev']) +}) + +const runner = (code: number | null, stdout = '') => async () => ({ code, stdout }) + +test('hasNvidiaGpu parses nvidia-smi -L output', async () => { + assert.equal(await hasNvidiaGpu(runner(0, 'GPU 0: NVIDIA RTX A4000 (UUID: GPU-1)\n')), true) + assert.equal(await hasNvidiaGpu(runner(0, '')), false) + assert.equal(await hasNvidiaGpu(runner(0, 'No devices were found')), false) + assert.equal(await hasNvidiaGpu(runner(9, 'GPU 0: x')), false) + assert.equal(await hasNvidiaGpu(runner(null)), false) +}) + +test('hasNvidiaGpu never throws and asks for a ~5 s timeout', async () => { + assert.equal(await hasNvidiaGpu(async () => { throw new Error('ENOENT') }), false) + let seen: unknown[] = [] + await hasNvidiaGpu(async (cmd, args, t) => { seen = [cmd, args, t]; return { code: 1, stdout: '' } }) + assert.deepEqual(seen, ['nvidia-smi', ['-L'], 5000]) +}) + +test('gpuEnvOverride reads _GPU', () => { + assert.equal(gpuEnvOverride({ TESTAPP_GPU: '0' }), false) + assert.equal(gpuEnvOverride({ TESTAPP_GPU: '1' }), true) + assert.equal(gpuEnvOverride({ TESTAPP_GPU: '' }), undefined) + assert.equal(gpuEnvOverride({}), undefined) +}) + +// ── wheelhouse ─────────────────────────────────────────────────────────────── + +function wheelhouseDir(manifest: unknown): string { + const dir = tempDir('wheelhouse-') + writeFileSync(join(dir, 'wheelhouse.json'), + typeof manifest === 'string' ? manifest : JSON.stringify(manifest)) + return dir +} + +const goodManifest = { + app: 'testapp', version: '1.2.3', platform: 'win_amd64', python: '3.13.12', extras: ['gpu'], +} + +test('readWheelhouse parses a valid manifest', () => { + assert.deepEqual(readWheelhouse(wheelhouseDir(goodManifest)), goodManifest) +}) + +test('readWheelhouse is null for missing, unparseable, or foreign manifests', () => { + assert.equal(readWheelhouse(emptyDir()), null) + assert.equal(readWheelhouse(wheelhouseDir('{not json')), null) + assert.equal(readWheelhouse(wheelhouseDir({ ...goodManifest, app: 'other' })), null) +}) + +test('findWheelhouse returns the first valid directory', () => { + const a = wheelhouseDir({ ...goodManifest, app: 'other' }) + const b = wheelhouseDir(goodManifest) + const c = wheelhouseDir({ ...goodManifest, version: '9' }) + const found = findWheelhouse([emptyDir(), a, b, c]) + assert.equal(found?.dir, b) + assert.equal(found?.manifest.version, '1.2.3') + assert.equal(findWheelhouse([emptyDir()]), null) +}) + +test('wheelhouseCandidates are ordered option, env, resources, userData, exec dir', () => { + const got = wheelhouseCandidates( + { wheelhouse: '/opt', resourcesPath: '/res', userData: '/ud' }, + { TESTAPP_WHEELHOUSE: '/env' }, + join('/bin', 'app.exe'), + ) + assert.deepEqual(got, ['/opt', '/env', join('/res', 'wheelhouse'), + join('/ud', 'wheelhouse'), join('/bin', 'wheelhouse')]) + // Unset option / env are skipped, not left as holes. + assert.deepEqual( + wheelhouseCandidates({ wheelhouse: null, resourcesPath: '/res', userData: '/ud' }, {}, '/bin/app'), + [join('/res', 'wheelhouse'), join('/ud', 'wheelhouse'), join('/bin', 'wheelhouse')]) +}) + +test('platformMatches is loose but os/arch aware', () => { + assert.equal(platformMatches('win_amd64', 'win32', 'x64'), true) + assert.equal(platformMatches('win_amd64', 'linux', 'x64'), false) + assert.equal(platformMatches('manylinux_2_28_x86_64', 'linux', 'x64'), true) + assert.equal(platformMatches('macosx_11_0_arm64', 'darwin', 'arm64'), true) + assert.equal(platformMatches('macosx_11_0_arm64', 'darwin', 'x64'), false) +}) + +test('fileUrl uses forward slashes with a three-slash drive form', () => { + assert.equal(fileUrl('C:\\wh\\python'), 'file:///C:/wh/python') + assert.equal(fileUrl('/wh/python'), 'file:///wh/python') +}) + +test('offlinePlan builds the python, venv, export, pip-install sequence', () => { + const plan = offlinePlan({ + wheelhouse: join('/wh'), manifest: { python: '3.13.12' }, + envDir: join('/ud', 'python-env'), userData: '/ud', extras: ['gpu'], + }) + assert.equal(plan.env.UV_OFFLINE, '1') + assert.equal(plan.env.UV_PYTHON_DOWNLOADS, 'automatic') + assert.ok(plan.env.UV_PYTHON_INSTALL_MIRROR.startsWith('file://')) + assert.ok(plan.env.UV_PYTHON_INSTALL_MIRROR.endsWith('/python')) + assert.deepEqual(plan.python, ['python', 'install', '3.13.12']) + assert.deepEqual(plan.venv, ['venv', '--python', '3.13.12', '--managed-python', + '--allow-existing', join('/ud', 'python-env')]) + assert.deepEqual(plan.export.slice(0, 7), + ['export', '--frozen', '--no-dev', '--no-emit-project', '--no-emit-workspace', + '--extra', 'gpu']) + assert.equal(plan.export.at(-1), plan.requirementsFile) + assert.equal(plan.requirementsFile, join('/ud', 'requirements.offline.txt')) + for (const flag of ['--offline', '--no-index', '--require-hashes']) { + assert.ok(plan.install.includes(flag), flag) + } + assert.equal(plan.install[plan.install.indexOf('--find-links') + 1], join('/wh')) + assert.equal(plan.install[plan.install.indexOf('-r') + 1], plan.requirementsFile) + assert.equal(plan.install.at(-1), venvPython(join('/ud', 'python-env'))) +}) + +test('checkWheelhouseExtras names the missing extra', () => { + const m = { ...goodManifest, extras: [] } + assert.throws(() => checkWheelhouseExtras(['gpu'], m), /built without the 'gpu' extra/) + checkWheelhouseExtras([], m) // gpu disabled → not effective → fine + checkWheelhouseExtras(['gpu'], goodManifest) +}) + /** A packaged layout whose managed env already matches the shipped lock, so * resolvePythonEnv() answers from disk without shelling out to uv. */ function packagedLayout(): { resourcesPath: string; userData: string; envDir: string } { - const root = mkdtempSync(join(tmpdir(), 'packaged-')) + const root = tempDir('packaged-') const resourcesPath = join(root, 'resources') const userData = join(root, 'userData') const projectDir = join(resourcesPath, 'python') diff --git a/de_shell/js/main/pythonEnv.ts b/de_shell/js/main/pythonEnv.ts index b1a9ebf..d3ad664 100644 --- a/de_shell/js/main/pythonEnv.ts +++ b/de_shell/js/main/pythonEnv.ts @@ -21,16 +21,31 @@ * path), so the working install path can never regress. macOS keeps the plain * sync (no CUDA pin there; PyPI torch already carries MPS). * + * Extras: `ShellConfig.pythonExtras` are always installed (`--extra X`) and + * `gpuExtras` only when an NVIDIA GPU is found (`hasNvidiaGpu`). The effective + * extras are folded into the lock stamp, so gaining a GPU or adding an extra + * re-syncs while an app with no extras keeps its bare-hash stamp (no re-sync on + * upgrade). + * + * Offline: when a WHEELHOUSE (see `readWheelhouse`) is found, the network is not + * touched at all — see `offlinePlan` for why this is venv + export + pip install + * rather than `uv sync`. + * + * No residue: every packaged uv call pins `UV_PYTHON_INSTALL_DIR` under + * userData, so uv's managed interpreters do not land in %APPDATA%\uv\python, + * turns off uv's ~/.local/bin launchers and registry entries, and requires a + * managed interpreter so the env never links to a Python the app does not own. + * * In development (no bundled payload) we fall back to `uv run` from the repo * root — exactly the previous behaviour. */ import { spawn } from 'child_process' import { existsSync, mkdirSync, readdirSync, readFileSync, writeFileSync } from 'fs' import { createHash } from 'crypto' -import { join } from 'path' +import { dirname, join } from 'path' // Extension spelled out so node:test can load this module without a bundler // (native type-stripping resolves relative imports literally). -import { shellConfig } from './config.ts' +import { envPrefix, shellConfig } from './config.ts' export interface ResolvedPython { cmd: string[] // argv to spawn, e.g. [pythonExe, '-m', 'spyde'] (module from ShellConfig) @@ -43,6 +58,12 @@ export interface EnsureOptions { projectRoot: string // repo root (dev) — holds pyproject.toml userData: string // app.getPath('userData') — writable venv lives here onProgress?: (line: string) => void + /** Force the GPU decision (true = add `gpuExtras`, false = never), skipping + * the `nvidia-smi` probe. Beats the `_GPU` env var. */ + gpu?: boolean + /** Explicit wheelhouse directory for an offline install; first candidate + * (see `wheelhouseCandidates`). null/undefined = search the defaults. */ + wheelhouse?: string | null } const isWin = process.platform === 'win32' @@ -137,6 +158,246 @@ export function readLockedTorchVersion(projectDir: string): string | null { return pick.version.split('+')[0] // "2.6.0+cu124" → "2.6.0" } +// ── GPU probe ──────────────────────────────────────────────────────────────── + +/** Runs a command and reports its exit code + stdout. Injectable so the probe's + * parsing is testable without nvidia-smi. A spawn failure may reject or return + * code null — `hasNvidiaGpu` treats both as "no GPU". */ +export type CommandRunner = ( + cmd: string, args: string[], timeoutMs: number, +) => Promise<{ code: number | null; stdout: string }> + +const defaultRunner: CommandRunner = (cmd, args, timeoutMs) => + new Promise((resolve) => { + let out = '' + let settled = false + const done = (code: number | null) => { + if (settled) return + settled = true + clearTimeout(timer) + resolve({ code, stdout: out }) + } + let proc: ReturnType + // A hung driver must not hang app start-up: kill after the timeout. + const timer = setTimeout(() => { try { proc?.kill() } catch { /* gone */ } done(null) }, timeoutMs) + try { + proc = spawn(cmd, args, { stdio: ['ignore', 'pipe', 'ignore'], windowsHide: true }) + } catch { done(null); return } + proc.stdout?.on('data', (b: Buffer) => { out += b.toString() }) + proc.on('error', () => done(null)) // nvidia-smi absent (ENOENT) → no GPU + proc.on('close', (code) => done(code)) + }) + +/** + * Is an NVIDIA GPU usable on this machine? `nvidia-smi -L` lists one + * `GPU 0: (UUID: …)` line per device and ships with the driver, so it + * answers "driver + device present" — the condition under which the CuPy-style + * GPU extras can actually work. Never throws: a missing binary, non-zero exit, + * timeout or output without a `GPU ` line is simply false (a CPU install is + * always a safe fallback). + */ +export async function hasNvidiaGpu(run: CommandRunner = defaultRunner): Promise { + try { + const { code, stdout } = await run('nvidia-smi', ['-L'], 5000) + return code === 0 && /^GPU /m.test(stdout ?? '') + } catch { return false } +} + +/** The `_GPU` override: '0'/'false' → false, '1'/'true' → true, else + * undefined (no override). */ +export function gpuEnvOverride(env: NodeJS.ProcessEnv = process.env): boolean | undefined { + const v = (env[`${envPrefix()}_GPU`] ?? '').trim().toLowerCase() + if (v === '1' || v === 'true') return true + if (v === '0' || v === 'false') return false + return undefined +} + +/** Effective extras, in config order, de-duplicated: `pythonExtras` plus + * `gpuExtras` when `gpu`. Pure. */ +export function effectiveExtras( + pythonExtras: string[] | undefined, gpuExtras: string[] | undefined, gpu: boolean, +): string[] { + return [...new Set([...(pythonExtras ?? []), ...(gpu ? (gpuExtras ?? []) : [])])] +} + +/** + * The stamp written to `.-lock-hash`. With NO extras it is the bare + * sha256 of uv.lock — byte-identical to what earlier versions wrote, so + * upgrading de-shell does not force every existing install to re-sync. With + * extras it is `+extras=` so a new extra, or a + * machine gaining a GPU, re-syncs (order-insensitive: config order is not + * semantic). + */ +export function lockStamp(lockHash: string, extras: string[]): string { + return extras.length ? `${lockHash}+extras=${[...extras].sort().join(',')}` : lockHash +} + +/** `['--extra','a','--extra','b']` for `uv sync` / `uv export`. */ +export function extraArgs(extras: string[]): string[] { + return extras.flatMap((e) => ['--extra', e]) +} + +/** Every `uv sync` argv in one place so the torch two-step and the plain + * fallback cannot drift apart on extras. `skipPackages` → `--no-install-package`. */ +export function buildSyncArgs( + projectArgs: string[], extras: string[], skipPackages: string[] = [], +): string[] { + return [ + 'sync', '--frozen', '--no-dev', ...projectArgs, ...extraArgs(extras), + ...skipPackages.flatMap((p) => ['--no-install-package', p]), + ] +} + +// ── Wheelhouse (offline install) ───────────────────────────────────────────── + +/** `wheelhouse.json`, written by the wheelhouse builder. The directory beside + * it holds a flat set of `*.whl` plus `python//`, a + * `uv python install` mirror. */ +export interface WheelhouseManifest { + app: string + version: string + platform: string // e.g. 'win_amd64' + python: string // full version, e.g. '3.13.12' + extras: string[] +} + +/** Parsed manifest, or null when missing/unparseable/for another app. */ +export function readWheelhouse(dir: string): WheelhouseManifest | null { + try { + const m = JSON.parse(readFileSync(join(dir, 'wheelhouse.json'), 'utf8')) + if (!m || typeof m !== 'object' || m.app !== shellConfig().appId) return null + return { + app: m.app, + version: String(m.version ?? ''), + platform: String(m.platform ?? ''), + python: String(m.python ?? ''), + extras: Array.isArray(m.extras) ? m.extras.map(String) : [], + } + } catch { return null } +} + +/** First candidate directory holding a valid manifest for this app. */ +export function findWheelhouse( + candidates: string[], +): { dir: string; manifest: WheelhouseManifest } | null { + for (const dir of candidates) { + const manifest = readWheelhouse(dir) + if (manifest) return { dir, manifest } + } + return null +} + +/** Where to look for a wheelhouse, most explicit first: the option, the + * `_WHEELHOUSE` env var, beside the bundled payload, in userData (copied + * there by hand), and next to the executable (a USB-stick-style install). */ +export function wheelhouseCandidates( + opts: Pick, + env: NodeJS.ProcessEnv = process.env, + execPath: string = process.execPath, +): string[] { + return [ + opts.wheelhouse, + env[`${envPrefix()}_WHEELHOUSE`], + join(opts.resourcesPath, 'wheelhouse'), + join(opts.userData, 'wheelhouse'), + join(dirname(execPath), 'wheelhouse'), + ].filter((d): d is string => !!d) +} + +/** Does a wheel platform tag (`win_amd64`, `manylinux_2_28_x86_64`, + * `macosx_11_0_arm64`) plausibly describe this machine? Deliberately loose — + * it only drives a warning; uv's hash check is the real guard. */ +export function platformMatches( + tag: string, platform: string = process.platform, arch: string = process.arch, +): boolean { + const t = tag.toLowerCase() + const osOk = platform === 'win32' ? t.startsWith('win') + : platform === 'darwin' ? t.startsWith('macosx') + : t.includes('linux') + const archs: Record = { + x64: ['amd64', 'x86_64'], arm64: ['arm64', 'aarch64'], ia32: ['win32', 'i686'], + } + return osOk && (archs[arch] ?? [arch]).some((a) => t.includes(a)) +} + +/** `file:///C:/x/python` (Windows) or `file:///x/python` (posix). */ +export function fileUrl(dir: string): string { + const p = dir.replace(/\\/g, '/') + return p.startsWith('/') ? `file://${p}` : `file:///${p}` +} + +export interface OfflinePlan { + /** Extra env for every offline uv call. */ + env: Record + /** argv (after `uv`) for each step, in order. */ + python: string[] + venv: string[] + export: string[] + install: string[] + /** Where `uv export` writes the hashed requirements. */ + requirementsFile: string +} + +/** + * The offline install, as argv — pure so it is testable without spawning uv. + * + * Why not `uv sync --offline --find-links`: for a frozen sync uv only looks for + * registry-locked wheels in its CACHE, never in find-links, so it cannot + * install from a wheelhouse. Instead: + * 1. `uv python install ` — the interpreter comes from the wheelhouse's + * python-build-standalone mirror (UV_PYTHON_INSTALL_MIRROR=file://…). + * It has to be its own step: under UV_OFFLINE `uv venv` refuses to fetch + * an interpreter even from a file:// mirror ("a managed Python download + * is available … but uv is set to offline mode"), while `uv python + * install` treats the mirror as the source it was told to use. + * Then `uv venv --managed-python` over the interpreter just installed. + * 2. `uv export --frozen` — reads only the lock (no network) and emits every + * third-party pin WITH hashes. + * 3. `uv pip install --offline --no-index --find-links + * --require-hashes` — lock-exact, hash-verified, no network. Torch is + * therefore lock-exact too (no per-machine two-step). + * The staged member/app wheels are then installed by `installAppWheel` as usual. + */ +export function offlinePlan(p: { + wheelhouse: string + manifest: Pick + envDir: string + userData: string + extras: string[] +}): OfflinePlan { + const requirementsFile = join(p.userData, 'requirements.offline.txt') + return { + env: { + UV_OFFLINE: '1', + UV_PYTHON_INSTALL_MIRROR: fileUrl(join(p.wheelhouse, 'python')), + UV_PYTHON_DOWNLOADS: 'automatic', + }, + python: ['python', 'install', p.manifest.python], + venv: ['venv', '--python', p.manifest.python, '--managed-python', + '--allow-existing', p.envDir], + export: ['export', '--frozen', '--no-dev', '--no-emit-project', '--no-emit-workspace', + ...extraArgs(p.extras), '-o', requirementsFile], + install: ['pip', 'install', '--offline', '--no-index', '--find-links', p.wheelhouse, + '--require-hashes', '-r', requirementsFile, '--python', venvPython(p.envDir)], + requirementsFile, + } +} + +/** Fail loudly when the wheelhouse lacks an extra we need. `gpuExtras` are only + * required when `gpu` is true — a user who set gpu=false never asked for them, + * and they are not in `extras` then. */ +export function checkWheelhouseExtras(extras: string[], manifest: WheelhouseManifest): void { + for (const e of extras) { + if (!manifest.extras.includes(e)) { + throw new Error( + `this wheelhouse was built without the '${e}' extra ` + + `(it has: ${manifest.extras.join(', ') || 'none'}). Rebuild it with '${e}', ` + + `or disable the GPU install with ${envPrefix()}_GPU=0 if '${e}' is a GPU extra.`, + ) + } + } +} + /** Resolve the command to launch the app's backend, creating the venv if needed. */ export async function resolvePythonEnv(opts: EnsureOptions): Promise { const cfg = shellConfig() @@ -157,17 +418,63 @@ export async function resolvePythonEnv(opts: EnsureOptions): Promise _GPU > nvidia-smi probe. The probe only runs + // when the app declared gpuExtras — otherwise there is nothing to decide. An + // explicit choice (option or env var) is strict; a probed one may be dropped + // when the install source cannot satisfy it (see the wheelhouse branch below). + const gpuExtras = cfg.gpuExtras ?? [] + const gpuExplicit = opts.gpu ?? gpuEnvOverride() + const gpu = gpuExtras.length === 0 ? false + : (gpuExplicit ?? await hasNvidiaGpu()) + let extras = effectiveExtras(cfg.pythonExtras, gpuExtras, gpu) + let stamp = lockStamp(lockHash, extras) + // The stamp written when a probed GPU was dropped because the wheelhouse was + // built without the GPU extras. Accepting it below is what stops an + // air-gapped GPU machine from re-syncing — online, so failing — on every + // launch; a new lock, or an explicit _GPU=1, re-evaluates. + const cpuOnly = effectiveExtras(cfg.pythonExtras, gpuExtras, false) + const gpuUnavailableStamp = `${lockStamp(lockHash, cpuOnly)}+gpu=unavailable` + + // Skip the sync if the venv already matches the shipped lock AND extras. + const onDisk = existsSync(stampFile) ? readSafe(stampFile) : null const upToDate = existsSync(pythonExe) && - existsSync(stampFile) && - readSafe(stampFile) === lockHash + (onDisk === stamp || + (gpu && gpuExplicit === undefined && onDisk === gpuUnavailableStamp)) if (!upToDate) { - await setupEnv(bundledProject, envDir, readAppVersion(bundledProject), opts.onProgress) + if (gpuExtras.length) { + opts.onProgress?.(gpu + ? `[env-setup] ${gpuExplicit ? 'GPU install requested' : 'NVIDIA GPU detected'} → extras: ${gpuExtras.join(', ')} +` + : `[env-setup] no NVIDIA GPU (or disabled) → skipping extras: ${gpuExtras.join(', ')} +`) + } + const found = findWheelhouse(wheelhouseCandidates(opts)) + if (found) { + // A probed GPU whose extras the wheelhouse lacks degrades to a CPU + // install rather than failing: the site still gets a working app, and + // the stamp remembers why. Anything else missing (an app extra, or a GPU + // the user asked for explicitly) fails in checkWheelhouseExtras. + const missingGpu = gpu && gpuExplicit === undefined && + gpuExtras.some((e) => !found.manifest.extras.includes(e)) + if (missingGpu) { + opts.onProgress?.( + `[env-setup] this wheelhouse has no GPU packages (${gpuExtras.join(', ')}); ` + + `installing without them. Set ${envPrefix()}_GPU=1 to require them. +`) + extras = cpuOnly + stamp = gpuUnavailableStamp + } + checkWheelhouseExtras(extras, found.manifest) + } + await setupEnv({ + projectDir: bundledProject, envDir, appVersion: readAppVersion(bundledProject), + extras, wheelhouse: found, onProgress: opts.onProgress, + }) try { mkdirSync(envDir, { recursive: true }) - writeFileSync(stampFile, lockHash, 'utf8') + writeFileSync(stampFile, stamp, 'utf8') } catch { /* a missing stamp just forces a re-sync next launch */ } } @@ -205,7 +512,10 @@ function readAppVersion(projectDir: string): string { * exits 1 (the first-launch crash). Absent in dev (marker not present) → the * env is unchanged and setuptools_scm resolves from the repo's git history. */ -function uvEnv(envDir: string, appVersion: string, projectDir?: string): NodeJS.ProcessEnv { +function uvEnv( + envDir: string, appVersion: string, projectDir?: string, + extraEnv: Record = {}, +): NodeJS.ProcessEnv { // Pretend-version env for setuptools_scm. The dist-name–scoped var is the // precise one; the plain SETUPTOOLS_SCM_PRETEND_VERSION is a harmless // belt-and-braces fallback in case the dist name ever changes. @@ -247,6 +557,25 @@ function uvEnv(envDir: string, appVersion: string, projectDir?: string): NodeJS. // Keep uv's own cache/tools next to the env so an air-gapped re-run is // self-contained and we never write into the read-only bundle. UV_CACHE_DIR: join(envDir, '..', 'uv-cache'), + // No residue outside userData: uv's managed interpreters default to + // %APPDATA%\uv\python (~/.local/share/uv/python), which an uninstall never + // cleans. Existing venvs keep working after this moves — their pyvenv.cfg + // records the absolute path of the interpreter they were created with. + UV_PYTHON_INSTALL_DIR: join(envDir, '..', 'python'), + // ...and nothing else of uv's leaks out either. `uv python install` (the + // offline path, and any automatic download) otherwise also drops a + // python3.X.exe launcher into ~/.local/bin and registers the interpreter + // under HKCU\Software\Python\Astral (PEP 514). Both outlive an uninstall + // and point into a folder it deletes. + UV_PYTHON_INSTALL_BIN: '0', + UV_PYTHON_INSTALL_REGISTRY: '0', + // Only ever build the env on an interpreter in that folder. Without it uv + // takes the first suitable Python it discovers -- one on PATH, or a + // launcher another tool left in ~/.local/bin -- and the venv's pyvenv.cfg + // then points at something this app neither owns nor removes, and which + // may vanish under it. Seen in testing: a venv linked to a stray launcher. + UV_MANAGED_PYTHON: '1', + ...extraEnv, } } @@ -257,6 +586,7 @@ function runUv( args: string[], appVersion: string, onProgress?: (line: string) => void, + extraEnv: Record = {}, ): Promise { const uv = join(projectDir, uvBinaryName()) // Fall back to a resolved PATH/per-user uv if not bundled (dev builds). @@ -264,7 +594,7 @@ function runUv( return new Promise((resolve, reject) => { const proc = spawn(uvCmd, args, { cwd: projectDir, - env: uvEnv(envDir, appVersion, projectDir), + env: uvEnv(envDir, appVersion, projectDir, extraEnv), stdio: ['ignore', 'pipe', 'pipe'], }) const relay = (b: Buffer) => onProgress?.(b.toString()) @@ -363,6 +693,7 @@ function installAppWheel( envDir: string, wheel: string, onProgress?: (line: string) => void, + offlineEnv?: Record, ): Promise { onProgress?.( `[env-setup] installing ${shellConfig().pythonDist} from wheel ` + @@ -370,8 +701,9 @@ function installAppWheel( ) return runUv( projectDir, envDir, - ['pip', 'install', '--no-deps', '--python', venvPython(envDir), wheel], - '', onProgress, + ['pip', 'install', ...(offlineEnv ? ['--offline'] : []), '--no-deps', + '--python', venvPython(envDir), wheel], + '', onProgress, offlineEnv, ) } @@ -387,12 +719,15 @@ function installAppWheel( * read-only source tree. In dev (no staged wheel) the project installs from * source as before. Logs which path ran to the progress stream. */ -async function setupEnv( - projectDir: string, - envDir: string, - appVersion: string, - onProgress?: (line: string) => void, -): Promise { +async function setupEnv(o: { + projectDir: string + envDir: string + appVersion: string + extras: string[] + wheelhouse: { dir: string; manifest: WheelhouseManifest } | null + onProgress?: (line: string) => void +}): Promise { + const { projectDir, envDir, appVersion, extras, wheelhouse, onProgress } = o const wheel = stagedAppWheel(projectDir) const memberWheels = stagedMemberWheels(projectDir, wheel) // With a staged wheel, tell uv sync NOT to touch the project (the app builds @@ -408,6 +743,29 @@ async function setupEnv( projectArgs.push('--no-install-package', wheelPackageName(memberWheel)) } + if (wheelhouse) { + const { dir, manifest } = wheelhouse + if (!platformMatches(manifest.platform)) { + onProgress?.(`[env-setup] WARNING: wheelhouse is for ${manifest.platform}, this machine is ${process.platform}/${process.arch}\n`) + } + if (appVersion && manifest.version && manifest.version !== appVersion) { + onProgress?.(`[env-setup] WARNING: wheelhouse is for version ${manifest.version}, app is ${appVersion}\n`) + } + onProgress?.(`[env-setup] offline install from wheelhouse ${dir}\n`) + const plan = offlinePlan({ wheelhouse: dir, manifest, envDir, userData: join(envDir, '..'), extras }) + mkdirSync(join(envDir, '..'), { recursive: true }) + await runUv(projectDir, envDir, plan.python, appVersion, onProgress, plan.env) + await runUv(projectDir, envDir, plan.venv, appVersion, onProgress, plan.env) + await runUv(projectDir, envDir, plan.export, appVersion, onProgress, plan.env) + await runUv(projectDir, envDir, plan.install, appVersion, onProgress, plan.env) + for (const memberWheel of memberWheels) { + await installAppWheel(projectDir, envDir, memberWheel, onProgress, plan.env) + } + if (wheel) await installAppWheel(projectDir, envDir, wheel, onProgress, plan.env) + onProgress?.('[env-setup] offline install complete\n') + return + } + const twoStep = (process.platform === 'win32' || process.platform === 'linux') && readLockedTorchVersion(projectDir) !== null @@ -420,7 +778,7 @@ async function setupEnv( // only torch-family package in the lock, so one exclusion covers it. await runUv( projectDir, envDir, - ['sync', '--frozen', '--no-dev', ...projectArgs, '--no-install-package', 'torch'], + buildSyncArgs(projectArgs, extras, ['torch']), appVersion, onProgress, ) // Step 2: torch resolved for this machine. @@ -442,7 +800,7 @@ async function setupEnv( } onProgress?.('[env-setup] running full locked uv sync\n') - await runUv(projectDir, envDir, ['sync', '--frozen', '--no-dev', ...projectArgs], appVersion, onProgress) + await runUv(projectDir, envDir, buildSyncArgs(projectArgs, extras), appVersion, onProgress) for (const memberWheel of memberWheels) { await installAppWheel(projectDir, envDir, memberWheel, onProgress) } diff --git a/de_shell/js/main/shellIpc.ts b/de_shell/js/main/shellIpc.ts new file mode 100644 index 0000000..485a123 --- /dev/null +++ b/de_shell/js/main/shellIpc.ts @@ -0,0 +1,45 @@ +/** + * shellIpc.ts — the updater and problem-report IPC, registered with one call + * each. Every app used to hand-wire these (and Ground Crew wired neither). + * + * Channels are `:update-*` and `:report-*`. Update STATUS is + * pushed on `:update-status` by updater.ts itself (it holds the window + * given to `initUpdater`), so there is no subscribe channel here — only the + * `update-info` invoke that seeds a late-mounting UI with the last status. + * Safe to call twice: handlers are replaced, not stacked. + */ +import { app, ipcMain } from 'electron' + +import { channel } from './config' +import { collectDiagnostics, reportingConfigured, submitReport } from './errorReport' +import { bindReportIpc, bindUpdaterIpc } from './shellIpcCore' +import { + checkForUpdates, downloadUpdate, getLastUpdateStatus, quitAndInstall, + readUpdateChannel, setUpdateChannel, updatesSupported, +} from './updater' + +export function registerUpdaterIpc( + opts: { onChannelChange?: (c: 'stable' | 'beta') => void } = {}, +): void { + bindUpdaterIpc(ipcMain, { + channel, + readChannel: readUpdateChannel, + setChannel: setUpdateChannel, + supported: updatesSupported, + status: getLastUpdateStatus, + appVersion: () => app.getVersion(), + check: checkForUpdates, + download: downloadUpdate, + quitAndInstall, + onChannelChange: opts.onChannelChange, + }) +} + +export function registerProblemReportIpc(): void { + bindReportIpc(ipcMain, { + channel, + configured: reportingConfigured, + collect: collectDiagnostics, + submit: submitReport, + }) +} diff --git a/de_shell/js/main/shellIpcCore.test.ts b/de_shell/js/main/shellIpcCore.test.ts new file mode 100644 index 0000000..7a4bf2a --- /dev/null +++ b/de_shell/js/main/shellIpcCore.test.ts @@ -0,0 +1,99 @@ +/** + * shellIpcCore.test.ts — channel names, idempotency and input validation for + * the updater / report IPC, against a fake ipcMain. + */ +import { test } from 'node:test' +import assert from 'node:assert/strict' + +import { + bindReportIpc, bindUpdaterIpc, REPORT_CHANNELS, UPDATER_CHANNELS, +} from './shellIpcCore.ts' +import type { IpcLike, UpdaterDeps } from './shellIpcCore.ts' + +function fakeIpc() { + const handlers = new Map unknown>() + const listeners = new Map void>>() + const ipc: IpcLike = { + handle(ch, fn) { + if (handlers.has(ch)) throw new Error(`second handler for ${ch}`) // Electron does this + handlers.set(ch, fn) + }, + on(ch, fn) { listeners.set(ch, [...(listeners.get(ch) ?? []), fn]) }, + removeHandler(ch) { handlers.delete(ch) }, + removeAllListeners(ch) { listeners.delete(ch) }, + } + const emit = (ch: string, ...a: unknown[]) => listeners.get(ch)?.forEach(f => f({}, ...a)) + return { ipc, handlers, listeners, emit } +} + +function updaterDeps(calls: string[]): UpdaterDeps { + return { + channel: n => `app:${n}`, + readChannel: () => 'stable', + setChannel: c => calls.push(`set:${c}`), + supported: () => true, + status: () => ({ state: 'idle' }), + appVersion: () => '1.2.3', + check: () => calls.push('check'), + download: () => calls.push('download'), + quitAndInstall: () => calls.push('install'), + onChannelChange: c => calls.push(`changed:${c}`), + } +} + +test('channel names are namespaced and stable (the preload spells the same ones)', () => { + assert.deepEqual(UPDATER_CHANNELS, { + info: 'update-info', check: 'update-check', download: 'update-download', + install: 'update-install', setChannel: 'update-set-channel', status: 'update-status', + }) + assert.deepEqual(REPORT_CHANNELS, { diagnostics: 'report-diagnostics', submit: 'report-submit' }) +}) + +test('updater actions reach the updater; info seeds a late UI', async () => { + const f = fakeIpc(); const calls: string[] = [] + bindUpdaterIpc(f.ipc, updaterDeps(calls)) + f.emit('app:update-check'); f.emit('app:update-download'); f.emit('app:update-install') + assert.deepEqual(calls, ['check', 'download', 'install']) + assert.deepEqual(await f.handlers.get('app:update-info')!({}), { + channel: 'stable', supported: true, status: { state: 'idle' }, appVersion: '1.2.3', + }) +}) + +test('registering twice neither throws nor doubles the work', () => { + const f = fakeIpc(); const calls: string[] = [] + bindUpdaterIpc(f.ipc, updaterDeps(calls)) + bindUpdaterIpc(f.ipc, updaterDeps(calls)) + f.emit('app:update-check') + assert.deepEqual(calls, ['check']) + bindReportIpc(f.ipc, { channel: n => `app:${n}`, configured: () => true, collect: async () => ({}), submit: async () => ({}) }) + bindReportIpc(f.ipc, { channel: n => `app:${n}`, configured: () => true, collect: async () => ({}), submit: async () => ({}) }) +}) + +test('set-channel ignores anything but stable and beta', () => { + const f = fakeIpc(); const calls: string[] = [] + bindUpdaterIpc(f.ipc, updaterDeps(calls)) + f.emit('app:update-set-channel', 'nightly') + f.emit('app:update-set-channel', { evil: 1 }) + f.emit('app:update-set-channel', 'beta') + assert.deepEqual(calls, ['set:beta', 'changed:beta']) +}) + +test('report: diagnostics reports canSend, submit sanitises its input', async () => { + const f = fakeIpc(); const seen: unknown[] = [] + bindReportIpc(f.ipc, { + channel: n => `app:${n}`, configured: () => false, + collect: async () => ({ problems: [] }), + submit: async i => { seen.push(i); return { sent: false } }, + }) + assert.deepEqual(await f.handlers.get('app:report-diagnostics')!({}), + { canSend: false, diagnostics: { problems: [] } }) + const submit = f.handlers.get('app:report-submit')! + await submit({}, { message: 'it broke', contact: ' a@b.c ' }) + await submit({}, { message: 5, contact: ' ' }) + await submit({}, null) + assert.deepEqual(seen, [ + { message: 'it broke', contact: 'a@b.c' }, + { message: '', contact: undefined }, + { message: '', contact: undefined }, + ]) +}) diff --git a/de_shell/js/main/shellIpcCore.ts b/de_shell/js/main/shellIpcCore.ts new file mode 100644 index 0000000..efc688e --- /dev/null +++ b/de_shell/js/main/shellIpcCore.ts @@ -0,0 +1,97 @@ +/** + * shellIpcCore.ts — the updater / problem-report IPC wiring with the Electron + * pieces injected, so it runs under node --test. shellIpc.ts binds it to the + * real `ipcMain` and the real updater.ts / errorReport.ts. + * + * Re-registering REPLACES the previous handlers rather than throwing (Electron + * throws on a second `handle` for one channel), which is what makes the public + * `register*` calls idempotent. + */ +export interface IpcLike { + handle(ch: string, fn: (event: unknown, ...args: any[]) => unknown): void + on(ch: string, fn: (event: unknown, ...args: any[]) => void): void + removeHandler(ch: string): void + removeAllListeners(ch: string): void +} + +/** Channel suffixes, joined to `:` by the caller. preload/index.ts + * spells the same names; shellIpc.test.ts pins them. */ +export const UPDATER_CHANNELS = { + info: 'update-info', + check: 'update-check', + download: 'update-download', + install: 'update-install', + setChannel: 'update-set-channel', + /** Main -> renderer push, sent by updater.ts itself. */ + status: 'update-status', +} as const + +export const REPORT_CHANNELS = { + diagnostics: 'report-diagnostics', + submit: 'report-submit', +} as const + +export interface UpdaterDeps { + channel: (name: string) => string + readChannel: () => 'stable' | 'beta' + setChannel: (c: 'stable' | 'beta') => void + supported: () => boolean + status: () => unknown + appVersion: () => string + check: () => void + download: () => void + quitAndInstall: () => void + /** Called after a valid channel change, e.g. to mirror it into the backend. */ + onChannelChange?: (c: 'stable' | 'beta') => void +} + +function replaceHandle(ipc: IpcLike, ch: string, fn: (e: unknown, ...a: any[]) => unknown) { + ipc.removeHandler(ch) + ipc.handle(ch, fn) +} +function replaceOn(ipc: IpcLike, ch: string, fn: (e: unknown, ...a: any[]) => void) { + ipc.removeAllListeners(ch) + ipc.on(ch, fn) +} + +export function bindUpdaterIpc(ipc: IpcLike, d: UpdaterDeps): void { + const ch = (k: keyof typeof UPDATER_CHANNELS) => d.channel(UPDATER_CHANNELS[k]) + // Seeds a dialog/card that mounts AFTER the startup check already fired. + replaceHandle(ipc, ch('info'), () => ({ + channel: d.readChannel(), + supported: d.supported(), + status: d.status(), + appVersion: d.appVersion(), + })) + replaceOn(ipc, ch('check'), () => d.check()) + replaceOn(ipc, ch('download'), () => d.download()) + replaceOn(ipc, ch('install'), () => d.quitAndInstall()) + replaceOn(ipc, ch('setChannel'), (_e, c: unknown) => { + // The renderer is untrusted input; anything but the two channels is ignored. + if (c !== 'stable' && c !== 'beta') return + d.setChannel(c) + d.onChannelChange?.(c) + }) +} + +export interface ReportDeps { + channel: (name: string) => string + configured: () => boolean + collect: () => Promise + submit: (input: { message: string; contact?: string }) => Promise +} + +export function bindReportIpc(ipc: IpcLike, d: ReportDeps): void { + // Reading diagnostics has no side effects: opening the dialog sends nothing. + replaceHandle(ipc, d.channel(REPORT_CHANNELS.diagnostics), async () => ({ + canSend: d.configured(), + diagnostics: await d.collect(), + })) + replaceHandle(ipc, d.channel(REPORT_CHANNELS.submit), (_e, input: unknown) => { + const i = (input ?? {}) as { message?: unknown; contact?: unknown } + return d.submit({ + message: typeof i.message === 'string' ? i.message : '', + contact: typeof i.contact === 'string' && i.contact.trim() ? i.contact.trim() : undefined, + }) + }) +} diff --git a/de_shell/js/packaging/build-wheelhouse.mjs b/de_shell/js/packaging/build-wheelhouse.mjs new file mode 100644 index 0000000..4f052b0 --- /dev/null +++ b/de_shell/js/packaging/build-wheelhouse.mjs @@ -0,0 +1,316 @@ +#!/usr/bin/env node +/** + * build-wheelhouse.mjs — build the OFFLINE wheelhouse for one target platform. + * + * node shell/packaging/build-wheelhouse.mjs --platform win_amd64 --out wheelhouse [flags] + * + * A normal packaged install runs `uv sync` against PyPI on first launch. A + * wheelhouse is the same install with the network taken out: every locked + * wheel for one platform, plus the python-build-standalone interpreter archive, + * so a lab machine with no internet can set up from a folder (or a zip on a + * USB stick). De-shell's main/pythonEnv.ts is the consumer; THE LAYOUT BELOW IS + * THAT CONTRACT: + * + * /wheelhouse.json {"app","version","platform","python","extras":[…]} + * /*.whl every locked wheel for the platform, flat + * /python// + * the interpreter, at the path uv's + * UV_PYTHON_INSTALL_MIRROR expects: the URL path + * after `…/releases/download/`, file name URL-decoded + * + * HOW. The lock is the source of truth, exactly as for the online install: + * 1. `uv export --frozen` writes the locked, hashed requirement set (no dev + * group, no project/workspace members — those are wheels the payload + * already carries, see bundle-python.mjs — plus the chosen extras). + * 2. Environment markers are evaluated HERE against the TARGET platform. pip + * downloading for a foreign platform still evaluates markers against the + * HOST, so a Linux CI runner would silently drop win32-only dependencies + * (colorama, pywin32 …) from the Windows wheelhouse. + * 3. `pip download --only-binary=:all: --no-deps --platform …` fetches the + * wheels. pip checks the hashes in the file, so the wheelhouse is exactly + * what the lock says, and `--require-hashes` works on the install side. + * A package with no wheel for the platform fails the build loudly; an + * sdist would have to be compiled on the user's machine, which is the very + * thing an offline install cannot do. + * 4. The interpreter archive URL comes from `uv python list`, so the archive + * is the one uv itself would have downloaded for that version. + * pip is run via `uv tool run pip`, so nothing needs installing. + * + * FLAGS (override the `deShell` block of electron/package.json, same keys as + * bundle-python.mjs): + * --platform win_amd64 | win_arm64 | macosx_11_0_arm64 | macosx_10_13_x86_64 | + * manylinux2014_x86_64 | manylinux2014_aarch64 REQUIRED + * --out output dir default: wheelhouse- + * --extra an optional-dependency extra to include (repeatable; or deShell.wheelhouseExtras) + * --no-extras none, even when deShell.wheelhouseExtras names some + * --python-version 3.13 or 3.13.12; default /.python-version + * --app-id, --version, --project-root, --electron-dir as bundle-python.mjs + * --app-name display name for the zip file default: deShell.productName, else appId + * --package `uv export --package` for a workspace member app + * --zip also write --wheels-.zip next to + * + * Node builtins only (fetch needs Node 18+). + */ +import { createWriteStream, existsSync, mkdirSync, readFileSync, readdirSync, rmSync, writeFileSync } from 'node:fs' +import { basename, dirname, join, resolve } from 'node:path' +import { execFileSync } from 'node:child_process' +import { Readable } from 'node:stream' +import { pipeline } from 'node:stream/promises' + +function log(msg) { console.log(`[build-wheelhouse] ${msg}`) } + +function parseFlags(argv) { + const out = { extra: [] } + for (let i = 0; i < argv.length; i++) { + const m = /^--([^=]+)(?:=(.*))?$/.exec(argv[i]) + if (!m) throw new Error(`unexpected argument ${argv[i]}`) + const [, key, inline] = m + const val = inline !== undefined ? inline : (['zip', 'no-extras'].includes(key) ? true : argv[++i]) + if (key === 'extra') out.extra.push(val) + else out[key] = val + } + return out +} + +// ── target platform ───────────────────────────────────────────────────────── + +/** + * What a pip platform tag means: the python-build-standalone os/arch/libc to + * fetch, the tags to hand pip (`--platform`, repeated: a manylinux machine + * accepts several generations, and a single tag would miss wheels published + * only as manylinux_2_28), and the values environment markers see. + */ +function describePlatform(tag) { + let m + if ((m = /^win_(amd64|arm64)$/.exec(tag))) { + const arm = m[1] === 'arm64' + return { + pbs: { os: 'windows', arch: arm ? 'aarch64' : 'x86_64' }, pip: [tag], + env: { sys_platform: 'win32', platform_system: 'Windows', os_name: 'nt', platform_machine: arm ? 'ARM64' : 'AMD64' }, + } + } + if ((m = /^macosx_\d+_\d+_(arm64|x86_64)$/.exec(tag))) { + const arm = m[1] === 'arm64' + return { + pbs: { os: 'darwin', arch: arm ? 'aarch64' : 'x86_64' }, pip: [tag], + env: { sys_platform: 'darwin', platform_system: 'Darwin', os_name: 'posix', platform_machine: m[1] }, + } + } + if ((m = /^manylinux\w*_(x86_64|aarch64)$/.exec(tag))) { + const a = m[1] + return { + pbs: { os: 'linux', arch: a, libc: 'gnu' }, + pip: [...new Set([tag, `manylinux2014_${a}`, `manylinux_2_28_${a}`])], + env: { sys_platform: 'linux', platform_system: 'Linux', os_name: 'posix', platform_machine: a }, + } + } + throw new Error(`unsupported --platform ${tag}`) +} + +// ── environment markers (PEP 508), evaluated for the target ──────────────── + +function evalMarker(marker, env) { + const toks = marker.match(/\(|\)|===|==|!=|<=|>=|~=|<|>|'[^']*'|"[^"]*"|[A-Za-z_][A-Za-z0-9_.]*/g) || [] + let i = 0 + const value = (t) => (/^['"]/.test(t) ? t.slice(1, -1) : t) + const isExtra = (t) => t === 'extra' + const vparts = (s) => (/^\d+(\.\d+)*$/.test(s) ? s.split('.').map(Number) : null) + function cmp(a, op, b) { + if (op === 'in') return b.includes(a) + if (op === 'not in') return !b.includes(a) + const pa = vparts(a); const pb = vparts(b) + let c + if (pa && pb) { + c = 0 + for (let k = 0; k < Math.max(pa.length, pb.length) && !c; k++) c = (pa[k] ?? 0) - (pb[k] ?? 0) + } else c = a < b ? -1 : a > b ? 1 : 0 + switch (op) { + case '==': case '===': return c === 0 + case '!=': return c !== 0 + case '<': return c < 0 + case '<=': return c <= 0 + case '>': return c > 0 + case '>=': return c >= 0 + case '~=': return c >= 0 && (pa && pb ? pa.slice(0, Math.max(1, pb.length - 1)).join('.') === pb.slice(0, Math.max(1, pb.length - 1)).join('.') : true) + default: throw new Error(`marker operator ${op}`) + } + } + function atom() { + if (toks[i] === '(') { i++; const v = orExpr(); i++; return v } + const l = toks[i++] + let op = toks[i++] + if (op === 'not') { op = 'not in'; i++ } + const r = toks[i++] + // `extra == 'x'`: true when x is one of the selected extras. + if (isExtra(l) || isExtra(r)) { + const name = value(isExtra(l) ? r : l) + const has = env.extras.includes(name) + return op === '!=' ? !has : has + } + const get = (t) => (/^['"]/.test(t) ? value(t) : String(env[t] ?? '')) + return cmp(get(l), op, get(r)) + } + function andExpr() { let v = atom(); while (toks[i] === 'and') { i++; v = atom() && v } return v } + function orExpr() { let v = andExpr(); while (toks[i] === 'or') { i++; v = andExpr() || v } return v } + return orExpr() +} + +/** + * Re-write an exported requirements file for the target: drop entries whose + * marker is false there, strip the markers that are kept (pip would otherwise + * re-evaluate them against the HOST), keep the --hash lines. + */ +function filterRequirements(text, env) { + const joined = text.replace(/\\\r?\n/g, ' ') + const out = [] + for (const raw of joined.split(/\r?\n/)) { + const line = raw.trim() + if (!line || line.startsWith('#')) continue + if (line.startsWith('-')) { out.push(line); continue } + const hashAt = line.indexOf(' --hash') + const req = hashAt < 0 ? line : line.slice(0, hashAt) + const hashes = hashAt < 0 ? '' : line.slice(hashAt).trim() + const [spec, marker] = req.split(/\s*;\s*/, 2) + if (marker && !evalMarker(marker, env)) continue + out.push([spec.trim(), ...hashes.split(/\s+(?=--hash)/).filter(Boolean)].join(' \\\n ')) + } + return out.join('\n') + '\n' +} + +// ── the interpreter ───────────────────────────────────────────────────────── + +/** Find the install_only CPython archive uv itself would download. */ +function findPythonArchive(want, pbs) { + const json = execFileSync('uv', ['python', 'list', '--all-platforms', '--all-arches', '--all-versions', + '--only-downloads', '--show-urls', '--output-format', 'json'], + { encoding: 'utf8', maxBuffer: 64 << 20 }) + const key = (v) => v.split('.').map(Number) + const newer = (a, b) => { const x = key(a); const y = key(b); for (let i = 0; i < 3; i++) if ((x[i] ?? 0) !== (y[i] ?? 0)) return (x[i] ?? 0) - (y[i] ?? 0); return 0 } + const hits = JSON.parse(json).filter((p) => + p.implementation === 'cpython' && p.variant === 'default' && p.url && + p.os === pbs.os && p.arch === pbs.arch && (!pbs.libc || p.libc === pbs.libc) && + (p.version === want || p.version.startsWith(`${want}.`)) && !p.version.includes('-')) + if (!hits.length) throw new Error(`uv knows no CPython ${want} download for ${pbs.os}/${pbs.arch}`) + hits.sort((a, b) => newer(b.version, a.version)) + return hits[0] +} + +/** The mirror-relative path: what follows `/releases/download/`, URL-decoded. */ +function mirrorPath(url) { + const marker = '/releases/download/' + const at = url.indexOf(marker) + if (at < 0) throw new Error(`cannot place ${url} under a mirror: no ${marker}`) + return decodeURIComponent(url.slice(at + marker.length)) +} + +async function download(url, dest) { + const res = await fetch(url) + if (!res.ok || !res.body) throw new Error(`GET ${url} -> ${res.status}`) + mkdirSync(dirname(dest), { recursive: true }) + await pipeline(Readable.fromWeb(res.body), createWriteStream(dest)) +} + +function zipDir(dir, zipPath) { + rmSync(zipPath, { force: true }) + if (process.platform === 'win32') { + // bsdtar ships with Windows 10+ and writes a zip for `-a` on a .zip name. + // Named by full path: under Git Bash `tar` is GNU tar, which has no -a. + const bsdtar = join(process.env.SystemRoot ?? 'C:\Windows', 'System32', 'tar.exe') + execFileSync(bsdtar, ['-a', '-c', '-f', zipPath, '-C', dir, '.'], { stdio: 'inherit' }) + return + } + try { + execFileSync('zip', ['-qr', zipPath, '.'], { cwd: dir, stdio: 'inherit' }) + } catch { + execFileSync('python3', ['-m', 'zipfile', '-c', zipPath, '.'], { cwd: dir, stdio: 'inherit' }) + } +} + +// ── main ──────────────────────────────────────────────────────────────────── + +const flags = parseFlags(process.argv.slice(2)) +const electronDir = resolve(flags['electron-dir'] ?? process.cwd()) +let pkg = {} +try { pkg = JSON.parse(readFileSync(join(electronDir, 'package.json'), 'utf8')) } catch { /* flags only */ } +const cfg = pkg.deShell ?? {} + +const appId = flags['app-id'] ?? cfg.appId +if (!appId) throw new Error('appId is required: pass --app-id or set deShell.appId in electron/package.json') +const version = flags.version ?? pkg.version +if (!version) throw new Error('no app version: pass --version or set "version" in electron/package.json') +const platform = flags.platform +if (!platform) throw new Error('--platform is required (e.g. win_amd64)') +const target = describePlatform(platform) +const projectRoot = resolve(electronDir, flags['project-root'] ?? cfg.projectRoot ?? '..') +// --extra replaces deShell.wheelhouseExtras rather than adding to it; --no-extras +// is the way to say "none" when the config names some (a CPU-only wheelhouse). +if (flags['no-extras'] && flags.extra.length) throw new Error('--no-extras and --extra are mutually exclusive') +const extras = flags['no-extras'] ? [] + : flags.extra.length ? flags.extra : (cfg.wheelhouseExtras ?? []) +const outDir = resolve(electronDir, flags.out ?? `wheelhouse-${platform}`) +const appName = (flags['app-name'] ?? cfg.productName ?? appId).replace(/\s+/g, '-') + +let pyWant = flags['python-version'] ?? cfg.pythonVersion +if (!pyWant) { + try { pyWant = readFileSync(join(projectRoot, '.python-version'), 'utf8').split(/\r?\n/)[0].trim() } catch { /* below */ } +} +if (!pyWant) throw new Error('no interpreter version: pass --python-version or commit a .python-version') +const archive = findPythonArchive(pyWant, target.pbs) +const pyFull = archive.version +const pyMinor = pyFull.split('.').slice(0, 2).join('.') +log(`app ${appId} ${version}, ${platform}, CPython ${pyFull}, extras [${extras.join(', ')}]`) + +rmSync(outDir, { recursive: true, force: true }) +mkdirSync(outDir, { recursive: true }) + +// 1. The locked requirement set. +const exportArgs = ['export', '--frozen', '--no-dev', '--no-emit-project', '--no-emit-workspace', + '--no-header', '--no-annotate'] +if (flags.package) exportArgs.push('--package', flags.package) +for (const x of extras) exportArgs.push('--extra', x) +const exported = execFileSync('uv', exportArgs, { cwd: projectRoot, encoding: 'utf8', maxBuffer: 64 << 20 }) +const reqText = filterRequirements(exported, { ...target.env, extras, + python_version: pyMinor, python_full_version: pyFull, + implementation_name: 'cpython', platform_python_implementation: 'CPython' }) +// A git / path / URL requirement has no hash and no wheel to download, and pip +// would fail on it with a message about hashing version-control repositories. +// Say what is actually wrong: a wheelhouse can only carry index-published +// packages, so such a source has to be released (or become a workspace member, +// which bundle-python ships as a wheel) first. +const unhashable = reqText.split(/\r?\n/) + .filter((l) => /^\s*[A-Za-z0-9._-]+(\[[^\]]*\])?\s*@\s*\S/.test(l) || /^\s*(-e\s|git\+|file:|\.{1,2}[\\/])/.test(l)) +if (unhashable.length) { + throw new Error('the lock has dependencies that are not on a package index, so they cannot go in a ' + + `wheelhouse:\n ${unhashable.map((l) => l.trim()).join('\n ')}\n` + + 'Publish them, or make them workspace members, first.') +} +const reqFile = join(outDir, '..', `.${basename(outDir)}.requirements.txt`) +writeFileSync(reqFile, reqText) + +// 2. The wheels. +try { + const pipArgs = ['tool', 'run', 'pip', 'download', '-r', reqFile, '-d', outDir, + '--only-binary=:all:', '--no-deps', '--python-version', pyMinor, '--implementation', 'cp'] + for (const p of target.pip) pipArgs.push('--platform', p) + execFileSync('uv', pipArgs, { stdio: 'inherit' }) +} finally { + rmSync(reqFile, { force: true }) +} +const wheels = readdirSync(outDir).filter((f) => f.endsWith('.whl')) +log(`${wheels.length} wheel(s)`) + +// 3. The interpreter, at the mirror layout. +const rel = mirrorPath(archive.url) +log(`python archive ${rel}`) +await download(archive.url, join(outDir, 'python', ...rel.split('/'))) + +// 4. The manifest pythonEnv.ts keys on. +writeFileSync(join(outDir, 'wheelhouse.json'), + JSON.stringify({ app: appId, version, platform, python: pyFull, extras }, null, 2) + '\n') + +if (flags.zip) { + const zipPath = join(dirname(outDir), `${appName}-${version}-wheels-${platform}.zip`) + zipDir(outDir, zipPath) + log(`zip -> ${zipPath}`) +} +log(`done -> ${outDir}`) diff --git a/de_shell/js/packaging/bundle-python.mjs b/de_shell/js/packaging/bundle-python.mjs new file mode 100644 index 0000000..c3436a5 --- /dev/null +++ b/de_shell/js/packaging/bundle-python.mjs @@ -0,0 +1,226 @@ +#!/usr/bin/env node +/** + * bundle-python.mjs — stage the Python sidecar payload for electron-builder. + * + * node shell/packaging/bundle-python.mjs [flags] (from the app's electron/ dir) + * + * Writes the exact layout de-shell's main/pythonEnv.ts reads at + * `/python`: + * + * pyproject.toml, uv.lock the project, for a reproducible first-launch `uv sync` + * .python-version pins the PACKAGED interpreter only (see below) + * .-version the release version, for setuptools_scm + * wheels/*.whl the app's wheel AND every workspace member's + * uv(.exe) the platform uv that runs the sync + * git/ optional portable git (MinGit) for git+https deps + * + * electron-builder ships the directory as an extraResource; the venv itself is + * NOT bundled (tiny installer, and uv picks the right wheels per machine) — + * unless the app also ships a wheelhouse, see build-wheelhouse.mjs. + * + * CONFIGURATION — flags win over the `deShell` block of the app's + * electron/package.json, which wins over the defaults: + * + * flag deShell key meaning + * --app-id appId the shell's appId (names the version marker) REQUIRED + * --python-dist pythonDist the app's distribution name (SCM pretend var) default: appId + * --project-root projectRoot the uv project, relative to electron/ default: .. + * --python-version pythonVersion interpreter for the packaged env default: /.python-version + * --mingit includeMingit stage electron/vendor/git/ (error if missing) + * --no-mingit do not stage git even when it exists + * --uv uvSource the uv binary to ship default: vendor/uv//uv, else PATH + * --out out staging dir, relative to electron/ default: resources/python + * --electron-dir the app's electron/ dir default: cwd + * + * No npm dependencies: node builtins only, so it runs before `npm ci` has + * anything to offer and from inside a site-packages tree. + */ +import { + cpSync, existsSync, mkdirSync, rmSync, copyFileSync, chmodSync, readdirSync, readFileSync, writeFileSync, +} from 'node:fs' +import { join, resolve } from 'node:path' +import { execFileSync } from 'node:child_process' + +const isWin = process.platform === 'win32' +const uvName = isWin ? 'uv.exe' : 'uv' + +function log(msg) { console.log(`[bundle-python] ${msg}`) } + +/** `--flag value` / `--flag=value` / bare `--flag` -> { flag: value|true }. */ +function parseFlags(argv) { + const out = {} + for (let i = 0; i < argv.length; i++) { + const m = /^--([^=]+)(?:=(.*))?$/.exec(argv[i]) + if (!m) throw new Error(`unexpected argument ${argv[i]}`) + const [, key, inline] = m + if (inline !== undefined) out[key] = inline + else if (['mingit', 'no-mingit'].includes(key)) out[key] = true + else out[key] = argv[++i] + } + return out +} + +function readJson(p) { + try { return JSON.parse(readFileSync(p, 'utf8')) } catch { return {} } +} + +const flags = parseFlags(process.argv.slice(2)) +const electronDir = resolve(flags['electron-dir'] ?? process.cwd()) +const pkg = readJson(join(electronDir, 'package.json')) +const cfg = pkg.deShell ?? {} + +const appId = flags['app-id'] ?? cfg.appId +if (!appId) throw new Error('appId is required: pass --app-id or set deShell.appId in electron/package.json') +const pythonDist = flags['python-dist'] ?? cfg.pythonDist ?? appId +const projectRoot = resolve(electronDir, flags['project-root'] ?? cfg.projectRoot ?? '..') +const outDir = resolve(electronDir, flags.out ?? cfg.out ?? join('resources', 'python')) +const uvOverride = flags.uv ?? cfg.uvSource +const includeMingit = flags['no-mingit'] ? false : (flags.mingit ? true : (cfg.includeMingit ?? null)) + +/** + * The release version to bake into the staged payload. + * + * The staged tree has NO `.git`, and an app whose pyproject derives its version + * from setuptools_scm would fail its first-launch `uv sync` with `LookupError: + * unable to detect version`. So the version is resolved HERE (CI has git and + * tags) and written to `.-version`; pythonEnv.ts exports it as + * SETUPTOOLS_SCM_PRETEND_VERSION_FOR_. electron/package.json's `version` + * wins: the release workflow verifies it equals the pushed tag, so it is the + * already-validated release version (and a plain X.Y.Z is valid PEP 440). + */ +function resolveVersion() { + if (typeof pkg.version === 'string' && pkg.version.trim()) return pkg.version.trim() + try { + const v = execFileSync('uv', ['run', 'python', '-m', 'setuptools_scm'], { + cwd: projectRoot, stdio: ['ignore', 'pipe', 'ignore'], + }).toString().trim() + if (v) return v + } catch { /* no git / no uv / not scm — leave the marker unwritten */ } + return null +} + +/** The `.python-version` pin: flag > deShell > the project's own file. */ +function resolvePythonVersion() { + const v = flags['python-version'] ?? cfg.pythonVersion + if (v) return String(v).trim() + try { + const first = readFileSync(join(projectRoot, '.python-version'), 'utf8').split(/\r?\n/)[0].trim() + if (first) return first + } catch { /* fall through */ } + throw new Error( + 'no interpreter version: pass --python-version, set deShell.pythonVersion, or commit a ' + + `.python-version in ${projectRoot}. (Unpinned, first-launch uv grabs the NEWEST CPython, ` + + "which may have no wheels for the app's dependencies yet.)") +} + +const version = resolveVersion() +const pythonVersion = resolvePythonVersion() + +// Fresh staging dir. +rmSync(outDir, { recursive: true, force: true }) +mkdirSync(outDir, { recursive: true }) + +// 1. Project metadata + lock (reproducible `uv sync`). +for (const f of ['pyproject.toml', 'uv.lock']) { + const src = join(projectRoot, f) + if (!existsSync(src)) throw new Error(`missing ${f} in ${projectRoot}`) + copyFileSync(src, join(outDir, f)) + log(`staged ${f}`) +} + +// Pin the interpreter for the PACKAGED env only. Without this, first-launch +// `uv sync` picks the newest CPython. We write `.python-version` ONLY into the +// payload, never at the project root: a root file would also hijack dev + the +// CI test matrix (every `uv run` re-creates the venv at that version). Scoped +// to the staged tree, shipped users get the pinned Python while dev/CI stay +// free to pick their own. +writeFileSync(join(outDir, '.python-version'), pythonVersion + '\n', 'utf8') +log(`staged .python-version = ${pythonVersion}`) + +// 2. PRE-BUILT wheels — the app's and every workspace member's — NOT source. +// Building from source on first launch is impossible when the app is +// installed read-only (e.g. C:\Program Files): setuptools' build_wheel runs +// egg_info, which writes `.egg-info` INTO the source dir -> "Access is +// denied", `uv sync` exits 1 (--no-editable does not help; a wheel build +// still writes egg_info into the tree). So we build HERE, where the repo is +// writable, and pythonEnv.ts installs the .whl files with `uv pip install +// --no-deps` after syncing the dependencies without the project itself, so +// NOTHING is built on the user's machine. --all-packages is what picks up +// the siblings that are path dependencies of the app. Pure-Python wheels are +// py3-none-any and serve every platform we ship. +const wheelsDir = join(outDir, 'wheels') +mkdirSync(wheelsDir, { recursive: true }) +const scmSuffix = pythonDist.toUpperCase().replace(/[^A-Z0-9]+/g, '_') +const buildEnv = version + ? { ...process.env, [`SETUPTOOLS_SCM_PRETEND_VERSION_FOR_${scmSuffix}`]: version, + SETUPTOOLS_SCM_PRETEND_VERSION: version } + : process.env +execFileSync('uv', ['build', '--all-packages', '--wheel', '--out-dir', wheelsDir], { + cwd: projectRoot, stdio: 'inherit', env: buildEnv, +}) +// `uv build` drops a catch-all .gitignore into --out-dir; it is not payload. +rmSync(join(wheelsDir, '.gitignore'), { force: true }) +const builtWheels = readdirSync(wheelsDir).filter((f) => f.endsWith('.whl')) +if (!builtWheels.length) throw new Error('uv build produced no wheels') +log(`staged ${builtWheels.length} wheel(s): ${builtWheels.join(', ')}`) + +// 3. The uv binary: --uv / deShell.uvSource, else a vendored copy at +// electron/vendor/uv//uv[.exe], else the `uv` on PATH (CI installs +// a pinned one, so that is what ships). +let uvSrc = uvOverride ? resolve(electronDir, uvOverride) : join(electronDir, 'vendor', 'uv', process.platform, uvName) +if (!existsSync(uvSrc)) { + if (uvOverride) throw new Error(`uv binary not found at ${uvSrc}`) + try { + uvSrc = execFileSync(isWin ? 'where' : 'which', ['uv']).toString().trim().split(/\r?\n/)[0] + log(`vendor/ uv not found; using uv from PATH: ${uvSrc}`) + } catch { + throw new Error('no uv binary: pass --uv, add electron/vendor/uv//uv, or install uv on PATH') + } +} +const uvDst = join(outDir, uvName) +copyFileSync(uvSrc, uvDst) +if (!isWin) chmodSync(uvDst, 0o755) +log(`staged ${uvName}`) + +// 3b. Vendored git (portable). Dependencies that are `git+https://…` sources +// are resolved by `git clone`, and a clean user machine often has NO git +// on PATH -> "Git operation failed" and first-launch setup dies. So an app +// with such deps ships a portable git (MinGit on Windows) in the payload; +// pythonEnv.ts prepends its cmd/ and bin/ to PATH for the sync. Staged from +// electron/vendor/git//. `includeMingit: true` makes absence an +// error (a shippable build must have it); unset stages it when present. +const vendoredGit = join(electronDir, 'vendor', 'git', process.platform) +if (includeMingit !== false && existsSync(vendoredGit)) { + const gitDst = join(outDir, 'git') + cpSync(vendoredGit, gitDst, { recursive: true }) + // Make the git executables runnable on posix (Windows ignores the mode). + if (!isWin) { + for (const sub of ['bin', 'libexec/git-core']) { + const d = join(gitDst, sub) + if (existsSync(d)) { + for (const f of readdirSync(d)) { + try { chmodSync(join(d, f), 0o755) } catch { /* best-effort */ } + } + } + } + } + log('staged vendored git') +} else if (includeMingit === true) { + throw new Error(`includeMingit is set but ${vendoredGit} does not exist — populate it (the release workflow does) before bundling`) +} else { + log(includeMingit === false + ? 'git not requested; NOT staging it' + : `vendor/git/${process.platform} not found; NOT staging git — first-launch ` + + "`uv sync` will need a git on the user's PATH if any dependency is a git source.") +} + +// 4. Version marker (see resolveVersion for why the staged tree needs it). +if (version) { + writeFileSync(join(outDir, `.${appId}-version`), version, 'utf8') + log(`staged .${appId}-version = ${version}`) +} else { + log(`WARNING: could not resolve a version — .${appId}-version NOT written; ` + + 'first-launch `uv sync` may fail with a setuptools_scm LookupError') +} + +log(`done -> ${outDir}`) diff --git a/de_shell/js/packaging/electron-builder.base.yml b/de_shell/js/packaging/electron-builder.base.yml new file mode 100644 index 0000000..a34279a --- /dev/null +++ b/de_shell/js/packaging/electron-builder.base.yml @@ -0,0 +1,109 @@ +# electron-builder.base.yml — the packaging config every de-shell app shares. +# An app's own electron-builder.yml extends it and adds only what is its own: +# +# extends: shell/packaging/electron-builder.base.yml +# appId: com.directelectron.groundcrew +# productName: Ground Crew +# publish: { provider: github, owner: directelectron, repo: de_ground_crew } +# mac: { icon: ../assets/icon.png } # see "icons" below +# win: { icon: ../assets/icon.png } +# linux: { icon: ../assets/icon.png } +# +# HOW PATHS RESOLVE. `shell/` is the link `python -m de_shell.js link electron` +# makes to the installed de_shell/js, so `shell/packaging/…` is this folder. +# electron-builder resolves `extends` against the extending file's directory, +# and every path INSIDE the merged config (directories, extraResources `from`, +# nsis.include, icons) against the PROJECT directory -- the app's electron/ -- +# not against this file. That is why every path below is written relative to +# electron/ and goes through `shell/`, and why none of them break when this +# file is read from site-packages through the link. (The one thing the link +# must not be is missing: the release workflow and the vite config both create +# it before electron-builder runs.) +# +# The installer is small on purpose: it ships uv, the app's wheels and the lock +# as extraResources (staged by bundle-python.mjs); the venv is built on first +# launch into the user's data dir, or from an offline wheelhouse +# (build-wheelhouse.mjs) on a machine with no network. + +copyright: © Direct Electron + +directories: + output: dist + buildResources: build + +# The compiled app (electron-vite output) + package manifest. node_modules are +# pulled in automatically by electron-builder for production deps. `shell/` is +# deliberately NOT listed: the TypeScript is compiled into out/ by the bundler, +# and a junction into site-packages must never be walked by the packager. +files: + - out/**/* + - package.json + - "!**/*.map" + +# The Python sidecar payload -- staged by bundle-python.mjs BEFORE the build. +# Lands at /python at runtime, which is where pythonEnv.ts reads +# it. (An app that stages elsewhere with --out must repeat this block.) +extraResources: + - from: resources/python + to: python + filter: ["**/*"] + +# Icons: ONE source image per app (a 512x512+ RGBA PNG), set per platform in the +# app's own config -- electron-builder generates the .icns and .ico from it at +# build time, so there is no hand-maintained icon set to drift. The icon paths +# are the app's, so they are not defaulted here. +mac: + category: public.app-category.education + # BOTH dmg (what users download by hand) AND zip. electron-updater CANNOT + # auto-update from a .dmg on macOS: it needs a .zip in latest-mac.yml. A + # dmg-only feed makes the updater download a dmg it cannot install. + target: [dmg, zip] + # Hardened runtime + entitlements are REQUIRED for notarization. The app owns + # build/entitlements.mac.plist; its sidecar loads third-party native wheels + # signed by other teams, so it needs com.apple.security.cs.disable-library-validation. + hardenedRuntime: true + gatekeeperAssess: false + entitlements: build/entitlements.mac.plist + entitlementsInherit: build/entitlements.mac.plist + # Notarize with the App Store Connect API key env (APPLE_API_KEY[_ID]/ + # APPLE_API_ISSUER/APPLE_TEAM_ID). NB: NO `identity: null` -- that would + # DISABLE signing. With no signing env (a local build, a fork) electron-builder + # logs "skipped macOS code signing" and produces an unsigned build instead of + # failing, so only CI with the certificate imported actually signs. + notarize: true + +# Windows signing is NOT configured here: it is opt-in per build. The release +# workflow passes -c.win.azureSignOptions.* (Azure Trusted Signing) only when +# the Azure secrets exist, so an unsigned local build never trips over it. +win: + target: [nsis] + +# Per-user NSIS installer: no admin, Start-Menu + desktop shortcuts, uninstaller, +# user-chosen install dir. +# +# installer.nsh replaces the stock "is the app still running?" check, whose +# single-process kill and instant re-check dead-ended a Windows auto-update in +# " cannot be closed. Please close it manually and click Retry", and adds +# the offer to remove the managed Python environment on uninstall. Read it +# before changing anything about how the app shuts down. +nsis: + oneClick: false + perMachine: false + allowToChangeInstallationDirectory: true + createDesktopShortcut: true + createStartMenuShortcut: true + include: shell/packaging/installer.nsh + +linux: + target: [AppImage] + category: Science + +# Where release artifacts live (GitHub Releases); electron-builder also emits +# the update-feed metadata (latest.yml / latest-mac.yml / latest-linux.yml) +# electron-updater reads at runtime. This block alone publishes nothing: only +# `--publish always` (the release workflow) does. owner/repo are the app's to +# set for a LOCAL build's app-update.yml; the release workflow overrides both +# (-c.publish.owner/.repo) with the repository it runs in, so publishing +# survives a repo transfer with no edits. +publish: + provider: github diff --git a/de_shell/js/packaging/installer.nsh b/de_shell/js/packaging/installer.nsh new file mode 100644 index 0000000..cb7405c --- /dev/null +++ b/de_shell/js/packaging/installer.nsh @@ -0,0 +1,210 @@ +; installer.nsh — the shell's NSIS customisations, shared by every de-shell app. +; electron-builder includes it through `nsis.include` (electron-builder.base.yml +; sets it); nothing in here names an app: the product comes from the defines +; electron-builder generates (PRODUCT_NAME, APP_FILENAME, APP_EXECUTABLE_FILENAME). +; +; Two things live here: +; +; 1. customCheckAppRunning — a replacement for electron-builder's "is the app +; still running?" check, which dead-ends in " cannot be closed". +; 2. customUnInstall — OFFERS to remove the managed Python environment and +; everything it downloaded, so an uninstall can leave no residue. +; +; This file has never been run through makensis by the person who wrote it +; (it was generalised from SpyDE's, which has shipped). Read the macros against +; app-builder-lib/templates/nsis/ before changing them. +; +; ─────────────────────────────────────────────────────────────────────────── +; 1. WHY customCheckAppRunning EXISTS +; ─────────────────────────────────────────────────────────────────────────── +; +; The stock check (app-builder-lib/templates/nsis/include/ +; allowOnlyOneInstallerInstance.nsh) asks PowerShell for every process whose +; executable path sits under $INSTDIR, stops them one at a time with +; Stop-Process, re-checks with no pause, and after two failed rounds ends in a +; " cannot be closed. Please close it manually and click Retry" box whose +; Retry re-runs the identical check. That is a dead end for the user: nothing +; they can do in the dialog changes the answer, and the only way out is to +; uninstall by hand. +; +; Four things make it trip during a real update: +; +; * Stop-Process ends ONE process. The app runs a Python sidecar which may run +; worker processes of its own, so killing a parent leaves children that are +; found again on the next round. +; * The install directory holds more than the app binary -- resources\python +; carries uv and possibly a vendored git -- so the path-prefix match can +; catch a helper the app is mid-way through running. +; * Re-checking with no delay counts processes that are already exiting. +; * MATCHING ON THE EXECUTABLE PATH MISSES THE PYTHON SIDECAR ENTIRELY. Its +; interpreter lives in the managed env under $APPDATA, not in $INSTDIR -- +; only its working directory was in the install dir. A working directory is +; an open handle, so an orphaned sidecar (and everything it spawned, which +; inherits that directory) blocks the install dir from being removed while +; staying invisible to any scan keyed on image path. That is why this pass +; also sweeps the app's own userData (python-env, and python: a venv's +; python.exe is a launcher that re-executes the uv-managed interpreter in +; there), and why the sidecar no longer runs from the install dir at all +; (de-shell's pythonEnv.ts). +; +; The userData sweep is keyed on $APPDATA\${APP_FILENAME}\ and nothing broader. +; Two de-shell apps are routinely installed side by side; updating one must not +; kill the other's sidecar. +; +; So: kill whole process TREES by pid, look outside $INSTDIR for the processes +; that hold it, wait between rounds, give the app several seconds to go before +; asking the user for anything, and make Retry actually retry the kill rather +; than just the question. +; +; THIS MACRO IS NOT THE ONLY SOURCE OF THAT DIALOG. electron-builder's +; uninstallOldVersion (app-builder-lib templates/nsis/include/installUtil.nsh) +; runs the PREVIOUS version's uninstaller and, if it returns non-zero five +; times, shows the very same "$(appCannotBeClosed)" -- with a Retry that only +; re-runs the uninstaller and never kills anything. There is no hook to override +; that loop, so the only way to stay out of it is to leave nothing holding the +; install directory by the time it runs. Hence the sweep above. +; +; The app side of the same race is in de-shell's main/updater.ts -- it tears the +; sidecar down and force-exits rather than trusting Electron's graceful quit to +; win. Both halves are needed: this macro cannot reach a sidecar that lives +; outside $INSTDIR, and the app cannot clean up after a crash it did not survive. + +Var deShellSetupPid + +!macro customCheckAppRunning + Push $0 + Push $R0 + Push $R1 + + ; Our own pid, so a tree-kill can never take down the installer itself. (The + ; stock macro's $pid is not declared when this override is in play.) + System::Call 'kernel32::GetCurrentProcessId() i .r0' + StrCpy $deShellSetupPid $0 + + StrCpy $R1 0 ; rounds spent asking the user + + deShellCloseRetry: + DetailPrint "Closing ${PRODUCT_NAME}..." + + ; One PowerShell pass: up to 12 rounds of "find everything holding the + ; install directory, kill each one's whole tree, wait". Exits 0 the moment + ; nothing is left, 1 if the directory is still busy after ~6 s. Two things + ; count as holding it: a process running out of $INSTDIR, and one running + ; out of THIS app's managed Python (python-env\ or python\ under + ; $APPDATA\${APP_FILENAME}\), which is the sidecar -- see the header. + ; The trailing backslash on each directory keeps a sibling install (...\App + ; vs ...\App-old) from matching. `$$` is NSIS's escape for a literal `$`, + ; so `$$_` reaches PowerShell as `$_`. + nsExec::Exec `"$SYSDIR\WindowsPowerShell\v1.0\powershell.exe" -NoProfile -NonInteractive -Command "$$dir='$INSTDIR\'; $$data='$APPDATA\${APP_FILENAME}\'; $$self=$deShellSetupPid; for($$round=0; $$round -lt 12; $$round++){ $$busy=@(Get-CimInstance Win32_Process -ErrorAction SilentlyContinue | Where-Object { $$p=$$_.Path; $$p -and $$_.ProcessId -ne $$self -and ($$p.StartsWith($$dir,'CurrentCultureIgnoreCase') -or ($$p.StartsWith($$data,'CurrentCultureIgnoreCase') -and ($$p -like '*\python-env\*' -or $$p -like '*\python\*'))) }); if($$busy.Count -eq 0){ exit 0 }; foreach($$victim in $$busy){ $$null = & taskkill.exe /PID $$victim.ProcessId /T /F 2>&1 }; Start-Sleep -Milliseconds 500 }; exit 1"` + Pop $R0 + + ; nsExec pushes "error" when the program could not be started at all -- a + ; machine with no PowerShell, or one where it is blocked by policy. Fall + ; back to killing the app's own image tree, which needs nothing but + ; taskkill. It exits 128 ("no tasks") when there was nothing to kill, so + ; both 0 and 128 mean the directory is clear afterwards. + ${if} $R0 == "error" + nsExec::Exec `"$SYSDIR\cmd.exe" /C taskkill /IM "${APP_EXECUTABLE_FILENAME}" /T /F /FI "USERNAME eq %USERNAME%"` + Pop $R0 + Sleep 1000 + ${if} $R0 == 128 + StrCpy $R0 0 + ${endif} + ${endif} + + ${if} $R0 == 0 + Goto deShellClosed + ${endif} + + ; Still busy. An update handoff is allowed one silent second round: the app + ; that spawned this installer may simply be slow to die. + IntOp $R1 $R1 + 1 + ${if} $R1 < 2 + Sleep 1000 + Goto deShellCloseRetry + ${endif} + + ; Genuinely stuck -- something is holding the install directory that we are + ; not allowed to end. Now the user's help is worth asking for, and Retry + ; runs the whole kill pass again rather than re-asking the same question. + MessageBox MB_RETRYCANCEL|MB_ICONEXCLAMATION "$(appCannotBeClosed)" /SD IDCANCEL IDRETRY deShellCloseRetry + Quit + + deShellClosed: + Pop $R1 + Pop $R0 + Pop $0 +!macroend + +; ─────────────────────────────────────────────────────────────────────────── +; 2. customUnInstall — "no residue" +; ─────────────────────────────────────────────────────────────────────────── +; +; Everything the app creates OUTSIDE the install directory sits in its userData +; ($APPDATA\${APP_FILENAME}, Electron's app.getPath('userData')), and is what +; de-shell's pythonEnv.ts put there: +; +; python-env\ the venv (can be many GB: scientific wheels) +; uv-cache\ uv's download/wheel cache +; python\ uv-managed interpreters +; requirements.offline.txt the offline install's requirement set +; wheelhouse\ an unpacked offline wheelhouse, if one was used +; +; Those are regenerable, so removing them is safe; the user's settings and logs +; (the rest of userData, and ~\.) are NOT touched -- reinstalling should +; not lose them. +; +; It is an OFFER, not a default: +; +; * An UPDATE runs the previous version's uninstaller silently, and the +; environment must survive it (re-creating it is a first-launch download). +; `/SD IDNO` is what makes every silent run answer No, so the update path +; and `Uninstall.exe /S` never delete anything. +; * Interactively the box defaults to No (MB_DEFBUTTON2) for the same reason: +; Enter must not be the destructive answer. +; +; Strings are English-only; the stock installer strings are localised by +; electron-builder, ours are not. +; +; Which directory? Electron names userData after app.name -- the productName +; when package.json has one -- while electron-builder derives APP_FILENAME from +; the same productName (sanitised). They agree in every de-shell app; if an app +; ever sets `nsis.shortcutName` or `electronBuilder` names that make them +; differ, ${APP_FILENAME} is what to change here. + +!macro deShellRemoveUserDataItem name + ; RMDir /r on a path that is a file does nothing, and Delete on a directory + ; does nothing, so one macro covers both without testing which it is. + RMDir /r "$APPDATA\${APP_FILENAME}\${name}" + Delete "$APPDATA\${APP_FILENAME}\${name}" +!macroend + +!macro customUnInstall + ; Per-machine uninstallers run in the all-users shell context, where $APPDATA + ; is the shared ProgramData -- not where Electron put userData. + SetShellVarContext current + + ${if} ${FileExists} "$APPDATA\${APP_FILENAME}\python-env\*.*" + ${orIf} ${FileExists} "$APPDATA\${APP_FILENAME}\uv-cache\*.*" + ${orIf} ${FileExists} "$APPDATA\${APP_FILENAME}\python\*.*" + ${orIf} ${FileExists} "$APPDATA\${APP_FILENAME}\wheelhouse\*.*" + MessageBox MB_YESNO|MB_ICONQUESTION|MB_DEFBUTTON2 \ + "Also remove ${PRODUCT_NAME}'s Python environment, package cache and downloaded Python?$\r$\n$\r$\nThis frees the disk space they use. Your settings and logs are kept.$\r$\n$\r$\nChoose No if you are only reinstalling or updating ${PRODUCT_NAME}." \ + /SD IDNO IDNO deShellKeepData + DetailPrint "Removing ${PRODUCT_NAME}'s Python environment..." + !insertmacro deShellRemoveUserDataItem "python-env" + !insertmacro deShellRemoveUserDataItem "uv-cache" + !insertmacro deShellRemoveUserDataItem "python" + !insertmacro deShellRemoveUserDataItem "requirements.offline.txt" + !insertmacro deShellRemoveUserDataItem "wheelhouse" + ${endif} + deShellKeepData: + + ; Put the context back the way electron-builder's uninstaller expects it. + ; $installMode only exists in assisted (non-oneClick) installers. + !ifndef ONE_CLICK + ${if} $installMode == "all" + SetShellVarContext all + ${endif} + !endif +!macroend diff --git a/de_shell/js/packaging/shell-link.d.mts b/de_shell/js/packaging/shell-link.d.mts new file mode 100644 index 0000000..a88f7cf --- /dev/null +++ b/de_shell/js/packaging/shell-link.d.mts @@ -0,0 +1,10 @@ +// Types for shell-link.mjs — plain JavaScript because npm's postinstall runs it +// before any bundler is involved; this file is what lets the vite config import +// it under `strict`. + +/** Where the installed de-shell keeps its TypeScript, by asking it. */ +export function shellJsDir(electronDir?: string): string + +/** Make `/shell` point at the installed shell's TypeScript and + * return the link path. Idempotent: an up-to-date link is left alone. */ +export function ensureShellLink(electronDir?: string, options?: { quiet?: boolean }): string diff --git a/de_shell/js/packaging/shell-link.mjs b/de_shell/js/packaging/shell-link.mjs new file mode 100644 index 0000000..b7bfea3 --- /dev/null +++ b/de_shell/js/packaging/shell-link.mjs @@ -0,0 +1,116 @@ +// shell-link.mjs — link the shell's TypeScript into this project. +// +// The shell ships its TypeScript INSIDE the de-shell wheel (de_shell/js: one +// folder per Electron target), so the JavaScript that speaks the sidecar +// protocol and the Python that speaks it move together — one `pip install -U` +// upgrades both, and two versions in one app cannot happen. The bundler and +// tsc need a fixed path, though, and site-packages is not one: it moves with +// the platform and the Python version, and an editable install points at a +// checkout instead. So this makes `electron/shell` a junction (a symlink off +// Windows) to wherever the installed package's `de_shell/js` is, and every +// alias and tsconfig path goes through it. +// +// Run by npm's postinstall, by `npm run shell:link` after a `uv sync` that +// changed the shell, and by electron.vite.config.ts on every build, which is +// what keeps it honest: a stale link is re-pointed, never trusted. +// +// This is the copy that ships IN de-shell (de_shell/js/packaging), so an app +// no longer vendors it. The bootstrap problem is real, though: the link does +// not exist before this runs, so an app reaches the script as +// `python -m de_shell.js link electron` (the stdlib twin, same semantics, in +// de_shell/js/__main__.py) or copies this file once. Imports through the link +// (`./shell/packaging/shell-link.mjs`) work after the first link exists. +import { execFileSync } from 'node:child_process' +import { + existsSync, lstatSync, realpathSync, rmdirSync, symlinkSync, unlinkSync, +} from 'node:fs' +import { join, resolve } from 'node:path' +import { fileURLToPath } from 'node:url' + +// The app's electron/ directory. This file lives inside the shell now, and +// import.meta.url resolves THROUGH the link into site-packages, so the script's +// own location says nothing about the app: default to where it was run from +// (npm scripts and postinstall run in electron/), and let callers that know +// better (the vite config passes __dirname) say so. +const DEFAULT_ELECTRON_DIR = process.cwd() + +/** The project's Python: the venv's own interpreter when it exists (fast, and + * no uv needed), else `uv run` from the locked environment. */ +function pythonCmd(projectDir) { + const venv = process.platform === 'win32' + ? join(projectDir, '.venv', 'Scripts', 'python.exe') + : join(projectDir, '.venv', 'bin', 'python') + return existsSync(venv) ? [venv] : ['uv', 'run', '--frozen', 'python'] +} + +/** + * Where the installed de-shell keeps its TypeScript, by asking it — or, when + * there is no Python environment to ask (a TypeScript-only CI job), the + * sibling checkout `../de-shell` that the uv path source points at anyway. + */ +export function shellJsDir(electronDir = DEFAULT_ELECTRON_DIR) { + const projectDir = resolve(electronDir, '..') + const [cmd, ...args] = pythonCmd(projectDir) + let out = '' + let why = '' + try { + out = execFileSync(cmd, [...args, '-m', 'de_shell.js'], + { cwd: projectDir, encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] }) + } catch (err) { + why = `${cmd}: ${err.message.split('\n')[0]}` + } + const dir = out.trim().split(/\r?\n/).pop() + if (dir && existsSync(join(dir, 'main', 'index.ts'))) return dir + const sibling = resolve(projectDir, '..', 'de-shell', 'de_shell', 'js') + if (existsSync(join(sibling, 'main', 'index.ts'))) return sibling + throw new Error( + (why ? `could not ask the project's Python where de-shell is (${why})` + : `de_shell.js answered ${JSON.stringify(dir)}, which has no main/index.ts`) + + `, and there is no checkout at ${sibling}. Run \`uv sync --extra tests\` first.`) +} + +/** Remove a stale LINK without ever following it (a junction is unlinked, + * rmdir on the platforms that insist). Anything that is not a link — a real + * directory someone copied into place, a file — is refused rather than + * deleted: a build script has no business removing the user's files. */ +function clearLink(link) { + let st + try { st = lstatSync(link) } catch { return } + if (!st.isSymbolicLink()) { + throw new Error(`${link} exists and is not a link; refusing to replace it. Remove it by hand.`) + } + try { unlinkSync(link) } catch { rmdirSync(link) } +} + +/** + * Make `/shell` point at the installed shell's TypeScript, and + * return the link path. Idempotent: an up-to-date link is left alone. + */ +export function ensureShellLink(electronDir = DEFAULT_ELECTRON_DIR, { quiet = false } = {}) { + const link = join(electronDir, 'shell') + const target = realpathSync(shellJsDir(electronDir)) + let current = null + try { + if (lstatSync(link).isSymbolicLink()) current = realpathSync(link) + } catch { /* nothing there yet */ } + if (current === target) return link + clearLink(link) + symlinkSync(target, link, process.platform === 'win32' ? 'junction' : 'dir') + if (!quiet) console.log(`[shell-link] electron/shell -> ${target}`) + return link +} + +// As a script: `node shell-link.mjs [electronDir] [--soft]` (electronDir +// defaults to the current directory). `--soft` (what postinstall passes) warns +// instead of failing when the Python environment is not there yet — an `npm +// install` before the first `uv sync` is a normal order of events, and the vite +// config re-tries with a hard error at build time. +if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { + const dir = process.argv.slice(2).find((a) => !a.startsWith('--')) + try { + ensureShellLink(dir ? resolve(dir) : DEFAULT_ELECTRON_DIR) + } catch (err) { + if (!process.argv.includes('--soft')) throw err + console.warn(`[shell-link] not linked yet: ${err.message}`) + } +} diff --git a/de_shell/js/preload/index.ts b/de_shell/js/preload/index.ts index c52cd22..11d2622 100644 --- a/de_shell/js/preload/index.ts +++ b/de_shell/js/preload/index.ts @@ -21,6 +21,35 @@ import { contextBridge, ipcRenderer, webUtils } from 'electron' * electron's types. */ export interface FileFilter { name: string; extensions: string[] } +/** Last-known updater state; mirrors @de/shell-main's UpdateStatus (the + * renderer cannot import main, so the shape is restated here). */ +export type BridgeUpdateStatus = + | { state: 'idle' } + | { state: 'checking' } + | { state: 'available'; version: string; releaseNotes?: string } + | { state: 'not-available' } + | { state: 'downloading'; percent: number } + | { state: 'downloaded'; version: string } + | { state: 'error'; message: string } + +export interface BridgeUpdateInfo { + channel: 'stable' | 'beta' + /** False for dev / unpackaged runs, which have no app-update.yml. */ + supported: boolean + status: BridgeUpdateStatus + appVersion: string +} + +export interface BridgeReportResult { + sent: boolean + eventId?: string + bundlePath?: string + error?: string +} + +/** What the renderer answers when first-run setup fails. */ +export type EnvSetupChoice = 'retry' | 'use-wheelhouse' | 'quit' + export interface ShellBridgeOptions { /** Matches @de/shell-main's `appId` — channels are `:`. */ appId: string @@ -112,6 +141,33 @@ export function createShellBridge(opts: ShellBridgeOptions) { saveFile: (filters?: FileFilter[], defaultPath?: string): Promise => ipcRenderer.invoke(channel('save-file'), filters, defaultPath), + /** Auto-update. Needs `registerUpdaterIpc()` in main. Status is pushed by + * main's updater; `info()` seeds a UI that mounts after the first push. */ + updater: { + info: (): Promise => ipcRenderer.invoke(channel('update-info')), + check: () => ipcRenderer.send(channel('update-check')), + download: () => ipcRenderer.send(channel('update-download')), + quitAndInstall: () => ipcRenderer.send(channel('update-install')), + setChannel: (c: 'stable' | 'beta') => ipcRenderer.send(channel('update-set-channel'), c), + onStatus: (cb: (status: BridgeUpdateStatus) => void) => + on<[BridgeUpdateStatus]>(channel('update-status'), cb), + }, + + /** Problem reporting. Needs `registerProblemReportIpc()` in main. */ + report: { + diagnostics: (): Promise<{ canSend: boolean; diagnostics: Record }> => + ipcRenderer.invoke(channel('report-diagnostics')), + submit: (input: { message: string; contact?: string }): Promise => + ipcRenderer.invoke(channel('report-submit'), input), + }, + + /** First-run environment setup. The progress/error messages themselves + * arrive as `{type: 'env_setup'}` on `onMessage` (route them with + * `toShellAction`); this is only the answer path back to `runEnvSetup`. */ + envSetup: { + answer: (choice: EnvSetupChoice) => ipcRenderer.send(channel('env-setup-answer'), choice), + }, + /** Escape hatch for an app's own channels, so it does not have to * re-implement the disposer discipline above. */ onChannel: on, @@ -128,3 +184,7 @@ export function exposeShellBridge( ...createShellBridge(opts), ...extra, }) } + +/** The bridge's shape, so an app types `window.` from this instead of + * hand-copying it (and drifting): `ShellBridge & { myOwnThing(): … }`. */ +export type ShellBridge = ReturnType diff --git a/de_shell/js/renderer/EnvSetupOverlay.tsx b/de_shell/js/renderer/EnvSetupOverlay.tsx new file mode 100644 index 0000000..d4cdf56 --- /dev/null +++ b/de_shell/js/renderer/EnvSetupOverlay.tsx @@ -0,0 +1,177 @@ +/** + * EnvSetupOverlay.tsx — the first-run "Setting up " card shown while main's + * `runEnvSetup` builds the Python environment (a download of well over a GB on + * a packaged first run). Without it the window sits blank for minutes, + * indistinguishable from a hang. + * + * Driven by `ShellState.envSetup` (shellReducer maps the `env_setup` messages): + * a spinner, phase pills, a step line with a real percentage when uv reports + * one (indeterminate otherwise), and a live log tail so it is always visibly + * moving. When setup FAILS the same card turns into a recovery prompt — the + * message plus Retry / "Install from a folder…" / Quit — because on an offline + * lab PC the user is the only one who can fix it. + */ +import React, { useEffect, useRef } from 'react' + +import type { EnvSetupApi } from './bridgeTypes' +import { logTail, phasePills, stepText } from './envSetupView' +import type { EnvSetupState } from './shellState' +import { buttons, colors, font, mono } from './theme' + +export interface EnvSetupOverlayProps { + setup: EnvSetupState + appName: string + /** The answer path back to main (`bridge.envSetup`). */ + envSetup: EnvSetupApi + /** Replaces the default first-launch blurb. */ + description?: React.ReactNode +} + +export function EnvSetupOverlay({ setup, appName, envSetup, description }: EnvSetupOverlayProps) { + const tailRef = useRef(null) + // Keep the log tail pinned to the newest line. + useEffect(() => { + const el = tailRef.current + if (el) el.scrollTop = el.scrollHeight + }, [setup.lines]) + + const failed = !!setup.error + const tail = logTail(setup.lines) + + return ( +
+
+
+ {!failed && } +
+ {failed ? `${appName} could not finish setting up` : `Setting up ${appName}`} +
+
+ + {failed ? ( +
{setup.error}
+ ) : ( +
+ {description ?? 'First launch only — downloading the analysis environment. ' + + 'This can take several minutes; every later launch is instant.'} + {setup.wheelhouse &&
Installing from {setup.wheelhouse}
} +
+ )} + + {!failed && ( + <> +
+ {phasePills(setup.phase).map((p) => ( + + {p.state === 'done' ? '✓ ' : ''}{p.label} + + ))} +
+
+ {stepText(setup)} +
+ + + )} + +
Setup log
+
+ {tail.length === 0 + ? Starting… + : tail.map((l, i) =>
{l}
)} +
+ + {failed && ( +
+ + + +
+ )} +
+
+ ) +} + +/** Indeterminate when percent is null; a filled bar when we have a %. */ +function ProgressBar({ percent }: { percent: number | null }) { + return ( +
+
+ +
+ ) +} + +function Spinner() { + return ( +
+ +
+ ) +} + +/** Keyframes ship inline (no stylesheet to load); duplicates are harmless. */ +function Keyframes() { + return ( + + ) +} + +const S: Record = { + scrim: { + position: 'fixed', inset: 0, zIndex: 99998, background: colors.scrim, + backdropFilter: 'blur(2px)', display: 'flex', alignItems: 'center', + justifyContent: 'center', padding: 24, userSelect: 'text', + }, + card: { + maxWidth: 560, width: '100%', display: 'flex', flexDirection: 'column', + padding: '26px 30px', borderRadius: 12, background: colors.bg, + border: '1px solid', boxShadow: '0 12px 40px rgba(0,0,0,0.5)', + color: colors.fg, fontFamily: font, + }, + header: { display: 'flex', alignItems: 'center', gap: 12, marginBottom: 6 }, + blurb: { fontSize: 13, lineHeight: 1.5, color: colors.muted, marginBottom: 16 }, + // Bounded: an arbitrarily long failure scrolls instead of growing the card. + error: { + fontSize: 13, lineHeight: 1.5, color: colors.danger, marginBottom: 12, + maxHeight: 120, overflowY: 'auto', overflowWrap: 'anywhere', whiteSpace: 'pre-wrap', + }, + pills: { display: 'flex', gap: 6, marginBottom: 14, flexWrap: 'wrap' }, + logLabel: { + marginTop: 16, fontSize: 10.5, color: colors.faint, textTransform: 'uppercase', + letterSpacing: 0.5, marginBottom: 5, + }, + log: { + height: 128, overflow: 'auto', background: colors.inset, + border: `1px solid ${colors.border}`, borderRadius: 6, padding: '7px 10px', + fontFamily: mono, fontSize: 11, lineHeight: 1.55, whiteSpace: 'pre-wrap', + wordBreak: 'break-word', + }, + actions: { display: 'flex', justifyContent: 'flex-end', gap: 8, marginTop: 16 }, +} diff --git a/de_shell/js/renderer/ReportProblemDialog.tsx b/de_shell/js/renderer/ReportProblemDialog.tsx new file mode 100644 index 0000000..cc31884 --- /dev/null +++ b/de_shell/js/renderer/ReportProblemDialog.tsx @@ -0,0 +1,155 @@ +/** + * ReportProblemDialog.tsx — "Report a Problem…". + * + * What the user writes is the smaller half of a report. The rest — OS, app and + * runtime versions, the last thing the updater said, the tail of the backend's + * output, plus whatever the host app adds — is collected by main + * (errorReport.ts) and shown here BEFORE anything is sent, because a report + * that quietly ships a machine description is a report people learn not to + * send. Nothing leaves the machine until Send is pressed, and a copy is always + * written to disk so an offline PC still produces something to email. + */ +import React, { useEffect, useState } from 'react' + +import type { ReportApi, ReportResult } from './bridgeTypes' +import { canSubmit, countProblems, outcomeModel, submitLabel } from './reportModel' +import type { ReportPhase } from './reportModel' +import { ShellModal } from './ShellModal' +import { buttons, colors, font, mono } from './theme' + +export interface ReportProblemDialogProps { + report: ReportApi + appName: string + onClose: () => void +} + +export function ReportProblemDialog({ report, appName, onClose }: ReportProblemDialogProps) { + const [message, setMessage] = useState('') + const [contact, setContact] = useState('') + const [canSend, setCanSend] = useState(false) + const [diagnostics, setDiagnostics] = useState | null>(null) + const [showDetails, setShowDetails] = useState(false) + const [phase, setPhase] = useState('writing') + const [result, setResult] = useState(null) + + useEffect(() => { + let cancelled = false + report.diagnostics().then((info) => { + if (cancelled) return + setCanSend(info.canSend) + setDiagnostics(info.diagnostics) + }).catch(() => { /* the dialog still works; details just stay empty */ }) + return () => { cancelled = true } + }, [report]) + + const submit = async () => { + setPhase('sending') + try { + setResult(await report.submit({ message, contact: contact.trim() || undefined })) + } catch (err) { + setResult({ sent: false, error: String(err) }) + } + setPhase('done') + } + + const finished = phase === 'done' && result + const problems = countProblems(diagnostics) + const ok = canSubmit(message, phase) + + return ( + Close + ) : <> + + + }> + {finished ? : ( + <> +

+ {canSend + ? `This goes straight to the ${appName} maintainers, with the details below attached.` + : 'This build has no reporting service configured, so the report will be saved ' + + 'to your computer for you to send on.'} +

+ +