diff --git a/.github/workflows/action-json-lint.yml b/.github/workflows/action-json-lint.yml index d5d98348..be77a99b 100644 --- a/.github/workflows/action-json-lint.yml +++ b/.github/workflows/action-json-lint.yml @@ -28,9 +28,9 @@ jobs: lint: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - - uses: actions/setup-python@v5 + - uses: actions/setup-python@v6 with: python-version: "3.12" @@ -41,8 +41,10 @@ jobs: python -m pip install --only-binary :all: --upgrade "pip==26.0.1" # The spec is a documented workflow input — pinned by default so a # lint run is reproducible, overridable so a caller can track their - # own release or a git ref. - python -m pip install "$AUTOCONTROL_REF" + # own release or a git ref. --only-binary still builds a git ref's + # own project (pip builds VCS requirements regardless) but runs no + # dependency's setup script. + python -m pip install --only-binary :all: "$AUTOCONTROL_REF" - name: Lint action JSON files shell: bash diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml deleted file mode 100644 index 24349702..00000000 --- a/.github/workflows/dev.yml +++ /dev/null @@ -1,118 +0,0 @@ -name: AutoControl Dev CI - -on: - push: - branches: [ "dev" ] - pull_request: - branches: [ "dev" ] - schedule: - - cron: "0 1 * * *" - -permissions: - contents: read - -jobs: - test: - runs-on: windows-2022 - strategy: - fail-fast: false - matrix: - python-version: [ "3.10", "3.11", "3.12", "3.13", "3.14" ] - - steps: - - uses: actions/checkout@v4 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - - - name: Install dependencies - run: | - python -m pip install --upgrade pip wheel - pip install -r dev_requirements.txt - pip install -e . - - # Screen tests - - name: Test Screen Size - run: python ./test/unit_test/screen/screen_test.py - - name: Test Screenshot - run: python ./test/unit_test/screen/screenshot_test.py - - name: Test Screen Get Pixel - run: python ./test/unit_test/screen/get_pixel_test.py - - name: Upload Screenshot Artifact - uses: actions/upload-artifact@v4 - if: always() - with: - name: screenshot_png_${{ matrix.python-version }} - path: test.png - if-no-files-found: ignore - - # Keyboard tests - - name: Test Keyboard Type - run: python ./test/unit_test/keyboard/keyboard_type_test.py - - name: Test Keyboard Write - run: python ./test/unit_test/keyboard/keyboard_write_test.py - - name: Test Keyboard Is Press - run: python ./test/unit_test/keyboard/keyboard_is_press_test.py - - name: Test Keyboard Hotkey - run: python ./test/unit_test/keyboard/hotkey_test.py - - # Mouse tests - # These three drive the real mouse and the real exit path on a hosted - # runner with no desk in front of it, so a failure says more about the - # runner's session than about the change under test. They are demo - # scripts (CLAUDE.md: the *_test.py files run on import), not the CI - # gate -- that is pytest-headless in quality.yml. - - name: Test Mouse Module - run: python ./test/unit_test/mouse/mouse_test.py - continue-on-error: true - - # Exception tests - - name: Test Exceptions - run: python ./test/unit_test/exception/auto_control_exception_test.py - - # Critical exit tests - - name: Test Critical Exit - run: python ./test/unit_test/critical_exit/critical_exit_test.py - continue-on-error: true - - name: Test Real Critical Situation - run: python ./test/unit_test/critical_exit/real_critical_test.py - continue-on-error: true - - # Record tests - - name: Test Record Module - run: python ./test/unit_test/record/record_test.py - - name: Test Total Record - run: python ./test/unit_test/total_record/total_record_test.py - - # Executor tests - - name: Test Execute Action - run: python ./test/unit_test/execute_action/execute_action_test.py - - # JSON tests - - name: Test JSON Module - run: python ./test/unit_test/json/json_test.py - - # Report generation tests - - name: Test Generate JSON Report - run: python ./test/unit_test/generate_report/json_report.py - - name: Test Generate HTML Report - run: python ./test/unit_test/generate_report/html_report_test.py - - # Argparse test - - name: Test Argparse - run: python ./test/unit_test/argparse/argparse_test.py - - # Callback test - - name: Test Callback Module - run: python ./test/unit_test/callback/callback_test.py - - # Project creation test - - name: Test Create Project - run: python ./test/unit_test/create_project_file/create_project_test.py - - # Info tests - - name: Test Get Mouse Info - run: python ./test/unit_test/get_info/mouse_info.py - - name: Test Get Keyboard Info - run: python ./test/unit_test/get_info/keyboard_info.py diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index d9a58a40..53e8615c 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -2,14 +2,14 @@ name: AutoControl Docker CI on: push: - branches: [ "dev", "main" ] + branches: [ "main" ] paths: - "docker/**" - "je_auto_control/**" - "pyproject.toml" - ".github/workflows/docker.yml" pull_request: - branches: [ "dev", "main" ] + branches: [ "main" ] paths: - "docker/**" - "je_auto_control/**" @@ -25,15 +25,15 @@ jobs: runs-on: ubuntu-22.04 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - name: Set up Docker Buildx # nosemgrep: yaml.github-actions.security.third-party-action-not-pinned-to-commit-sha.third-party-action-not-pinned-to-commit-sha - uses: docker/setup-buildx-action@v3 # NOSONAR githubactions:S7637 + uses: docker/setup-buildx-action@v4 # NOSONAR githubactions:S7637 - name: Build image (no push) # nosemgrep: yaml.github-actions.security.third-party-action-not-pinned-to-commit-sha.third-party-action-not-pinned-to-commit-sha - uses: docker/build-push-action@v5 # NOSONAR githubactions:S7637 + uses: docker/build-push-action@v7 # NOSONAR githubactions:S7637 with: context: . file: docker/Dockerfile @@ -51,15 +51,15 @@ jobs: runs-on: ubuntu-22.04 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - name: Set up Docker Buildx # nosemgrep: yaml.github-actions.security.third-party-action-not-pinned-to-commit-sha.third-party-action-not-pinned-to-commit-sha - uses: docker/setup-buildx-action@v3 # NOSONAR githubactions:S7637 + uses: docker/setup-buildx-action@v4 # NOSONAR githubactions:S7637 - name: Rebuild image (cached) # nosemgrep: yaml.github-actions.security.third-party-action-not-pinned-to-commit-sha.third-party-action-not-pinned-to-commit-sha - uses: docker/build-push-action@v5 # NOSONAR githubactions:S7637 + uses: docker/build-push-action@v7 # NOSONAR githubactions:S7637 with: context: . file: docker/Dockerfile @@ -115,15 +115,15 @@ jobs: runs-on: ubuntu-22.04 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - name: Set up Docker Buildx # nosemgrep: yaml.github-actions.security.third-party-action-not-pinned-to-commit-sha.third-party-action-not-pinned-to-commit-sha - uses: docker/setup-buildx-action@v3 # NOSONAR githubactions:S7637 + uses: docker/setup-buildx-action@v4 # NOSONAR githubactions:S7637 - name: Build the Wayland verification image # nosemgrep: yaml.github-actions.security.third-party-action-not-pinned-to-commit-sha.third-party-action-not-pinned-to-commit-sha - uses: docker/build-push-action@v5 # NOSONAR githubactions:S7637 + uses: docker/build-push-action@v7 # NOSONAR githubactions:S7637 with: context: . file: docker/Dockerfile.wayland @@ -161,15 +161,15 @@ jobs: runs-on: ubuntu-22.04 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - name: Set up Docker Buildx # nosemgrep: yaml.github-actions.security.third-party-action-not-pinned-to-commit-sha.third-party-action-not-pinned-to-commit-sha - uses: docker/setup-buildx-action@v3 # NOSONAR githubactions:S7637 + uses: docker/setup-buildx-action@v4 # NOSONAR githubactions:S7637 - name: Build the EIS verification image # nosemgrep: yaml.github-actions.security.third-party-action-not-pinned-to-commit-sha.third-party-action-not-pinned-to-commit-sha - uses: docker/build-push-action@v5 # NOSONAR githubactions:S7637 + uses: docker/build-push-action@v7 # NOSONAR githubactions:S7637 with: context: . file: docker/Dockerfile.eis @@ -200,15 +200,15 @@ jobs: runs-on: ubuntu-22.04 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - name: Set up Docker Buildx # nosemgrep: yaml.github-actions.security.third-party-action-not-pinned-to-commit-sha.third-party-action-not-pinned-to-commit-sha - uses: docker/setup-buildx-action@v3 # NOSONAR githubactions:S7637 + uses: docker/setup-buildx-action@v4 # NOSONAR githubactions:S7637 - name: Build the portal verification image # nosemgrep: yaml.github-actions.security.third-party-action-not-pinned-to-commit-sha.third-party-action-not-pinned-to-commit-sha - uses: docker/build-push-action@v5 # NOSONAR githubactions:S7637 + uses: docker/build-push-action@v7 # NOSONAR githubactions:S7637 with: context: . file: docker/Dockerfile.portal @@ -247,15 +247,15 @@ jobs: runs-on: ubuntu-22.04 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - name: Set up Docker Buildx # nosemgrep: yaml.github-actions.security.third-party-action-not-pinned-to-commit-sha.third-party-action-not-pinned-to-commit-sha - uses: docker/setup-buildx-action@v3 # NOSONAR githubactions:S7637 + uses: docker/setup-buildx-action@v4 # NOSONAR githubactions:S7637 - name: Build the X11 verification image # nosemgrep: yaml.github-actions.security.third-party-action-not-pinned-to-commit-sha.third-party-action-not-pinned-to-commit-sha - uses: docker/build-push-action@v5 # NOSONAR githubactions:S7637 + uses: docker/build-push-action@v7 # NOSONAR githubactions:S7637 with: context: . file: docker/Dockerfile.x11 @@ -295,15 +295,15 @@ jobs: runs-on: ubuntu-22.04 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - name: Set up Docker Buildx # nosemgrep: yaml.github-actions.security.third-party-action-not-pinned-to-commit-sha.third-party-action-not-pinned-to-commit-sha - uses: docker/setup-buildx-action@v3 # NOSONAR githubactions:S7637 + uses: docker/setup-buildx-action@v4 # NOSONAR githubactions:S7637 - name: Build the seat verification image # nosemgrep: yaml.github-actions.security.third-party-action-not-pinned-to-commit-sha.third-party-action-not-pinned-to-commit-sha - uses: docker/build-push-action@v5 # NOSONAR githubactions:S7637 + uses: docker/build-push-action@v7 # NOSONAR githubactions:S7637 with: context: . file: docker/Dockerfile.seat @@ -352,15 +352,15 @@ jobs: runs-on: ubuntu-22.04 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - name: Set up Docker Buildx # nosemgrep: yaml.github-actions.security.third-party-action-not-pinned-to-commit-sha.third-party-action-not-pinned-to-commit-sha - uses: docker/setup-buildx-action@v3 # NOSONAR githubactions:S7637 + uses: docker/setup-buildx-action@v4 # NOSONAR githubactions:S7637 - name: Build the ydotool verification image # nosemgrep: yaml.github-actions.security.third-party-action-not-pinned-to-commit-sha.third-party-action-not-pinned-to-commit-sha - uses: docker/build-push-action@v5 # NOSONAR githubactions:S7637 + uses: docker/build-push-action@v7 # NOSONAR githubactions:S7637 with: context: . file: docker/Dockerfile.ydotool diff --git a/.github/workflows/platform-smoke.yml b/.github/workflows/platform-smoke.yml index 3d7abbdb..c5c89bfb 100644 --- a/.github/workflows/platform-smoke.yml +++ b/.github/workflows/platform-smoke.yml @@ -2,9 +2,9 @@ name: Platform smoke on: push: - branches: ["main", "dev"] + branches: ["main"] pull_request: - branches: ["main", "dev"] + branches: ["main"] permissions: contents: read @@ -50,8 +50,8 @@ jobs: python-version: "3.10" runs-on: ${{ matrix.os }} steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 + - uses: actions/checkout@v5 + - uses: actions/setup-python@v6 with: python-version: ${{ matrix.python-version }} - run: python -m pip install -e . # NOSONAR githubactions:S8544 # reason: installs the checked-out project itself, there is no upstream version to lock @@ -75,7 +75,7 @@ jobs: FailureBundleOptions, create_failure_bundle; create_failure_bundle('platform-smoke.zip', options=FailureBundleOptions(screenshot=False))" - - uses: actions/upload-artifact@v4 + - uses: actions/upload-artifact@v6 if: always() with: name: platform-smoke-${{ matrix.os }}-${{ matrix.python-version }} @@ -88,7 +88,7 @@ jobs: timeout-minutes: 30 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 # The X11 backend was gated on sys.platform being linux/linux2, so it # refused to load on a FreeBSD desktop that runs the same X server, the @@ -165,9 +165,9 @@ jobs: runs-on: macos-14 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - - uses: actions/setup-python@v5 + - uses: actions/setup-python@v6 with: python-version: "3.12" diff --git a/.github/workflows/quality.yml b/.github/workflows/quality.yml index 4d0c0814..ce142176 100644 --- a/.github/workflows/quality.yml +++ b/.github/workflows/quality.yml @@ -7,9 +7,9 @@ name: AutoControl Code Quality on: push: - branches: [ "dev", "main", "stable" ] + branches: [ "main" ] pull_request: - branches: [ "dev", "main", "stable" ] + branches: [ "main" ] workflow_dispatch: permissions: @@ -22,16 +22,16 @@ jobs: permissions: contents: read steps: - - uses: actions/checkout@v4 - - uses: actions/dependency-review-action@v4 + - uses: actions/checkout@v5 + - uses: actions/dependency-review-action@v5 lint: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: "3.12" cache: "pip" @@ -45,10 +45,10 @@ jobs: security: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: "3.12" cache: "pip" @@ -81,10 +81,10 @@ jobs: - { os: macos-14, python-version: "3.10" } - { os: macos-14, python-version: "3.14" } steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: ${{ matrix.python-version }} cache: "pip" @@ -120,15 +120,26 @@ jobs: # snapshot of `je_auto_control/` straight into site-packages and masks # the editable install for any sub-package the snapshot doesn't include # (admin, usb, remote_desktop, vision, …). - - name: Install the project itself + # + # `[webrtc]` is part of the install and is load-bearing for what this job + # measures. Eleven modules under `utils/remote_desktop` raise ImportError + # at module level without `aiortc`/`av` — 2,090 statements that were a + # hard 0% here no matter what anyone wrote. Worse than the number: the + # tests that cover the WebRTC host's auth, TLS, tokens and file transfer + # were already written and `importorskip`ped straight past on every + # square, so they ran on developer machines and nowhere else. Measured on + # this tree, one variable changed: 513 of those statements are covered by + # tests that exist today. The extra is NOT added to `typing-stable-api` + # below — that gate must not depend on what is installed. + - name: Install the project itself, with the WebRTC extra shell: bash - run: pip install -e . # NOSONAR githubactions:S8544 githubactions:S8541 # reason: installs the checked-out project itself, so there is no upstream version to lock and no third-party setup script to run + run: pip install -e ".[webrtc]" # NOSONAR githubactions:S8544 githubactions:S8541 # reason: installs the checked-out project itself, so there is no upstream version to lock and no third-party setup script to run - name: Install the test tooling shell: bash # Quoted: `--only-binary :all:` puts a colon-space inside the # scalar, which YAML reads as a mapping and refuses. - run: "pip install --only-binary :all: ruff==0.15.22 bandit==1.9.4 pytest==9.1.1 pytest-timeout==2.4.0 pytest-rerunfailures==15.1 coverage==7.15.4 PySide6==6.11.1" + run: "pip install --only-binary :all: ruff==0.15.22 bandit==1.9.4 pytest==9.1.1 pytest-timeout==2.4.0 pytest-rerunfailures==15.1 coverage==7.15.4 PySide6==6.11.1 radon==6.0.1" # Paths come from `testpaths` in pyproject.toml. Do NOT pass an explicit # path here: an argument overrides testpaths, which previously meant the @@ -166,7 +177,7 @@ jobs: - name: Upload coverage report if: always() - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v6 with: name: coverage-${{ matrix.os }}-${{ matrix.python-version }} path: coverage.xml @@ -180,8 +191,8 @@ jobs: typing-stable-api: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 + - uses: actions/checkout@v5 + - uses: actions/setup-python@v6 with: python-version: "3.12" - run: pip install -e . # NOSONAR githubactions:S8541,githubactions:S8544 # reason: installs the checked-out project itself, there is no upstream version to lock and the build must run diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index f02adcee..33db068b 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -15,8 +15,8 @@ jobs: id-token: write attestations: write steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 + - uses: actions/checkout@v5 + - uses: actions/setup-python@v6 with: python-version: "3.12" - run: "python -m pip install --only-binary :all: build==1.5.0 twine==6.2.0" @@ -49,7 +49,7 @@ jobs: - uses: actions/attest-build-provenance@v2 with: subject-path: "dist/*" - - uses: actions/upload-artifact@v4 + - uses: actions/upload-artifact@v6 with: name: python-distributions path: dist/ @@ -71,7 +71,7 @@ jobs: permissions: id-token: write steps: - - uses: actions/download-artifact@v4 + - uses: actions/download-artifact@v7 with: name: python-distributions path: dist/ diff --git a/.github/workflows/stable.yml b/.github/workflows/stable.yml index bccd9f48..944af954 100644 --- a/.github/workflows/stable.yml +++ b/.github/workflows/stable.yml @@ -2,9 +2,9 @@ name: AutoControl Stable CI on: push: - branches: [ "main", "stable" ] + branches: [ "main" ] pull_request: - branches: [ "main", "stable" ] + branches: [ "main" ] schedule: - cron: "0 1 * * *" @@ -24,9 +24,9 @@ jobs: python-version: [ "3.10", "3.11", "3.12", "3.13", "3.14" ] steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: ${{ matrix.python-version }} @@ -44,7 +44,7 @@ jobs: - name: Test Screen Get Pixel run: python ./test/unit_test/screen/get_pixel_test.py - name: Upload Screenshot Artifact - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v6 if: always() with: name: screenshot_png_${{ matrix.python-version }} @@ -133,13 +133,13 @@ jobs: permissions: contents: write steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 with: fetch-depth: 0 token: ${{ secrets.GITHUB_TOKEN }} - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: "3.12" diff --git a/CHANGELOG.md b/CHANGELOG.md index 18911307..aa05a237 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,15 +1,976 @@ # Changelog This file records user-visible compatibility changes. Detailed development -notes remain in `WHATS_NEW.md`. +notes are recorded in `docs/updates/` (index: `docs/updates/README.md`). The format follows Keep a Changelog. Until 1.0, breaking changes are permitted only when documented here with a migration path. +New entries go under `## Unreleased`. The version bump on `main` is automated +and does not touch this file, so after a release tag appears, move the entries +it shipped into a `## [x.y.z] - date` section of their own; the tag's +`CHANGELOG.md` shows which ones those are (`git show vX.Y.Z:CHANGELOG.md`). + ## Unreleased ### Added +- **Failure-bundle manifests record `error_type`** beside the redacted + message, which is empty for exceptions such as `TimeoutError()`. +- **Config-sync routes on the signaling server.** `GET` / `PUT + /config/{user_id}` serve the buckets `ConfigSyncClient` pushes and pulls + (in memory, secret-checked, 1 MiB per bucket); the client had no server + to talk to before. + +- **`approval_gate(db=None)`.** The approval gate the `AC_approval_*` + commands use: file-backed with `db`, otherwise one per process. + +- **`adaptive_mean` / `adaptive_gaussian` preprocessing steps.** They take + `block_size` / `c`, which no step used before. + +- **`SoftAssertionsFailed`.** The exception `SoftAssertions.assert_all` + raises; exported from the package. + +- **State-machine `if_image_found` guard.** Fires once the template is on + screen (`"welcome.png"` or `{"image": ..., "detect_threshold": ...}`). + Other `if_*` keys, which used to fire unconditionally, now raise + `StateMachineError`. + +- **`WorkQueue.get_next(stale_after_s=...)`** (and the same optional + argument on `AC_queue_next` / `ac_queue_next`) reclaims an item a crashed + performer left in progress for that many seconds. + +- **`ac_rrule_next`, `ac_rrule_occurrences` and `ac_format_date` now declare + the string format they parse.** Their `dtstart` / `now` / `value` properties + carry `"format": "date-time"` (or `"date"`) in the tool's input schema, which + the descriptions already said in prose and the schema did not. A client + generating values from the schema alone used to produce a plain string and + get a `ValueError` out of `datetime.fromisoformat`. + +### Changed + +- **Golden-image capture (`take_golden` / `compare_to_golden`) reads its + region in mouse coordinates**, like every other capture; region goldens + taken on a scaled display need re-taking. +- **`ShellManager.exec_shell` accepts `command=`** as well as + `shell_command=`, matching the Script Builder and the docs; the new + `command_args()` is how every shell entry point turns a command into what + `subprocess` receives. + +- **`replay_timeline` / `run_sequence` refuse unknown ops and bad speeds.** + An unknown op raises instead of being skipped, and `speed` must be a + positive number. + +- **`WorkQueue.complete` / `fail` require an `in_progress` item.** An unknown + id or an item in another state raises, so finished work is never requeued + and a stale performer cannot overwrite a newer outcome. + +- **`AgentTrace.to_otel` returns OTLP/JSON spans** (trace and span ids, + nanosecond times, integer enums, typed attributes) instead of flat dicts + with `duration_s`. Cost records refuse negative token counts. + +- **`AssetStore.set` validates the type and value** it is given, and a + string `tags` argument is one tag, not a list of letters. + +- **Plugins cannot replace built-in commands by default.** + `register_plugin_commands` and `load_plugins` skip (and log) a name that + already belongs to a built-in or user command; pass `allow_override=True` + to replace one deliberately. + +- **Impossible rate-limit requests raise.** `TokenBucket` and + `SlidingWindowLimiter` refuse `n <= 0` or above capacity / limit, and + `CredentialBroker.lease` refuses a TTL that is not a finite positive number. + +- **Pseudo-localization padding counts visible text only**, and + `check_catalog` compares printf conversions and argument names rather + than whole ICU blocks. + +- **Colour-match scores are RMS colour distance.** `match_color` scores are + `1 -` the root-mean-square HSV distance, hue compared round the colour + wheel; `min_score` thresholds tuned on the old metric may need adjusting. + `AC_image_hash` refuses an unknown `algo`, `hamming_distance` refuses + hashes of different sizes, and `upscale` refuses a non-positive scale. + +- **A failed soft-assert batch is an assertion failure.** It raises + `SoftAssertionsFailed` (an `AutoControlAssertionException` and still an + `AutoControlActionException`), so suites score it *failed* and lenient runs + no longer swallow it. + +- **`raise_on_error=True` reaches into nested bodies.** Loops, branches and + macros run by a strict list are strict too, so a failure inside them + raises instead of being recorded; lenient runs are unchanged. + +- **JSONPath refuses what it cannot read.** `json_query` raises `ValueError` + for an unsupported filter, an unterminated `[` or a stray character (an + unsupported filter used to match every element). Filters take nested + fields (`@.a.b`) and existence tests (`[?(@.k)]`), `true` no longer equals + `1`, and a bare key may contain `-`. + +- **More MCP tools are destructive.** Tools that run action lists or code, + send input, delete data, send data off the machine or loosen a security + control carry `destructiveHint: true`, so + `JE_AUTOCONTROL_MCP_CONFIRM_DESTRUCTIVE=1` asks before them. + `ac_export_sarif`, `ac_compliance_report` and `ac_assert_visual` are no + longer read-only. A `tools/call` argument the schema does not declare is + refused with `-32602`. + +- **Secret-key matching is by word.** `scan_secrets` and `redact_config` no + longer treat keys that merely contain `pass` or `token` (`bypass_proxy`, + `tokenizer`) as secrets, and now do treat `apiKey`, `cookie`, `sessionId` + and `Authorization` as secrets. A redaction bounding box with no known + coordinate keys raises `ValueError`. + +- **An empty window title is an error.** `find_window`, `focus_window`, + `close_window_by_title` and the other title lookups raise + `AutoControlActionException` for a blank or non-string title; `""` used to + match every window. + +- **`je_auto_control run` exits 1 when any action failed.** It used to exit + 0 whenever the file loaded. Migration: a pipeline that relied on the old + status can ignore it (`|| true`). + +- **Chat-ops `/screenshot` takes a file name, not a path.** The PNG is + written into the router context's `screenshot_dir` (default: a + `je_auto_control_chatops` folder in the temp directory); anyone in the + channel could previously choose any path. Migration: set `screenshot_dir`. + +- **`AC_shell_to_var` on Windows.** A string command is passed to + `CreateProcess` as written; quoted arguments used to arrive with their + quotes still on. Output is decoded with the new `encoding` argument, + defaulting to the locale's encoding rather than always UTF-8. Migration: + pass `"encoding": "utf-8"` for a program that writes UTF-8. + +- **Passphrase-encrypted action files are salted.** `encrypt_action_file` + with a passphrase now derives the key with scrypt and a random per-file + salt, and writes `ACENC1:` + salt + token; it used one unsalted SHA-256. + Older files still decrypt. Migration: none, but a file written by this + version cannot be decrypted by an older one. + +- **`max_runs` must be at least 1.** `Scheduler.add_job` and + `add_cron_job` raise `ValueError` for `max_runs=0` or below, which used to + run the job once. Migration: pass `max_runs=1`. + +- **Importing the package no longer sets the root logger to DEBUG.** A + library must not reconfigure its host's logging, and that one line sent + every third-party logger's DEBUG records to whatever handlers the host + application had configured. `autocontrol_logger` now carries the DEBUG + level itself, so this package's own records are unchanged. Migration: a + program that relied on the side effect should call + `logging.getLogger().setLevel(logging.DEBUG)` itself. + +- **Per-user state paths are resolved when used, not at import.** Nine + defaults under `~/.je_auto_control/` (the action signing and encryption + keys, the host fingerprint and known hosts, the host service config, the + WebRTC inbox, and the A/B locator, cost and self-healing logs) were fixed + when their module was imported, so setting `HOME` / `USERPROFILE` + afterwards had no effect on them. The class attributes + `ABStore.DEFAULT_PATH`, `CostStore.DEFAULT_PATH` and + `HealEventLog.DEFAULT_PATH` are replaced by the functions + `default_stats_path()`, `default_cost_log_path()` and + `default_heal_log_path()` in the same modules. Migration: call the + function where the attribute was read. + +- **The log file moved out of the current directory.** Importing the package + opened `AutoControlGUI.log` relative to the cwd, so every process that + imported it — including every pytest run on a machine where it is installed, + through the `pytest11` plugin — left a log wherever it started. The file is + now `~/.je_auto_control/logs/AutoControlGUI.log`, or whatever + `JE_AUTOCONTROL_LOG_FILE` names when the file is first opened, so a + `conftest.py` can still redirect it after the plugin imported the package + (`os.devnull` turns it off). Because every + process shares it, lines carry the process id (`time | pid | logger | level + | message`), and instead of growing without limit it is moved to `.1` once + past 10 MB, at the moment a process opens it. The file is opened on the + first record rather than at import, and the `Load Windows Setting` / + `Load Linux x11 Setting` / `Load Linux Wayland Setting` / `Load MacOS + Setting` lines every import used to log are gone, so importing the package + writes no file. A file that cannot be opened (read-only home or cwd) is + replaced by `os.devnull` with one `RuntimeWarning`; it used to make + `import je_auto_control` fail. + Migration: to keep the old location, set + `JE_AUTOCONTROL_LOG_FILE=AutoControlGUI.log`; changing the working + directory before the import no longer redirects the file. + +### Security + +- **HTTP cassettes no longer record credential headers**, and a `match_on` + field they cannot compare raises instead of matching every request. +- **JWT decoding rejects characters outside base64url** (which made tokens + malleable) and a `NaN` expiry that never expired, and `encode_jwt` no longer + lets extra headers override `alg`. +- **The admin console no longer sends its bearer token to a redirect target**, + bounds each host's response by `timeout_s` as a whole and by size, and + reports broadcast labels that name no host. +- **The USB passthrough ACL fails closed when its signature is deleted** once a + signing key exists, keeps a damaged file aside instead of overwriting it, + and refuses malformed vendor/product ids that never matched. +- **`AC_list_plugins` / `AC_load_plugins` and the matching MCP tools load only + the `je_auto_control.commands` entry-point group**; another group such as + `console_scripts` ran every installed tool's `main()`. +- **Remote desktop: failed logins no longer use up the host's client slots**, + a view-only viewer can no longer set the clipboard or write files, and an + IP allowlist whose every entry is invalid admits nobody instead of + everyone. +- **Anonymous `initialize` requests can no longer evict an MCP HTTP session in + use**, and `DELETE` off the `/mcp` path no longer ends a session. +- **The REST API checks the token before reading a POST body**, and a valid + token is never locked out by other clients' failed attempts from the same + IP. +- **The RBAC user store refuses a token another user already has**, and no + longer replaces every user when its file is damaged. +- **The audit log's hash chain catches a forged row whose hash was cleared + and the deletion of its oldest rows**, and `clear()` leaves an + `audit_log_cleared` event instead of an empty, clean-looking log. +- **`JE_AUTOCONTROL_REQUIRE_SIGNED_ACTIONS` covers DAG nodes that run on a + remote host**; an unsigned action file was dispatched to it. +- **Keys, passwords and tokens given to actions are masked** in the executor + log and result record (signing, encryption and JWT keys; any `password`, + `passphrase`, `token`...), and the MCP audit file masks them at any depth, + including inside `actions` lists. +- **The `discovery` extra requires zeroconf 0.149.16**, which fixes the + 2026 mDNS memory and CPU exhaustion advisories reachable from the local + network. + +- **The `signaling` extra requires Starlette 1.0.1** (CVE-2026-48710, + "BadHost": a crafted `Host` header made middleware see another path than + the router served). The signaling server's pre-body guard also decides by + the routed path, so it holds on older Starlette too. + +- **MCP over HTTP: a web page could drive the machine, and one session could + confirm another's destructive call.** With no token configured (the default) + the server accepted cross-site browser requests — a `text/plain` POST needs + no CORS preflight — so any page the user opened could run tools; it now + refuses a non-loopback `Origin`, and a non-loopback `Host` when bound to + loopback (DNS rebinding). `JE_AUTOCONTROL_MCP_ALLOWED_ORIGINS` admits + specific browser origins. Replies to server-sent prompts were matched by a + sequential id alone, so any client could accept a confirmation shown to + another; they are now bound to the session the prompt went to and the ids + are random. With `JE_AUTOCONTROL_MCP_CONFIRM_DESTRUCTIVE=1`, a destructive + call that had no stream to ask on ran unconfirmed; it is refused. A + non-ASCII bearer token crashed the request thread in both the MCP and REST + servers (never counted toward lockout); it is refused. + +- **USB passthrough: three ways a viewer got past the ACL.** Vendor and + product ids were compared as strings but parsed by the backend with + `int(x, 16)`, so `0x1050`, `01050` or `10_50` missed a `1050` deny rule and + opened the device anyway; ids are now normalised to four lowercase hex digits + (an optional `0x` prefix is accepted, anything else is refused). A viewer + that omitted the serial skipped rules written for one serial and got + whichever matching device the backend found first; that is refused when the + ACL has a serial rule for the device. The WinUSB backend logged a requested + serial and ignored it; it now refuses to open by serial. Transfer `length` + and `timeout_ms` from the wire are bounded (control 0–65535, bulk/interrupt + up to 1 MiB, timeout up to 60 s), where a single request could make the host + allocate a gigabyte. + +- **A WebRTC viewer could be approved without ever sending the token.** The + public `approve_pending_viewer` guard read `not pending and authenticated`, + so for a session whose viewer had sent nothing both were false and the + approval went through, cancelling the auth deadline and accepting input. + Only a viewer that presented the token and is waiting on the user can be + approved now; the token is compared with `hmac.compare_digest` instead of + `!=`, which leaked how many leading characters matched. + +- **The `pdf` extra now requires `pypdf>=6.16.1`** (was `>=4.0`). + `extract_pdf_text`, `pdf_metadata` and `assert_pdf_text` open whatever PDF + they are given, and every pypdf before 6.16.1 can be driven into an + infinite loop or unbounded memory by a malformed file (unterminated inline + images, repeated bad cross-reference entries, large `/ToUnicode` streams or + CID width ranges, `TreeObject.insert_child`, outlines, XForm objects). + Migration: `pip install -U "je_auto_control[pdf]"`. +- **`requirements.txt` states `pillow>=12.3.0` directly.** It already + resolved 12.3.0 through the `je_auto_control>=0.0.216` floor; the direct + line is for the dependency graph, which still reported Pillow 12.2.0 for + this file from before that floor existed. +- **`uv.lock` moves anyio 4.13.0 → 4.14.2, cryptography 49.0.0 → 50.0.1 and + pypdf 6.13.3 → 6.19.0.** anyio (through `starlette`, `[signaling]` extra) + fixes a TLS host-name encoding flaw that allowed certificate spoofing. + cryptography 50 fixes a PKCS#7 EnvelopedData decryption oracle; this + package does no PKCS#7 decryption, so the `>=48.0.1` floor is unchanged. + +### Fixed + +- **Work queue and durable state**: claims are numbered so a stale performer + cannot settle a reclaimed item, and repeatedly abandoned items fail instead + of looping; a failed `run_resumable` step is retried rather than skipped; + idempotency keys can be released; the dedup window and outbox are + thread-safe; S3 `list()` stays in its prefix and reads every page. +- **Agent requests are bounded**: each model call times out after 120 s and + only the three newest screenshots are resent, so long runs stay under the + request size limit; `export_*_tools(only=[])` offers no tools instead of all + of them; an empty LLM plan raises `LLMPlanError`; VLM coordinates are no + longer read out of longer numbers or accepted off the screenshot. +- **Computer use reaches the API**: the `computer` tool is sent under its + beta (it was rejected on every request), now `computer_20251124` on + `claude-opus-5` by default; model-chosen scrolls and waits are bounded and a + scroll goes to the model's coordinate. +- **`python -m je_auto_control -d` runs a directory's files in sorted order** + and no longer follows links out of it, and the legacy entry point prints its + error to stderr; `je_auto_control run` exits 1 for a failed `AC_run_suite` + and accepts `--dry-run --var` with loop variables; `validate` agrees with + `run` on wrapped and empty files; `start-server --port 0` reports the real + port; the pytest plugin no longer ends the session when a failure screenshot + cannot be taken, nor screenshots skipped tests. +- **Screen recordings play back at their real length** (frames are paced to + the declared fps), stop adding an `AC_screenshot` record per frame, and an + unusable frame rate, codec or path raises instead of leaving a recorder + that writes nothing; iOS `find_element` honours `timeout_s`. +- **`AC_run_dag` node actions are placeholder-expanded once**, a nested + `AC_execute_action` failure reaches the enclosing `AC_try`, and a top-level + `AC_break` or a dry run no longer overwrites or collapses the record of a + repeated identical action. +- **`AC_parallel` reports a failed branch under `raise_on_error`** (so `AC_try` + and `AC_retry` see it), counts branch failures for the caller, keeps the + macro recursion limit across branches, and validates a JSON-string + `branches` before running any of it. +- **The versioned store (`AC_cas_*`) keeps versions monotonic across a + reload and serialises concurrent writers**, and a config-bundle import whose + write fails leaves the live file in place. +- **Every malformed JWT raises `JwtError`**, so `AC_jwt_decode` answers + `{"ok": false}` instead of failing, and it accepts `algorithms` given as one + name. +- **The admin console keeps a damaged or partly unreadable host file** + instead of dropping entries on the next save, and the USB passthrough + viewer caps a message that never sends EOF. +- **USB/IP answers a libusb error without an errno with `-EIO`** instead of + dropping the client connection. +- **`match_masked`, `match_masked_all`, `match_subpixel`, `match_auto` and + `detect_scale` return screen coordinates** when given a region or when the + virtual desktop starts at a negative x, instead of frame-local ones. +- **Remote desktop: a damaged trust list, known_hosts or address book is + moved aside instead of being overwritten**, an upload interrupted by a + disconnect or a host stop leaves no `.part` file, and stopping the relay + ends its paired sessions. +- **The socket server reads indented (multi-line) JSON commands whole**, and + the socket driver docs' client example sends the newline terminator it + needs. +- **The MCP HTTP transport releases the state of a session dropped while a + request for it was running.** +- **A corrupt audit database no longer stops the REST server from + starting**; it runs without the audit hook, as intended. +- **Two `SecretManager`s on one vault keep each other's changes**, and a + malformed vault raises `SecretStoreError` instead of `KeyError` or + `ValueError`. +- **`UserAuthError` and `CredentialBrokerError` derive from + `AutoControlException`**, so containment boundaries catch them. +- **Replayed wheel-up scrolls up on X11 and Wayland**, the cleanup release + after a failed replay step stays at the cursor instead of `(0, 0)`, gesture + waypoints round to the nearest pixel, and a NaN key-hold duration is + refused before the key goes down. +- **macOS and Linux hotkeys no longer retry a combo that failed on every + tick**, logging an error ten times a second; `bind()` rejects a key macOS + cannot take up front, as it already did on Windows. +- **The webhook server reads chunked request bodies** instead of running the + script on an empty body, accepts a lower-case `bearer` scheme, and answers + 500 when the run history is unavailable instead of dropping the connection. +- **A trigger removed by an earlier trigger's script in the same pass no + longer runs**; an infinite poll interval no longer kills the trigger or + observer thread; one failing observer rule no longer stops the others; and + `callback_function` returns `None` for any trigger error, as documented. +- **Email triggers fire a failing script once per message**, not on every + poll, and record it as an error; IMAP connections time out after 30 s + instead of hanging the watcher; a body in an unknown charset is kept. +- **`write_step_video` renders a generator of steps** instead of an empty + video, and a trajectory rubric given one action name as a string checks + that name rather than its letters. +- **Codegen output always compiles and runs as the JSON would**: keyword or + non-identifier flow names, NaN / Infinity values, `AC_execute_action` + arguments, `{"auto_control": [...]}` files and Robot test names that read as + headers or comments are all handled. +- **Rate limiters refuse a NaN or infinite rate, capacity or window**, which + let every request through or spun the waiter; `RetryBudget` refuses a NaN + deadline; the shared `LoopGuard` is thread-safe; `plan_repair` returns no + tactics for a negative `max_attempts`. +- **Cron jobs fire once per slot through the hour repeated when clocks fall + back**, instead of on every scheduler tick; and a run of a job removed + while in flight no longer counts against a new job registered under the + same id. +- **`AC_validate_json` reports an invalid regular expression instead of + aborting the script**, detects a `$ref` cycle through a sub-schema, applies + the keywords beside a `$ref`, keeps `true` and `1` apart inside containers, + checks `multipleOf` exactly for integers and treats `1` and `1.0` as + duplicates for `uniqueItems`. +- **Action files saved with a UTF-8 BOM run.** `validate` accepted them but + every runner and the action linter refused them; a `.env` file with a BOM + no longer loses its first key. + +- **A NaN timeout raises `ValueError` instead of polling forever** in + `AC_wait_image`, `AC_wait_pixel`, `expect_poll`, the app-idle, IME, lock + and window waits and OCR `wait_for_text`; the MCP image and pixel waits no + longer report an immediate timeout for one. + +- **Quoted command lines work on Windows.** `AC_shell_command`, + `AC_exec_shell_to_var` and `ac_shell` pass the string to `CreateProcess` + as written instead of re-quoting `shlex` tokens that still held their + quotes. +- **Keep-awake requests nest.** Releasing one restores the state it replaced, + so a second `keep_awake_on()` or a nested `keep_awake()` no longer lets the + machine sleep. +- **The Win32 clipboard setters free their memory when setting fails.** +- **`FilePathTrigger` fires when the file is created** and when it is replaced + by one with an older timestamp. +- **`wait_until_file` does not accept a directory**, and every smart wait + refuses a NaN timeout or poll interval. +- **A missing psutil is an error, not a failed assertion**, and + `ac_kill_process` reports a process that exits mid-kill instead of raising. +- **`.env` values containing U+2028, U+0085, `\v` or `\f` survive a round trip.** + +- **The signaling server refuses unauthenticated requests before reading + them.** A wrong `X-Signaling-Secret` gets 401 and an oversized or + length-less POST 413 / 411 before the body is buffered; the MCP HTTP + transport accepts a lower-case `bearer` scheme. + +- **Replays happen where they were recorded and never leave keys held.** + Recorded presses, releases and scrolls move to their recorded position; + recording gaps keep their total length; a failing step in `run_sequence`, + `replay_timeline`, `tween_drag` or `drag_path` releases held keys and + buttons. Windows accepts `ctrl` and every platform accepts `left` / + `right` / `middle` as button names. + +- **Sagas roll back.** `run_saga` / `AC_run_saga` ran steps leniently, so a + failing step was never noticed and nothing was compensated; a compensation + that raises is reported in `compensation_errors`. One failing device no + longer aborts the device matrix, a DAG node raising any exception fails + that node, and an observer rule removed mid-poll no longer fires. + +- **Metrics, SARIF, SBOM and version checks follow their specs.** Labelled + Prometheus metrics no longer render a bogus unlabelled series, partial + label sets and non-ASCII names are refused and HELP text is escaped. SARIF + levels are normalised, lint issues point at the right 1-based line and + file locations are URIs. SBOM purls are normalised and percent-encoded. + The vulnerability scan orders PEP 440 and SemVer pre-releases before their + release. W3C multi-tenant `tracestate` keys are kept. Step videos resize + mismatched frames, fail on an unwritable path and leave the caller's frame + untouched. + +- **Stores keep what they are given.** Two skill libraries or element + repositories on one file no longer lose each other's saves (a corrupt file + still raises rather than being erased); asset commands without `db` share + one store; a config-bundle entry without content no longer empties its + file, and imported files are written 0600; agent memory recalls non-ASCII + words; action files with a UTF-8 BOM load. + +- **Plugins load reliably.** One plugin file that fails to import no longer + stops the rest of its directory, `@dataclass` plugins load, a non-function + entry-point value is skipped instead of aborting discovery half-way, and + the MCP plugin watcher keeps a tool another file still defines. + +- **Approvals and leases enforce what they promise.** Approval commands + without `db` share one gate (a token was forgotten between commands), an + anonymous approver is refused, and a lease TTL of NaN or infinity is + rejected instead of never expiring. +- **Retry, rate-limit and breaker edge cases.** Backoff caps huge attempt + numbers instead of overflowing, `RetryPolicy` caps its first sleep, the + sliding-window wait is long enough, asctime `Retry-After` dates are GMT, + the loop guard reports the longest stuck pattern, re-created CAS keys never + reuse a version, an interrupted half-open trial no longer jams the + breaker, and idempotency claims are atomic with the TTL starting at + completion. + +- **Text and clipboard helpers handle real-world input.** Long + near-identical strings fuzzy-match again; RTF round-trips characters + beyond the BMP, lone CRs and Word's fallback escapes; a file-drop list + with an empty path is refused and parsing stops at its terminator; + pseudo-localization keeps printf, HTML and ICU placeholders; `slugify` + inserts the separator literally; a null CSV cell is empty. + +- **Image utilities see the colours and files they are given.** Grayscale + no longer swaps red and blue for PIL images and screen grabs, palette + images are read by colour, deskew works on dark themes, and image paths + may contain non-ASCII characters. A red glyph can be colour-matched, colour + regions accept any PIL mode, a golden of another size is a mismatch rather + than an error, and zero-area elements get no mark. +- **An approval `extension` cannot leave `approvals_dir`.** + +- **Test reports and suites count what happened.** A setup failure is + counted in the JUnit totals and reported to Allure; `assert_http` scores a + read timeout or dropped connection as a failed assertion instead of + crashing; `assert_eventually` refuses a NaN timeout (an endless loop); one + malformed case or quarantine entry no longer aborts the suite; string tags + are one tag; runs with equal timestamps list newest first; + `critical_steps(top=0)` is empty. + +- **Failures inside nested blocks reach `AC_try`, `AC_retry` and strict + callers.** A failure inside an `AC_loop`, `AC_if_*` branch or macro was + swallowed at the block boundary. `AC_try` / `AC_retry` now also catch + arithmetic and lookup errors and let the macro depth limit reach the + top-level record; a planned `AC_break` is recorded instead of raised. +- **A variable's value is never expanded as a placeholder.** Commands that + run a nested action list (`AC_execute_action`, `AC_circuit_call`, + `AC_run_saga`, ...) expanded it twice, so a value containing + `${secrets.NAME}` was resolved from the vault. + +- **VEX no longer suppresses findings it does not cover.** `apply_vex` + matches products by package name instead of substring, honours the + statement's aliases, and lets a later statement supersede an earlier one. +- **Parsing fixes for `.env`, data sources and HTTP headers.** Quoted `.env` + values drop a trailing comment and may span lines; `dump_dotenv` output + parses back unchanged. CSV/JSON data sources skip a UTF-8 BOM. A past + cookie `Expires` deletes the cookie and a nameless cookie is ignored; + quoted `Cache-Control` and `Link` parameters stay whole; `rel` is + case-insensitive; SSE handles a `\r\n` split across chunks and a BOM. + +- **JSON Patch, JSONPath and unified diffs follow their specs.** JSON Patch + `add` accepts an index equal to the array length and no longer shares its + value with the patch; `move` checks its source. `apply_unified` places + `-N,0` insertion hunks correctly, keeps body lines that start with `---` / + `+++` and skips `\ No newline` markers. `three_way_merge` reports two + insertions at one point as a conflict and applies an identical change once. + +- **MCP read-only mode and the confirmation gate hold.** `ac_bulkhead_run` + and `ac_run_chaos` no longer run action lists in read-only mode; file + writers and `ac_assert_http` with a mutating method are out of it; read-only + tools no longer create a database at a missing `db` path; `resources/read` + serves only the `*.json` files `resources/list` shows. + +- **Secrets stay out of reports, bundles and logs.** Secret keys are matched + by word (`apiKey`, `db_password`, `sessionId` count; `tokenizer` does not); + list and tuple items and numbers under a secret key are checked; JWTs and + credentials URLs are found; free-text redaction masks prefixed keys, + Basic/Token/Digest `Authorization` and URL passwords. Failure bundles mask + `AC_secret_*` arguments, re-raise the block's own error when the bundle + cannot be written, and drop a truncated log's partial first line. + Screenshot redaction reads `x/y/width/height` boxes and handles palette, + grayscale and bilevel images. + +- **Accessibility, OCR and window lookups match what was asked.** A blank + `contains` name no longer matches every element; a blank window title is + refused instead of matching (and closing) the first window; Linux + single-control lookups honour `window_title`; saving and restoring a window + layout no longer reads or moves the wrong window when titles overlap or + repeat; `show_window` no longer foregrounds a window it was told to + minimise or show without activating; `wait_for_window` / `wait_for_text` + with `timeout=0` look once; OCR text matching normalises Unicode. + +- **Android input cannot run shell commands on the device.** Text typed with + `AC_android_text` is shell-quoted, and `AC_android_key` accepts only key + names and codes; `$(...)`, quotes or `;` in either used to reach the + device shell. +- **Android and iOS device errors are contained.** uiautomator2, adbutils + and facebook-wda errors (no device, several devices, an invalid session) + are raised as `UIAutomatorUnavailableError` / `IOSUnavailableError` + instead of aborting the rest of a script. + +- **Image location is accurate.** `locate_image_center` / `locate_and_click` + return the best-scoring match instead of the first position over the + threshold (which was a few pixels off), an identical template now matches + at the default threshold of 1.0, and `locate_all_image(draw_image=True)` + works. A missing template file or a threshold outside 0..1 raises + `ImageNotFoundException` naming the problem. +- **Mouse and keyboard input.** Scrolling at a point on a secondary monitor + reaches it instead of the primary monitor's edge; a coordinate that is not + a number or is out of range raises `AutoControlMouseException` instead of + moving somewhere else; on Windows a keycode past 16 bits is refused instead + of pressing a different key. + +- **Computer-use agent actions work.** Clicks, double and triple clicks, + drags, waits and held keys were translated into calls the executor could + not make, so each failed as a step error; they now map to real commands. +- **Agent backends run only the tools they offered.** A model reply naming + any other `AC_*` command (a shell command, for instance) raises + `AgentBackendError` instead of executing; unparsable or non-object OpenAI + arguments raise instead of running the tool with `{}`; computer-use + coordinates are clamped to the display. +- **VLM location.** Provider errors (rate limit, timeout) are handled like + other request failures instead of escaping, and a point outside the + requested region is "not found" instead of clicked. + +- **Circuit breaker, config sync and ACME.** A half-open circuit breaker + admits one trial call at a time instead of every concurrent caller, and is + safe to share between threads. Config sync rejects a malformed server reply + and reports a dropped connection as `ConfigSyncError`; the ACME client + reports an unreachable CA as `AcmeError` and a bad CSR as `JwsError`; and + `renewal_due` accepts a naive `now`. + +- **DAG, state machine, recurrence rules and suites.** A DAG node whose + actions fail is failed (its dependants are skipped), and a slow node no + longer holds back unrelated ready nodes. State-machine `after` guards wait + for their timer, counted from state entry, and reaching the final state on + the last allowed step succeeds. YEARLY RRULEs without BYMONTH cover the + whole year as RFC 5545 specifies, a rule that can never match ends instead + of overflowing, and invalid INTERVAL / COUNT / BYMONTH / BYMONTHDAY values + are rejected. A data-driven suite no longer leaves its row variable set, + and an `OverflowError` or `ZeroDivisionError` from an action is recorded + instead of aborting the script. + +- **Remote desktop keeps serving after bad input.** A malformed INPUT + message or WebSocket frame no longer kills the host's receive thread while + its viewer keeps a client slot, nor the viewer's thread without an error + callback; a slow or silent peer no longer blocks other viewers from + connecting (each connection is handshaken on its own thread); and a + signaling timeout or hang-up is reported as `SignalingError`. + +- **Generated code cannot run what an action file smuggles in.** Codegen + emits a parameter as a keyword argument only when its name is a plain + identifier, and the Robot target carries the actions base64-encoded. + Pytest code generated with `failure_bundle=True` now runs (it imported the + wrong module as `ac`). +- **USB/IP, TLS keys, signaling and relay.** A USB/IP client can only send + URBs to the device it imported; TLS private keys are written atomically and + 0600 from creation; the signaling server compares its secret in constant + time and caps live sessions (503 when full); and the relay frees the slot + of a parked peer that disconnected. + +- **Approval gate, asset store and locator-repair store across processes.** + Each change re-reads the file under a lock file, so processes sharing it no + longer overwrite each other; an approval request can be decided only once. + +- **Recording and hotkeys.** A recording that cannot start no longer + replaces the output file with `[]`; starting a second recording stops the + first input hook instead of leaking it. Hotkeys on punctuation keys + (`ctrl+.`, `ctrl+[`) register those keys rather than Delete or the Windows + key; a combo Windows cannot register is refused by `bind` and no longer + retried 20 times a second; and an error in the run history or an injected + executor no longer ends the hotkey listener. +- **Legacy `python -m je_auto_control`.** A missing or invalid action file + is reported as a log line instead of a traceback (the exit status was + already 1), and a `-d` path that is not a directory is an error. + +- **JSON-file stores.** The flaky-test quarantine, the remote-desktop trust + list and known hosts, and the RBAC user store are replaced atomically + (readers never see a partial file) and load empty instead of raising when + the file is not UTF-8 or has the wrong shape. A failed first read no longer + makes the A/B locator store overwrite its counts, and a torn last line in + the cost or self-healing log no longer swallows the next record. + +- **SQLite-backed stores.** Two dispatchers can no longer enqueue the same + work-item reference; `WorkQueue.fail` on an unknown id raises instead of + reporting a requeue; the work queue, checkpoint store and agent memory + close their connections; a run-history or audit-log database error is an + `AutoControlException` (`HistoryStoreError`, `AuditLogError`) instead of a + `sqlite3.Error` that ended the hotkey listener thread; and two processes + writing one audit log keep its hash chain valid. + +- **Credentials no longer follow a redirect to another host.** The HTTP + client drops `Authorization` and cookies when a redirect changes host. The + Jira, Linear and GitHub failure-hook backends and the Slack bot now obey + the egress policy and refuse redirects, and a non-object JSON reply is a + failed call instead of an exception that stopped the Slack poll loop. + +- **Remote-desktop file transfers cannot leave partial files or destroy the + original.** Both receivers write to a `.part` file and rename it into place + only when exactly the announced number of bytes arrived; excess or missing + data fails the transfer. The WebRTC inbox refuses Windows device names + (`nul`, `COM1.txt`) and names ending in a dot or space, and a malformed + envelope or destination fails the transfer instead of killing the + connection's receive thread. +- **One misbehaving admin host no longer fails every host's poll**, and an + address book whose `hosts` is not a list loads empty instead of raising. + +- **A failing data step no longer aborts the script.** SQLite, CSV, regex, + PDF and malformed-HTTP errors from `AC_sql_to_var`, `AC_assert_db`, + `AC_for_each_row`, `AC_transform_var`, `AC_pdf_to_var` and the HTTP + commands are recorded like any other failed action; `AC_otp_to_var` + refuses `step <= 0` instead of dividing by zero. +- **HTTP redirects obey the egress policy.** Only the first URL was checked, + so an allowed host could redirect a request anywhere, including `ftp://`. + +- **Reports.** A recorded control character (an ANSI colour code, a stray + `\x01`) no longer breaks the XML report or makes a JUnit file unreadable; + such characters appear as U+FFFD. A report that cannot be written raises + `AutoControlHTMLException`, `AutoControlGenerateJsonReportException` or + `XMLException` instead of being logged and skipped, and reports are written + atomically. Exception text is no longer wrapped in an extra pair of quotes. + +- **Vault passphrases and secret values no longer reach logs or results.** + The arguments of `AC_secret_*` commands are shown as `***` in the + executor's log lines and in the keys of the record it returns (a secret + command's key changes accordingly), and the socket server no longer logs + the command text. +- **Changing the vault passphrase cannot lose secrets.** The vault is + rewritten once, atomically, instead of being deleted and refilled. + `SecretStoreError` is now also an `AutoControlException`. +- **Config-bundle imports keep every backup.** Two imports in the same + second no longer overwrite the first `.bak` file. + +- **`JE_AUTOCONTROL_REQUIRE_SIGNED_ACTIONS` covers every way a file runs.** + Only `execute_files` checked it: the CLI's `run`, the scheduler, triggers, + webhooks, hotkeys, the MCP `execute_action_file` tool and the GUI ran + unsigned files with enforcement on, and `execute_files` itself verified one + read of the file and parsed another. All of them now load through the new + `read_executable_action_json`, which verifies and parses the same bytes. +- **Signing and encryption keys.** Key files are created 0600 in one step and + never overwritten by a concurrent process; a key file shorter than 32 + bytes, which an interrupted first run could leave empty, is refused instead + of being used as an empty HMAC key. An empty explicit key or passphrase is + refused. + +- **Cron expressions follow standard cron, and a job with no next run stops.** + With both day fields restricted a day now matches if *either* does + (`0 0 1 * 1` is the 1st and every Monday, as in Vixie cron and croniter; + it was the Mondays that fall on the 1st), `7` is accepted as Sunday, and + `5/15` is `5,20,35,50` rather than `5`. `0 0 29 2 *` failed to find its next + run whenever the next leap day was over a year away, and the scheduler then + fired that job on every tick; the search covers eight years, and a job whose + next run cannot be computed is removed and logged. Migration: an expression + with both day fields restricted fires on more days than before — write one + of them as `*` to keep the old intersection. +- **`AllOf` triggers no longer lose a cron minute, file change or sequence + step to a false sibling.** The edge child was checked first and spent its + event before a later condition failed; edges are now checked last. +- **E-mail triggers leave mail unread with `mark_seen=False`.** The fetch + itself set `\Seen`; it uses `BODY.PEEK[]`. +- **A webhook whose script fails with any exception answers 500.** Types + outside four caught ones were recorded as a success and dropped the + connection without a reply. + +- **USB passthrough claims stalled, leaked and outlived their viewer.** After + the first 16 transfers every request failed with "credit exhausted" — the + host granted the viewer more credit with each reply but never counted it + itself — and each failure counted as abuse. Claims were never released when + the viewer's channel closed or the host stopped, leaving the device open and + its kernel drivers detached (a claimed keyboard or mouse stayed gone). After + 65,534 claims the id counter wrapped onto a live claim and orphaned its + handle. + +- **Multi-viewer WebRTC host: sessions that outlived their viewer.** A + viewer waiting on the Accept/Reject dialog was torn down by the 5-second + auth deadline, so the user approved a dead session. A session whose peer + connection failed or closed stayed registered — `session_count` only grew + and screen capture never stopped once the last viewer left. An offer that + failed (consent refused, timeout) left a session the caller never learned + the id of, and the host-service daemon left one behind on every answer that + did not arrive within 300 s. All four now end the session. + +- **Flow control: seven ways a script did something other than it said.** + A failed assertion inside an `AC_parallel` branch, or in an `AC_retry` that + ran out of attempts, was wrapped into an ordinary error and swallowed under + `raise_on_error=False`; it now propagates like any other assertion. + `AC_break` / `AC_continue` with no enclosing loop escaped `execute_action` + entirely and skipped the rest of the script; they are now a recorded failure + ("outside a loop"). A macro that calls itself recursed until Python's limit + (or, with two self-calls, ran exponentially long); calls now fail past + `MAX_MACRO_DEPTH` (50) and the failure is recorded at the top level. An + empty macro, `AC_assert_duration` body or `AC_parallel` branch is a no-op, + as empty bodies are everywhere else, instead of an error. `AC_wait_image` / + `AC_wait_pixel` with `timeout` 0 now look once instead of never. + +- **A USB watcher stopped while it was still taking its first inventory + forgot that inventory.** The poller discarded its priming enumeration + whenever `stop()` had been called, not only when a newer `start()` had + replaced it, so a following `poll_once()` reported every connected device + as newly added — or not, depending on which side of the enumeration the + stop landed. It now discards it only when superseded. + +- **On Windows the key name `down` pressed F17 instead of the Down arrow.** + The Windows `keyboard_keys_table` was built from every constant in + `win32_vk.py`, including the `MOUSEEVENTF_*` and `KEYEVENTF_*` flags, and + `down` was `MOUSEEVENTF_XDOWN` (0x80), which is `VK_F17`. It is now `VK_DOWN`, + as on Linux and macOS. The 18 other names that were flags rather than keys + are gone from the table, so a script using one fails with "unknown key" + instead of pressing whatever key shares its value (`middledown` pressed + space, `move` and `xbutton1` sent the left-button code): `absolute`, + `eventf_extendedkey`, `eventf_keyup`, `eventf_scancode`, `eventf_unicode`, + `hwheel`, `leftdown`, `leftup`, `middledown`, `middleup`, `move`, + `rightdown`, `rightup`, `xbutton1`, `xbutton2`, `vktovsc`, `wheel`, `xup`. + Migration: click mouse buttons through the mouse API (`mouse_x1`, + `mouse_x2`, `mouse_left`…); `vk_xbutton1` / `vk_xbutton2` remain for the + side-button virtual keys. + +- **Recording on Windows lost touchpad scrolling up and multiplied it down.** + A precision touchpad reports the wheel in fractions of a notch (typically + ±30 of 120), and the recorder floored each event separately: `30 // 120` is + 0 and `-30 // 120` is -1. Scrolling up vanished from the recording and + scrolling down replayed about four times too far. The hook now carries the + remainder until it makes a whole notch, and drops it when the direction + reverses. + +- **The scheduler could start a job again while it was still running.** A job + is rescheduled only after it finishes, so until then it still looks due. A + single loop cannot overlap itself, but after a `stop()` whose join timed out + inside a long job, the new run's loop saw the job as due and started a second + copy. `Scheduler` now tracks the jobs it is executing and skips them. + +- **A remote desktop viewer from a stopped host could attach to the restarted + one.** `RemoteDesktopHost`, `RemoteDesktopRelay` and `RemoteDesktopViewer` + had the same shared-event restart as the services below. On the host it + mattered most: the accept loop performs the auth handshake (up to 60 s) + before re-checking the stop flag, so a handshake begun before `stop()` and + finished after the next `start()` passed that check and attached a viewer + authenticated against the old run to the new one. On the viewer, a receiver + outliving `disconnect()` marked the *next* connection as disconnected and + reported its own closing socket through that connection's `on_error`. Each + run now has its own event, and a receiver only touches its own connection. + +- **Restarting a background service could leave the old loop running beside + the new one.** Sixteen services — the scheduler, trigger engine, e-mail + triggers, hotkey daemon, screen observer, popup watchdog, clipboard history, + resource profiler, accessibility recorder, MCP plugin watcher, folder sync, + ACME renewal, USB loopback, USB/IP server, the macOS recorder tap and the + Slack bot — stopped by setting an event and joining with a timeout, and + started by calling `clear()` on that same event. When the join timed out + because the loop was inside a long iteration (a scheduled job, a hotkey + action), the next `start()` cleared the event the old loop was waiting for + and it resumed, untracked: every scheduled job ran twice, every hotkey fired + twice. Each run now gets its own event, passed to its loop, so a stopped run + stays stopped. + +- **Restarting the USB hotplug watcher could leave a second poller running.** + `UsbHotplugWatcher.stop()` waited only 2 s for the poller, but one + enumeration (PowerShell `Get-PnpDevice`, `lsusb`, `system_profiler`) may + take up to its 10 s subprocess timeout, so `stop()` could return — and + `AC_usb_watch_stop` report `running: false` — while the thread was still + running. A following `start()` then called `clear()` on the same stop + event, and the old poller carried on beside the new one, untracked, for the + life of the process. Each run now gets its own event, a poller stopped + mid-enumeration no longer overwrites a newer run's snapshot, and `stop()` + waits for up to the subprocess timeout plus one second, logging a warning + if the poller is still running after that. + `usb_devices.SUBPROCESS_TIMEOUT_S` is now public (it was `_SUBPROCESS_TIMEOUT_S`). + +- **A failing `hotkey()` or `type_keyboard()` left keys held down.** Both + press and then release with nothing protecting the gap, and their + `except (OSError, RuntimeError, AttributeError, TypeError, ValueError)` does + not cover `AutoControlKeyboardException` — the error `press_keyboard_key` / + `release_keyboard_key` actually raise for an unknown key name, an unsupported + platform or a backend failure. So `hotkey(["ctrl", "shift", "esc"])` failing + on `esc` left `Ctrl` and `Shift` down on the real keyboard, changing the + meaning of every later click and keystroke. The release now runs from + `finally`: only keys that were actually pressed and not yet released are + released, in reverse order, and a release that fails during that cleanup is + logged rather than raised, so the caller still sees the original error. + +- **The Windows recorder dropped mouse side buttons.** Playback has always + accepted `mouse_x1` / `mouse_x2`, but the low-level hook keyed its button + table by message id, and `WM_XBUTTONDOWN` / `WM_XBUTTONUP` are one id for + both buttons (which one is in the high word of `mouseData`). Side clicks were + therefore never recorded, and a macro replayed without them with no warning. + `stop_record_timeline()` now yields `mouse_down` / `mouse_up` events with + `button` `"x1"` / `"x2"`, and `replay_timeline()` maps them to `mouse_x1` / + `mouse_x2`. An unrecognised side-button value is dropped rather than guessed, + because the replay side falls back to the left button for a name it does not + know. The legacy down-events-only queue (`stop_record()`) still omits them: + there is no `AC_mouse_x1` command to put in it. + +- **Changing a hotkey's combo on X11 left the old key grabbed for the life of + the daemon.** `LinuxHotkeyBackend._sync_one` dropped the previous + registration from its own table without calling `ungrab_key`, so the *old* + combo stayed grabbed on the X server: it was swallowed from every + application, fired nothing, and `_ungrab_all` could not release it at + shutdown because it no longer knew about it. Rebinding `ctrl+alt+k` to + something else made `ctrl+alt+k` dead system-wide until the process exited. + The Windows backend has always unregistered at the same point; the X11 one + now does too. Unaffected on Windows and macOS. + +- **A window closing mid-call let a COM error escape every Windows + accessibility read.** `comtypes` reports a provider failure as `COMError`, + which derives straight from `Exception` — the reason + `windows_query._uia_errors()` exists — but only the two tree-walking guards + in `backends/windows_backend.py` used that tuple. The other 37, covering + every control pattern (`get_value`, `invoke`, `toggle`, `read_table`, the + text and grid reads, …), named `(OSError, AttributeError, …)` and therefore + contained none of them. An application that stopped responding, or a window + that closed between the search that found an element and the call that read + it, raised `COMError` out of the `ac_*` tool or `AC_*` command instead of + answering `None` / `False` / `[]`, and past the executor's + `AutoControlException` boundary. All 37 now use the same tuple. This only + widens what is caught: no call that used to succeed behaves differently. + +- **The WebRTC viewer ended every clean disconnect with an unhandled task + exception.** `WebRTCDesktopViewer._consume_video` caught + `(OSError, RuntimeError)`, but aiortc signals the end of a track by raising + `MediaStreamError`, which derives straight from `Exception` and so matched + neither. Nothing awaits that task, so the normal end of a session — the host + stopping its screen share, or the connection closing — reached the console as + asyncio's "Task exception was never retrieved" traceback instead of the + "video stream ended" line the host's own drain loop already logged. The + stream is unaffected either way; only the logging changes. + +- **A `null` in a remote-desktop entry's `tags` became a tag named `"None"`.** + `AddressBook.set_tags()` cleaned its input with `str(t).strip()`, and + `str(None)` is the non-empty string `"None"`, so a JSON `null` in the array — + what a client sends for an omitted tag — was stored as a tag and then listed + by `all_tags()` alongside the real ones. Nulls are now dropped. Tags that + were already stored this way stay until the entry's tags are set again. + +## [0.0.222] - 2026-08-23 + +### Changed + +- **`mouse_scroll()` rejects a scroll direction the platform has no axis for.** + A name outside `special_mouse_keys_table` used to be passed down to the + backend unchanged, which meant `int('scroll_upp')` on Wayland and uinput and + an Xlib failure on X11 — deep in the backend, with the offending name nowhere + in the message. It now raises `AutoControlCantFindKeyException` naming the + direction, the same answer the button table has always given for an unknown + button name. Windows and macOS are unaffected: they have a single wheel axis + and never read the direction. + +- `je_auto_control.stop_record()` returns an empty list where it used to + return `None`. It has always been annotated `-> list`, but the failure path + fell off the end of the function, so a caller that did not write + `stop_record() or []` iterated over `None` and raised in its own code + instead. `stop_record_timeline()` already returned `[]` on the same + failure; the two now agree. + +- `je_auto_control.mouse_scroll()` reports its return type as + `Tuple[int, Union[int, str]]`. The value has not changed — X11 and Wayland + still hand back the backend axis code the direction name resolved to, and + every other platform the name itself — the signature just no longer claims + it is always a `str`. + +- The Windows screen backend's `size()` returns a `tuple`, not a `list`. + The macOS, X11 and Wayland backends all returned tuples already, and the + public `screen_size()` has always been annotated `Tuple[int, int]`; every + caller unpacks the two values, so nothing that used it needs changing. + +### Fixed + +- **Typing text through the key-event route raised `AttributeError` on the + three platforms that cannot do it.** `type_unicode_keys()` (and + `AC_type_unicode_keys` / `ac_type_unicode_keys`) called the backend's + `type_unicode_unit` outright, and only Windows has one, so macOS, X11 and + Wayland raised an exception from outside the `AutoControlException` family + that the executor, the background poll loops and the request handlers each + catch in one `except` — it escaped every containment boundary in the + project. It now raises `AutoControlKeyboardException` pointing at + `type_unicode_text()`, which picks a route that works on any platform. + +- **A backend that could not report the cursor aborted the script instead of + raising what the API promises.** `press_mouse` / `release_mouse` / + `click_mouse` with an omitted `x` or `y` unpacked `get_mouse_position()` + without checking it for `None`, so a backend that answers "I don't know" + raised `TypeError` from the unpacking — outside the + `AutoControlMouseException` family every containment boundary catches. It + now raises `AutoControlMouseException`. `mouse_scroll` reached the same + unpacking through `_scroll_to` and now skips the pre-move instead, which is + the graceful degradation its own comment already documented for backends + that cannot report the cursor. + +- **`je_auto_control.windows.message.window_message` could not be imported + at all.** It did `from ...windows_window_manage import FindWindowW`, and + that module has no such name — `FindWindowW` is a method on its private + `user32` handle — so importing `window_message` raised `ImportError` on + every Windows machine. It now calls the module's public + `get_one_window_hwnd`, which is also the one that declares HWND-width + argtypes rather than letting ctypes truncate a 64-bit handle to `c_int`. + +- **Importing the Win32 input backend no longer writes into + `ctypes.wintypes`.** `win32_ctype_input` set `wintypes.ULONG_PTR = + wintypes.WPARAM` on the standard library's own module. Nothing in this + package ever read it back, so the only effect the assignment could have was + on some other library in the same process asking `ctypes.wintypes` whether + it has `ULONG_PTR`. + +- **Stopping an X11 recording that was never started raised instead of + returning nothing.** The X11 listener's `stop_record()` handed back the + `None` its queue attribute was constructed with, and the recorder one frame + up reads `.queue` off that result, so `stop_record()` without a preceding + `record()` produced an `AttributeError` that the wrapper caught and logged + as a failure. It now returns an empty queue, so the public `stop_record()` + returns the empty list it documents. + +- **`check_key_is_press()` passed `None` to the backend for an unknown key + name.** A name the virtual-key table has no entry for became `None` and was + handed to the platform backend anyway: a `TypeError` on Windows and a silent + `False` on X11 — that is, "no, it is not pressed" for a key that does not + exist. It now logs the lookup failure and returns `None`, which is the + documented "could not answer" value. + +## [0.0.221] - 2026-08-20 + +### Added + - **Windows on arm64 installs.** `opencv-python`, `cryptography` and `je_open_cv` now carry the environment marker `sys_platform != 'win32' or platform_machine != 'ARM64'`, because none of @@ -22,6 +983,11 @@ only when documented here with a migration path. missing wheel rather than a bare `ModuleNotFoundError`. Python 3.11 is the floor there, since CPython publishes no official Windows arm64 build for 3.10. + +## [0.0.220] - 2026-08-20 + +### Added + - **The macOS recorder works.** `record()`, `stop_record()`, `stop_record_timeline()`, the `AC_record*` commands, the `ac_record_*` MCP tools and `je_auto_control record` all run on macOS now; they used to refuse @@ -33,69 +999,201 @@ only when documented here with a migration path. `AutoControlRecordException` naming the permission — where the facade's `record()` logs it, as it does every other backend's start failure — rather than starting a session that silently records nothing. + - `je_auto_control.utils.input_macro.recorder_base` — the platform-neutral half of recording: `timeline()`, `legacy_action_queue()` and the `InputRecorder` base the Windows and macOS recorders now share. `timeline` keeps working when imported from `je_auto_control.windows.record.win32_input_hook`, where it used to live. + - Cross-platform window management. The 23 `AC_*` window commands and their MCP tools now work on macOS and Linux/X11 as well as Windows, through a backend seam (`je_auto_control.wrapper.window_backends`). Wayland remains unsupported: the protocol does not let a client enumerate or move another application's windows. + - Linux accessibility backend over AT-SPI2 (`je_auto_control.utils.accessibility.backends.linux_backend`), with no new dependency. Serves both X11 and Wayland sessions. + - `je_auto_control.utils.platform_id` — one place that classifies the operating system family, and the BSDs are now one of them. FreeBSD, OpenBSD, NetBSD and DragonFly route to the X11 backend instead of raising "unknown operating system". + - `AutoControlUnsupportedOperationException`, raised when a platform backend cannot perform an operation. It subclasses both `AutoControlException` and `NotImplementedError`, so existing `except NotImplementedError` handlers are unaffected while the executor's containment boundaries now catch it. + - `je_auto_control.utils.dbus_client` — the D-Bus client, moved out of `linux_wayland/` so `utils/` can use it. The old path re-exports it. -- Stable, headless `je_auto_control.api` façade. -- Portable `autocontrol.failure-bundle/v1` diagnostic archives and CLI command. -- Public API lifecycle, capability matrix, security policy, coverage and type - checking configuration. -- Unicode text entry by key injection: `type_unicode_keys`, `type_unicode_text`, - `plan_unicode_keys`, `unicode_keys_supported` (commands - `AC_type_unicode_keys` / `AC_type_unicode_text`, MCP tools - `ac_type_unicode_keys` / `ac_type_unicode_text`), on Windows backend - primitives `press_unicode` / `release_unicode` / `type_unicode_unit`. -- Cross-word OCR matching helpers `find_spans` / `group_lines`. -- `monitor_layout.grab_logical` / `logical_virtual_rect` / `logical_scale` / - `needs_rescale` — screen capture in the coordinate space the mouse uses. -- `find_image` / `find_image_multi` accept `all_screens` and `screen_region`. -- `AutoControlFlatTemplateException` (a subclass of `AutoControlScreenException`) - for a template with too little variation to locate. -- Accessibility search scoping and matching: `window_title` on - `list_accessibility_elements` / `find_accessibility_element` / - `click_accessibility_element` / `control_get_state`, a `contains` substring - mode with exact-name ranking, `find_accessibility_elements`, - `accessibility_status`, `control_get_state`, and `rank_by_name` (commands - `AC_a11y_find_all` / `AC_control_get_state`, MCP `ac_a11y_find_all` / - `ac_control_get_state`). The accessibility GUI tab gains a window filter. -- `AccessibilityElement.enabled`. -- `stop_record_timeline` (`AC_stop_record_timeline`, - `ac_record_stop_timeline`): the recording as press *and* release, wheel - movement and `delta_ms`, ready for `replay_timeline`. -- `utils/input_reach`: `input_desktop_available`, `input_reaches_system` - (`AC_input_reachable`, `ac_input_reachable`) — whether input this process - sends can actually arrive. The second probe presses F13 to find out. -- `utils/keyboard_layout`: `char_table`, `layout_char_table`, `vk_to_char`, - `foreground_keyboard_layout` — which character each key produces on the - active layout, with a US fallback. -- Window management gains the primitives it was missing: - `minimize_window_by_title`, `foreground_window`, `window_rect` and - `move_window_by_title` (`AC_minimize_window`, `AC_foreground_window`, - `AC_window_rect`, `AC_move_window`; `ac_minimize_window`, - `ac_foreground_window`, `ac_window_rect`). `list_windows` takes - `titled_only`, and `move_window_by_title` keeps the window's current size - when width/height are omitted. +- **MCP sessions over HTTP.** `initialize` now mints an `Mcp-Session-Id` and + returns it as a response header. A client that echoes it keeps one + dispatcher scope — the capabilities it advertised, and the slots its + in-flight calls occupy — across every connection it opens, instead of one + scope per TCP connection. `GET /mcp` with `Accept: text/event-stream` and a + valid session id opens the standing server-to-client SSE stream (one per + session; a second gets 409), `DELETE /mcp` with the id terminates the + session, and a server request is answered by `POST`ing an ordinary JSON-RPC + response on any connection. Sessions are swept after ten minutes untouched + and capped at 128. `je_auto_control.utils.mcp_server.http_sessions` holds + the registry. + +- **`JE_AUTOCONTROL_MCP_CONFIRM_DESTRUCTIVE=1` now works over HTTP** — for a + client that echoes `Mcp-Session-Id` and holds the `GET` stream open. It + previously fired only on stdio: the prompt needs a server-to-client channel + bound to the scope that received `initialize`, and a connection-keyed scope + never survived to the `tools/call`. A client that does neither still cannot + be prompted and its destructive calls still proceed, exactly as for a stdio + client that never advertised `elicitation`; that fallback is documented and + is not a substitute for the bearer token, the `127.0.0.1` bind or + `JE_AUTOCONTROL_MCP_READONLY`. + +### Changed + +- The MCP HTTP transport answers `GET /mcp` differently. It used to return + `405` with `{"error": "GET stream not supported"}` for every request; it now + serves the session's SSE stream when the request carries + `Accept: text/event-stream` and a valid `Mcp-Session-Id`, and still returns + `405` when the `Accept` header does not ask for a stream. A request — of any + method — carrying an `Mcp-Session-Id` the server does not know is refused + with `404` rather than served under a fresh scope, which is the signal to + re-run `initialize`. `DELETE /mcp` without a session header is still + accepted as a no-op, so clients that never adopt sessions are unaffected. + +- The default run-history database is created when it is first written to, + not while `je_auto_control` is being imported. `HistoryStore` opens its + connection (and makes its parent directory) on first use, so merely + importing the package no longer creates + `~/.je_auto_control/run_history.sqlite`. Every method behaves as before; + a store that was never used and then closed simply never touched the + disk. + +- **The sign of `scroll_value` picks the scroll direction on every platform.** + Windows and macOS have always read it that way; X11 and Wayland took the + direction from `scroll_direction` alone and used `abs(scroll_value)`, so + `mouse_scroll(-3)` — code written and tested against the Windows convention — + scrolled *down* three notches on Linux instead of up, with no exception and + no warning. `scroll_direction` now names the direction a **positive** count + takes, and a negative count reverses it, on all four backends. + + *Migration.* Code that passed a negative `scroll_value` to Linux or Wayland + and relied on the magnitude alone now scrolls the opposite way. Take + `abs()` at the call site to keep the old behaviour: + `mouse_scroll(abs(value), scroll_direction="scroll_down")`. Code that passed + a positive count is unaffected, as is every Windows and macOS caller. + +- **`import je_auto_control` no longer imports OpenCV, NumPy, Pillow, + `je_open_cv` or `cryptography`.** They are imported by the functions that use + them. The facade pulled all five in at module scope, so a platform without + wheels for them — a FreeBSD desktop, for one — could not use the input + automation half of the package at all, though it needs none of them. Nothing + moves in the public API and the packages remain hard dependencies; what + changes is *when* a missing one is reported, which is now at the first image + or encryption call rather than at import. `test_facade_import_is_light.py` + keeps it that way. + +- **`macos_record_error_message` now names a permission, not a platform.** It + read "Cannot use recorder on macOS", which described a limitation that no + longer exists; it now names the Accessibility grant that recording actually + needs, and is raised from the event tap rather than from the wrapper. + +### Fixed + +- The MCP HTTP transport no longer tries to drain a request body it has + already read. Any `4xx` decided *after* the body was parsed — the new + unknown-session `404` and duplicate-stream `409`, and the pre-existing + "body must be UTF-8" `400` — called `_drain_body()`, which then blocked + reading bytes that were gone until the 30-second socket timeout, pinning + that worker and logging a `ConnectionAbortedError` traceback when the peer + closed first. The drain is now skipped once the body is consumed, and a + peer that has already vanished ends it quietly instead of raising. + +- **A rejected config bundle aborted the rest of the script.** Five + framework errors still inherited `Exception` directly — + `ConfigBundleError`, the USB passthrough `ProtocolError`, + `SessionError` and `UsbClientError`, and the work queue's + `BusinessError` — and the containment boundaries all catch the + `AutoControlException` family, so none of them caught these. A + malformed bundle passed to `AC_config_import` therefore raised straight + past the executor's per-action boundary and killed every remaining + action, even under `raise_on_error=False`; `AC_usb_remote_devices` and + `AC_usb_remote_open` had the same path through `UsbClientError`. All + five derive from `AutoControlException` now, so they are recorded as a + failed action like every other framework error. `LoopBreak`, + `LoopContinue` and the MCP dispatcher's private error carrier stay + outside the family deliberately — they are control flow, not failure. + +- **`import je_auto_control` needed a Python built with `sqlite3`, and + FreeBSD's is not.** `sqlite3` is in the standard library but not in every + build of it: CPython links it against a system library, and FreeBSD ships + the result as the separate `databases/py-sqlite3` package. Ten subsystems + imported it at module scope — run history, checkpoints, the work queue, + agent memory, the remote-desktop audit log, SQL data sources, and the + error tuples in the REST, chat-ops and MCP containment boundaries — and + all ten are reachable from the facade, so the whole package failed to + import on a stock FreeBSD, mouse and keyboard included. They go through + `je_auto_control.utils.sqlite_support` now, which fails at the first call + that opens a database rather than at import, and raises + `AutoControlUnsupportedOperationException` — the type the GUI tabs, the + REST handler and the executor already report as "not available here" — + instead of an `ImportError` none of them catch. `run_diagnostics()` lists + `sqlite3` among the optional dependencies, so the gap is visible without + reading a traceback. + +- **`mouse_scroll` did nothing at all on the BSDs.** It matched Windows, then + macOS, then a literal `["linux", "linux2"]`, so a FreeBSD, OpenBSD, NetBSD or + DragonFly caller fell off the end of the chain: no backend call, no + exception, no log line. It asks `platform_id.is_x11_unix()` now, and an + unrecognised platform raises `AutoControlMouseException` instead of returning + as though it had scrolled. + +- **A recorded timeline replayed nothing.** `replay_timeline`'s dispatch table + held the `run_sequence` DSL's vocabulary (`press` / `click` / `key`) and the + recorders emit their own (`key_down` / `mouse_up` / `scroll`), and the two + were disjoint — so `stop_record_timeline()` fed to `replay_timeline()`, the + pipeline both the docstrings and the `ac_record_stop_timeline` tool + prescribe, matched no handler, replayed an empty session, and still returned + every event as played. The recorder ops dispatch now, and the wheel reads + `delta` as well as `value` (reading only `value` fell back to the default of + one notch, so a three-notch scroll down replayed as one notch the other + way). Affects every platform, not only macOS. + +- macOS: recorded mouse coordinates were mirrored vertically. The listener + read `NSEvent.mouseLocation()`, whose origin is the bottom-left of the + display, while every replay posts into the top-left space `osx_mouse` uses — + so a click recorded near the top of the screen replayed near the bottom. It + now reads `CGEventGetLocation`, which is already in the space the replay + posts into. + +- macOS: modifier keys were not recorded at all. macOS sends no key-down for + Shift, Control, Option or Command, only a `flagsChanged` event carrying the + new flag set, so a recording could not say a modifier was held across the + actions that followed. They are reconstructed from the flags now. + +- macOS: `write()` typed a space instead of a backspace, because `"\b"` had + no route in the macOS key table and fell through to the space fallback. + +- macOS: USB enumeration returned `apple_vendor_id` in `vendor_id`, a field + documented as a four-hex-digit string. A value that is not a hex id is now + `None`; the device is still listed and `manufacturer` still names the vendor. + +- Linux/X11: `window_rect` returned the client area rather than the frame, + disagreeing with Win32's `GetWindowRect` by the window decorations. + +- Linux/X11: `move_window_by_title` configured the client window directly, + which under a reparenting window manager positions it in the wrong + coordinate space. It now goes through `_NET_MOVERESIZE_WINDOW`. + +- The D-Bus client could not marshal or demarshal signed integers, so any + protocol using them (AT-SPI extents among them) failed to decode. + +## [0.0.219] - 2026-08-19 + +### Added + - Window ownership: `foreground_window_process_id` and `window_process_id` (`AC_foreground_window_pid`, `AC_window_pid`; `ac_foreground_window_pid`, `ac_window_pid`; two Script Builder specs), on the Windows backend @@ -104,6 +1202,7 @@ only when documented here with a migration path. of" — the process id can. Unavailable reads as `None` (`{"pid": 0}` on the JSON surfaces) rather than a bare `0`, which a caller could otherwise match against a process list and hit the System Idle Process. + - Windows by owning process: `windows_for_process_id` and `minimize_windows_for_process` (`AC_windows_for_pid`, `AC_minimize_windows_for_pid`; `ac_windows_for_pid`, @@ -111,6 +1210,7 @@ only when documented here with a migration path. application cannot be addressed by title — its windows are named after whatever they display and several of its processes have no window at all — so ownership is the stable key. + - Input posted to a window without focusing it: `post_key_to_window` and `post_click_to_window` (`AC_post_key_to_window`, `AC_post_click_to_window`; `ac_post_key_to_window`, `ac_post_click_to_window`; two Script Builder specs), @@ -124,11 +1224,7 @@ only when documented here with a migration path. focused edit accepted it. Both return whether the messages were queued, and posting remains best effort — applications reading raw input or checking the foreground ignore posted messages. -- `utils/url_canon` reaches its delivery surfaces: `canonicalize_url`, - `normalize_url`, `urls_equal`, `build_query` and `parse_query` are exported - from the facade, with `AC_canonicalize_url` / `AC_normalize_url` / - `AC_urls_equal`, the matching `ac_*` MCP tools, and three Script Builder - specs. The module and its tests already existed; only the wiring is new. + - `JE_AUTOCONTROL_WAYLAND_POINTER_ACCEL` — how an operator declares what the library cannot read back. `flat` says pointer acceleration is off for the ydotoold device, so an absolute move through the ydotool fallback is exact @@ -136,123 +1232,21 @@ only when documented here with a migration path. land somewhere else; unset (or any unrecognised value, which says so and falls back) keeps the existing warn-once-and-move behaviour. The libei path is absolute at the protocol level and is not affected either way. -- **MCP sessions over HTTP.** `initialize` now mints an `Mcp-Session-Id` and - returns it as a response header. A client that echoes it keeps one - dispatcher scope — the capabilities it advertised, and the slots its - in-flight calls occupy — across every connection it opens, instead of one - scope per TCP connection. `GET /mcp` with `Accept: text/event-stream` and a - valid session id opens the standing server-to-client SSE stream (one per - session; a second gets 409), `DELETE /mcp` with the id terminates the - session, and a server request is answered by `POST`ing an ordinary JSON-RPC - response on any connection. Sessions are swept after ten minutes untouched - and capped at 128. `je_auto_control.utils.mcp_server.http_sessions` holds - the registry. -- **`JE_AUTOCONTROL_MCP_CONFIRM_DESTRUCTIVE=1` now works over HTTP** — for a - client that echoes `Mcp-Session-Id` and holds the `GET` stream open. It - previously fired only on stdio: the prompt needs a server-to-client channel - bound to the scope that received `initialize`, and a connection-keyed scope - never survived to the `tools/call`. A client that does neither still cannot - be prompted and its destructive calls still proceed, exactly as for a stdio - client that never advertised `elicitation`; that fallback is documented and - is not a substitute for the bearer token, the `127.0.0.1` bind or - `JE_AUTOCONTROL_MCP_READONLY`. - -### Removed - -- `je_auto_control.linux_wayland._detect.WAYLAND_GDBUS` is gone, along with the - `gdbus` probe it named: the desktop-portal capture tier no longer shells out - to any binary. `_detect` is a private module and nothing else referenced the - constant. -- **Breaking — `je_auto_control.windows.listener` is gone**, with its - `Win32KeyboardListener` and `Win32MouseListener` classes. Recording moved to - `windows/record/win32_input_hook.py`, after which nothing in the package or - the test suite referenced them. -- **Breaking — `je_auto_control.utils.clipboard.clipboard_image` is gone.** Its - two functions were duplicates of the ones in - `je_auto_control.utils.clipboard.clipboard`, under identical names but with a - different `set_clipboard_image` signature, so importing the wrong module - failed at runtime and only for one of the two argument types. Import from - `je_auto_control.utils.clipboard` (or the top-level facade) instead; the - surviving function accepts both PNG bytes and a file path. ### Changed -- **`mouse_scroll()` rejects a scroll direction the platform has no axis for.** - A name outside `special_mouse_keys_table` used to be passed down to the - backend unchanged, which meant `int('scroll_upp')` on Wayland and uinput and - an Xlib failure on X11 — deep in the backend, with the offending name nowhere - in the message. It now raises `AutoControlCantFindKeyException` naming the - direction, the same answer the button table has always given for an unknown - button name. Windows and macOS are unaffected: they have a single wheel axis - and never read the direction. -- `je_auto_control.stop_record()` returns an empty list where it used to - return `None`. It has always been annotated `-> list`, but the failure path - fell off the end of the function, so a caller that did not write - `stop_record() or []` iterated over `None` and raised in its own code - instead. `stop_record_timeline()` already returned `[]` on the same - failure; the two now agree. -- `je_auto_control.mouse_scroll()` reports its return type as - `Tuple[int, Union[int, str]]`. The value has not changed — X11 and Wayland - still hand back the backend axis code the direction name resolved to, and - every other platform the name itself — the signature just no longer claims - it is always a `str`. -- The Windows screen backend's `size()` returns a `tuple`, not a `list`. - The macOS, X11 and Wayland backends all returned tuples already, and the - public `screen_size()` has always been annotated `Tuple[int, int]`; every - caller unpacks the two values, so nothing that used it needs changing. -- The MCP HTTP transport answers `GET /mcp` differently. It used to return - `405` with `{"error": "GET stream not supported"}` for every request; it now - serves the session's SSE stream when the request carries - `Accept: text/event-stream` and a valid `Mcp-Session-Id`, and still returns - `405` when the `Accept` header does not ask for a stream. A request — of any - method — carrying an `Mcp-Session-Id` the server does not know is refused - with `404` rather than served under a fresh scope, which is the signal to - re-run `initialize`. `DELETE /mcp` without a session header is still - accepted as a no-op, so clients that never adopt sessions are unaffected. -- The default run-history database is created when it is first written to, - not while `je_auto_control` is being imported. `HistoryStore` opens its - connection (and makes its parent directory) on first use, so merely - importing the package no longer creates - `~/.je_auto_control/run_history.sqlite`. Every method behaves as before; - a store that was never used and then closed simply never touched the - disk. - -- **The sign of `scroll_value` picks the scroll direction on every platform.** - Windows and macOS have always read it that way; X11 and Wayland took the - direction from `scroll_direction` alone and used `abs(scroll_value)`, so - `mouse_scroll(-3)` — code written and tested against the Windows convention — - scrolled *down* three notches on Linux instead of up, with no exception and - no warning. `scroll_direction` now names the direction a **positive** count - takes, and a negative count reverses it, on all four backends. - - *Migration.* Code that passed a negative `scroll_value` to Linux or Wayland - and relied on the magnitude alone now scrolls the opposite way. Take - `abs()` at the call site to keep the old behaviour: - `mouse_scroll(abs(value), scroll_direction="scroll_down")`. Code that passed - a positive count is unaffected, as is every Windows and macOS caller. -- **`import je_auto_control` no longer imports OpenCV, NumPy, Pillow, - `je_open_cv` or `cryptography`.** They are imported by the functions that use - them. The facade pulled all five in at module scope, so a platform without - wheels for them — a FreeBSD desktop, for one — could not use the input - automation half of the package at all, though it needs none of them. Nothing - moves in the public API and the packages remain hard dependencies; what - changes is *when* a missing one is reported, which is now at the first image - or encryption call rather than at import. `test_facade_import_is_light.py` - keeps it that way. -- **`macos_record_error_message` now names a permission, not a platform.** It - read "Cannot use recorder on macOS", which described a limitation that no - longer exists; it now names the Accessibility grant that recording actually - needs, and is raised from the event tap rather than from the wrapper. - **The `xdg-desktop-portal` capture tier no longer needs `gdbus` installed.** It speaks D-Bus directly, so `linux_wayland.portal.is_available()` now reports whether a session bus address is set rather than whether the `gdbus` binary is on `PATH`. This widens where the last-resort tier runs; the install hint in the "no capture tool found" error and the `screen_capture` diagnostics check were reworded to match. + - **`LibeiBackend.scroll()` sends whole wheel clicks, not raw detent counts.** libei measures discrete scroll in 120ths of a click, so the previous call asked for 1/120th of the scroll requested and libei logged it as a client bug. Measured against a real EIS server (`docker/eis_verify.py`). + - **Wayland `mouse.scroll()` goes through libei where libei is up, instead of always shelling out to ydotool.** Motion, buttons and keys already did; scroll was held back because its sign was a guess. The two paths count @@ -262,16 +1256,19 @@ only when documented here with a migration path. `wl_pointer` frame (positive is down) — so the vertical axis is negated on the way to libei and the horizontal one is not. No API change: scrolling on a libei host no longer needs ydotool or a uinput daemon at all. + - **A libei emission that a live backend refuses now falls back to the CLI, as `libei`'s own docstring already claimed it did.** Only the *connection* degraded; a compositor that paused a device, or a session that ended between two calls, raised out of `set_position` / `press_key` / `hotkey` instead of reaching ydotool. A chord refused part-way releases the keys it already pressed before handing over, so no modifier is left held. + - **`LibeiUnavailable` derives from `AutoControlException`** (as well as `RuntimeError`, which existing probes catch). It was a bare `RuntimeError`, so it escaped every `except AutoControlException` containment boundary — the executor, the poll loops, the request handlers and the GUI slots. + - **A libei session that completed its handshake is released instead of abandoned.** `ei_unref` segfaults on libei 1.3.901 only for a context whose backend opened and whose handshake never progressed; with an EIS peer to @@ -288,6 +1285,7 @@ only when documented here with a migration path. nothing was executed. `/execute_file` answers the same way for a path that is unreadable or holds something that is not an action list. A client that keyed off `500` to detect a bad request must key off `400` instead. + - **Windows clipboard calls wait out a clipboard another process is holding open** instead of failing immediately. Only one process may have it open at a time, so `RuntimeError: OpenClipboard failed` used to escape whenever @@ -306,67 +1304,11 @@ only when documented here with a migration path. the key sender now matches the window title as a *substring* (it required an exact title before, via `FindWindowW`), and the mouse sender accepts a title string as well as the hwnd it always took. + - `save_window_layout` now snapshots only titled windows (its documented behaviour). Untitled entries could never be restored — `restore_window_layout` addresses a window by title and skips blank ones — so they only inflated the saved count, by roughly half on a real desktop. -- `set_clipboard_image` accepts PNG bytes **or** a path to any Pillow-readable - image, and `get_clipboard_image` / `set_clipboard_image` are now exported - from `je_auto_control.utils.clipboard` and the top-level facade, with - `AC_clipboard_get_image` / `AC_clipboard_set_image` commands. They were - previously reachable only through MCP and the GUI, not `execute_action`. -- **Breaking — `close_window_by_title` / `AC_close_window` / `ac_close_window` - now actually close the window** (they post `WM_CLOSE`). They previously - *minimised* it: the Win32 call underneath is named `CloseWindow` but - minimises, and the wrapper inherited both the call and the wrong promise, so - every caller asking to close a window silently got a minimise instead. The - old behaviour is available unchanged as `minimize_window_by_title` / - `AC_minimize_window` / `ac_minimize_window`. -- `focus_window` restores a window that is minimised before bringing it to the - front — focusing a minimised window used to do nothing visible. A maximised - window is left maximised (`SW_RESTORE` would have un-maximised it). -- `show_window_by_title` no longer calls `SetForegroundWindow` after `SW_HIDE`; - hiding a window and then pulling it forward are contradictory. -- Releases are prepared from version tags and use PyPI Trusted Publishing. -- The USB/IP server binds `127.0.0.1` by default (least-privilege). Exporting - the attached device to the LAN now requires an explicit `host="0.0.0.0"`. -- `write` no longer raises on a character missing from the virtual-key table - where the backend can inject Unicode; it types that character instead. -- `find_text_matches` returns runs of consecutive word boxes, so a target split - across boxes now matches. Results are merged boxes covering the whole run - (union rectangle, minimum confidence) rather than one box per word. -- `find_image` / `find_image_multi` search every monitor by default and return - virtual-desktop coordinates, which are negative when a monitor sits left of or - above the primary. Pass `all_screens=False` for the previous primary-only - behaviour. -- `match_template` / `match_template_all` capture every monitor and return - screen coordinates. A hit found inside a `region` previously came back in - region-local coordinates; it is now offset by the region's origin. Matches - against a caller-supplied `haystack` are unchanged (image-local). -- `match_template` / `match_template_all` refuse an almost-single-colour - template instead of returning an arbitrary position. -- `element_matches` accepts a friendly role name (`"button"`) as well as the - raw `"ControlType_50000"` the Windows backend reports. -- `AccessibilityBackend.list_elements` takes `window_title`; in-tree backends - accept it, and the facade only forwards it when set, so an out-of-tree - backend keeps working until someone asks for scoping. -- `AccessibilityElement.to_dict()` gains an `enabled` key. -- The Windows recorder captures through one low-level hook - (`Win32InputHook`) instead of the two listeners. `record` / `stop_record` - keep their behaviour and return shape. -- An unscoped `list_accessibility_elements` walks one top-level window at a - time in z-order, node by node, and stops at `max_results`, instead of one - uninterruptible `FindAll` over the whole desktop. Results are therefore - ordered front-most window first, and a small `max_results` no longer - reaches windows further back. -- The UIAutomation object is created from `CUIAutomation8` as - `IUIAutomation2` with a bounded `ConnectionTimeout` where available, so an - application that never answers UIA can no longer stall a search for a - minute. Falls back to `CUIAutomation` / `IUIAutomation` otherwise. -- `find_accessibility_elements` / `AC_a11y_find_all` / `ac_a11y_find_all`: - `max_results` now caps the matches returned (default 50) and the new - `scan_limit` caps how many elements are examined (default 1500). Callers - that passed `max_results` expecting a scan bound should pass `scan_limit`. ### Deprecated @@ -375,133 +1317,14 @@ only when documented here with a migration path. `DeprecationWarning` and delegate to the working implementation; see Changed for the behaviour that changes. -- New integrations should avoid the eager, historical top-level import surface - and import stable entry points from `je_auto_control.api`. - -### Fixed +### Removed -- **Typing text through the key-event route raised `AttributeError` on the - three platforms that cannot do it.** `type_unicode_keys()` (and - `AC_type_unicode_keys` / `ac_type_unicode_keys`) called the backend's - `type_unicode_unit` outright, and only Windows has one, so macOS, X11 and - Wayland raised an exception from outside the `AutoControlException` family - that the executor, the background poll loops and the request handlers each - catch in one `except` — it escaped every containment boundary in the - project. It now raises `AutoControlKeyboardException` pointing at - `type_unicode_text()`, which picks a route that works on any platform. -- **A backend that could not report the cursor aborted the script instead of - raising what the API promises.** `press_mouse` / `release_mouse` / - `click_mouse` with an omitted `x` or `y` unpacked `get_mouse_position()` - without checking it for `None`, so a backend that answers "I don't know" - raised `TypeError` from the unpacking — outside the - `AutoControlMouseException` family every containment boundary catches. It - now raises `AutoControlMouseException`. `mouse_scroll` reached the same - unpacking through `_scroll_to` and now skips the pre-move instead, which is - the graceful degradation its own comment already documented for backends - that cannot report the cursor. -- **`je_auto_control.windows.message.window_message` could not be imported - at all.** It did `from ...windows_window_manage import FindWindowW`, and - that module has no such name — `FindWindowW` is a method on its private - `user32` handle — so importing `window_message` raised `ImportError` on - every Windows machine. It now calls the module's public - `get_one_window_hwnd`, which is also the one that declares HWND-width - argtypes rather than letting ctypes truncate a 64-bit handle to `c_int`. -- **Importing the Win32 input backend no longer writes into - `ctypes.wintypes`.** `win32_ctype_input` set `wintypes.ULONG_PTR = - wintypes.WPARAM` on the standard library's own module. Nothing in this - package ever read it back, so the only effect the assignment could have was - on some other library in the same process asking `ctypes.wintypes` whether - it has `ULONG_PTR`. -- **Stopping an X11 recording that was never started raised instead of - returning nothing.** The X11 listener's `stop_record()` handed back the - `None` its queue attribute was constructed with, and the recorder one frame - up reads `.queue` off that result, so `stop_record()` without a preceding - `record()` produced an `AttributeError` that the wrapper caught and logged - as a failure. It now returns an empty queue, so the public `stop_record()` - returns the empty list it documents. -- **`check_key_is_press()` passed `None` to the backend for an unknown key - name.** A name the virtual-key table has no entry for became `None` and was - handed to the platform backend anyway: a `TypeError` on Windows and a silent - `False` on X11 — that is, "no, it is not pressed" for a key that does not - exist. It now logs the lookup failure and returns `None`, which is the - documented "could not answer" value. -- The MCP HTTP transport no longer tries to drain a request body it has - already read. Any `4xx` decided *after* the body was parsed — the new - unknown-session `404` and duplicate-stream `409`, and the pre-existing - "body must be UTF-8" `400` — called `_drain_body()`, which then blocked - reading bytes that were gone until the 30-second socket timeout, pinning - that worker and logging a `ConnectionAbortedError` traceback when the peer - closed first. The drain is now skipped once the body is consumed, and a - peer that has already vanished ends it quietly instead of raising. +- `je_auto_control.linux_wayland._detect.WAYLAND_GDBUS` is gone, along with the + `gdbus` probe it named: the desktop-portal capture tier no longer shells out + to any binary. `_detect` is a private module and nothing else referenced the + constant. -- **A rejected config bundle aborted the rest of the script.** Five - framework errors still inherited `Exception` directly — - `ConfigBundleError`, the USB passthrough `ProtocolError`, - `SessionError` and `UsbClientError`, and the work queue's - `BusinessError` — and the containment boundaries all catch the - `AutoControlException` family, so none of them caught these. A - malformed bundle passed to `AC_config_import` therefore raised straight - past the executor's per-action boundary and killed every remaining - action, even under `raise_on_error=False`; `AC_usb_remote_devices` and - `AC_usb_remote_open` had the same path through `UsbClientError`. All - five derive from `AutoControlException` now, so they are recorded as a - failed action like every other framework error. `LoopBreak`, - `LoopContinue` and the MCP dispatcher's private error carrier stay - outside the family deliberately — they are control flow, not failure. -- **`import je_auto_control` needed a Python built with `sqlite3`, and - FreeBSD's is not.** `sqlite3` is in the standard library but not in every - build of it: CPython links it against a system library, and FreeBSD ships - the result as the separate `databases/py-sqlite3` package. Ten subsystems - imported it at module scope — run history, checkpoints, the work queue, - agent memory, the remote-desktop audit log, SQL data sources, and the - error tuples in the REST, chat-ops and MCP containment boundaries — and - all ten are reachable from the facade, so the whole package failed to - import on a stock FreeBSD, mouse and keyboard included. They go through - `je_auto_control.utils.sqlite_support` now, which fails at the first call - that opens a database rather than at import, and raises - `AutoControlUnsupportedOperationException` — the type the GUI tabs, the - REST handler and the executor already report as "not available here" — - instead of an `ImportError` none of them catch. `run_diagnostics()` lists - `sqlite3` among the optional dependencies, so the gap is visible without - reading a traceback. -- **`mouse_scroll` did nothing at all on the BSDs.** It matched Windows, then - macOS, then a literal `["linux", "linux2"]`, so a FreeBSD, OpenBSD, NetBSD or - DragonFly caller fell off the end of the chain: no backend call, no - exception, no log line. It asks `platform_id.is_x11_unix()` now, and an - unrecognised platform raises `AutoControlMouseException` instead of returning - as though it had scrolled. -- **A recorded timeline replayed nothing.** `replay_timeline`'s dispatch table - held the `run_sequence` DSL's vocabulary (`press` / `click` / `key`) and the - recorders emit their own (`key_down` / `mouse_up` / `scroll`), and the two - were disjoint — so `stop_record_timeline()` fed to `replay_timeline()`, the - pipeline both the docstrings and the `ac_record_stop_timeline` tool - prescribe, matched no handler, replayed an empty session, and still returned - every event as played. The recorder ops dispatch now, and the wheel reads - `delta` as well as `value` (reading only `value` fell back to the default of - one notch, so a three-notch scroll down replayed as one notch the other - way). Affects every platform, not only macOS. -- macOS: recorded mouse coordinates were mirrored vertically. The listener - read `NSEvent.mouseLocation()`, whose origin is the bottom-left of the - display, while every replay posts into the top-left space `osx_mouse` uses — - so a click recorded near the top of the screen replayed near the bottom. It - now reads `CGEventGetLocation`, which is already in the space the replay - posts into. -- macOS: modifier keys were not recorded at all. macOS sends no key-down for - Shift, Control, Option or Command, only a `flagsChanged` event carrying the - new flag set, so a recording could not say a modifier was held across the - actions that followed. They are reconstructed from the flags now. -- macOS: `write()` typed a space instead of a backspace, because `"\b"` had - no route in the macOS key table and fell through to the space fallback. -- macOS: USB enumeration returned `apple_vendor_id` in `vendor_id`, a field - documented as a four-hex-digit string. A value that is not a hex id is now - `None`; the device is still listed and `manufacturer` still names the vendor. -- Linux/X11: `window_rect` returned the client area rather than the frame, - disagreeing with Win32's `GetWindowRect` by the window decorations. -- Linux/X11: `move_window_by_title` configured the client window directly, - which under a reparenting window manager positions it in the wrong - coordinate space. It now goes through `_NET_MOVERESIZE_WINDOW`. -- The D-Bus client could not marshal or demarshal signed integers, so any - protocol using them (AT-SPI extents among them) failed to decode. +### Fixed - **Wayland: an absolute mouse move through the ydotool fallback counted from the wrong origin.** `ydotool mousemove --absolute` emits no absolute event — @@ -514,6 +1337,7 @@ only when documented here with a migration path. Measured against a real wlroots session consuming the real ydotool device (`docker/Dockerfile.seat`, the new `seat-verification` job). Layouts whose outputs all sit at non-negative positions are unaffected. + - **Wayland: the same call is only pixel-accurate where pointer acceleration is off.** The displacement ydotool sends is relative motion, so the compositor accelerates it — libinput's default adaptive profile moves the @@ -537,6 +1361,7 @@ only when documented here with a migration path. D-Bus itself on a single connection, subscribing to the request path it predicts before it calls. No API changed; a path that always failed now works. + - **On Wayland, a monitor placed left of or above the primary one made every capture path read the wrong pixels.** The compositor lays its outputs out on one plane, and that plane starts at a negative coordinate as soon as an @@ -608,16 +1433,165 @@ only when documented here with a migration path. `rich_clipboard`, `clipboard_rich_formats`, `clipboard_files` and `clipboard_formats` all go through it. +## [0.0.218] - 2026-08-16 + +### Added + +- Unicode text entry by key injection: `type_unicode_keys`, `type_unicode_text`, + `plan_unicode_keys`, `unicode_keys_supported` (commands + `AC_type_unicode_keys` / `AC_type_unicode_text`, MCP tools + `ac_type_unicode_keys` / `ac_type_unicode_text`), on Windows backend + primitives `press_unicode` / `release_unicode` / `type_unicode_unit`. + +- Cross-word OCR matching helpers `find_spans` / `group_lines`. + +- `monitor_layout.grab_logical` / `logical_virtual_rect` / `logical_scale` / + `needs_rescale` — screen capture in the coordinate space the mouse uses. + +- `find_image` / `find_image_multi` accept `all_screens` and `screen_region`. + +- `AutoControlFlatTemplateException` (a subclass of `AutoControlScreenException`) + for a template with too little variation to locate. + +- Accessibility search scoping and matching: `window_title` on + `list_accessibility_elements` / `find_accessibility_element` / + `click_accessibility_element` / `control_get_state`, a `contains` substring + mode with exact-name ranking, `find_accessibility_elements`, + `accessibility_status`, `control_get_state`, and `rank_by_name` (commands + `AC_a11y_find_all` / `AC_control_get_state`, MCP `ac_a11y_find_all` / + `ac_control_get_state`). The accessibility GUI tab gains a window filter. + +- `AccessibilityElement.enabled`. + +- `stop_record_timeline` (`AC_stop_record_timeline`, + `ac_record_stop_timeline`): the recording as press *and* release, wheel + movement and `delta_ms`, ready for `replay_timeline`. + +- `utils/input_reach`: `input_desktop_available`, `input_reaches_system` + (`AC_input_reachable`, `ac_input_reachable`) — whether input this process + sends can actually arrive. The second probe presses F13 to find out. + +- `utils/keyboard_layout`: `char_table`, `layout_char_table`, `vk_to_char`, + `foreground_keyboard_layout` — which character each key produces on the + active layout, with a US fallback. + +- Window management gains the primitives it was missing: + `minimize_window_by_title`, `foreground_window`, `window_rect` and + `move_window_by_title` (`AC_minimize_window`, `AC_foreground_window`, + `AC_window_rect`, `AC_move_window`; `ac_minimize_window`, + `ac_foreground_window`, `ac_window_rect`). `list_windows` takes + `titled_only`, and `move_window_by_title` keeps the window's current size + when width/height are omitted. + +- `utils/url_canon` reaches its delivery surfaces: `canonicalize_url`, + `normalize_url`, `urls_equal`, `build_query` and `parse_query` are exported + from the facade, with `AC_canonicalize_url` / `AC_normalize_url` / + `AC_urls_equal`, the matching `ac_*` MCP tools, and three Script Builder + specs. The module and its tests already existed; only the wiring is new. + +### Changed + +- `set_clipboard_image` accepts PNG bytes **or** a path to any Pillow-readable + image, and `get_clipboard_image` / `set_clipboard_image` are now exported + from `je_auto_control.utils.clipboard` and the top-level facade, with + `AC_clipboard_get_image` / `AC_clipboard_set_image` commands. They were + previously reachable only through MCP and the GUI, not `execute_action`. + +- **Breaking — `close_window_by_title` / `AC_close_window` / `ac_close_window` + now actually close the window** (they post `WM_CLOSE`). They previously + *minimised* it: the Win32 call underneath is named `CloseWindow` but + minimises, and the wrapper inherited both the call and the wrong promise, so + every caller asking to close a window silently got a minimise instead. The + old behaviour is available unchanged as `minimize_window_by_title` / + `AC_minimize_window` / `ac_minimize_window`. + +- `focus_window` restores a window that is minimised before bringing it to the + front — focusing a minimised window used to do nothing visible. A maximised + window is left maximised (`SW_RESTORE` would have un-maximised it). + +- `show_window_by_title` no longer calls `SetForegroundWindow` after `SW_HIDE`; + hiding a window and then pulling it forward are contradictory. + +- `write` no longer raises on a character missing from the virtual-key table + where the backend can inject Unicode; it types that character instead. + +- `find_text_matches` returns runs of consecutive word boxes, so a target split + across boxes now matches. Results are merged boxes covering the whole run + (union rectangle, minimum confidence) rather than one box per word. + +- `find_image` / `find_image_multi` search every monitor by default and return + virtual-desktop coordinates, which are negative when a monitor sits left of or + above the primary. Pass `all_screens=False` for the previous primary-only + behaviour. + +- `match_template` / `match_template_all` capture every monitor and return + screen coordinates. A hit found inside a `region` previously came back in + region-local coordinates; it is now offset by the region's origin. Matches + against a caller-supplied `haystack` are unchanged (image-local). + +- `match_template` / `match_template_all` refuse an almost-single-colour + template instead of returning an arbitrary position. + +- `element_matches` accepts a friendly role name (`"button"`) as well as the + raw `"ControlType_50000"` the Windows backend reports. + +- `AccessibilityBackend.list_elements` takes `window_title`; in-tree backends + accept it, and the facade only forwards it when set, so an out-of-tree + backend keeps working until someone asks for scoping. + +- `AccessibilityElement.to_dict()` gains an `enabled` key. + +- The Windows recorder captures through one low-level hook + (`Win32InputHook`) instead of the two listeners. `record` / `stop_record` + keep their behaviour and return shape. + +- An unscoped `list_accessibility_elements` walks one top-level window at a + time in z-order, node by node, and stops at `max_results`, instead of one + uninterruptible `FindAll` over the whole desktop. Results are therefore + ordered front-most window first, and a small `max_results` no longer + reaches windows further back. + +- The UIAutomation object is created from `CUIAutomation8` as + `IUIAutomation2` with a bounded `ConnectionTimeout` where available, so an + application that never answers UIA can no longer stall a search for a + minute. Falls back to `CUIAutomation` / `IUIAutomation` otherwise. + +- `find_accessibility_elements` / `AC_a11y_find_all` / `ac_a11y_find_all`: + `max_results` now caps the matches returned (default 50) and the new + `scan_limit` caps how many elements are examined (default 1500). Callers + that passed `max_results` expecting a scan bound should pass `scan_limit`. + +### Removed + +- **Breaking — `je_auto_control.windows.listener` is gone**, with its + `Win32KeyboardListener` and `Win32MouseListener` classes. Recording moved to + `windows/record/win32_input_hook.py`, after which nothing in the package or + the test suite referenced them. + +- **Breaking — `je_auto_control.utils.clipboard.clipboard_image` is gone.** Its + two functions were duplicates of the ones in + `je_auto_control.utils.clipboard.clipboard`, under identical names but with a + different `set_clipboard_image` signature, so importing the wrong module + failed at runtime and only for one of the two argument types. Import from + `je_auto_control.utils.clipboard` (or the top-level facade) instead; the + surviving function accepts both PNG bytes and a file path. + +### Fixed + - `write` failing a whole string on the first character outside the 192-entry virtual-key table — on a US layout that includes `, . / : ? ! _ + @ %` and every CJK character, so URLs and non-English text could not be typed at all. + - OCR locating text that the engine split across word boxes (`Save As`, `另存新檔`), which previously reported "not found" for text plainly on screen. + - Template matching never finding a target on a second monitor, and returning coordinates offset by the physical-vs-logical pixel difference on a mixed-DPI desktop (measured ~116 px) and by the virtual-desktop origin. + - Template images failing to load from a path containing non-ASCII characters (`cv2.imread` returns `None` there, which surfaced as "could not read image"). + - `list_windows` handing back `LP_c_long` pointer objects instead of integer hwnds, so `int(hwnd)` raised `ValueError` and a listed window could not be used in any follow-up Win32 call. The `EnumWindows` callback declared its @@ -625,22 +1599,60 @@ only when documented here with a migration path. `windows_window_manage` declares `argtypes`/`restype` so a 64-bit handle is not truncated to 32 bits. This also un-breaks the `ac_list_windows` MCP tool, whose handler called `int(hwnd)`. + - Accessibility listing truncating to `max_results` *before* filtering, so an element past the cap could never be found however specific the filter. + - `control_get_value` returning a password field's value when a custom-drawn control puts plaintext in ValuePattern instead of masking it. + - The recorder leaking one thread per session: its listener pumped `GetMessage` once and `stop_record` never woke it, so the thread stayed blocked forever. + +## [0.0.217] - 2026-07-23 + +No compatibility changes. + +## [0.0.216] - 2026-07-23 + +### Added + +- Stable, headless `je_auto_control.api` façade. + +- Portable `autocontrol.failure-bundle/v1` diagnostic archives and CLI command. + +- Public API lifecycle, capability matrix, security policy, coverage and type + checking configuration. + +### Changed + +- Releases are prepared from version tags and use PyPI Trusted Publishing. + +- The USB/IP server binds `127.0.0.1` by default (least-privilege). Exporting + the attached device to the LAN now requires an explicit `host="0.0.0.0"`. + +### Deprecated + +- New integrations should avoid the eager, historical top-level import surface + and import stable entry points from `je_auto_control.api`. + +### Fixed + - macOS cursor position and omitted-coordinate clicks on Retina / HiDPI displays (pixel-vs-point display-height mismatch). + - Remote-desktop relay hang on Linux + CPython 3.14 when one paired peer disconnected (a cross-thread `shutdown()` no longer wakes a blocked `recv()`). + - `AC_expect_poll` crashing on a not-ready value instead of continuing to poll; `AC_parallel` branch variable-scope isolation; malformed `run_suite` specs now report a clean error instead of aborting. + - Windows Interception backend send-to-window click silently no-opping. + - Wayland partial-coordinate `mouse_scroll` raising instead of degrading. + - Action-file save now raises `AutoControlJsonActionException` (not a raw `UnicodeEncodeError`) on non-encodable text; non-ASCII USB/IP busid no longer kills the client thread; SQLite connections are closed; USB ACL removal is diff --git a/CLAUDE.md b/CLAUDE.md index d8e97752..3b38a419 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -24,7 +24,7 @@ Layering: entry points (`cli.py`, `gui/`, socket / REST / MCP servers) → execu ```bash pip install -r dev_requirements.txt # dev deps -pip install -e .[gui] # + GUI extra +pip install -e .[gui,webrtc] # + GUI and WebRTC extras python -m pytest test/unit_test/headless # headless unit tests python -m pytest test/integrated_test/ # cross-module workflows python -m coverage run -m pytest # the suite WITH coverage (see below) @@ -41,6 +41,13 @@ have their import-time lines recorded as never executed: measured, that is suite). `test/unit_test/headless/test_coverage_measurement.py` holds CI to the correct spelling. +**Measure it with the `[webrtc]` extra installed**, which is why it is in the +line above. Eleven modules under `utils/remote_desktop` raise `ImportError` at +module level without `aiortc`/`av` — 2,090 statements, about 4 points — and the +tests covering the WebRTC host's auth, TLS, tokens and file transfer +`importorskip` straight past. `quality.yml` installs the extra so the floor is +measured against the same tree a developer sees. + `pyproject.toml` pins `python_files = ["test_*.py"]` on purpose: the `*_test.py` files under `test/unit_test/` are manual demo scripts whose module bodies drive the real mouse and keyboard on import. Never loosen that setting. ## Feature Delivery Rules @@ -91,7 +98,7 @@ The map is only useful while it matches the tree, so **update it in the same cha Anything agreed but not done — deferred follow-ups, known gaps, half-delivered features, decisions waiting on the maintainer — is recorded in [Progress.md](Progress.md), not left in chat history or buried in a commit message. - **Write the entry when you defer the work**, in the same change that created the gap. Each entry states its status (`TODO` / `WIP` / `BLOCKED` / `DECIDE`), what is missing, and where in the tree. -- **Open items only.** Delete the entry when the work lands; shipped work is described in `WHATS_NEW.md` and compatibility changes in `CHANGELOG.md`. `Progress.md` is not a changelog. +- **Open items only.** Delete the entry when the work lands; finished work is recorded as an entry in `docs/updates/` (index and query commands: `docs/updates/README.md`) and compatibility changes in `CHANGELOG.md`. `Progress.md` is not a changelog. - A feature that reaches only some of the delivery surfaces above belongs here until the rest land. ## Coding Standards @@ -116,7 +123,7 @@ Anything agreed but not done — deferred follow-ups, known gaps, half-delivered Cyclomatic complexity ≤ 10 · cognitive complexity ≤ 15 · function ≤ 75 lines · parameters ≤ 7 · nesting ≤ 4 · file ≤ 750 lines · line ≤ 120 chars · no duplicated block ≥ 10 lines. -**What actually enforces these.** `quality.yml` runs ruff and bandit only, so line length is the only limit a CI job rejects. Complexity is measured by `radon cc -nc` in the pre-commit list below and read by a human. The file-length limit is enforced by nobody — treat this section as a review standard, not a gate, and do not describe it as CI-enforced. +**What actually enforces these.** `quality.yml` has five jobs — `lint` (ruff), `security` (bandit), `pytest-headless` (the suite plus the coverage floor), `typing-stable-api` (mypy) and `dependency-review`. Of the limits in this section, line length is rejected by ruff (`[tool.ruff] line-length = 120` with `E501`; it exempts a line ending in a pragma, which cannot wrap), and the file-length limit by `test/unit_test/headless/test_file_length_budget.py`, which reads the exemption list in `Progress.md` and fails on a file over the limit that is not listed, on a listed file that grew past its recorded ceiling, and on a row whose file is now under the limit. Cyclomatic complexity is measured with the same `radon` the pre-commit list below names, by `test/unit_test/headless/test_complexity_budget.py`; the whole package was one function over the limit when that gate went in. Cognitive complexity, function length, parameter count and nesting depth are still review standards rather than gates. **Scope of the file-length limit.** It applies to: @@ -152,6 +159,21 @@ These tools own the generic rules (bare `except`, mutable defaults, unused names Suppressions need an inline justification — `# noqa: # reason: ` or `# nosec B404 # reason: `. Blanket file- or module-level suppressions are forbidden. +A broad `except` (`Exception`, `BaseException`, bare) that swallows rather than re-raises needs `# reason:` on its own `except` line. `test/unit_test/headless/test_broad_except_reasons.py` fails CI on one that does not — the linters cannot: CI runs `ruff` with its default rules (no `BLE`) and does not run pylint. + +## Stage commits, `Progress.md`, `docs/updates/` and `architecture.md` + +Workspace rule shared by every repository under `D:\Codes` (full text: `D:\Codes\CLAUDE.md`). + +- **Commit at every stage.** A stage is the smallest piece of work that leaves the repository consistent and passes this project's checks (definition of done, tests, lint): one finished `Progress.md` item, or one self-contained step of a larger one. Commit it before starting the next stage, before switching to another repository, and before the session ends. Do not leave work uncommitted across sessions; if a stage cannot be finished, commit the consistent part and record the rest in `Progress.md`. + - Stage only the files that stage touched (`git add `, never `git add -A`), follow this file's commit-message rules, and never add AI attribution. + - Committing is not pushing: push or open a PR only as this project's branch flow says or when asked. +- **`Progress.md`** (repository root, tracked) holds outstanding work only: no finished items, no history, no rules. +- **`docs/updates/`** records finished work: one batch file per month (`YYYY-MM.md`), one entry per piece of work headed `## U-YYYYMMDD-NN · date · title · #tags`, and an index with query commands in `docs/updates/README.md`. When a `Progress.md` item is done, delete it and add a `#done` entry plus its index row in the same commit. +- **`architecture.md`** (repository root) is the short architecture overview: layers, entry points, main flows, extension points, cross-project boundaries. Update it in the same commit whenever a change alters any of those. `architecture_explore.md` stays the detailed per-module map under its own rule in this file. +- **Cross-project contracts** are listed in `architecture.md` §6: what other repositories rely on here (CLI flags, import paths, constructor arguments, file layouts) and what this repository relies on elsewhere. `test/unit_test/headless/test_cross_project_contracts.py` pins the ones §6 lists (legacy CLI flags as a real child process, the facade and internal names each consumer imports), but it only knows what §6 knows: never rename or remove one without changing its consumers in the same round, and update §6 and that test whenever a contract is added or changes. +- Here the progress file is `Progress.md` (see "Outstanding work goes in `Progress.md`" above); its 750-line exemption list stays there. `WHATS_NEW.md` now only points to `docs/updates/`, and compatibility changes still go to `CHANGELOG.md`. + ## Commit Conventions - Concise messages focused on **why**, not what. Imperative mood: `Add image threshold parameter validation`, `Fix mouse scroll direction on macOS`, `Remove deprecated screen capture fallback`. @@ -173,10 +195,11 @@ Suppressions need an inline justification — `# noqa: # reason: ` seven `AdminConsoleTab`s this way, and they detonated inside the nested modal `exec()` of `test_usb_acl_prompt.py`, killing the interpreter with rc 3221226505 (0xC0000409) — a `__fastfail`, so no traceback, no faulthandler - output, and nothing after it in the suite ran. Note the failure is invisible - to CI: `test_usb_acl_prompt.py` needs the optional `webrtc` extra (`av`, - `aiortc`), which CI does not install, so CI skips it and only developers with - that extra installed see the crash. + output, and nothing after it in the suite ran. `test_usb_acl_prompt.py` needs + the optional `webrtc` extra (`av`, `aiortc`) and skips without it; the + `pytest-headless` job installs `.[webrtc]`, so CI runs it on every square + and a leak like this fails CI instead of only a developer's machine. Keep + the extra in that install for this reason as well as for coverage. ## Key Conventions diff --git a/Progress.md b/Progress.md index 153e4c4d..19743455 100644 --- a/Progress.md +++ b/Progress.md @@ -1,7 +1,8 @@ # Progress -**只記未完成的事。** 已出貨的內容寫進 [WHATS_NEW.md](WHATS_NEW.md)，相容性變更寫進 -[CHANGELOG.md](CHANGELOG.md)；完成的項目從本檔移除，不累積歷史。 +**只記未完成的事。** 完成的工作記在 [docs/updates/](docs/updates/README.md)（每月一個批次檔， +索引與查詢指令在它的 README），相容性變更寫進 [CHANGELOG.md](CHANGELOG.md)；完成的項目 +從本檔移除，同一個 commit 在 `docs/updates/` 補一筆 `#done` 條目，不在這裡累積歷史。 狀態標記： @@ -18,22 +19,24 @@ `CLAUDE.md` §Size and complexity limits 規定:超標檔案只能列在這裡,列不進來的就是缺陷。 清單上的檔案**可以改、可以變短,但不得再變長**——要再長就得先拆。 -行數為 2026-08-19 實測（`len(text.splitlines())`）；`webrtc_panel.py` 於 -2026-08-22 拆出 `advanced_group.py` 後降到 2,545，上限跟著往下走。 +行數為實測（`len(text.splitlines())`）；`webrtc_panel.py` 於 2026-08-22 拆出 +`advanced_group.py`、2026-09-23 拆出 `trusted_group.py` 後降到 2,530，上限跟著往下走。 +**這張表現在有測試在守**：`test/unit_test/headless/test_file_length_budget.py` 比對本表與樹， +超標未列、列上的檔案變長、或已經縮到線內卻還留著的列，都會紅。 | 檔案 | 行數 | 為何還沒拆 | | --- | ---: | --- | -| `utils/mcp_server/tools/_handlers.py` | 4,789 | 676 個 MCP 工具的處理函式本體。與 `_factories.py`（表）不同,這裡是邏輯,應該依主題拆成 `_handlers/` 套件（input／screen／window／file／agent…）。拆點清楚,純粹是量大。 | -| `gui/remote_desktop/webrtc_panel.py` | 2,545 | 單一 Qt 面板,但已含連線、監視器選擇、頻寬自適應、麥克風、錄影五組互動狀態。應拆成 panel + 各控制器。 | -| `utils/accessibility/backends/windows_backend.py` | 915 | 已拆出 `windows_query.py`（170）與 `windows_state.py`（98）。剩下的是同一套 UIA COM 生命週期管理,再拆會把 `CoInitialize`／介面釋放的配對邏輯切散。 | +| `utils/mcp_server/tools/_handlers_executor_bridge.py` | 1,448 | 2026-09-23 拆 `_handlers.py` 時新建。252 個純委派（中位數 3 行）：`from action_executor import _x` 再 `return _x(...)`,沒有分支。**不套用 flat data tables 條款**——那一條講的是「一個對照表或清單」,這裡是 252 個函式定義。再切下去只能照 MCP 工廠領域分（159 個領域）,那會把同一種委派散進十幾個檔,而它們之間沒有語意邊界。規則照舊:只准變短。 | +| `gui/remote_desktop/webrtc_panel.py` | 2,530 | 單一 Qt 面板,但已含連線、監視器選擇、頻寬自適應、麥克風、錄影五組互動狀態。應拆成 panel + 各控制器。 | +| `utils/accessibility/backends/windows_backend.py` | 801 | 已拆出 `windows_query.py`（176）、`windows_state.py`（98）與 `windows_reads.py`（142,2026-09-23）。剩下的是同一套 UIA COM 生命週期管理,再拆會把 `CoInitialize`／介面釋放的配對邏輯切散。 | **本質豁免（依 `CLAUDE.md` 的「flat data tables」條款,不算既有豁免）**: -`utils/mcp_server/tools/_factories.py`（8,972,MCP 工具註冊表）、 -`utils/executor/action_executor.py`（8,125,`AC_*` 分派表）、 +`utils/mcp_server/tools/_factories.py`（8,975,MCP 工具註冊表）、 +`utils/executor/action_executor.py`（8,131,`AC_*` 分派表）、 `gui/script_builder/command_schema.py`（5,051,每個 `AC_*` 的參數 schema）、 `je_auto_control/__init__.py`（1,970,門面 re-export）、 `gui/language_wrapper/{english,japanese,traditional_chinese,simplified_chinese}.py` -（1,316／1,203／1,189／1,188,語系字串表）。 +（1,318／1,205／1,191／1,190,語系字串表；2026-09-22 實測）。 ### 2026-08-19 決議:上表的實測行數就是新的上限 @@ -42,11 +45,13 @@ **維護者已於 2026-08-19 拍板:接受實測數字當新基準**——不為了回到舊數字而去拆 `_handlers.py`（4,789）與 `webrtc_panel.py`。上表的行數即是各自的新上限, 規則不變:只准變短,再變長就得先拆。 +（`_handlers.py` 後來還是拆了:2026-09-22 拆出 QA 主題,2026-09-23 再拆出九個主題模組, +本體降到 522 行、離開上表。見 `docs/updates/` 的 U-20260922-05 與 U-20260923-09。） 同一批裡有六個檔案在 2026-08-19 已經拆回線內、從表上移除,做法寫在 -[WHATS_NEW.md](WHATS_NEW.md)。 +commit `46f4cd5` 的說明裡（`docs/updates/` 沒有對應條目：舊的 `WHATS_NEW.md` 從沒記過這件事）。 -行數沒有任何 CI 在把關（`quality.yml` 只跑 ruff 與 bandit,而 ruff 只管行寬), +行數沒有任何 CI 在把關（`quality.yml` 的五個 job 裡只有 ruff 管到這一節的限制,而它只管行寬), 所以這張表只會在有人手動實測時才會被發現對不上——上次就是。 --- @@ -122,7 +127,7 @@ pip install --dry-run --only-binary=:all: --platform win_arm64 --python-version 當成了「容器做不到的事」**。portal 是 D-Bus 介面,誰佔住那個名字誰就是 portal; 「會吃 libinput 裝置的 seat」是 wlroots 的 `WLR_BACKENDS=headless,libinput` 加 `LIBSEAT_BACKEND=builtin` 加 `SEATD_VTBOUND=0`（第四個條件是 udev 要比 ydotoold 早起 -來)。都已經是 CI job 了,見下面「已經有答案的」與 [WHATS_NEW.md](WHATS_NEW.md)。 +來)。都已經是 CI job 了,見 [docs/updates/2026-08.md](docs/updates/2026-08.md) 的 U-20260819-02（原本在這裡的「已經有答案的」一節）與 U-20260818-01、U-20260819-01。 **下次要往這裡加「需要一台 VM／真桌面」之前,先問這件事到底是誰做不到。** @@ -141,63 +146,168 @@ pip install --dry-run --only-binary=:all: --platform win_arm64 --python-version bus 上跑過,三種都得在自己的時限內收斂。至於真的 mutter 對話框長什麼樣、真人猶豫 三十秒會不會撞到別的東西,那是 mutter 的事,CI 裡沒有人可以去按它。 -### 已經有答案的（都在 CI 裡,做法見 WHATS_NEW） +**緩解**:驗不到的擷取部分有逃生門——`JE_AUTOCONTROL_WAYLAND_CAPTURE_COMMAND` 讓操作者 +直接指定自己的擷取指令（`{output}` 會被換成暫存 PNG 路徑）,優先於所有偵測。 -五個 job 都在 GitHub runner 上跑過了（2026-08-19,PR #481）。`modprobe uinput evdev` -在 runner 上載得起來,`systemd-udevd` 在容器裡也收得到 kernel uevent——這兩件事原本 -只在本機（Docker Desktop 的 WSL2 kernel）驗過,曾經記在上面當待辦,現在有答案了。 -job 一律寫成模組載不起來就明講失敗,不會靜默跳過,所以哪天 runner 的 kernel 變了會 -當場紅掉。 +--- -| 面向 | 怎麼驗的 | job | -| --- | --- | --- | -| 擷取路徑 | 真的 wlroots 合成器（sway headless,兩個上不同純色的 output）,27 項 × 2 種版面 | `wayland-verification` | -| libei 協定層 | 真的 `libeis.so.1` server 在 Unix socket 上,20 項 | `eis-verification` | -| RemoteDesktop portal 交握 | 真的 `dbus-daemon` + 真的 `liboeffis`,對面是自己實作的 portal,`ConnectToEIS` 交出通往真 libeis 的活 fd,20 項 | `portal-verification` | -| ydotool CLI | 真的 uinput 裝置,直接讀回 `/dev/input/eventN`,12 項 | `ydotool-verification` | -| ydotool 的絕對移動落在哪 | 真的 wlroots session 吃真的 ydotool 裝置（`headless,libinput` + builtin seat）,游標位置從 `grim -c` 的像素讀回,14 項 × 2 種版面 | `seat-verification` | - -擷取那一列的第二種版面是**負原點**:`output HEADLESS-1 position -1280 0`, -也就是「第二台螢幕在主螢幕左邊」的桌面。sway headless 收這個座標,grim 也收負的 -`-g`,所以這件事根本不必等 GNOME VM——原本記在這裡說測不到,是把「合成器做得到的事」 -當成了「容器做不到的事」。跑起來當場抓到三個真的錯:`size()` 回的是版面右緣不是寬度、 -非 grim 層級的裁切用版面座標去裁一張以版面原點為 (0,0) 的圖、`grab_logical()` 一律回 -原點 (0,0) 所以比對到的座標整個偏掉。修法見 [WHATS_NEW.md](WHATS_NEW.md)。 - -portal 那一列是同一個錯誤犯第二次的結果,而它抓到的東西比前一次更嚴重: -`portal.py` 那條「先開 `gdbus monitor`、再用 `gdbus call` 發請求」的路 -**在任何真的 bus 上都不可能成功**——portal 的 `Response` 是**指名送給發出呼叫的那條 -連線**,兩個 gdbus 行程是兩條連線,監聽的那條永遠不是收件人。在真的 `dbus-daemon` 上 -量到的就是這樣:呼叫看得到,回答永遠等不到,每次都走到 30 秒逾時。修法見 -[WHATS_NEW.md](WHATS_NEW.md)。 - -五者都不需要合成器以外的東西,更不需要 GNOME VM。libei 這一層驗掉的包含 -capability enum 值與 variadic `ei_seat_bind_capabilities`、event-type enum 值、 -`start_emulating` → 事件 → `frame` 的實際上線內容、live context 的 teardown -安全性（原本每個行程漏一個 context + 一個 fd,已修）、以及絕對指標的座標空間 -（region offset 讀得回來且含在座標裡、region 外的移動被靜靜丟掉、負原點的版面要 -正規化）。portal 這一層驗掉的是四個呼叫的順序與 client 自己預測的 request path、 -`SelectDevices` 收到的裝置遮罩（也就是使用者被要求同意的範圍）、交回來的 fd 真的 -承載得起一個 EI session,以及六種拒絕路徑各自都要 fail closed。ydotool 這一層驗掉的是 -`click` 位元遮罩、拆邊的 press／release、`mousemove --absolute` 的實際上線內容、 -捲動正負號與軸向,以及 `mouse`／`keyboard` 自己組出來的 argv。seat 這一層驗掉的是 -`--absolute` 到底相對於哪裡（版面左上角,不是版面座標的 `(0, 0)`)、關掉加速度後 -一像素對一像素、沒轉換的 `(0, 0)` 會打到隔壁螢幕、`set_position` 減掉的正好是原點、 -以及預設 profile 下的 2 倍加速。 - -### 一件關於發行版的事實,會影響使用者拿到什麼 - -- **`liboeffis` 是獨立的二進位套件,`libei1` 不會把它帶進來。** Debian trixie - **有** `liboeffis1`（1.3.901-1,`liboeffis.so.1`,連 libsystemd 的 sd-bus）—— - 這裡原本寫「Debian trixie 沒有」,是錯的,已實測更正。Arch（1.6.0）與 Fedora 也有。 - 但因為它不是 `libei1` 的相依,只裝 libei 的機器上 portal 快速路徑仍然是關閉的, - `connect()` 會退到 `$XDG_RUNTIME_DIR/eis-0` socket,GNOME／KDE 不開那個 socket - → 退回 ydotool。**所以要用 libei 快速路徑,`liboeffis` 得自己裝。** -- 而那條退路本身,在同一批發行版上原本是壞的——0.1.x 對本專案送的 argv 回傳 0 - 卻不送任何事件。已於 2026-08-19 擋掉,見 CHANGELOG 與 WHATS_NEW;此處無待辦。 +## 鍵盤與滑鼠 wrapper 的輸入修正：等 Jeffrey_RPA 批次停下 -**緩解**:驗不到的擷取部分有逃生門——`JE_AUTOCONTROL_WAYLAND_CAPTURE_COMMAND` 讓操作者 -直接指定自己的擷取指令（`{output}` 會被換成暫存 PNG 路徑）,優先於所有偵測。 +`BLOCKED` — Jeffrey_RPA 以 editable install 載入這個工作樹，正式批次（`webrunner_novelai.py`）與 Discord bot 正在跑，並且經 `_gui_control.py` 呼叫 `ac.write`、`ac.hotkey`、`ac.mouse_scroll`；下面每一項都會改變它打出來的字或滾動方向，依工作區規則在它執行期間不動 + +2026-09-24 稽核用假後端重現： + +- **大寫字母打成小寫**：`wrapper/auto_control_keyboard.py:234` `write()` 在 Windows 送的是與小寫相同的虛擬鍵（`_platform_windows.py` 的表裡 `"A"` 與 `"a"` 同一個碼），沒有按 Shift，`"Hi"` 打成 `hi`；X11 很可能一樣。做法：需要 Shift 的字元改走 `_write_char_via_unicode`，或包一層 Shift 按下／放開。 +- **`is_shift` 在 Windows 與 X11 無效**：`auto_control_keyboard.py:73`、`:104` 只在 macOS 把它傳下去，其他平台直接忽略，docstring 卻寫「是否同時按下 Shift」。做法：在這一層按住 `keyboard_keys_table["shift"]`，`finally` 放開。 +- **`"\r\n"` 按兩次 Enter**：`write()` 把 `\r` 與 `\n` 都對到 `return`，從檔案讀進來的 Windows 換行每行多一個空行。做法：迴圈前把 `\r\n` 換成 `\n`。 +- **X11 預設滾動方向與 Windows／macOS 相反**：`wrapper/auto_control_mouse.py` `mouse_scroll(..., scroll_direction="scroll_down")`，正值在 X11 往下、其他平台往上，與 docstring「一份寫法各平台通用」不符。做法：預設改 `scroll_up`，或改 docstring 講清楚（重播路徑已在 U-20260924-14 明確傳 `scroll_up`）。 +- **`mouse_scroll` 的 NaN 座標被悄悄夾到桌面邊緣**：`auto_control_mouse.py` 的夾限在 `_coordinate()` 驗證之前，`mouse_scroll(3, x=nan, y=100)` 移到 `(-1920, 100)` 才滾；`set_mouse_position(nan, …)` 則正確丟例外。做法：夾限前先過 `_coordinate()`。 +- **座標截斷而非四捨五入**：`set_mouse_position(-0.6, 10.9)` 得到 `(0, 10)`，註解寫的是「rounded point」。做法：`int(round(value))`。 + +同一次稽核的影像與 OCR 部分也在它的路徑上（Discord bot 的 `!find_image`／`!find_text`），一併等： + +- **非 ASCII 路徑與灰階樣板**：`cv2_utils/template_detection.py:126` 經 `je_open_cv` 的 `cv2.imread` 讀樣板，`測試\t.png` 讀不到；2-D 陣列或 PIL `"L"` 樣板丟出 `cv2.error`，不在 `wrapper/auto_control_image.py` 的例外清單裡。做法：路徑改走 `cv2_utils/image_file.read_image`，2-D 直接用，`cv2.error` 包成 `ImageNotFoundException`。 +- **部分超出螢幕的 `screen_region` 被補黑**：`monitor_layout/logical_frame.py:143` 沒有先和畫面取交集，PIL `crop` 補零，可能回傳螢幕外的命中；寬或高為負時丟裸 `ValueError`。做法：先取交集（回傳裁過的原點），非正的寬高丟框架例外。 +- **OCR 跨框比對漏掉從長框中段開始的字串**：`ocr/text_span.py:330` 的視窗超過「目標長度＋40」就整個丟掉最左框，即使目標從那框開始；`"Save As"` 在長句框之後就找不到。做法：只有剩下的部分仍不短於目標時才丟左框。 +- **負座標的中心點差一**：`wrapper/auto_control_image.py:48`、`:73` 的 `int((x1 + x2) / 2)` 向零截斷。做法：`(x1 + x2) // 2`。 + +**解除條件**：Jeffrey_RPA 沒有批次在跑（`webrunner.pid` 的行程不在、Discord bot 停止）；改完在 Jeffrey_RPA 跑 `test/test_je_facade.py`。 + +--- + +## RBAC 還沒接到 REST API 與 MCP server + +`DECIDE` — 要不要把 `utils/rbac` 接上兩個伺服器，以及現有單一共用 token 怎麼過渡（維護者拍板） + +`utils/rbac/users.py` 有使用者、角色與權杖驗證（2026-09-24 已補上：壞檔不覆寫、權杖不得重複），但沒有任何程式 +import 它：`rest_api/rest_auth.py` 與 `mcp_server/http_transport.py` 都只比對一個共用 token，稽核 log 也沒有 +`user_id`。模組 docstring 已改成照實描述。 + +**做法**：`RestAuthGate.check` 改成先查 `UserStore.authenticate`、再依路由對應的 `Capability` 呼叫 `can()`； +MCP 的 bearer 比對同理；稽核寫入帶上 `user_id`。 + +**要先想清楚**：沒有任何使用者時是否退回共用 token（相容現有部署）；viewer／operator／admin 各能呼叫哪些路由與工具。 + +--- + +## 能執行動作的人也能替檔案簽章 + +`DECIDE` — `JE_AUTOCONTROL_REQUIRE_SIGNED_ACTIONS` 要防的是誰（維護者拍板） + +`AC_sign_action_file` 用預設的個人金鑰簽章，所以凡是能透過 socket、REST 或 MCP 執行動作的人，都能先簽一個檔再用 +`AC_execute_files` 執行它；內嵌的動作清單本來就不驗簽。現在的強制簽章只擋得住「能改檔案、但不能執行動作」的人。 + +**選項**：簽章指令在強制模式下只准本機 CLI 使用；或簽章金鑰與執行權限分開保存（簽章端不在執行端）。 + +--- + +## USB passthrough viewer 以種類配對回覆，逾時的回覆會交給下一個請求 + +`DECIDE` — 協定要不要加請求編號（線上格式改動，新舊版本相容要一起想） + +`utils/usb/passthrough/viewer_client.py:413`（`_on_opened`）與 `:466`（`_complete_pending`）只按 OPEN／LIST／claim +配對回覆，回覆沒有序號。請求逾時後，對同一種類的下一個請求會拿到遲到的舊回覆：`open(aaaa)` 逾時、`open(bbbb)` +收到 `aaaa` 的 OPENED，claim 綁錯裝置；bulk 讀逾時後，下一次傳輸拿到上一次的資料。host 接受最長 60 秒的 +`timeout_ms`，client 預設 10 秒就放棄，正常使用就會遇到（2026-09-24 稽核重現）。 + +**選項**：在 payload 加一個由 client 產生、host 原樣帶回的請求編號（舊 host 不帶就退回現在的配對）；或逾時後把該 +claim 標成需排空，丟掉下一個回覆——但 host 若根本沒回，會丟掉正確的回覆。 + +--- + +## Admin console 廣播的 `ok` 只代表 HTTP 200 + +`TODO` — 讓遠端 `/execute` 的動作失敗也能回報成失敗 + +`utils/admin/admin_client.py`（`_execute_one`）在 host 回 200 時一律 `ok: True`；遠端 `/execute` 以 +`raise_on_error=False` 執行，動作失敗只出現在結果內容裡（例如 `{"execute: [...]": "TypeError(...)"}`）。 +`utils/dag/runner.py:270` 的遠端節點因此把失敗的節點算成成功，本機路徑早已用 `raise_on_error=True` 修正過。 + +**做法**：REST `/execute` 接受並轉交 `raise_on_error`（失敗時回非 200 或 `ok: false`），admin client 與 DAG 遠端 +節點帶上它；同時更新 REST 的 OpenAPI 描述與 `architecture.md` §6（其他工具也會呼叫 `/execute`）。 + +--- + +## 全域 executor 的變數會留到下一次執行 + +`DECIDE` — 每次頂層執行要不要有自己的變數範圍（行為改動，維護者拍板） + +`execute_action_with_vars`（`utils/executor/action_executor.py`）把變數種進全域 `executor` 後從不清除，REST、MCP、 +socket server 的執行也都用同一個 `executor`；`for_each` 的迴圈變數與巨集參數同樣留著。下一次執行裡的 `${user}` +會安靜地取到前一個呼叫者的值，而不是報 `Unknown variable`（2026-09-24 稽核重現）。模組文件把這個範圍描述成 +「共用」，所以有人可能依賴它在執行之間傳值。 + +**選項**：`execute_action_with_vars` 與各伺服器入口每次開一個新的 `VariableScope`（`AC_set_var` 在單次執行內照舊）； +或保留共用，但在伺服器入口清空，並在文件寫明。 + +**附帶**：`AC_circuit_call`、`AC_bulkhead_run`、`AC_run_chaos`、`AC_run_dag` 的巢狀動作跑在全域 `executor` 上， +在 `AC_parallel` 分支裡因此用到父層的變數範圍，而不是分支自己的。 + +--- + +## 舊式 CLI（`-e`／`-d`／`--execute_str`）在動作失敗時仍然結束碼 0 + +`DECIDE` — 要不要讓舊式入口也以結束碼 1 回報動作失敗（跨專案契約，PyBreeze 與 TestPioneer 以子程序呼叫） + +`je_auto_control/__main__.py` 執行完不看 `recorded_failures()`；`je_auto_control run` 在 `cli.py` 已經會回 1。 +同一個會失敗的腳本，`run` 回 1，`-e`、`-d`、`--execute_str` 回 0（2026-09-24 稽核重現）。 + +**要先確認**：PyBreeze（`AI_CONTEXT.md` §5）與 TestPioneer 的 `parallel_run` 怎麼解讀這個結束碼——若把非 0 當成 +「無法執行」而非「有動作失敗」，改了會讓它們把一次有失敗步驟的執行回報成錯誤。改的話兩邊的 `architecture.md` §6 +與相容性測試要一起更新。 + +--- + +## Computer use 改走 GA 的 `computer_toolset_20260801` + +`TODO` — 換成新的工具形式需要改 agent 迴圈，不只是換一個 tool 型別 + +`utils/agent/backends/anthropic_computer_use.py` 現在以 beta 送 `computer_20251124`（2026-09-24 修正：原本沒帶 beta， +每個請求都被 API 拒絕）。GA 的 `computer_toolset_20260801` 不需要 beta，但每個動作是一個名稱為成員名的 `tool_use` +（`screenshot`、`left_click`…），可能一回合好幾個，每個 `tool_result` 都要帶回 `"toolset_name": "computer"`； +截圖要先縮到模型的影像上限內。Claude Opus 5.5 只接受這個形式。 + +**做法**：`_decision_from_computer_action` 改讀區塊的 `name`，一回合允許多個呼叫並逐一回覆，`_ingest_history` 帶上 +`toolset_name`；在 `claude-opus-5`（兩種都接受）上測過再換預設。 + +**附帶**：`AC_run_agent backend="openai"` 送出全部約 740 個工具，超過 OpenAI Chat Completions 的 128 個上限， +所以一定失敗——與「`AC_run_agent` 預設工具集」那一條 DECIDE 一起決定。 + +--- + +## Idempotency 的 `release` 還沒有執行器指令 + +`TODO` — 只有 headless API，JSON 腳本與 MCP 還放不掉失敗的鍵 + +`utils/idempotency/idempotency.py` 的 `IdempotencyStore.release()` 讓工作失敗的 `in_progress` 鍵可以重跑，但 +`action_executor.py` 的 `_idempotency_begin`／`_idempotency_complete` 旁邊沒有對應的 `AC_idempotency_release`， +而執行器的具名儲存沒有 TTL，所以腳本裡工作失敗的鍵仍然永遠是 `in_progress`。 + +**做法**：加 `AC_idempotency_release`、`ac_idempotency_release` 與 Script Builder 的 **Flow** 指令，並重量指令數 +（`test_doc_counts.py` 會要求 README 三份與 `architecture_explore.md` 一起改）。 + +--- + +## pytest11 進入點會把整個門面拉進每一次 pytest + +`DECIDE` — 要不要把進入點搬到一個精簡的頂層模組（打包層的改動，維護者拍板） + +`pyproject.toml` 的 `pytest11` 進入點指向 `je_auto_control.utils.pytest_plugin.plugin`。 +外掛模組本身很輕（只 import pytest，fixture 裡才 import 本套件），但它是**套件的子模組**， +所以 Python 會先跑 `je_auto_control/__init__.py`——量到 **1,355 個模組**。機器上任何一個 +安裝了本套件的環境，每一次 pytest 啟動都付這筆成本（Jeffrey_RPA 因此在 `pytest.ini` 用 +`-p no:je_auto_control` 擋掉它）。pytest 官方文件建議的形狀正是「進入點指向只 import pytest +的精簡模組」。 + +改法：新增頂層模組（例如 `je_auto_control_pytest.py`，`[tool.setuptools] py-modules`）， +進入點改指它，`utils/pytest_plugin/plugin.py` 轉為 re-export 以維持 +`pytest_plugins = ["je_auto_control.utils.pytest_plugin"]` 這條路。 + +**為什麼要拍板**：(1) 這是發佈產物的改動，會在 site-packages 多一個頂層名字； +(2) 進入點改了要重裝才生效（本機的 editable 安裝、CI 的 `pip install -e .`）； +(3) `test/unit_test/headless/test_coverage_measurement.py` 的前提會改變——它現在釘住 +「外掛載入時門面已經在 `sys.modules` 裡」，改完就不成立，那份說明與測試要一起改寫 +（CI 仍可繼續用 `coverage run -m pytest`）。 --- @@ -220,131 +330,106 @@ capability enum 值與 variadic `ei_seat_bind_capabilities`、event-type enum --- -## 兩個門檻：mypy 那半到終點了，覆蓋率那半是量錯了 +## `cryptography` 的安全下限要不要拉到 50 -`DECIDE` — 兩半都做完了，只剩「下一個覆蓋率目標是多少」要維護者拍板 +`DECIDE` — 要不要用 Intel Mac 的預編 wheel 換掉一個本套件沒用到的漏洞範圍 -原本這一條記的是兩個只存在於 `pyproject.toml` 註解裡、沒有任何機制的承諾。 -2026-08-21 把**機制**補上了（做法見 [WHATS_NEW.md](WHATS_NEW.md)），兩半也都走完了： -型別契約的豁免清單 2026-08-22 清空，平台縫最後兩個名稱（`keyboard`／`mouse`） -2026-08-23 拿到合約；覆蓋率那半發現不是爬得不夠，是量測起點錯了，修正後地板 -從 50 提到 69。 +`pyproject.toml` 的 `cryptography>=48.0.1` 仍包含 GHSA-g6cj-pr64-35w5（high,`>=44.0.0, <50.0.0`, +PKCS#7 EnvelopedData 解密的 Bleichenbacher oracle）的範圍。本套件沒有呼叫 PKCS#7 解密 +（用的是 Fernet，以及 aiortc 的 DTLS），所以目前不受影響。`uv.lock` 已鎖在 50.0.1。 -**這一條還留著，是因為只剩一個問題要維護者回答：下一個覆蓋率目標值是多少。** -另外兩節留著是因為它們記的那幾個坑之後還會踩到。 +**為什麼要拍板**:49.0.0 起上游不再發 `macosx_10_9_universal2` wheel，只剩 `macosx_11_0_arm64`。 +下限拉到 `>=50.0.0` 之後，Intel Mac 上的 `pip install` 要從原始碼編譯（得先裝 Rust 工具鏈）。 +CI 只有 macos-14(arm64)，量不到這一點。重新檢查（不需要機器）: -### 覆蓋率：目標 70 其實早就到了，是量錯了 +```bash +pip install --dry-run --only-binary=:all: --platform macosx_10_9_x86_64 \ + --python-version 3.12 --target /tmp/probe 'cryptography>=50' +``` + +--- -`DECIDE` — 地板已經設成修正後矩陣的最低那一格（69）；**下一個目標值要維護者定** +## Viewer 端要不要把 host 推來的檔案關在一個目錄裡 -`fail_under` 一度從 35 提到 50，理由寫在 `pyproject.toml`。**那兩個數字都低了大約 -24 點**，而原因不在測試，在量測的起點： +`DECIDE` — 這是改一個已寫進文件的功能，由維護者決定 -`quality.yml` 用的是 `pytest --cov`，而本套件註冊了 `pytest11` entry point。 -pytest 在載入外掛時就會 import `je_auto_control.utils.pytest_plugin.plugin`—— -要 import 那個子模組，Python 必須先執行 `je_auto_control/__init__.py`，也就是門面， -連帶把好幾百個模組拉進來。`pytest-cov` 是**在那之後**才開始量的，所以那幾百個模組的 -import 期程式碼（`def` 行、類別本體、常數、兩張大分派表）全部被記成「從沒執行過」。 +`host.send_file_to_viewers(source, dest_path)` 由 **host** 指定 viewer 機器上的完整路徑 +（`docs/source/{Eng,Zh}/doc/new_features/new_features_doc.rst` 的範例是 `/tmp/from_host.bin`）， +viewer 端的 `FileReceiver`（`utils/remote_desktop/file_transfer.py`）照單全收：`expanduser`、 +建立父目錄、寫入。也就是被控端可以在控制端機器的任何可寫位置放檔案。模組說明的 +「trusted token holders == trusted users」只涵蓋 host 端；viewer 連上一台被入侵的 host 時沒有這層保護。 -2026-08-23 實測，同一套測試、同一份 `[tool.coverage.run]` 設定，**只差開始的時機**： +**做法**：`FileReceiver` 加 `base_dir`，viewer（`viewer.py` 的 `_ensure_file_receiver`、GUI 的 +`viewer_panel.py`）預設給一個下載目錄，只保留相對路徑並拒絕跳出 `base_dir`；host 端維持現狀。 -| 量法 | 總覆蓋率 | -| --- | ---: | -| `pytest --cov=je_auto_control` | 52.22% | -| `coverage run -m pytest` | **72.05%** | +**為什麼要拍板**：`dest_path` 的語意會從「viewer 上的絕對路徑」變成「viewer 下載目錄裡的相對路徑」， +現有腳本與文件範例都要跟著改。 -差 11,962 個 statement。受害最深的正好是最大的幾個檔：`action_executor.py` +786、 -`_handlers.py` +684、門面自己 +369、`_factories.py` +209。 +--- -**一個註冊了 pytest 外掛的套件，沒辦法用 `pytest --cov` 量自己。** -`quality.yml` 已經改成 `coverage run -m pytest`（先於 pytest 載入任何東西）， -`test/unit_test/headless/test_coverage_measurement.py` 把這件事釘住——因為兩種寫法的 -差別在綠色的建置裡看不出來：改回去會白送 24 點，而每一格照樣是綠的。 +## Config sync 刪掉的項目會在下次同步時回來 -修正後的九宮格已經量出來了（2026-08-23，本 PR 的 run）： +`TODO` — 同步格式要加 tombstone，伺服器端與舊版客戶端的相容要一起想 + +`utils/config_sync/client.py` 的 `ConfigBucket.remove()` 直接把項目從本機 dict 拿掉；`merge_buckets` 把 +「只有遠端有」的項目照收，所以 `remove()` 之後 `sync()` 會把它從伺服器拿回來（2026-09-23 稽核重現）。 + +**做法**：`remove()` 留下 `{"deleted": True, "last_modified": now}`，merge 照一般 last-write-wins 比較， +合併完再把 tombstone 從對外的檢視濾掉；過了保留期（例如 30 天）才真正清掉。 + +**要先想清楚**：已經在跑的舊版客戶端看不懂 `deleted`，會把 tombstone 當成一般項目；伺服器是否要認得它。 + +--- + +## `AC_run_agent` 預設把每個 AC_* 指令都交給模型 + +`DECIDE` — 預設工具集要不要排除高風險指令 + +`utils/executor/action_executor.py` 的 `_run_agent` 以 `export_anthropic_tools()` / `export_openai_tools()` +不帶 `only=` 建立 backend，所以模型拿得到 `AC_shell_command`、`AC_execute_process`、`AC_android_shell`、 +`AC_add_package_to_executor`、`AC_run_agent`、`AC_computer_use`、`AC_execute_action` 等指令。 +2026-09-23 已讓 backend 拒絕「沒有提供的工具」，但提供的清單本身就包含這些； +螢幕上的內容（網頁、文件）若誘導模型呼叫 shell，目前不會被擋。 + +**做法**：`_run_agent` 預設排除上述類別，另加一個 opt-in 參數（例如 `allow_system_commands`） +讓需要的人明確打開；MCP `ac_run_agent` 與 Script Builder 的欄位同步。 + +**為什麼要拍板**：這會縮小既有的 agent 能力，依賴它跑 shell 的腳本會改變行為。 + + +--- + +## MCP 工具的檔案路徑參數要不要限制在工作區根目錄裡 + +`DECIDE` — 限制範圍與預設值由維護者決定 + +MCP 工具的檔案參數（`path`、`file_path`、`db`、`image_path`、`golden_path`、`output_path`… 約 100 個） +不受任何根目錄限制；只有 `resources/read` 關在 `roots/list` 的根目錄裡。完整模式下這不是新的權限 +（`ac_execute_actions` 本來就能做任何事），但 `JE_AUTOCONTROL_MCP_READONLY=1` 的部署仍能讀到根目錄外的 +任意檔案：例如 `ac_load_dotenv` 會把任何檔案解析成 KEY=VALUE 回給模型，`ac_read_document`、 +`ac_extract_pdf_text` 也一樣。2026 年 MCP 伺服器通報最多的一類就是這種路徑越界。 + +**做法**：在 `utils/mcp_server/tools/_factories.py` 的 schema 裡把真正是檔案路徑的屬性標上 +`"format": "path"`（不能照名字判斷：`ac_json_query` 的 `path` 是 JSON 路徑，`template`／`source`／ +`target` 有時是檔案有時不是），`server.py` 的 `_prepare_tool_call` 在設定了根目錄時先 `realpath` +再檢查是否落在根目錄內，不在就回 `-32602`。 + +**為什麼要拍板**：根目錄從哪來（新的環境變數、沿用 `roots/list`、或兩者），唯讀模式要不要預設開啟； +預設開啟會讓現有讀取工作區外檔案的用法失效。 + +--- + +## 無障礙錄製器沒辦法追蹤焦點 + +`TODO` — 各平台後端都缺「目前焦點元素」的查詢 + +`utils/accessibility/recorder.py` 的 `_default_fetcher` 只能呼叫 `find_accessibility_element(app_name=...)`， +拿到的是掃描到的第一個元素，不是焦點所在的元素；`AccessibilityElement` 也沒有焦點欄位。所以焦點從 +一個欄位移到另一個欄位時，錄製器偵測不到（2026-09-23 稽核重現；docstring 已改成照實描述）。 + +**做法**：在 `utils/accessibility/backends/base.py` 加 `focused_element(app_name)`，Windows 用 UIA +`GetFocusedElement`、Linux 用 AT-SPI 的 `STATE_FOCUSED`、macOS 用 `AXFocusedUIElement`，再讓 +`_default_fetcher` 改用它；無法取得時退回現在的行為。 + +**要先想清楚**：三個後端都要能在 CI 上用假物件測；macOS 的 AX 呼叫需要 TCC 權限（CI runner 有）。 -| | 最低 | 最高 | -| --- | --- | --- | -| 修正前（`pytest --cov`） | 50.26%（ubuntu-22.04／3.10） | 51.69%（windows-2022／3.14） | -| 修正後（`coverage run`） | **69.67%**（ubuntu-22.04／3.14） | 70.97%（windows-2022／3.12） | - -地板因此設成 **69**——取最低那一格往下取整，與當初 50 取自 50.26% 是同一個慣例。 -`[tool.coverage.report]` 的 `precision` 也從預設的 0 提到 2：預設精度下九格全部印 -「70%」，而它們其實是 69.67 到 70.97，害得這次的地板得去 XML artifact 裡撈。 -順帶把 `fail_under` 的容差從一整個百分點縮到 0.01。 -地板只有一個家（`pyproject.toml` 的 `fail_under`），`quality.yml` 不再另外抄一份。 - -Windows 是高的那一角，因為門面 import 進來的是**它自己那個平台的後端**； -換句話說剩下的那 30 點裡，有一部分是任何單一平台都拿不到的。 - -**還要決定的**：70 是舊的目的地，現在等於已經到了，**下一個目標值該由維護者定**。 -真正還低的是哪幾塊，現在有實測（本機 Windows／3.14，修正後）： -`utils/remote_desktop` 35%、`utils/mcp_server` 34%（`_handlers.py` 自己 10%）、 -`utils/executor` 41%、`utils/accessibility` 29%、`wrapper/window_backends` 10%。 -這四塊的共同形狀是「一大堆薄轉接函式包著已經測過的無頭函式」，所以往上爬的方式 -是走註冊表逐一驅動，而不是一支一支手寫測試。 - -### mypy：整包把關，**豁免清單已經清空** - -`TODO` → **完成（2026-08-22）** - -範圍不再是兩條路徑，而是**整包減去一張只准變少的清單** -（`test/verify/typing_contract_exempt.txt`）。差別在於預設值：路徑清單只有人想到才會長， -新模組預設在圈外；現在新模組**預設就在契約裡**。 - -**2026-08-22 那張清單降到零**：`je_auto_control/` 的 1,018 個檔案在 -win32／linux／darwin 三個目標上全部乾淨。清掉 136 個模組的過程與每一群的做法寫在 -[WHATS_NEW.md](WHATS_NEW.md)；這裡只留下之後還用得到的五件事： - -* **反覆出現的五種形狀**：mixin 讀取宿主的成員（用類別本體裡的 - `if TYPE_CHECKING:` 宣告，執行期會被剝掉）、`self._x = None` 沒有標注 - （mypy 會把屬性的型別判成 `None`）、`callable` 被當成型別用、 - `x: SomeType = None` 的隱含 Optional、以及掉了長度的 tuple。 -* **攔截用的 tuple 必須標成 `Tuple[Type[BaseException], ...]`**，而且要收成一個 - 模組常數——`except (A, B, *TUPLE)` 的星號解包 mypy 跟不進 `except`。 -* **`# type: ignore` 只有當它是那一行的第一個註解時才生效**（已實測），所以有 - `# nosec` 的行要把它放前面。 -* **`cv2` 的 stub 會隨版本變**：`pyproject.toml` 把它列在「ship no stubs 的基礎相依」 - 底下，但 opencv-python 有附 `.pyi`，閘門會去讀。實測 4.13.0：`MSER_create`、 - `ORB_create`、`VideoWriter_fourcc` 執行期都在、stub 裡都沒有。`>=4.8,<6` 範圍內 - 版本一換，判定就可能跟著動——與 numpy 那條註解同一類的坑。 -* **要讓 mypy 剪掉一個分支，整條條件都得是它讀得懂的**：`sys.platform == "..."` - 與 `.startswith("...")` 算，`in [...]` 不算，而只要裡面**混進一個函式呼叫** - （`is_windows()`），`or`／`and` 整條就變成未知、兩邊都會被檢查。所以 - `platform_wrapper` 那種「問 `platform_id` 才知道綁哪個後端」的分支**沒辦法** - 讓自己被剪掉——它綁的三種形狀互不相容，一個型別蓋不住。做法是那兩個名稱進來時 - 先落在私有的 `Any` 上、出去時才標合約：後端那一側在 `_platform_*.py` 被檢查， - 呼叫端那一側在 `auto_control_*.py` 被檢查，中間那一接頭本來就沒有東西可查。 - 細節見 `wrapper/backend_contract.py` 的 docstring。 - -清單現在只有標頭、沒有任何條目。**它變長就是退步**，`typing_contract_verify.py` -會在有人讓它變長時紅掉。 - -#### 已拍板（2026-08-22）：Win32 ctypes 表面用 28 個逐行抑制解決 - -原本這裡是一條 `DECIDE`，寫的是「`windows/` 底下 8 個模組」。重新實測後是 -**16 個模組**，而且**一半不在 `windows/` 底下**（`utils/trash/`、`utils/app_idle/`、 -`utils/file_assoc/`、`utils/idle_keepawake/`、`utils/lock_session/`、 -`utils/session_guard/`、`utils/usb/passthrough/key_provider.py`、 -`gui/main_window.py`）——這一點直接否掉了原本推薦的那一條（照目錄決定用哪個平台量， -分不到這八個）。 - -**維護者選了逐行 `# type: ignore` 附理由**，實際只用了 **28 行**（原本估的 58 -是把同一行在 linux 與 darwin 各算了一次）。做法見 [WHATS_NEW.md](WHATS_NEW.md)， -兩件必須實測的事記在這裡免得再踩： - -* **mypy 只認每一行的第一個註解**——接在既有 `# nosec` 後面的 `# type: ignore` - 完全不生效（已實測）。所以有 `# nosec` 的那兩行，marker 放前面、兩個理由併成一句。 -* 有九行放不進 120 字元，是**改寫**而不是把理由砍到看不懂：括號換行時 marker 跟著 - 左括號走，兩處先把值取出來成區域變數（DPAPI 的 `last_error`、input hook 的 - `kernel32`），讀起來比原本的一行式更清楚。 - -十六個模組事後都在真的 Windows 機器上重新 import 並實際呼叫過 -（`dpapi_available()`、`_windows_locked()`、`check_key_is_press`）—— -只有型別檢查器驗過的改寫等於沒人驗過。 - -有一件事別再踩：**這個閘門的判定不能隨環境浮動**。裝了 `[gui]`／`[webrtc]` 的開發機 -與乾淨的 `pip install -e .` 曾經對 38 個模組看法不同（36 個 Qt 模組只在 PySide6 -*不在*時才過關，2 個只在 babel／pytest 不在時才失敗）。修法是把所有非基礎相依的 -第三方模組壓成 `Any`；其中 `follow_imports = "skip"` 對 `.pyi` 無效、必須同時開 -`follow_imports_for_stubs`，正是 numpy 那條註解早就寫過的坑。 diff --git a/README.md b/README.md index 0114f4df..03ee31d7 100644 --- a/README.md +++ b/README.md @@ -75,6 +75,12 @@ sudo apt-get install cmake libssl-dev OCR, VLM, and LLM backends (`pytesseract`, `easyocr`, `paddleocr`, `anthropic`, `openai`) are loaded on demand — install whichever you actually use. +**Log file:** the library logs to `~/.je_auto_control/logs/AutoControlGUI.log`, +created on the first record (importing alone writes nothing) and shared by every +process on the account (appended to, one process id per line, moved to `.1` +past 10 MB). Set `JE_AUTOCONTROL_LOG_FILE` to write elsewhere, or to +`os.devnull` to turn the file off. + --- ## 60-second quick start @@ -120,11 +126,12 @@ je_auto_control run flow.json --dry-run # list the steps without touching th ```bash pip install je_auto_control[gui] -python -m je_auto_control # or: je_auto_control.start_autocontrol_gui() +python -c "import je_auto_control; je_auto_control.start_autocontrol_gui()" ``` Record a flow, edit it in the visual Script Builder, and save it as the same JSON -format the CLI runs. +format the CLI runs. (`python -m je_auto_control` is the legacy action-file runner — `-e`, `-d`, +`-c`, `--execute_str` — not the GUI.) --- @@ -155,7 +162,7 @@ desktop app; tab commands live in the window's **Actions** menu. | Scheduler (interval + cron) | `default_scheduler` | — | Scheduler | | Global hotkeys | `default_hotkey_daemon` | — | Hotkeys | | Event triggers | `default_trigger_engine` | `AC_email_trigger_add` | Triggers, Webhooks, Email | -| Window management *(Windows)* | `list_windows`, `focus_window` | `AC_focus_window`, `AC_snap_window` | Window Manager | +| Window management *(Windows, macOS, X11)* | `list_windows`, `focus_window` | `AC_focus_window`, `AC_snap_window` | Window Manager | | Clipboard (text + image) | `get_clipboard`, `set_clipboard`, `get_clipboard_image`, `set_clipboard_image` | `AC_clipboard_get`, `AC_clipboard_set`, `AC_clipboard_get_image`, `AC_clipboard_set_image` | — | | Remote desktop | `RemoteDesktopHost`, `RemoteDesktopViewer` | `AC_start_remote_host`, `AC_remote_connect` | Remote Desktop | | USB enumeration & passthrough | `list_usb_devices`, `enable_usb_passthrough` | `AC_usb_*` (16 commands) | USB Devices, USB Share | @@ -189,8 +196,9 @@ je_auto_control version ``` `--var name=value` is parsed as JSON when possible (`count=10` becomes an int), -otherwise kept as a string. The legacy `python -m je_auto_control -e file.json` -entry point still works. +otherwise kept as a string. `run` exits 1 when any action failed (the run +still goes on to the end), so a CI step fails with it. The legacy +`python -m je_auto_control -e file.json` entry point still works. --- @@ -329,8 +337,10 @@ export JE_AUTOCONTROL_WAYLAND_CAPTURE_COMMAND="mycapture --png {output}" Wayland forbids global input recording for unprivileged clients — set `JE_AUTOCONTROL_LINUX_DISPLAY_SERVER=x11` to record on an X11 session. Window -management is currently Windows-only and raises a clear `NotImplementedError` -elsewhere. Opt-in driver-level backends (`JE_AUTOCONTROL_WIN32_BACKEND=interception`, +management works on Windows, macOS (pyobjc) and X11, including XWayland; on a pure +Wayland session, whose protocol hides other clients' windows, `list_windows()` returns +an empty list and every window action raises `AutoControlUnsupportedOperationException` +saying why. Opt-in driver-level backends (`JE_AUTOCONTROL_WIN32_BACKEND=interception`, `JE_AUTOCONTROL_LINUX_BACKEND=uinput`, ViGEm virtual gamepad) exist for apps that ignore synthetic input, and fall back silently when the driver is absent. @@ -345,7 +355,7 @@ ignore synthetic input, and fall back silently when the driver is absent. | [architecture_explore.md](architecture_explore.md) | Every module's responsibility, layer by layer. | | [docs/CAPABILITY_MATRIX.md](docs/CAPABILITY_MATRIX.md) | Capability × platform matrix. | | [docs/API_LIFECYCLE.md](docs/API_LIFECYCLE.md) | Stable-API and deprecation policy. | -| [WHATS_NEW.md](WHATS_NEW.md) | Per-release notes. | +| [docs/updates/](docs/updates/README.md) | Update log: release notes and finished work, one file per month (formerly `WHATS_NEW.md`). | | [CHANGELOG.md](CHANGELOG.md) | Compatibility changelog. | | [SECURITY.md](SECURITY.md) | Security policy and reporting. | diff --git a/README/README_zh-CN.md b/README/README_zh-CN.md index 8f153676..0f273a8a 100644 --- a/README/README_zh-CN.md +++ b/README/README_zh-CN.md @@ -65,6 +65,11 @@ sudo apt-get install cmake libssl-dev OCR、VLM 与 LLM 后端（`pytesseract`、`easyocr`、`paddleocr`、`anthropic`、`openai`） 都是按需加载——只装你实际会用到的。 +**日志文件：** 库写到 `~/.je_auto_control/logs/AutoControlGUI.log`，第一条记录时 +才创建（只 import 不会写任何文件），同一个账户的所有进程共用（追加写入、每行带进程 ID， +超过 10 MB 就改名为 `.1`）。要写到别处就设置 +`JE_AUTOCONTROL_LOG_FILE`，设为 `os.devnull` 则不写文件。 + --- ## 60 秒上手 @@ -110,10 +115,11 @@ je_auto_control run flow.json --dry-run # 只列出步骤，不会真的动 ```bash pip install je_auto_control[gui] -python -m je_auto_control # 或：je_auto_control.start_autocontrol_gui() +python -c "import je_auto_control; je_auto_control.start_autocontrol_gui()" ``` 录制一段流程、在可视化 Script Builder 里编辑，然后存成 CLI 能直接执行的同一种 JSON 格式。 +（`python -m je_auto_control` 是旧式的动作文件执行器——`-e`、`-d`、`-c`、`--execute_str`——不会打开 GUI。） --- @@ -144,7 +150,7 @@ python -m je_auto_control # 或：je_auto_control.start_autocontrol_gui | 调度（间隔 + cron） | `default_scheduler` | — | Scheduler | | 全局热键 | `default_hotkey_daemon` | — | Hotkeys | | 事件触发 | `default_trigger_engine` | `AC_email_trigger_add` | Triggers、Webhooks、Email | -| 窗口管理 *(仅 Windows)* | `list_windows`、`focus_window` | `AC_focus_window`、`AC_snap_window` | Window Manager | +| 窗口管理 *(Windows、macOS、X11)* | `list_windows`、`focus_window` | `AC_focus_window`、`AC_snap_window` | Window Manager | | 剪贴板（文本 + 图片） | `get_clipboard`、`set_clipboard`、`get_clipboard_image`、`set_clipboard_image` | `AC_clipboard_get`、`AC_clipboard_set`、`AC_clipboard_get_image`、`AC_clipboard_set_image` | — | | 远程桌面 | `RemoteDesktopHost`、`RemoteDesktopViewer` | `AC_start_remote_host`、`AC_remote_connect` | Remote Desktop | | USB 枚举与直通 | `list_usb_devices`、`enable_usb_passthrough` | `AC_usb_*`（16 个命令） | USB Devices、USB Share | @@ -177,7 +183,7 @@ je_auto_control version ``` `--var name=value` 会尽量以 JSON 解析（`count=10` 会变成整数），否则视为字符串。 -旧版 `python -m je_auto_control -e file.json` 入口仍然可用。 +`run` 只要有任何动作失败就以 1 退出（仍会跑完整份脚本），CI 步骤会随之失败。旧版 `python -m je_auto_control -e file.json` 入口仍然可用。 --- @@ -298,8 +304,9 @@ export JE_AUTOCONTROL_WAYLAND_CAPTURE_COMMAND="mycapture --png {output}" ``` Wayland 禁止非特权客户端进行全局输入录制——若要录制，请设置 -`JE_AUTOCONTROL_LINUX_DISPLAY_SERVER=x11` 并在 X11 会话下运行。窗口管理目前仅 -Windows 有实现，其他平台会抛出明确的 `NotImplementedError`。对于会忽略合成输入的应用， +`JE_AUTOCONTROL_LINUX_DISPLAY_SERVER=x11` 并在 X11 会话下运行。窗口管理支持 +Windows、macOS（pyobjc）与 X11（含 XWayland）；纯 Wayland 会话的协议不让客户端看到其他程序的窗口， +所以 `list_windows()` 返回空列表，其余窗口操作一律抛出带原因的 `AutoControlUnsupportedOperationException`。对于会忽略合成输入的应用， 可选用驱动层后端（`JE_AUTOCONTROL_WIN32_BACKEND=interception`、 `JE_AUTOCONTROL_LINUX_BACKEND=uinput`、ViGEm 虚拟手柄）；驱动未安装时会自动回退到原有行为。 @@ -314,7 +321,7 @@ Windows 有实现，其他平台会抛出明确的 `NotImplementedError`。对 | [architecture_explore.md](../architecture_explore.md) | 逐层记录每个模块的职责。 | | [docs/CAPABILITY_MATRIX.md](../docs/CAPABILITY_MATRIX.md) | 能力 × 平台对照矩阵。 | | [docs/API_LIFECYCLE.md](../docs/API_LIFECYCLE.md) | 稳定 API 与弃用策略。 | -| [WHATS_NEW.md](../WHATS_NEW.md) | 各版本更新说明。 | +| [docs/updates/](../docs/updates/README.md) | 更新记录：各版本说明与完成的工作，每月一个文件（原 `WHATS_NEW.md`）。 | | [CHANGELOG.md](../CHANGELOG.md) | 兼容性变更记录。 | | [SECURITY.md](../SECURITY.md) | 安全策略与报告方式。 | diff --git a/README/README_zh-TW.md b/README/README_zh-TW.md index 7c15911a..cf0f28f3 100644 --- a/README/README_zh-TW.md +++ b/README/README_zh-TW.md @@ -65,6 +65,11 @@ sudo apt-get install cmake libssl-dev OCR、VLM 與 LLM 後端（`pytesseract`、`easyocr`、`paddleocr`、`anthropic`、`openai`） 都是按需載入——只裝你實際會用到的。 +**記錄檔：** 函式庫寫到 `~/.je_auto_control/logs/AutoControlGUI.log`，第一筆記錄時 +才建立（只 import 不會寫任何檔），同一個帳號的所有行程共用（附加寫入、每行帶行程 ID， +超過 10 MB 就改名成 `.1`）。要寫到別處就設定 +`JE_AUTOCONTROL_LOG_FILE`，設成 `os.devnull` 則不寫檔。 + --- ## 60 秒上手 @@ -110,10 +115,11 @@ je_auto_control run flow.json --dry-run # 只列出步驟，不會真的動 ```bash pip install je_auto_control[gui] -python -m je_auto_control # 或：je_auto_control.start_autocontrol_gui() +python -c "import je_auto_control; je_auto_control.start_autocontrol_gui()" ``` 錄製一段流程、在視覺化 Script Builder 裡編輯，然後存成 CLI 能直接執行的同一種 JSON 格式。 +（`python -m je_auto_control` 是舊式的動作檔執行器——`-e`、`-d`、`-c`、`--execute_str`——不會開 GUI。） --- @@ -144,7 +150,7 @@ python -m je_auto_control # 或：je_auto_control.start_autocontrol_gui | 排程（間隔 + cron） | `default_scheduler` | — | Scheduler | | 全域熱鍵 | `default_hotkey_daemon` | — | Hotkeys | | 事件觸發 | `default_trigger_engine` | `AC_email_trigger_add` | Triggers、Webhooks、Email | -| 視窗管理 *(僅 Windows)* | `list_windows`、`focus_window` | `AC_focus_window`、`AC_snap_window` | Window Manager | +| 視窗管理 *(Windows、macOS、X11)* | `list_windows`、`focus_window` | `AC_focus_window`、`AC_snap_window` | Window Manager | | 剪貼簿（文字 + 影像） | `get_clipboard`、`set_clipboard`、`get_clipboard_image`、`set_clipboard_image` | `AC_clipboard_get`、`AC_clipboard_set`、`AC_clipboard_get_image`、`AC_clipboard_set_image` | — | | 遠端桌面 | `RemoteDesktopHost`、`RemoteDesktopViewer` | `AC_start_remote_host`、`AC_remote_connect` | Remote Desktop | | USB 列舉與直通 | `list_usb_devices`、`enable_usb_passthrough` | `AC_usb_*`（16 個指令） | USB Devices、USB Share | @@ -177,7 +183,7 @@ je_auto_control version ``` `--var name=value` 會盡量以 JSON 解析（`count=10` 會變成整數），否則視為字串。 -舊版 `python -m je_auto_control -e file.json` 進入點仍然可用。 +`run` 只要有任何動作失敗就以 1 結束（仍會跑完整份腳本），CI 步驟會跟著失敗。舊版 `python -m je_auto_control -e file.json` 進入點仍然可用。 --- @@ -299,8 +305,9 @@ export JE_AUTOCONTROL_WAYLAND_CAPTURE_COMMAND="mycapture --png {output}" ``` Wayland 禁止非特權用戶端進行全域輸入錄製——若要錄製，請設定 -`JE_AUTOCONTROL_LINUX_DISPLAY_SERVER=x11` 並在 X11 session 下執行。視窗管理目前僅 -Windows 有實作，其他平台會拋出明確的 `NotImplementedError`。對於會忽略合成輸入的應用程式， +`JE_AUTOCONTROL_LINUX_DISPLAY_SERVER=x11` 並在 X11 session 下執行。視窗管理支援 +Windows、macOS（pyobjc）與 X11（含 XWayland）；純 Wayland session 的協定不讓用戶端看到別的程式的視窗， +所以 `list_windows()` 回傳空清單，其餘視窗操作一律拋出帶原因的 `AutoControlUnsupportedOperationException`。對於會忽略合成輸入的應用程式， 可選用驅動層後端（`JE_AUTOCONTROL_WIN32_BACKEND=interception`、 `JE_AUTOCONTROL_LINUX_BACKEND=uinput`、ViGEm 虛擬手把）；驅動未安裝時會自動退回原本行為。 @@ -315,7 +322,7 @@ Windows 有實作，其他平台會拋出明確的 `NotImplementedError`。對 | [architecture_explore.md](../architecture_explore.md) | 逐層記錄每個模組的職責。 | | [docs/CAPABILITY_MATRIX.md](../docs/CAPABILITY_MATRIX.md) | 能力 × 平台對照矩陣。 | | [docs/API_LIFECYCLE.md](../docs/API_LIFECYCLE.md) | 穩定 API 與棄用政策。 | -| [WHATS_NEW.md](../WHATS_NEW.md) | 各版本更新說明。 | +| [docs/updates/](../docs/updates/README.md) | 更新紀錄：各版本說明與完成的工作，每月一個檔（原 `WHATS_NEW.md`）。 | | [CHANGELOG.md](../CHANGELOG.md) | 相容性變更記錄。 | | [SECURITY.md](../SECURITY.md) | 安全政策與回報方式。 | diff --git a/WHATS_NEW.md b/WHATS_NEW.md index dcc58e16..08ecc56c 100644 --- a/WHATS_NEW.md +++ b/WHATS_NEW.md @@ -1,3659 +1,25 @@ # What's New — AutoControl -## What's new (2026-08-21) - -### Two Quality Gates That Had Been Standing Still - -`Progress.md` carried an entry called "two thresholds we agreed to climb and -never did". Both were promises that lived only in a `pyproject.toml` comment, -with nothing anywhere that would ever move them. Both now have a mechanism. - -**Coverage: the floor was 15 points below what the tests already earn.** -`fail_under = 35` was the first measured baseline and then never moved while the -suite grew past it. Every square of the nine-way matrix is over 50% — measured -on the run for PR #484, from 50.26% (ubuntu-22.04 / 3.10) to 51.69% -(windows-2022 / 3.14) — so CI would have passed a change that deleted a third of -the tests without a word. The floor is now **50**, taken from the lowest square -rather than from one machine, and the comment states the rule that was missing: -this is a ratchet, raised whenever the suite has earned it, not a target to -admire. 70 is still the destination. - -**mypy: the scope was two directories, and could only ever grow by hand.** -The job checked `je_auto_control/api` and `je_auto_control/utils/failure_bundle` -— 4 files — while a comment promised the rest would join "the contract" later. -A scope written as a path list never grows on its own, and every new module -lands outside it by default. - -So the scope is inverted. mypy now checks **the whole package**, and the modules -that do not pass yet are named in `test/verify/typing_contract_exempt.txt`. -**862 of 1,017 files are inside the contract today**, a new module is inside it -the moment it is written, and the list may only shrink: -`test/verify/typing_contract_verify.py` fails just as loudly when a listed -module starts passing (delete the line) as when an unlisted one stops. Both -directions were checked by breaking them. - -**And it checks three platforms, not one.** mypy resolves `sys.platform` -branches against a single target, so the Ubuntu-only job never looked inside -the Windows, macOS or platform-gated code — three of the four backends. That -blind spot was real and is now measured: 13 modules fail only when the target is -Linux, 3 only when it is Windows. The script runs `win32`, `linux` and `darwin` -and unions the results, so the Windows and macOS backends are type-checked from -the Ubuntu runner. - -**The part that took the work: the gate has to mean one thing.** A first -measurement taken on a dev checkout disagreed with a bare `pip install -e .` -about **38 modules** — 36 Qt modules that pass only because PySide6 is *absent*, -and 2 that fail only because `babel` and `pytest` are. Committing that list -would have reddened CI on arrival and mis-listed the other 36. A gate that flips -on `pip install` is not a gate, so every third-party module outside the base -dependency set is now forced to `Any`. `ignore_missing_imports` alone was not -enough — it still lets mypy read the real package when it *is* installed — and -neither was `follow_imports = "skip"`, which is silently ignored for `.pyi` -files: PySide6 ships inline stubs, so `follow_imports_for_stubs` was required -too. That is the same trap the numpy override in `pyproject.toml` had already -paid for and written down. A dev checkout with `[gui]` and `[webrtc]` and a -clean base install now produce identical results. - -The remaining modules cluster — `utils/remote_desktop` (13), `gui` (11) — and -`Progress.md` records clearing them one cluster at a time. The first cluster is -below. - -### The Platform Seam Now Says What a Backend Is - -`wrapper/platform_wrapper.py` is the Strategy hub: it imports exactly one -backend and re-exports eight names, and everything above it is written against -those names rather than against a platform. Nothing said what they *were*. - -Measured, that had two consequences, and neither was theoretical: - -- **mypy bound every name to the Windows backend, on every target.** The - branches are `if is_windows() / elif is_macos() / …`, which a type checker - cannot resolve, so it read all of them and kept the first — so the layer above - the seam was checked against Win32 signatures even when the target was Linux - or macOS, and it reported "`recorder` has type `OSXRecorder`, expected - `Win32Recorder`" for the *correct* code on the way past. -- **A backend could omit a member and nobody would say so** until a call site - three layers up failed on the user's machine. - -The eight names are now declared before the branches bind them, three of them -with protocols in the new `wrapper/backend_contract.py` — `ScreenBackend`, -`KeyboardCheckBackend`, `RecorderBackend` — and each `_platform_*` assembly -module annotates what it assigns. A backend that does not answer the seam's -questions now fails in its own file, naming the missing member. That is not -hypothetical either: turning it on immediately reported that the Windows -`screen.size()` returns a `list` where the other three backends return a -`tuple` and where the public `screen_size()` promises a tuple. Fixed, and every -caller only ever unpacked the two values. - -`keyboard` and `mouse` stayed `Any` for now, which is what mypy had already -inferred for them: macOS takes `is_shift` on `press_key` and orders its mouse -calls `(x, y, button)` where Windows and X11 take the button alone, and a -Windows mouse "keycode" is a tuple of three event flags where the others are an -int. One protocol cannot describe both. They got three each on 2026-08-23; -that entry is below. - -Clearing the cluster fixed four bugs the types had been hiding, all of the same -shape — a value that could be `None` reaching something that could not take one. -They are listed in `CHANGELOG.md`. Along the way four modules outside the -cluster (`utils/cv2_utils/screen_grabber`, `utils/executor/mouse_aliases`, -`utils/pytest_plugin/keywords`, `utils/vision/vlm_api`) went green on their own: -they had been failing on the seam's accidental types, not on their own code. -**145 modules left, 872 of 1,017 files inside the contract.** - -Re-measuring `architecture_explore.md` for this change turned up one row the -measuring tool had never been able to read: §8's `wrapper/` row carried prose -in its 行數 cell, so it parsed as a one-column row — the tool wrote the *line* -count into the 檔案數 column, and left the directory out of the named subtotal, -which meant 其餘模組 counted it a second time. The row is now two plain numbers -(19 files, 3,293 lines), the note it was carrying has moved to §5.2 as the -`wrapper/window_backends/` row that section had been missing entirely, and -其餘模組 no longer double-counts 3,293 lines. - -### The Coverage Number Had Been 24 Points Low, and Not Because of the Tests - -`fail_under` went from 35 to 50 two days ago because the matrix was measured -and 35 had been left behind. **Every number in that measurement was about 24 -points low**, and the reason is not in the test suite at all. - -`quality.yml` measured with `pytest --cov`, and this package registers a -`pytest11` entry point. pytest imports -`je_auto_control.utils.pytest_plugin.plugin` while it loads plugins — and to -import that submodule Python must first execute `je_auto_control/__init__.py`, -the facade, which pulls in several hundred modules. pytest-cov starts measuring -*after* plugin loading, so all of those modules had their import-time lines — -`def` lines, class bodies, constants, the two big dispatch tables — recorded as -never executed. - -Measured on one machine, same suite, same `[tool.coverage.run]` config, the -only difference being when measurement starts: - -| how | total | -| --- | ---: | -| `pytest --cov=je_auto_control` | 52.22% | -| `coverage run -m pytest` | **72.05%** | - -11,962 statements, and the files hit hardest were the biggest ones: -`action_executor.py` +786, `_handlers.py` +684, the facade itself +369, -`_factories.py` +209. 70 was the destination `Progress.md` had been aiming at -and describing as twenty points away; the tests had already passed it. - -**A package that registers a pytest plugin cannot measure itself with -`pytest --cov`.** `quality.yml` now runs `coverage run -m pytest`, which starts -before pytest loads anything, writes the XML before enforcing the floor so a -failing square still uploads what it was short of, and no longer carries a -second copy of the floor: `fail_under` in `pyproject.toml` is the only one. -`test/unit_test/headless/test_coverage_measurement.py` pins all three, because -the difference between the two spellings is invisible in a green build — -reverting gives back 24 points and every job still passes. - -The corrected matrix, measured on this PR's own run: - -| | lowest | highest | -| --- | --- | --- | -| before (`pytest --cov`) | 50.26% (ubuntu-22.04 / 3.10) | 51.69% (windows-2022 / 3.14) | -| after (`coverage run`) | **69.67%** (ubuntu-22.04 / 3.14) | 70.97% (windows-2022 / 3.12) | - -So the floor is **69**, floored from the lowest square exactly as 50 was -floored from 50.26. `precision` goes to 2 in the same commit, because at the -default all nine squares printed "70%" while actually running 69.67 to 70.97 — -the floor had to be read out of the XML artifacts rather than the log — and -because coverage lets a total within `10**-precision` of the floor pass, which -at precision 0 is a whole point of slack under something called a ratchet. - -Windows is the high corner because the facade imports *its own* platform's -backend, so part of the remaining 30 points is unreachable from any single -square. What is genuinely low is now measured rather than assumed: -`utils/remote_desktop` 35%, `utils/mcp_server` 34% (`_handlers.py` alone at -10%), `utils/executor` 41%, `utils/accessibility` 29%, -`wrapper/window_backends` 10% — four subsystems with the same shape, a great -many thin adapters wrapping headless functions that are already tested, which -is a registry to walk rather than a pile of tests to hand-write. - -### The Last Two Names on the Platform Seam, and the Three Bugs Behind Them - -`keyboard` and `mouse` were the two exports the seam still typed as `Any`, and -the two called most. They are typed now — not with one protocol each but with -three, because their call shape is genuinely platform-specific: `Win32*` -(SendInput and the Interception driver), `Darwin*` (Quartz) and `X11Unix*` -(XTest, uinput, Wayland and the BSDs, which share one shape). Each backend -module is checked against *its own* protocol on every target, so the macOS -mouse is verified from an Ubuntu runner, and `KeyboardBackend` / `MouseBackend` -alias whichever pair matches the target — which is what makes a caller checked -against the signature it will really reach. `mouse.press_mouse(x, y, button)` -is only type-correct on darwin, and only there is it the branch mypy walks into. - -**The seam module itself cannot be typed, and that is the point.** -`platform_wrapper` picks its backend by asking `platform_id`, and a call to a -function is something mypy cannot resolve — so its branches are not pruned, and -all four are read on every target. With one type on `mouse` that means three -mutually incompatible shapes assigned to one name: measured, 9 errors across -the three targets, all of them correct. So those two names now land on private -`Any`s on the way in and take their contract on the way out. Both ends are -still checked — the backend in `_platform_*.py`, the caller in -`auto_control_*.py` — and the joint in the middle never had anything to check -that the two ends do not already cover. - -Turning it on reported three things that were wrong rather than untyped: - -- **An unknown scroll direction was handed to the backend as a string.** - `special_mouse_keys_table.get(name, name)` fell back to the *name* when the - axis table had no such entry, and all three backends behind that branch take - an int: `int('scroll_upp')` on Wayland and uinput, an Xlib error on X11, and - in every case the offending name nowhere in the message. The button table had - refused an unknown name at this boundary since it was written; the axis table - did not. It does now, with the name in the exception. -- **`type_unicode_unit` was called on backends that do not have it.** - `text_unicode._default_sink` called it outright, so the three platforms - without Unicode injection raised `AttributeError` — outside the - `AutoControlException` family that the executor, the poll loops and the - request handlers each catch in one `except`, so it escaped every containment - boundary in the project. It now asks the way `unicode_keys_supported()` - already did and raises `AutoControlKeyboardException` saying which route to - use instead. The seam does not promise the member; only Windows has it. -- **The resolved button code was being narrowed back to what came in.** - `press_mouse`/`release_mouse`/`click_mouse` assigned `mouse_preprocess`'s - result over their own `Union[int, str]` parameter, so the platform button - code — a tuple of three Win32 flags, or an int — was typed as the *name* the - caller passed. It lands in its own local now. - -**One branch in `auto_control_mouse` spells OS names, and only one.** Windows -and macOS take `scroll(value)` where X11 and Wayland take `scroll(value, axis)`, -so unless one side is pruned each signature fails against the other's call — -and a `platform_id` call prunes nothing. That branch is `sys.platform ==` four -times over, exactly `is_windows()`'s names plus `is_macos()`'s, and -`test_wrapper_seam_contract.py` now pins the behaviour on each of the six -platform names so the list cannot quietly lose one. Everything else in the file -still asks which input stack it is. - -All sixteen backend modules already satisfied their protocols — the four -assembly modules plus the Interception and uinput alternatives — which is the -one result worth stating plainly: the shapes were consistent, nothing said so, -and now something does. The exemption list is still empty, on all three targets. -The seam was re-exercised on real Windows afterwards (both input backends -selected, cursor read, position set, a real key press and release, a real -scroll); a rewrite only a type checker has seen is a rewrite nobody has seen. - -### The Wayland Cluster: Four Modules, One Invariant Nobody Had Written Down - -The next cluster off the typing list is `linux_wayland` — `libei`, `_detect`, -`capture`, `screen` — and thirty-three of its forty errors were one sentence -repeated: `Item "None" of "BoundSymbols | None" has no attribute ei_…`. - -`LibeiBackend` holds its resolved entry points as `Optional[BoundSymbols]` -(`None` on a host without libei) and read them straight off that attribute at -every call site. Every one of those sites is in fact reached only after a guard -— `connect()` refuses an unavailable backend, `_emit` refuses a disconnected -one — so nothing was broken here. But the guarantee lived in the call graph -rather than anywhere a reader or a checker could see it, and what a new call -site that skipped the guard would raise is `AttributeError`: not an -`AutoControlException`, and therefore straight through every containment -boundary in the framework. The entry points now come through one `_api` -property that raises `LibeiUnavailable` — which is what this module's own -docstring says every failure in it raises. `_teardown` is the deliberate -exception: it runs from an `except BaseException` handler, so it narrows the -attribute itself rather than risking a raise that would replace the real -failure with a complaint about the symbol table. - -The other three modules were each one honest disagreement. `_detect`'s two -environment probes were annotated `dict` while both of them default to -`os.environ`, which is a `Mapping` — so one of them could not legally pass its -own environment to the other. `capture._write_to_temp_png` declared a writer -returning `None` while every caller passes one returning the tool's stdout, -which it discards because what it reads is the file. And `screen.get_pixel` -handed Pillow's `getpixel` union — a float for mode `F`, `None` for an empty -band — to callers that unpack three ints, though `grab_image` has always -converted to RGB first. **141 modules left, 876 of 1,017 files inside the -contract.** - -### The Three Backends That Only Needed Four Sentences - -`linux_with_x11` (3 modules) and `osx` (1) went the same day, and between them -they held five errors — but one was a live bug of a shape this branch has now -fixed several times. - -`KeypressHandler.record_queue` was assigned `None` in `__init__` with no -annotation, so its *type* was `None`: `record()` could not legally fill it, and -`stop_record()` promised a `Queue` while able to hand back the `None` it was -constructed with. Stopping a recording that was never started therefore reached -`x11_linux_record`, which reads `.queue` off the result — `AttributeError`, one -frame away from where the mistake was. It now returns an empty queue, which is -what "nothing was recorded" looks like and what `stop_record()` on the public -API already returns for the same case. - -`osx_keyboard.press_key` takes `int | str` and sends a string to -`special_key`. Testing `keycode in special_key_table` narrows the string case -*into* that branch but leaves `int | str` outside it — so a name the table does -not know fell through to `normal_key` and reached Quartz as a keycode. A string -only ever names a special key here, so that is now what the test asks, and an -unknown name gets `special_key`'s "Unknown special key" rather than a pyobjc -type error three frames down. - -The last one is a checker fact rather than a code fact: `uinput/_device` opens -`/dev/uinput` with the POSIX-only `O_NONBLOCK`, and the contract checks this -package against a Windows target too, where the `os` stub does not declare it. -The flag moved into a `sys.platform` branch mypy can prune, which states the -Linux-only-ness rather than silencing the question. **137 modules left, 880 of -1,017 files inside the contract.** - -### A Module Nobody Could Import, and a Line That Edited the Standard Library - -The Windows cluster turned out to hold the two most interesting findings on -this branch, and neither is a typing nicety. - -`je_auto_control.windows.message.window_message` did -`from ...windows_window_manage import FindWindowW`. That module has no such -name — `FindWindowW` is a method on its *private* `user32` handle — so -importing `window_message` raised `ImportError`, on every Windows machine, -since whenever the name was moved. Nothing noticed because the only importer -in the tree is a manual test. It now calls that module's public -`get_one_window_hwnd`, which is also the one carrying the argtypes that keep a -64-bit HWND from being truncated to `c_int`. - -`win32_ctype_input` ran `wintypes.ULONG_PTR = wintypes.WPARAM` at import — a -write into the standard library's own module namespace. Nothing in this package -reads `ULONG_PTR` back; measured, the name appears exactly once in the tree, on -that line. So the only thing the assignment could do was answer for some other -library in the same process that asked `ctypes.wintypes` whether it has -`ULONG_PTR`. Deleted. - -The same file also carried annotations that were wrong rather than merely -unhelpful: `_fields_: tuple` redeclares a ctypes `ClassVar` as an instance -variable, and `ctypes.POINTER` and `user32.SendInput` were used as types when -one is a function and the other a value. Dropping all three leaves exactly what -mypy infers, which was right all along. - -**Every module under `je_auto_control/windows/` now type-checks on the Windows -target.** Eight of them stay on the exemption list anyway, for a reason that is -not about them: the gate checks the package against Linux and macOS targets -too, where typeshed does not declare the Win32-only corner of `ctypes` -(`windll`, `WinDLL`, `WINFUNCTYPE`, `WinError`, `get_last_error`). Three -remedies were measured, one of them ruled out — pruning the module body makes -every *importer* fail with `has-type` instead — and `Progress.md` carries the -comparison as a `DECIDE`, because the cleanest of them changes what the gate -means rather than what the code says. **136 modules left, 881 of 1,017 files -inside the contract.** - -### The Typing Contract's Exemption List Is Empty - -`je_auto_control` type-checks clean on all three targets — win32, linux and -darwin — with nothing exempted. The list that started this branch at 155 modules -now holds a header and no entries, and `typing_contract_verify.py` fails if it -ever grows again. - -The last module was `gui/remote_desktop/webrtc_panel.py`, and it was blocked by -its own size rather than by its types. Seven of its errors came from -`_build_advanced_group(panel: TranslatableMixin, …)`, a free function that -*writes* five widget attributes back onto the panel it is handed — none of which -`TranslatableMixin` has. Writing that contract down needs a Protocol, and the -file was sitting exactly on the 2,555-line cap it may only shrink from. - -So the builder moved out, which is what `Progress.md` had said that file owed -anyway: `gui/remote_desktop/advanced_group.py` now holds the shared -STUN/TURN group, the `AdvancedGroupHost` protocol naming what it reads and what -it sets, and the hardware-codec row as its own function. The panel is 2,545 -lines — under its cap for the first time — and both panels were rebuilt -offscreen afterwards to confirm the STUN default, the TURN fields and the -host-only codec picker all still arrive where they did. - -Ten more errors in that file were the pattern the whole sweep kept meeting: a -handle that is `None` until the session starts. `_produce_offer`, -`_trust_session_viewer`, `_answer_and_push`, `_produce_answer` and the folder -sync all reached through `self._multi_host` / `self._viewer` without asking. They -go through `_require_multi_host()` / `_require_viewer()` now, which raise a -translated "start hosting first" / "connect to a host first" instead of an -`AttributeError` on `None` — two new keys in all four language catalogues. - -**0 modules left. 1,018 of 1,018 files inside the contract.** - -### Thirteen Small Modules, and a Stub That Disagrees With the Library - -Past the big clusters the list is a long tail: thirteen modules of two to six -errors each, cleared in one pass. Three recurring shapes, all of them cheap: - -- **`callable` used as a type.** It is the builtin *function*, so mypy reads - every call through the annotated value as calling something not callable. - `plugin_loader` had it six times, both hotkey backends once each. -- **`x: SomeType = None` defaults**, which PEP 484 prohibits and - `no_implicit_optional` rejects: the three `window_zorder` drivers. -- **A tuple that lost its length.** `tuple(r)` and `cv2.boundingRect(...)` are - `tuple[int, ...]`, and the fields they feed promise four ints. Spelling the - four out is both the fix and the documentation. - -Two are worth naming on their own: - -**`_StabilityTracker` read `now - self._since` on a path where `_since` could -only be non-`None` because of what an earlier call did.** True today, invisible -to a reader, and one refactor away from a `TypeError` in the poll loop. It -binds the value and treats "no start time" as "not stable yet". - -**`act_when_ready` passed `report.point` to a callback that requires a point.** -`point` is `None` whenever the target is invisible; the guard above it tests -`report.actionable`, which implies visible — again true, again only through the -call graph. The point is now checked where it is used. - -**And the cv2 stub disagrees with the cv2 that ships with it.** -`text_regions` calls `cv2.MSER_create`, which exists in every supported OpenCV -at runtime but is absent from the `.pyi` opencv-python installs (measured on -4.13.0: `hasattr(cv2, "MSER_create")` is `True`, the name is not in the stub). -That is one justified `type: ignore` — and a note for whoever next reads the -mypy config: cv2 is listed there under "base dependencies that ship no stubs", -but it does ship one, so the gate reads it and its verdict can move with the -OpenCV version inside the `>=4.8,<6` pin. - -**56 modules left, 961 of 1,017 files inside the contract.** - -### `normalize_url` Had Never Worked, on Either Surface - -The MCP cluster came off next, and the gate found a command that could not -succeed. `AC_normalize_url` and `ac_normalize_url` both forward to -`url_canon.normalize_url`, and both passed `drop_fragment=` — the name they -expose to callers. The function's parameter is `strip_fragment`. Every call, -with or without that flag, raised -`TypeError: normalize_url() got an unexpected keyword argument 'drop_fragment'`: -in the executor, in the MCP tool, and from the Script Builder field that feeds -them. The outward name is unchanged (it is in the action schema and the tool -registry); the two call sites now pass it through under the name the callee -uses. Measured before and after: the MCP tool returns -`{"url": "https://example.com/b"}` where it used to return an error. - -The rest of the cluster was the shapes this branch keeps meeting: - -- **`ClientRequestMixin` borrowed seven attributes from `MCPServer`** and listed - all seven in its docstring; that list is a `TYPE_CHECKING` declaration now. -- **Two catch tuples again.** `_DISPATCH_ERRORS` and `_TOOL_INVOKE_ERRORS` are - the containment boundary for the whole stdio loop, and neither was typed as a - tuple of exception classes, so all three `except` sites were errors. -- **`_dispatch` fed an `Optional[str]` method name to `dict.get`.** A JSON-RPC - request with no `method` now takes the not-found branch explicitly, with the - same `-32601` response body it produced by falling through. -- **The subscription callback was a default-argument lambda** - (`lambda u=uri: …`), which mypy cannot infer against a `Callable[[], None]` - parameter. `functools.partial` binds `uri` the same way and says the type. - -Two public return annotations were also wrong in the safe direction: -`set_mouse_position` and `hotkey` are declared `... | None` but every path -either returns the tuple or raises. Narrowing them is what let the MCP handlers -stop indexing an Optional. `get_mouse_position` keeps its `| None` — the Windows -backend really does return that. - -**69 modules left, 948 of 1,017 files inside the contract.** - -### The GUI Cluster: Three Real Failures Behind the Mixin Noise - -Thirteen of the fourteen `gui` modules came off the list. Most of the eighty-nine -errors were the mixin shape already fixed twice on this branch — six tab mixins -read `self._tr`, `self._translate` and `self.timer` off a host they never -declared, and every one of them said so in its own docstring -("Requires the host widget to expose…"). Those docstrings are now -`if TYPE_CHECKING:` declarations, stripped at runtime. - -Underneath them were three things that fail for a user, not for a checker: - -- **A pixel assertion with one coordinate reported the wrong problem.** - `assertions_tab` called - `assert_pixel(*_parse_ints(self._xy.text())[:2], _parse_ints(self._rgb.text()), …)`. - Type `5` instead of `5,6` and the star-unpack contributes one argument, so the - RGB list binds to `y`, `match=` and `raise_on_fail=` collide with the - positional slots, and the user sees a `TypeError` about duplicate keyword - arguments. The count is checked first now, and the message names what is - missing. -- **`_get_mouse_pos` unpacked a value the Windows backend really does return - as `None`.** `win32_ctype_mouse_control.position()` returns `None` when - `GetCursorPos` fails — which is what happens on a locked or secure desktop — - and the tab did `x, y = get_mouse_position()`. The existing `except TypeError` - caught it and showed "cannot unpack non-sequence NoneType object". It raises - `AutoControlException` with a sentence instead. -- **`multi_language_wrapper` typed its listener list `List[callable]`.** - `callable` is the builtin *function*, not a type, so mypy read every - `listener(language)` call as calling something that is not callable. It is - `List[Callable[[str], None]]` now. - -`recording_edit.editor` came along with them: both of its optional parameters -were written `end: int = None`, which PEP 484 prohibits and `no_implicit_optional` -rejects. - -**`webrtc_panel.py` is the one that stayed**, and its reason is now in -`Progress.md` rather than in nobody's head. Seven of its twenty-seven errors -come from `_build_advanced_group(panel: TranslatableMixin, …)`, a free function -that *writes* five widget attributes back onto the panel — none of which -`TranslatableMixin` has. The correct type is a Protocol naming what it reads and -writes, and the file is sitting exactly on its 2,555-line cap, which may only -shrink. The real fix is the split that file already owes: `_build_advanced_group` -is a shared widget-group builder that does not belong in the panel module, and -moving it out settles the length and the type in one go. - -**73 modules left, 944 of 1,017 files inside the contract.** - -### The WinUSB Backend Was One Failed DLL Load Away From Never Recovering - -With the ctypes surface settled, the two clusters behind it came off: -`utils/usb/passthrough` and `utils/clipboard`. Both were the same mistake told -two ways — a handle whose declared type could not do what the code asks of it — -and both hid a real defect behind it. - -**`winusb_backend` published its three DLL handles one at a time.** `_load_dlls` -assigned `_setupapi`, then `_winusb`, then `_kernel32`, guarded by -`if _setupapi is not None: return`. If loading `winusb.dll` raised — which is -exactly what happens on a machine where no device is bound to WinUSB and the -DLL is absent — `_setupapi` was already set, so the guard short-circuited every -later attempt and every call site got -`AttributeError: 'NoneType' object has no attribute 'WinUsb_Initialize'` -instead of the retry the guard was written to allow. The three handles now come -back from one loader as a `NamedTuple`, published only after all three load. -Two smaller ones went with it: a device enumerated without an interface path is -skipped rather than passed to `CreateFileW` as `None`, and a `WinUsb_Initialize` -that reports success with a null handle is now a failure rather than an -`Optional[int]` handed to the handle wrapper. - -**`clipboard_api()` returned `Tuple[object, object]`.** `object` has no -attributes, so all twelve `user32.OpenClipboard` / `kernel32.GlobalLock` calls -through it were type errors — on a module whose entire docstring is about -getting these prototypes right once. A ctypes library resolves every symbol -through `__getattr__`, so `Any` is the only honest promise, and it is what the -signature says now. - -Both were exercised against the real thing on Windows afterwards: a clipboard -text round trip, `clipboard_formats()` against a live clipboard, and the WinUSB -backend enumerating an actual bound device. **86 modules left, 931 of 1,017 -files inside the contract.** - -### The Win32 ctypes DECIDE, Settled — and It Was Twice the Size It Said - -`Progress.md` carried a `DECIDE` about eight modules under -`je_auto_control/windows/` that pass on `--platform win32` and fail on the other -two targets for one reason: typeshed declares `windll`, `WinDLL`, `WINFUNCTYPE`, -`WinError` and `get_last_error` on Windows only. Re-measuring it — by diffing -the three platform runs and keeping the modules whose *entire* non-win32 error -set is that one surface — turned up **sixteen** modules, not eight, and half of -them are nowhere near `windows/`: `utils/trash/`, `utils/app_idle/`, -`utils/file_assoc/`, `utils/idle_keepawake/`, `utils/lock_session/`, -`utils/session_guard/`, `utils/usb/passthrough/key_provider.py` and -`gui/main_window.py`. That killed the option the entry had recommended — -"measure a platform module on its own platform" cannot be a directory rule when -half the affected modules are not in a platform directory. - -The maintainer picked the suppression route, and it came to **28 lines, not the -58 the entry projected**: 58 counted the same source line once for Linux and -once for macOS. Each carries its own reason, none is blanket, and the runtime is -untouched. - -Two things had to be measured rather than assumed. **mypy honours -`# type: ignore` only as the first comment on the line** — a trailing one after -an existing `# nosec` is silently ignored — so on the two lines that already had -a `# nosec B607` the marker goes first and the two justifications merge into one -`# reason:`. And nine lines could not hold the marker inside the 120-char limit, -so they were reformatted rather than shortened into meaninglessness: an opening -paren takes the comment (`ctypes.WinDLL( # type: ignore[…]`), and two sites -hoist a value into a local first — `last_error = ctypes.get_last_error()` in the -DPAPI wrapper, `kernel32 = ctypes.windll.kernel32` in the input hook — which -reads better than the one-liner did. - -All sixteen were re-imported and exercised on a real Windows machine afterwards -(`dpapi_available()`, `_windows_locked()`, `check_key_is_press`), because a -reformat that only a type checker verifies is a reformat nobody verified. -**92 modules left, 925 of 1,017 files inside the contract.** - -### Fifteen More Modules, and Four Errors That Were Wrong Rather Than Untyped - -The accessibility backends, the observability trio, the three triggers, -`chatops.router`, `rest_api.rest_server`, `mcp_server.http_transport` and -`element_repository` came off the list together, because they kept running into -the same handful of causes. - -**Thirty-four of the forty-three accessibility errors were one missing -annotation.** `AccessibilityBackend._unsupported` raises for every action a -backend cannot perform, but it declared no return type — so mypy read the calls -as expressions that might fall through, and reported "missing return statement" -in all thirty-four methods that end with one. It is annotated `NoReturn` now, -which is what its body has always done. - -**A catch tuple that is not typed as one catches nothing, as far as mypy is -concerned.** `_uia_errors()` returns the exception classes a UIA call can raise -— including `comtypes`' `COMError`, which inherits from `Exception` and from -nothing else, so an `except (OSError, AttributeError)` never contained it. The -tuple was annotated `Tuple[type, ...]`, which is not "a tuple of exception -classes", so all five `except UIA_ERRORS` sites were errors. Same shape in -`rest_server` and `chatops.router`, where `except (…, *SQLITE_ERRORS)` unpacks -a tuple mypy cannot follow into an `except`; both now name the whole set as one -annotated module constant, the way `mcp_server._protocol` already did. - -**`parse_content_length` never took the type it declared.** Its parameter said -`Mapping[str, str]`; all three callers pass `self.headers` from a -`BaseHTTPRequestHandler`, which is an `email.message.Message` — not a mapping -over its keys, and case-insensitive about header names, which is the property -that makes `Content-length` work. It takes a one-method `HeaderLookup` protocol -now, which is what it actually uses and what the callers actually have. - -Four fixes are behaviour: - -- **`Gauge` and `Histogram` borrowed `Counter._labels_key` by assignment** - (`_labels_key = Counter._labels_key`), so a label typo in a gauge was - validated by a method whose `self` was declared to be a counter. The rule is - identical for all three, so it moved to `_MetricBase` — which also gives - `MetricRegistry.render()` a `render` to call on the base it iterates. -- **`_search_uids` decoded each IMAP UID at two later call sites and not at the - third.** UIDs are now decoded once where they arrive, so `_fetch_message`, - `_mark_seen` and `_seen_uids` all speak the same type. The stub in - `test_email_trigger.py` had pinned one exact `uid()` call shape - (`args[1]`); it now normalises arguments the way `imaplib._command` does — - skip `None`, ASCII-encode `str` — so it stands in for the library instead of - for one caller. -- **`ElementRepository` handed a stored locator straight to the accessibility - API as `**kwargs`.** A repository file is user-editable, so a field that is - not a filter surfaced as a `TypeError` about keyword arguments from inside - the backend. `_require` now rejects unknown fields by name, and the three - filters are passed explicitly. -- **`_AtspiConnection._call` had no bus to call outside its `with`.** It raises - `DBusError` naming the mistake rather than an `AttributeError` on `None`. - -`_process_name` in the Windows accessibility backend also stopped being checked -against Linux and macOS: it is a `kernel32` round trip, and now says so with a -`sys.platform` guard mypy can prune, in place of a bare `if process_id <= 0`. -**108 modules left, 909 of 1,017 files inside the contract.** - -### The Biggest Remaining Cluster: Thirteen Modules Under `utils/remote_desktop` - -`Progress.md` named this one as the next step and as the largest group left -(13 modules, 169 errors). It came off in one pass, and the errors sorted into -exactly two shapes. - -**Nine of the thirteen were `self._x = None` with no annotation.** mypy infers -the attribute's *type* as `None` from that line, so every later assignment is -"incompatible types" and every later use is "None has no attribute …". Several -of them even carried the intended type in a trailing comment -(`self._files_receiver = None # Optional[FileTransferReceiver]`) — the fact was -known, just written somewhere no checker reads. Those comments are now -annotations, with the classes imported under `TYPE_CHECKING` so the lazy runtime -imports that keep startup cheap are untouched. Where an attribute is assigned -and then used through a closure — `_wire_files_channel` in both the host and the -viewer — the receiver is bound to a local first, because a closure re-reads the -attribute and no narrowing survives that. - -**The other four were mixins reading attributes they do not own.** -`MediaNegotiationMixin`, `ViewerAuthMixin` and `FrameProductionMixin` are halves -of a host class split for readability, and each one's docstring already listed -what it borrows from the class it is mixed into — `_pc`, `_config`, `_send_ctrl`, -`_spawn_bg`, `_shutdown`, `_clients`… That list is now a declaration: an -`if TYPE_CHECKING:` block in the class body naming each borrowed attribute and -method. The block is stripped at runtime, so a stub in it cannot shadow what the -host actually binds — which a plain class-body `def` would risk for any future -mixin sibling that does not define it. - -Three findings in the batch are behaviour, not annotation: - -- **`WebRTCLoopBridge._run` read the loop off shared state.** `start()` set - `self._loop` and spawned a thread whose target then read `self._loop` back to - run it. The loop is now passed to the thread as an argument, and `start()` - returns it, so `submit()` and `call_soon()` hand a value they hold rather than - re-reading an `Optional`. Same behaviour, one less cross-thread read. -- **`_get_cursor_position` was invisible to the platform pruner.** It did - `import sys as _sys` *inside* the function and branched on `_sys.platform`, - which mypy does not treat as a platform test — so the Win32 branch was - type-checked against Linux and macOS too. The import moved to module scope, - which is what makes `ctypes.windll` a Windows-only fact rather than an error. -- **`totp` caught `base64.binascii.Error`.** That attribute exists only because - `base64` imports `binascii` itself; nothing declares it, and it would vanish - with a stdlib refactor. `binascii` is now imported by name. - -Two dicts also stopped being dicts of `object`: `manifest.json`'s entry rows and -the four `BANDWIDTH_PRESETS` are `TypedDict`s, so `preset["fps"]` is an `int` -without a cast and the manifest's shape is stated where it is written rather -than inferred from three literals. **123 modules left, 894 of 1,017 files inside -the contract.** - -### The macOS Grid Cell That Failed on a Test's Own Race - -`pytest-headless (macos-14, 3.14)` went red on -`test_modified_file_is_pushed_again`, asserting one push and seeing two. The -engine was right and the test was not: it edited the watched file in place and -*then* pushed its mtime forward, so between `write_text` and `os.utime` the file -briefly carried a third, intermediate mtime. A poll tick landing in that window -legitimately pushes twice — once for the intermediate value and once for the -final one. The new content is now staged outside the watch dir and swapped in -with `os.replace`, so one edit is one event. - -The same file guessed in the other direction too: every test slept a fixed 0.4s -hoping the baseline snapshot had been taken, while `FolderSyncEngine.start()` -returns as soon as the worker thread is spawned. On a slow runner the test's own -edit could land *in* the baseline and be pushed never. `wait_until_ready()` makes -the handshake observable — and it is not test-only scaffolding: any caller that -drops files right after `start()` has exactly that race. - -## What's new (2026-08-20) - -### Three Tests That Had Been Skipped Since They Were Written - -`test_r3_gui_thread_marshal.py` carried three `@pytest.mark.skip`s whose own -reason said what they needed: *"needs subprocess isolation (see -test_actions_menu_gui) … skip until then."* They covered real wiring — that a -file received on a WebRTC worker thread reaches the GUI thread through a -queued signal rather than a thread-affine `QTimer.singleShot`, and that the -admin console's thumbnail poll deletes its `QThread` each tick instead of -leaking one per interval. - -Skipping them was the right call at the time: building the WebRTC panel or the -admin console and then tearing a worker `QThread` down aborts the *shared* -pytest process under offscreen Qt. Because `deleteLater` is a no-op until an -event loop runs, the abort does not even land in the test that caused it — it -detonates inside some later, unrelated file, with no traceback. - -**They now run, in their own process.** One probe performs all three checks, -writes a JSON verdict per check, and `os._exit(0)`s without teardown — the same -shape `test_actions_menu_gui` has used for the full tab set. Each verdict is -`ok`, `failed: …` or `unavailable: …`, so a machine without the `[webrtc]` -extra (CI's `pytest-headless`, among others) reports a skip rather than a -failure, while a machine that has it actually checks the wiring. - -The checks have teeth, which was verified rather than assumed: deleting the -one line `thread.finished.connect(thread.deleteLater)` from -`admin_console_tab.py` turns the third verdict into `failed: the QThread -outlived finish`, and leaves the other two green. - -The headless suite now runs end to end with no `--ignore` flags — 4,815 -passing, and the only remaining skips are optional-dependency and -platform gates. No "skip until then" is left in it. - -### Windows arm64 Was Never a Code Problem - -The entry for this said `BLOCKED`, and that was half right. Two dependencies -publish no `win_arm64` wheel and still do not: `opencv-python` has none in any -version, and `cryptography` stopped at 46.0.3 while this project's floor of -`>=48.0.1` is a security floor (GHSA-537c-gmf6-5ccf) that cannot be lowered. -So `pip install` fell back to building OpenCV from source, CMake could not -configure for ARM64, and twelve minutes later the runner failed. That much was -measured, and it is why the runner left the matrix. - -What was not measured is the half that mattered. **Nothing in the package -needs either wheel at import time.** Blocking `cryptography`, `cv2`, -`je_open_cv`, `numpy` and `PIL` in a subprocess, `import je_auto_control` -still binds all 1,238 public names, and the executor, the MCP tool registry, -the CLI, `api.generate_code` and `api.create_failure_bundle` all import and -run. The blocker lived entirely in `pyproject.toml`'s dependency list. - -**So the fix is a marker, not an architecture.** Three requirements carry -`sys_platform != 'win32' or platform_machine != 'ARM64'` — `opencv-python`, -`cryptography`, and `je_open_cv`, which is pure Python but depends on OpenCV -and would otherwise drag it back in through the side door. Pillow is -deliberately not marked: it has always shipped `win_arm64` wheels and the -earlier note calling it a blocker was a guess. `windows-11-arm` is back in -`platform-smoke.yml`, on 3.14 only, because CPython's official Windows arm64 -builds start at 3.11. - -**What Windows arm64 gives up is now said out loud, in the error itself.** -`find_image*`, the OpenCV `screenshot()`, the secret vault, action-file -encryption, ACME/TLS and encrypted recording raise a message naming the -missing wheel and the platform, instead of a bare `ModuleNotFoundError` that -reads like a broken install. Two new accessors in -`utils/cv2_utils/optional.py` cover the two doors every image path goes -through; the other seventy-odd lazy `import cv2` sites are left alone on -purpose, because wrapping them buys the caller nothing it can act on. - -A deliberately unglamorous test guards the whole thing: -`test/unit_test/headless/test_arm64_dependency_markers.py` reads -`pyproject.toml` and evaluates the marker against five platforms, so removing -it — or "tidying" Pillow into it — fails loudly rather than costing an arm64 -user a working feature. - -### The Platforms This Project Claims, Now Measured - -The suite ran on `windows-2022` alone for its whole life, plus one Linux -container run. macOS got two commands and nothing else. Wayland had five jobs -reading input back off a real peer; X11 — the older and more widely deployed -of the two Linux paths — had none, and every X11 assertion in the suite was -made against a mock of `python-Xlib`. - -**The suite now runs where the project says it runs.** `pytest-headless` -became an OS matrix: Windows keeps all five Pythons, Linux and macOS carry the -two ends of the range. Linux runs under a real Xvfb rather than Qt's offscreen -platform, because the X11 backend opens a display at import time and offscreen -would hide exactly the breakage this exists to find. It found two real macOS -defects on the first run: - -- `write("\b")` had no key route on macOS, so it fell through to the space - fallback and typed a space where a backspace was asked for. X11 and Wayland - both carry the raw character; macOS was the one that did not. -- `system_profiler` reports a *symbolic* vendor id for Apple's own devices — - `apple_vendor_id`, not a number — and that went straight into a field - documented as four hex digits. Its leading `a` is a valid hex digit, so a - lenient parse turns it into `000a`. - -**X11 input is read back out of a real client.** A new `x11-verification` job -runs against a real Xvfb server with a real window manager, taking ground -truth from other codebases than the subject: `xev`, a real X client that -prints every event delivered to its window; ImageMagick's `import` against a -root painted two asymmetric colours; `xdotool` and `xdpyinfo`. The assertion -worth naming is `synthetic NO` — `XSendEvent` traffic arrives with `YES` and -is discarded by most toolkits, so a backend that quietly stopped driving real -input would still pass any check that only counted events. - -**macOS turned out to be fully testable in CI, contrary to the usual -assumption.** A `macos-14` runner grants *both* Screen Recording and -Accessibility: capture returns real pixels rather than the black rectangle a -refusal produces, `CGEventPost` moves the cursor and the move reads back -exactly, and the AX walk returns real elements. That was measured first and -asserted second, and the probe still refuses to pass while its expectations -table is empty. - -### Five Errors That Every Boundary Missed - -The exception hierarchy is flat so that the executor, the poll loops, the -request handlers and the GUI slots can each contain the whole family in one -`except`. Round 3 reparented the family for that reason; five classes were -missed, and one of them was reachable from an action list. - -`AC_config_import` with a malformed bundle raised `ConfigBundleError`, which -inherits `Exception` directly, so the executor's per-action clause — which -lists `AutoControlException` and the builtins — did not catch it. The error -went past the boundary and took every remaining action with it, under -`raise_on_error=False`, where the contract is that a failed action is -*recorded*. Measured, not reasoned: a two-action list lost its second action. -`AC_usb_remote_devices` and `AC_usb_remote_open` had the same path through -`UsbClientError`. - -All five now derive from `AutoControlException`. What keeps the next one out is -not a list of the five: `test_exception_family_is_flat.py` walks the package -with `ast` and fails on *any* class inheriting `Exception` directly, against a -three-entry allowlist that has to state why. `LoopBreak` and `LoopContinue` are -on it because they are control flow — a family handler swallowing a `break` -would be the mirror-image bug — and the MCP dispatcher's private error carrier -because it never leaves the dispatcher that raised it. The allowlist is checked -in both directions, so a stale entry fails too. - -### A BSD Found the Same Mistake in a Second Place - -The FreeBSD VM was added to prove the X11 backend drives a real BSD. It failed -before it got there, and what it failed on was not a wheel: **FreeBSD's -`python311` has no `sqlite3`.** The module is in the standard library but not -in every build of it — CPython links it against a system library, and FreeBSD -packages the result separately as `databases/py-sqlite3`. - -Ten subsystems imported it at module scope: run history, checkpoints, the work -queue, agent memory, the remote-desktop audit log, SQL data sources, and the -`except` tuples that keep a database error from killing the REST handler -thread, the chat-ops poll loop and the MCP transport. Every one of them is -reachable from the facade, so `import je_auto_control` failed outright — on a -machine where moving a mouse needs no database at all. This is the same shape -as the OpenCV/Pillow finding one commit earlier, from a source that reasoning -about wheels would never reach: the standard library is not the same size on -every platform. - -**The ten now go through `je_auto_control/utils/sqlite_support.py`.** -`require_sqlite3()` returns the module or raises -`AutoControlUnsupportedOperationException` — deliberately the type the platform -backends already raise for something they cannot do, so the GUI tabs, the REST -handler and the executor report "not available here" instead of dying on an -`ImportError` that none of them catch. `SQLITE_ERRORS` and -`SQLITE_OPERATIONAL_ERRORS` are tuples rather than classes, so -`except (ValueError, *SQLITE_ERRORS)` stays a valid handler that catches -exactly the right amount — nothing — where nothing can raise them. - -That left one thing still opening a database during import: `HistoryStore` -connected in its constructor, and `default_history_store` is built while the -facade is importing. It connects on first use now, which is also why -`import je_auto_control` no longer creates `~/.je_auto_control/` as a side -effect of being imported. - -Three things keep it fixed. `test_sqlite_is_optional.py` blocks `_sqlite3` in a -subprocess — exactly what FreeBSD reports — and requires the facade to import -anyway, with the error tuples empty. The FreeBSD job asserts the module is -*absent* on the VM, so a future image that happens to ship `py311-sqlite3` -turns the job red rather than quietly retiring the property it was added to -test. And `run_diagnostics()` lists `sqlite3` among the optional dependencies, -so an operator sees the gap as a line in a report instead of a traceback. - -### The macOS Recorder Was Written, Unreachable, and Wrong - -`OSXRecorder` had been a complete implementation for as long as -`wrapper/_platform_osx.py` had said `recorder = None`, and that was not an -oversight. The listener called `NSApplication.sharedApplication()` **at import -time**, and stopping a recording meant `AppHelper.runEventLoop()` — a loop -that never returns to its caller. Wiring it up would have put both on the path -of `import je_auto_control`, which is a regression, not a fix. So `record()`, -`stop_record()` and `je_auto_control record` all refused on macOS with -"Cannot use recorder on macOS", and the capability matrix said `unavailable`. - -**The premise was wrong: a `CGEventTap` needs a run loop, not an -application.** Create the tap on a dedicated thread, add its source to *that* -thread's run loop, and pump the loop in short `CFRunLoopRunInMode` slices so a -stop flag is honoured between them. Nothing touches AppKit, nothing runs at -import, and `record()` returns immediately. The macOS hotkey backend had been -driving a tap exactly this way in the same tree the whole time. - -The tap is **listen-only**, which is load-bearing rather than a detail: a -recorder that consumed events would swallow the very input it is recording, so -the user's clicks would stop working the moment recording started. - -Two defects were sitting in that code, and only a Mac could show either: - -- **Recorded clicks were mirrored vertically.** Coordinates came from - `NSEvent.mouseLocation()`, whose origin is the **bottom-left** of the - display, while every replay posts into the top-left space `osx_mouse` uses. - A click recorded near the top of the screen replayed near the bottom, with - no error anywhere — the same silent shape as `write("\b")` typing a space. - It reads `CGEventGetLocation` now, which is already the replay's space. -- **Modifiers were not recorded at all.** macOS sends no key-down for Shift, - Control, Option or Command; it sends one `flagsChanged` event carrying the - new flag set. A recording therefore could not say a modifier was held across - the actions that followed — which is one of the two reasons the timeline - exists. They are reconstructed from the flag bits now. - -**The half that is not platform-specific stopped being copied.** Everything -after the capture — the down-events-only queue the executor is handed, the -`delta_ms` timeline a replay consumes, the mouse-only and keyboard-only -filters — is one implementation in -`utils/input_macro/recorder_base.py`, and both backends subclass it. A second -hand-written copy of that shaping would have diverged silently, and the way it -would surface is a recording made on one OS replaying wrongly on the other. - -**And the recording had nowhere to go.** Verifying the new backend end to end -turned up a defect that was never macOS-specific: `replay_timeline`'s dispatch -table held the `run_sequence` DSL's vocabulary — `press`, `click`, `key` — -while every recorder emits its own — `key_down`, `mouse_up`, `scroll`. The two -sets were **disjoint**. So `stop_record_timeline()` handed to -`replay_timeline()`, which is the pipeline the docstrings and the -`ac_record_stop_timeline` tool description both prescribe, matched no handler -at all: it replayed an empty session and returned the full event count as -played. The one op that did match, `scroll`, read a key the recorder does not -write, so it fell back to a single notch in the default direction. Both are -fixed, on every platform. - -**It is verified on a real window server, not against a fake.** The -`macos-capabilities` job posts a move, a click and a keypress through the -public API while recording, and asserts they come back out of the tap with the -release and with the coordinates they were posted at. Decoding is unit-tested -separately against genuine `CGEventCreate*` events, which needs no -Accessibility grant — so the parts that can be tested without a permission -are, and the one part that cannot is where the permission is measured. - -### Window Management Is No Longer Windows-Only - -It was: the facade branched on `sys.platform` and raised everywhere else, -leaving 23 `AC_*` commands and their MCP tools dead on macOS and Linux. It now -goes through a backend seam — Win32, EWMH over `python-Xlib` on X11, Quartz -plus the accessibility API on macOS, and a null fallback that lists nothing -and refuses actions with a reason. - -Two things only a real window manager could show up were wrong first time: - -- **The rectangle is the frame, not the client.** Win32's `GetWindowRect` - returns the frame, and every caller is written against that, so reporting - the client area was off by the decorations on X11 alone — silently, and by - a different amount per window manager. -- **A move has to go through `_NET_MOVERESIZE_WINDOW`.** Under a reparenting - window manager a client's own x/y are relative to its frame, so a direct - `ConfigureWindow` asks in the wrong coordinate space. Asking openbox for - (300, 220) that way landed the window at (302, 260). - -Refusals now raise a class that is both an `AutoControlException` and a -`NotImplementedError`. The GUI tabs and the REST handler already catch the -latter to say "not on this platform"; the executor catches the former, and a -bare `NotImplementedError` slipped past every containment boundary — aborting -a whole script where one action should have been reported as failed. - -### Linux Has an Accessibility Backend - -It had none — the selector fell through to the null one while the capability -matrix claimed "backend tests" for Linux X11. The new backend speaks -**AT-SPI2**, which is a D-Bus protocol rather than a library, and that is what -makes it reachable without a new dependency: `pyatspi` and -`gi.repository.Atspi` are distribution packages built against the system -introspection data and cannot be installed into a virtual environment. - -The D-Bus client written for the portal handshake moved from `linux_wayland/` -to `utils/dbus_client/` to make that possible, and verifying the backend -against a real bus and a real GTK application immediately found a gap in it: -**it could not demarshal signed integers.** The portal never needed one, and -AT-SPI reports a component's extents as four *signed* values, because a window -on a monitor left of or above the primary one is at a negative coordinate — so -the backend could read a tree but not where anything in it was. - -Because AT-SPI is a bus rather than a display protocol, this is the one -capability where Wayland is not the restricted case: the same bus serves both -Linux sessions. - -### The BSDs, and arm64 - -`platform_wrapper` refused to start on anything that was not -win32/cygwin/msys, darwin or linux/linux2, and each of the seven X11 backend -modules carried its own copy of the same Linux-only guard — so a FreeBSD, -OpenBSD or NetBSD desktop, which runs the same X server and the same -`python-Xlib`, could not import the package at all. `sys.platform` was being -compared against literal lists in over a hundred places, so the fix is one -place that decides: `utils/platform_id`, whose `is_x11_unix()` asks the -question those guards were always trying to ask. - -A `freebsd` job boots a real FreeBSD 14 VM inside the runner. It could at -first only check the *decision* — that `sys.platform` really reads `freebsd14`, -and that the classification every relaxed guard asks answers correctly on it — -because importing anything under `je_auto_control` ran the facade, and the -facade imported OpenCV and cryptography at module scope. `utils/platform_id` -had to be loaded by file path to get even that far. See below: that turned out -to be the wrong thing to work around, and the job now drives the whole backend. - -`ubuntu-22.04-arm` joins the smoke matrix and passes; `macos-14` was already -arm64. (**Superseded the same day** — see "Windows arm64 Was Never a Code -Problem" at the top of this file: the runner is back, because the blocker -was the dependency list rather than the code.) `windows-11-arm` was tried -and removed, and re-measuring turned up a -**second** blocker the first pass had missed: opencv-python publishes no -`win_arm64` wheel in any version, and cryptography stopped publishing one after -46.0.3 — while this project's floor is `>=48.0.1`, a security floor -(GHSA-537c-gmf6-5ccf) that cannot be lowered to reach a wheel. Pillow, named -alongside OpenCV in the original entry, ships `win_arm64` wheels and was never -part of the problem. None of that needs an arm64 machine to check: `pip -install --dry-run --only-binary=:all: --platform win_arm64` answers it in -seconds, and `Progress.md` records the command next to the finding. - -### The Facade Insisted on OpenCV to Move a Mouse - -`Progress.md` recorded the missing BSD coverage as needing "a machine with the -dependency set on it, not a different CI trick". That entry was making the -mistake its own Wayland section warns about three lines further up: **asking -what the environment could not do, instead of asking who actually could not do -it.** It was not FreeBSD that could not run the X11 backend. It was this -package, which imported five image and crypto wheels before it would let you -move a pointer. - -The measurement was small. Ten modules on the facade's import path pulled in -OpenCV, NumPy, Pillow, `je_open_cv` or `cryptography` at module scope, across -about sixty call sites — while most of `utils/` had been importing OpenCV -lazily all along, with the docstrings to say so. Those ten now do the same. The -type annotations that referenced Pillow moved under `TYPE_CHECKING`, and the -two public `ImageSource` aliases keep Pillow in the union as a forward -reference, so nothing changes for a caller or a type checker. - -What `import je_auto_control` needs now is `defusedxml`, plus `python-Xlib` on -an X11 platform. Both are pure Python. The heavy five are still hard -dependencies and still install by default; the difference is that a platform -with no wheel for them can now use the half of the package that never needed -them. `test/unit_test/headless/test_facade_import_is_light.py` blocks all five -in a subprocess and imports the facade anyway, because this is a property one -convenience import silently undoes and every runner with wheels keeps passing. - -### The BSD Job Drives Real Input Now, and Found the Defect That Was Waiting - -With the facade light, the FreeBSD VM needs python-Xlib, defusedxml and an X -server — an install measured in seconds, where the ports build for OpenCV had -not finished after fifty minutes. So `test/verify/freebsd_verify.py` runs the -backend rather than the guard, and takes its ground truth from the X server -answering for itself: `query_pointer` for where the cursor is, its button mask -for which buttons the server believes are down, and `query_keymap` — the bitmap -of every physically-held key — for whether an injected key press really landed. -That last one is why no second process is needed here; the Linux -`x11-verification` job already reads events back out of `xev`, and what a BSD is -uniquely needed to answer is whether this code drives the same server on a -different kernel. - -It also maps a real X window that has asked for button events, because one -defect could not be caught any other way: **`mouse_scroll` did nothing at all -on a BSD.** It matched Windows, then macOS, then a literal -`["linux", "linux2"]` — one of the hundred-odd hand-written platform lists -`platform_id` exists to replace, and one that had been missed — so a BSD caller -fell off the end of the chain with no backend call, no exception and no log -line. A wheel event never shows up in the pointer mask, since X11 delivers a -scroll as a press *and* release of button 4/5/6/7 too fast to sample, so only a -client reading the event queue can see it happen or not happen. - -### `mouse_scroll` Means the Same Thing on Every Platform - -This had been sitting in `Progress.md` as a `DECIDE`, and the maintainer -settled it: **the sign of `scroll_value` reverses the direction everywhere, and -`scroll_direction` names the direction a positive count takes.** - -Windows and macOS had always read the sign. X11 encodes direction as a button -rather than a signed delta, so it took the direction from `scroll_direction` -and discarded the sign — deliberately, because a negative count used to make -`range()` empty and scroll nothing at all. The cost was portability with no -symptom to debug: `mouse_scroll(-3)`, written and tested on Windows, scrolled -three notches *down* on Linux instead of up. Wayland had the same `abs()` in -`_wheel_deltas` and the same result. - -Both now turn a negative count back into the opposite direction — a button swap -on X11, a sign on the Wayland delta. `docker/x11_verify.py` pins it against a -real X server through `xev`, `freebsd_verify.py` pins it on a BSD, and -`test_scroll_sign_is_portable.py` pins it on every runner without needing a -display. The migration note for anyone who was relying on the magnitude alone -is in `CHANGELOG.md`. - -### The Destructive-Action Prompt Reached Only One of the Two Transports - -`JE_AUTOCONTROL_MCP_CONFIRM_DESTRUCTIVE=1` is documented as gating *every* -destructive MCP tool behind a confirmation prompt, with one stated caveat: the -client has to advertise the `elicitation` capability. Measured against a real -`HttpMCPServer` with a real destructive tool, the gate fired on stdio and -**never fired over HTTP** — not in any of the four combinations of plain POST -or SSE, one connection or two. - -Two independent reasons, and the second is why keeping the connection open did -not help. A plain `POST` is one request and one response, so its connection -scope had no server→client channel and the prompt had nothing to travel down. -An SSE `POST` did have one, but `_dispatch_sse` sets `close_connection`, so the -scope keyed on that TCP connection was forgotten before the next request — and -the `elicitation` capability advertised at `initialize` went with it. The call -then took the "client cannot be prompted" branch, logged one INFO line, and ran -the tool. An operator who set that variable on an HTTP-exposed server was -getting no confirmation at all. - -**The transport now has the identity MCP actually specifies.** `initialize` -mints an `Mcp-Session-Id` and returns it as a response header; a client that -echoes it keeps one dispatcher scope across every connection it opens. `GET` -with `Accept: text/event-stream` opens the standing server→client stream that -server-initiated traffic belongs on, and answering a server request is an -ordinary `POST` matched back to the waiting call by id. `DELETE` terminates. -The dispatcher itself needed no change — it already scoped capabilities and -active-call slots on an opaque `connection_id`, so a session id simply takes -that slot, and the per-connection isolation guarded by -`test_r3_mcp_connection_isolation` is preserved verbatim: a session is just an -identity that outlives a socket. - -So the confirmation now round-trips over HTTP, and the test that proves it uses -four separate connections — one that initialized, one holding the stream, one -carrying the `tools/call`, one carrying the decline — none of which shared a -socket with the handshake. Declining blocks the tool; accepting runs it. - -Measuring it turned up a second way through that was worth pinning: an SSE -`POST` carrying a session id needs no standing stream at all, because its own -response stream is already a server-to-client channel. The `elicitation/create` -goes out ahead of the result on the same socket the call arrived on. Both paths -now have a test. - -What did *not* change is the fallback: a client that ignores the session header, -or that only ever sends plain JSON `POST`s with no stream open, has given the -server nowhere to ask, and its destructive calls still proceed — exactly as they -do for a stdio client that never advertised `elicitation`. That is now a -documented boundary with a test on each side of it rather than an accident — but -it is still a boundary, so the bearer token, the `127.0.0.1` bind and -`JE_AUTOCONTROL_MCP_READONLY` remain the controls that do not depend on the -client behaving. - -Sessions are bounded in both directions: swept after ten minutes untouched, and -the registry evicts the least recently seen once it holds 128. Dropping a -session — however it goes — releases the dispatcher state held under its id, -which is the same release a closing socket used to perform, moved to the -identity that actually owns that state. Because every `initialize` mints a -session, including for the many clients that ignore the header and never come -back, evicting one that was never used after its handshake is logged as routine; -the warning is saved for evicting a session someone was holding, which is the -one that means the cap is too low. - -The new refusals also exposed an old assumption in the transport. Every `4xx` -runs `_drain_body()` first, so the client can read the response before the -socket closes — but the new unknown-session `404` and duplicate-stream `409` are -decided *after* the body has been parsed, and so was the pre-existing "body must -be UTF-8" `400`. The drain then went looking for bytes that were already gone -and blocked until the thirty-second read timeout, pinning that worker and -printing a `ConnectionAbortedError` traceback whenever the peer closed first. -It now skips the drain once the body is consumed, and treats a peer that has -already vanished as nothing left to be courteous to. - -## What's new (2026-08-19) - -### Two Wayland Judgement Calls, Settled - -Two items had been sitting in `Progress.md` marked `DECIDE`: not missing work, -missing decisions. They turned out to be the same problem twice — a compositor -setting the library can *measure* but cannot *read*, and therefore must stop -pretending to know. - -- **Pointer acceleration is now something the operator declares, and the - library believes.** The measurement stands: `ydotool mousemove --absolute` - sends relative motion, libinput's default adaptive profile doubles it, and no - client can read the factor back. What was undecided was what to do about it — - keep warning and move anyway, refuse outright, or let the operator say. - Refusing outright would have taken `set_position` away from every Wayland - machine without `liboeffis`, so the answer is a declaration: - `JE_AUTOCONTROL_WAYLAND_POINTER_ACCEL=flat` means acceleration is off for the - ydotoold device, and the move goes out silently and exactly; `=strict` means - refuse the move rather than let a click land somewhere else; unset keeps - today's warn-once-then-move, so nothing that works now stops working. An - unrecognised value falls back to the warning *and says that it did* — a typo - in a shell profile must not quietly promote a move to trusted-exact. The - whole gate is on the ydotool path; libei is absolute at the protocol level. -- **The software cursor in a Wayland capture is documented, not worked - around.** `seat-verification` measured it in passing: no capture in this - project passes `grim -c`, so none asks for the pointer, and the pointer is in - the image anyway. The reason is not ours — wlroots draws a *software* cursor - whenever the backend has no cursor plane, and a software cursor is composited - into the output buffer, which is exactly the buffer `wlr-screencopy` hands - back. Headless is permanently in that state; so is any real desktop running - `WLR_NO_HARDWARE_CURSORS=1`. Windows' BitBlt and the X11 path never include - the pointer, so this is a Wayland-only inconsistency, and with the pointer - resting on its target a locator, a template match or an OCR read sees a - pointer-shaped hole in the middle of it. Both ways out — move the pointer - away and back, or mask around it — need to know where the pointer is, and - Wayland does not let a client read that; an in-process guess goes stale the - moment the user touches their own mouse, and masking the wrong place is worse - than a visible cursor. So it is written down instead: in the capability - matrix, in all three READMEs, and in the diagnostics bundle, where the - `screen_capture` check now carries `cursor_may_be_captured` so the report - that explains a failed locator names the reason. `seat-verification` asserts - the behaviour as measured, so if wlroots ever honours `overlay_cursor` for - software cursors, CI goes red and tells us. - -### A Compositor That Consumes Input Was Three Environment Variables Away - -`ydotool mousemove --absolute` is not absolute. It emits no absolute event: -it sends `INT32_MIN` on both axes to drive the cursor into whatever corner the -compositor clamps to, then sends the target as a relative displacement. What -that corner *is*, and what the compositor does to the displacement on the way, -were the last two open questions on the Wayland input path — and both were -recorded as needing a VM running a desktop that consumes libinput devices. - -- **They needed no VM.** wlroots takes `WLR_BACKENDS=headless,libinput`: the - outputs stay virtual while the input half is the real libinput backend. - libseat's builtin backend opens the devices without logind, and - `SEATD_VTBOUND=0` stops it reaching for a VT no container owns. The fourth - requirement is the one that is easy to miss — libinput enumerates through - udev rather than through `/dev`, so `systemd-udevd` has to be running before - ydotoold creates its device. With those four in place, ydotoold's uinput - device is an ordinary seat device, and `grim -c` composites the cursor into - a screenshot, so the compositor answers in layout coordinates. -- **The corner is the layout's top-left, not layout `(0, 0)`.** Those are the - same point only while every output sits at a non-negative position. On the - layout every desktop with a monitor left of the primary one has, they differ - by the layout origin — so on a `-1280` layout an untranslated request for - layout `(0, 0)` put the cursor on the *other monitor*, 1,280 pixels away. - `mouse.set_position` now subtracts `layout_origin()` before handing the - coordinate to ydotool, which is the same correction the capture path already - applies; the lookup both input paths share moved into - `linux_wayland/_layout.py` so libei and ydotool cannot drift apart on it. -- **And pointer acceleration scales the rest.** The displacement is relative - motion, so libinput accelerates it: measured against a real wlroots session, - the default adaptive profile lands the cursor exactly twice as far from that - corner as asked, because `--absolute` sends both of its events in one frame - and the velocity saturates the profile. With `accel_profile flat` and - `pointer_accel 0` the same call is pixel-exact. ydotool's own `--help` has - said "You need to disable mouse speed acceleration for correct absolute - movement" all along; the backend now logs that caveat once per process - instead of letting a click land silently in the wrong place. Nothing else - can be done from inside the library — the factor is the compositor's - setting, not something a caller can read back. -- **A new `seat-verification` job holds all of it.** `docker/Dockerfile.seat` - runs the two layouts the capture image runs, and 14 checks each: that sway - really is holding the ydotool device, that `--absolute (0, 0)` draws the - cursor flush into the layout's first pixel, that with acceleration off the - move is one pixel per pixel, that an untranslated `(0, 0)` misses the - monitor it names, that `set_position` subtracts exactly the origin and lands - on the pixel it was given on both monitors, and that the acceleration factor - is the 2x this was measured at. Nothing in it depends on the cursor theme: - every claim is a difference between two captures, which the image's offset - from its hotspot cancels out of. -- **One thing it found on the way.** On this compositor a `grim` capture that - asks for no cursor overlay contains one anyway, because wlroots draws a - *software* cursor whenever the backend has no cursor plane — always on - headless, and on any session where the driver refuses one or the user set - `WLR_NO_HARDWARE_CURSORS=1`. Every locator, template match and OCR read goes - through that capture, so the pointer punches a pointer-shaped hole in - whatever it is sitting on. The check records the behaviour as measured, and - the decision was to document it rather than work around it — see the - capture section below. - -### The Screen-Capture Portal Could Never Have Worked, and a Real Bus Said So - -- **`xdg-desktop-portal` answers with a signal directed at the connection that - called it.** `Screenshot` returns a *request handle*, not an image; the image - arrives later as `org.freedesktop.portal.Request::Response`, addressed to the - caller's unique bus name. The bus routes a directed message to its - destination and nowhere else, so no match rule on any other connection can - make it arrive somewhere else. -- **The old implementation was two connections.** It started `gdbus monitor` in - one subprocess, made the call from a second `gdbus` invocation, and read the - monitor's stdout with a pair of regular expressions. Each `gdbus` invocation - opens its own connection under its own unique name, so the process listening - was never the process addressed. Measured against a real `dbus-daemon`: the - monitor sees the call go past and prints nothing else, and the capture runs - out its full 30-second timeout, every time. The only listener that can see a - directed signal is a full bus monitor — `dbus-monitor`, which asks the bus - for `BecomeMonitor` — and needing permission to observe every message on the - user's session bus is a poor price for a fallback screenshot. -- **The tier now speaks D-Bus itself, on one connection.** - `linux_wayland/_dbus_client.py` is a session-bus client in the standard - library alone: connect, SASL EXTERNAL authentication, `Hello`, `AddMatch`, - one method call, then read until the matching signal arrives. It is - deliberately not a general binding — no properties, no introspection, no - object export, no descriptor passing (liboeffis still does the one call that - needs that). `portal.py` subscribes to the request path it predicts from its - own unique name *before* it calls, and follows the returned handle as well - when a portal ignores `handle_token`. -- **Which also removes a dependency rather than adding one.** The tier used to - need `gdbus` (glib2) installed; it now needs nothing but a session bus, so - the last-resort capture path is available on strictly more desktops than - before. The install hint and the diagnostics check say so. -- **Verified end to end on a real bus.** A new `portal-verification` job runs a - real `dbus-daemon`, a real portal implementation and the real client: the - capture comes back as PNG bytes that decode to the pixels the portal painted, - at a percent-escaped path with a space in it, and the portal's file is gone - afterwards. Every way a portal ends without an image is driven too — a - dismissed dialog, a dialog left open, a success carrying no URI, a URI that - is not a local file — and each has to fail closed on AutoControl's own clock. - -### The RemoteDesktop Portal Handshake Never Needed a GNOME VM Either - -- **The portal is a D-Bus interface, not a compositor feature.** Reaching libei - on GNOME and KDE means `CreateSession` → `SelectDevices` → `Start` → - `ConnectToEIS`, ending in an EIS file descriptor passed over the bus. That - was recorded as unverifiable without a GNOME VM because - `xdg-desktop-portal-wlr` implements ScreenCast and Screenshot but not - RemoteDesktop — which confuses "no container ships one" with "no container - can host one". Whatever owns `org.freedesktop.portal.Desktop` and answers - those four calls *is* the portal, as far as `liboeffis` is concerned. -- **So the verification owns the name itself.** `docker/portal_server.py` is a - real D-Bus service on a private session bus, and its `ConnectToEIS` hands - back a live connection to the same real `libeis` server the `eis` image uses. - The real `liboeffis` runs the real handshake; the descriptor that comes out - carries a real EI session; and the key presses, absolute motion and button - edges emitted through it are recorded by an independent implementation at the - far end. -- **What it settles.** That the four calls arrive in the prescribed order at the - request paths the client predicted; that `SelectDevices` is asked for keyboard - and pointer and nothing wider, so the grant a user consents to is the one this - backend needs and `OEFFIS_DEVICE_DEFAULT` is not the `= 0` all-devices - sentinel; that the descriptor is a live socket the caller owns and must close, - which is what makes handing it to `ei_setup_backend_fd` — a function that - takes ownership — correct rather than a double close. -- **And every refusal.** A dismissed consent dialog, a dialog left open, a - withheld descriptor, a session the portal closes, a portal too old to have - `ConnectToEIS` at all, and no portal on the bus: each has to come back as a - refusal on this project's own clock rather than a hang or a silent downgrade. - The `OEFFIS_EVENT_CLOSED` branch had never had a peer able to drive it; it - does now. -- **What is still not claimed.** The consent dialog as a dialog. Nobody - dismisses anything in CI, so what a real mutter dialog looks like, and how - long a real person leaves it open, stays mutter's business. What a dialog - *produces* — a grant, a refusal, silence — is all exercised. - -### One Packaging Fact That Was Recorded Wrong - -- **Debian trixie does ship `liboeffis`.** `Progress.md` said it did not, and - concluded that the libei fast path was effectively off across Debian and - Ubuntu. Measured: `liboeffis1` 1.3.901-1 is in trixie/main, providing - `liboeffis.so.1`. What is true, and what actually matters to a user, is that - it is a *separate binary package* which `libei1` does not depend on — so - installing libei alone still leaves the portal route off and `connect()` - falls back to the `eis-0` socket that GNOME and KDE do not open. Install - `liboeffis` to get the fast path. - -### A Monitor Left of the Primary Broke Every Wayland Capture Path - -- **Wayland has no per-monitor screen, and the one it does have need not start - at `(0, 0)`.** The compositor lays every output out on a single plane, and - that plane starts at a negative coordinate the moment an output sits left of - or above the origin — which is what "my second monitor is on the left" means - to a compositor. sway's headless backend accepts `output HEADLESS-1 position - -1280 0`, so this is now a layout CI can stand up: two 1280x720 outputs, one - 2560x720 capture, top-left pixel at x=-1280. -- **`screen.size()` was returning the layout's right edge, not its width.** It - computed `max(x + width)` over the outputs, which is 1280 on that layout - while `grab_image()` returns a frame 2560 wide. Everything that composes the - two believed the smaller number: the mss-shaped shim's monitor list (and so - `enumerate_monitors`), the screen recorder, the WebRTC host and the MCP - monitor grab all asked for a rectangle half the size of the desktop and - reported it as the whole screen. -- **The crop for the tiers that cannot take a region cropped in the wrong - space.** Only grim accepts a geometry; gnome-screenshot, spectacle, the - portal and `JE_AUTOCONTROL_WAYLAND_CAPTURE_COMMAND` all hand back the whole - layout and AutoControl crops afterwards. That crop used layout coordinates - against an image whose origin is the layout origin, so asking for - `[-1275, 5, -1175, 55]` — a rectangle on the left-hand monitor — asked - Pillow for a box 1275 px left of the frame and got black padding. -- **And a match found on that monitor was reported on the wrong one.** - `grab_logical`, the capture behind template search, OCR and visual match, - reads its origin from `GetSystemMetrics`, which says nothing off Windows — - so it returned `(0, 0)` and every hit came back 1280 px to the right of - where it was seen. That reads as "the click lands on the wrong screen" - rather than as a failure to find, which is the worse of the two. -- **Fixed at the seam, not at the call sites.** The Wayland backend publishes - `layout_origin()`; `size()` returns the bounding box's *size*; `grab_image` - subtracts the origin before cropping; and - `screen_grabber.backend_layout_origin()` is what `grab_logical` and the mss - shim ask, so a backend that captures its own screen can say where that - capture starts. Backends the generic libraries can already see (Windows, - macOS, X11) publish nothing and are unchanged — the origin is only ever - asked for, never guessed. -- **Verified against a real compositor, both ways round.** The - `wayland-verification` job now runs its 27 checks twice: once with the - outputs side by side from the origin, once with the left-hand one at - x=-1280. The second run is what pins the negative case end to end — grim's - negative `-g`, the reported size, `layout_origin()`, the mss shim's monitor - rectangle, `grab_logical`'s origin, and the fallback crop driven through the - operator override pointed at grim so a *real* whole-layout PNG goes through - the code path that has to shift it. - -### The Same Layout Problem, on the Input Side — a Move libei Was Dropping in Silence - -- **libei discards an absolute motion that lands in no region, and reports - nothing about it.** No return code, no event, no error the caller can see — - `ei_device_pointer_motion_absolute` simply does not put the event on the - wire. `set_position` then returned as though the pointer had moved. Measured - against a real EIS peer, not inferred: with the device offering one - `(0, 0, 1920, 1080)` region, `(1919, 1079)` arrives and `(1920, 1080)` - produces no server-side event at all. -- **And the space those regions live in need not be the layout's.** A region's - offset is a `uint32`, so no compositor *can* advertise one left of or above - the origin — while this project's layout space starts at `layout_origin()` - and goes negative the moment a monitor sits left of the primary. That is the - exact desktop the capture half was just fixed for. The two halves therefore - disagreed by the origin, which is the case where `get_pixel(x, y)` and - `set_position(x, y)` name different pixels — and the pointer that would have - gone to the wrong monitor instead went nowhere, quietly. -- **`LibeiBackend` now reads the device's regions and maps the point into - them.** `ei_device_get_region` and the four `ei_region_get_*` getters are - bound; `_region_point` sends a covered coordinate unchanged, retries an - uncovered one normalised by the layout origin, and refuses what neither - covers. A device that declared no region accepts anything and is passed - through untouched, which is measured too — that is the common single-monitor - case, and it costs nothing. -- **A refusal is the useful outcome, not a failure.** It is a - `LibeiUnavailable`, so `_select_input.emitted` hands the move to the ydotool - path exactly as it already does for a paused device. libei is documented as - the fast path and never the only one; the bug was that a dropped move never - reached the fallback because nothing knew it had been dropped. The frame is - not sent on a refusal either — nothing was buffered, and a frame there would - commit whatever the previous emission left on the device. -- **The layout origin is only consulted when a point misses.** It costs a - `wlr-randr` subprocess, so it stays off the path every ordinary mouse move - takes, and it answers `(0, 0)` on GNOME and KDE — which is the right answer - there rather than a fallback, because those compositors normalise the layout - themselves. -- **Five new checks in the `eis-verification` job, against the real - protocol.** That the client reads back the offsets the compositor - advertised; that a region at `x=1280` takes `1380` for a point 100 px into - it rather than `100`; that libei still drops an out-of-region motion without - a word — the measurement the whole guard rests on, so a future libei that - clamps instead says so; that AutoControl refuses such a move rather than - losing it; and that `(-1280, 10)` on a layout starting at `-1280` reaches - the server as `(0, 10)`. The job now runs 20 checks. -- **What this does not settle.** ydotool's `mousemove --absolute` has an - origin of its own — it clamps to the compositor's top-left corner and sends - the target as a relative delta — and whether that corner is the layout - origin needs a compositor that consumes libinput devices. That is what the - `seat-verification` job later built, and it answered this: the origin is - the layout's top-left corner, not the layout coordinate `(0, 0)`. - -### The ydotool Path Was Reporting Success While Doing Nothing - -- **`apt install ydotool` — the hint this backend printed — installs a - version whose command line cannot run it, and which says so by exiting - zero.** ydotool 1.0 replaced the whole CLI, and every argument the Wayland - backend builds arrived in that release: `mousemove --absolute`, `mousemove - --wheel`, hex `click` bitmasks (which is what lets a press and a release be - sent separately, and therefore what makes drag possible), and `key - CODE:STATE` taking numeric evdev codes. Debian bookworm, Ubuntu 22.04 and - Ubuntu 24.04 all still ship 0.1.8 under that name. Measured against a real - uinput device, 0.1.8 answers `click 0x40` with **no events and exit code - 0**, and answers `mousemove --absolute` with `unrecognised option` — also - **exit code 0**. The backend runs ydotool with `check=True`, so a non-zero - status was the only thing that would have raised. On those distributions a - script clicked nothing, typed nothing and moved nothing, and every call - reported success. -- **The legacy CLI is now refused before anything is sent.** - `linux_wayland/_ydotool_cli.py` classifies the installed ydotool once per - process — mouse and key dispatch cannot afford a subprocess per event — and - raises with the three routes out: a 1.0+ package, a source build, or - `JE_AUTOCONTROL_LINUX_DISPLAY_SERVER=x11`. Neither series implements - `--version` and 1.x will not answer `--help` without its daemon running, so - the probe reads the one thing both print with no daemon and no side - effects: the no-argument command list. A version it does not recognise is - allowed through rather than blocked, so a future release that changes that - banner cannot be locked out by a stale detector. -- **Both install hints were wrong in a second way.** Debian trixie ships no - `ydotool` package at all. The hints now name the distributions that do. - -### ydotool Never Needed a Desktop to Verify — Only a Reader - -- **The gap both verification images recorded turned out to be the wrong - gap.** `Dockerfile.wayland` and `Dockerfile.eis` each closed by saying - ydotool "needs /dev/uinput and a seat that consumes it", and `Progress.md` - filed that behind building a GNOME VM. A seat is what makes an injected - event *arrive somewhere*. It is not what makes one *observable*: ydotoold - creates an ordinary uinput device, the kernel publishes it as - `/dev/input/eventN`, and reading that node returns the exact `input_event` - structs ydotool wrote. No compositor, no session, no VM. -- **`docker/Dockerfile.ydotool` and `docker/ydotool_verify.py` do that, in - CI, in twelve checks.** They settle what had only ever been asserted - against mocks: `0xc0` / `0xc1` / `0xc2` really are BTN_LEFT / BTN_RIGHT / - BTN_MIDDLE; the split edges `0x40` and `0x80` really do send a press with - no release and a release with no press, which is the entire basis of - `press_mouse` and drag; `key 30:1 30:0` really does carry numeric evdev - codes; and **the wheel signs are measured rather than assumed** — `-y 1` - reaches the kernel as `REL_WHEEL +1`, `-y -1` as `-1`, and `-x 2` as - `REL_HWHEEL +2` with the axes not swapped. That last one is the assumption - `Progress.md` had flagged as untested since the scroll work landed. -- **The twelfth check drives the backend's own functions rather than a - hand-written argv**, so "what ydotool does with this command line" and - "what AutoControl sends" are joined rather than merely adjacent. -- **`mousemove --absolute` does not emit absolute events**, which is worth - knowing before trusting it. ydotool 1.x has no ABS axes on its device: it - sends `INT32_MIN` on both relative axes first, relies on the compositor - clamping that to the top-left corner, and then sends the target as a - relative delta. So `set_position` lands on the requested pixel *because of - that clamp*. The kernel side is now pinned; the clamp is the compositor's - behaviour and stays open. -- **The container gets `/dev/uinput` and character major 13, not - `--privileged`.** ydotoold creates its input node *after* the container - starts, which `--device` cannot cover, so the job grants - `--device-cgroup-rule 'c 13:* rmw'` and nothing else. - -### Scrolling on Wayland Stops Needing a uinput Daemon - -Motion, buttons and keys had already moved onto libei. Scroll was the one -input left shelling out to `ydotool` for every notch, and `Progress.md` said -why: the sign was a guess, and a scroll that goes the wrong way fails -silently. It is wired now, and the guess has been replaced with two -independent readings plus a measurement. - -- **The two paths count wheel detents in opposite directions.** This - repository's `wayland_scroll_direction_*` constants are in the kernel's - `REL_WHEEL` frame, because that is what ydotool writes into `/dev/uinput`: - positive is up. libei is in the `wl_pointer` / libinput frame, where - positive is down — libinput's own evdev reader negates `REL_WHEEL` to get - there, and the other libei sender that documents its sign (enigo) passes a - "positive scrolls down" value straight through to `scroll_discrete`. - Horizontal needs no flip: `REL_HWHEEL` and libinput both count right as - positive. So the vertical axis is negated on the way to libei and the - horizontal one is not, which is the decision `Progress.md` was waiting on. -- **The flip is checked against the real EIS server, negative value and - all.** A fifteenth check in `docker/eis_verify.py` drives the *public* - `mouse.scroll()` — not the backend method the earlier check drives — and - reads back `(0, -120)` for up, `(0, 120)` for down and `(120, 0)` for - right. It is also the only place a negative discrete value reaches the - wire; the earlier check only ever sent a positive one, so a marshalling - fault on the sign had nowhere to show up. -- **A refused emission now falls back to the CLI, which is what the code - always claimed.** `libei`'s module docstring says every failure raises - `LibeiUnavailable`, "which `keyboard` / `mouse` already treat as *use the - ydotool CLI*". Only the *connection* was treated that way. Once a backend - was handed over, a compositor that paused a device — or a session that - ended between two calls — raised straight out of `set_position`, - `press_key` or `hotkey`. Routing scroll through libei would have added a - fourth way for a script to die on a path that has a working fallback - sitting next to it, so the fallback was made real: a chord refused - part-way releases what it already pressed before handing over, so no - modifier is left held, and a button whose *release* is refused is released - by ydotool rather than staying down for the rest of the session. -- **`LibeiUnavailable` was escaping every containment boundary.** It - inherited `RuntimeError` alone, and `CLAUDE.md` is explicit that a - framework error which is not an `AutoControlException` "silently escapes - every boundary" — the executor, the background poll loops, the request - handlers, the GUI slots. It now inherits both, so the probes that catch - `RuntimeError` keep working and the boundaries finally see it. - -## What's new (2026-08-18) - -### The libei Input Path Now Has Something to Talk To - -The Wayland capture path was verified against a real compositor; the *input* -path was not, and was recorded as needing a GNOME VM. It does not. libeis is -the server side of libei's own protocol, Debian packages it, and the two -libraries will talk to each other over a plain Unix socket — so -`docker/eis_server.py` runs a real EIS implementation and -`docker/eis_verify.py` drives AutoControl's real sender against it. No -compositor, no desktop session, 14 checks, wired into CI. - -- **Discrete scroll was off by a factor of 120.** libei measures discrete - scroll in 120ths of a wheel click — the same convention as Windows' - `WHEEL_DELTA` — and `scroll()` was passing raw detent counts, so one click - asked for 1/120th of a scroll. libei says so at runtime ("suspicious - discrete event value 1, did you mean 120?"), which no mock was ever going to - print. `scroll(0, 1)` now arrives at the server as `(0, 120)`: one click, - right axis, right sign. This was the path `Progress.md` left deliberately - unwired because the *sign* was a guess; the sign turned out to be the - smaller half of the question. -- **The teardown no longer leaks a context per process.** `ei_unref` - segfaults on libei 1.3.901 — but only on a context whose backend opened and - whose handshake never progressed. With a peer to complete a handshake - against, the live case is finally testable, and it is safe. Teardown now - releases the devices and the context normally and abandons only the state - that actually crashes, instead of abandoning every opened backend on the - suspicion that it might. -- **The values a mock cannot check are checked.** The server offers six - capabilities and reads back what the client actually bound: exactly the four - AutoControl asks for, so both the `EI_DEVICE_CAP_*` bitmask and the variadic - `ei_seat_bind_capabilities` marshalling are right. Key codes, absolute - coordinates and button codes are read off the wire and compared. Every - emission is confirmed to carry a frame, and every device to open an - emulation transaction first — libei drops events from one that has not. -- **Two things measured but not ours to fix**, recorded rather than papered - over: `eis_device_pause()` puts nothing on the wire for a sender client on - libeis 1.3.901, so the client's `DEVICE_PAUSED` handling still has no peer - to exercise it; and the `start_emulating` sequence number does not survive - the trip (an explicit 4242 reads back as 0), so AutoControl's counter cannot - be checked from the far side. The check is written so that a libeis which - starts sending pauses will fail loudly if the client ignores them. - -### The Architecture Map's Line Counts Are Measured Again - -- **The map quoted the same subsystem at two different sizes.** `CLAUDE.md` - says every figure in `architecture_explore.md` is measured, but nothing - checked it, and two counting conventions had grown up side by side: the §5.4 - theme tables and the §5.4.17 file tables counted a phantom trailing line — - `len(text.split("\n"))` reports one line more than a file that ends in a - newline actually has, and one more *per file* for a package — while §1's - totals and the §8 appendix counted correctly. So `utils/executor/` was 8,811 - lines in one section and 9,001 in another, and the §8 column did not add up - to its own total. On top of that about fifty rows were simply stale, several - `####` headings were hundreds of lines out (`linux_wayland/` was still - quoted at 10 files / 1,093 lines against a real 14 / 2,235), and one theme - table had gained two subpackages its summary line never heard about. -- **413 figures re-measured** on one convention — `len(text.splitlines())`, - what `wc -l` reports and what `CLAUDE.md`'s own over-750-lines snippet - counts. §5.4, §5.4.17 and §8 now agree with each other for every subsystem, - and §8's rows sum to its stated total. -- **`test_doc_line_counts.py` is both the gate and the fix.** It fails CI when - any quoted line count stops matching the tree, naming the offending lines, - and rewrites all of them in place with `--fix`. The line counts were the one - part of the map with no gate — the command, MCP-tool, subpackage and example - counts already had `test_doc_counts.py` — which is exactly why they were the - part that drifted. - -### The Clipboard No Longer Fails Because Another Application Was Copying - -- **One process at a time may hold the Windows clipboard open, and every - clipboard call in AutoControl gave up the instant one did.** Explorer, - Office and every browser own the clipboard for a few milliseconds at a time - while they copy; `OpenClipboard` returns false for that whole window and all - six call sites — text, image, HTML, RTF, CSV, file drops, format - enumeration — turned it straight into `RuntimeError: OpenClipboard failed`. - Measured on a live desktop with a second process copying in a loop: about - one open in a thousand failed, which is a script that dies for no reason the - operator can see or reproduce. Win32 documents this as the condition to - retry, and a library whose job is driving a machine that other applications - are busy on cannot treat "somebody else was copying" as an error. -- **`win32_clipboard_api.open_clipboard()` is now the single place that opens - it**, waiting out a busy clipboard for roughly 200 ms before reporting - failure, and closing it however the block ends. The three modules that - hand-rolled the open/close pair — including the two that predated the shared - module — go through it, so the retry cannot be forgotten at a new call site. -- **The clipboard round-trip tests no longer depend on what the rest of the - machine is doing.** They exercise the real Win32 calls, which is the only - place the four historical writer bugs could ever be seen, so faking the - backend would have deleted the coverage instead of stabilising it. They now - read the Win32 clipboard sequence number instead: unchanged between the - write and the read means nothing else wrote in that window, so the assertion - is about AutoControl's code and nothing else. Verified against a process - making 4,112 competing clipboard writes during the run. - -### A Misspelled Command Name Is a 400, Not a 500 - -- **`POST /execute` answered `500 {"error": "execute_action failed"}` for an - `AC_*` name that does not exist**, which is the same answer it gives when - the server itself breaks. A client could not tell a typo in its own request - from an outage, and the message did not say which name was unrecognised. -- **Every command name is now checked before anything runs**, and an - unrecognised one comes back as `400` listing *all* of them in - `unknown_commands` — nested flow-control bodies included — so a client fixes - every typo in one round trip. `POST /execute_file` answers the same way for - a file that is unreadable, is not an action list, or names an unknown - command. The OpenAPI spec documents both, and states that a rejected request - executed nothing. -- Validation and collection share one traversal in `action_schema`, so there - is still exactly one definition of where a nested action list may hide. - -### A Segfault in the libei Teardown, and the Binding Checked Against the Real Library - -- **`ei_unref` crashes the process on libei 1.3.901 once a backend is open, - and the fallback path ran straight into it.** Every failure mode of the - libei handshake ends in `_teardown()`, so on any host where libei is - installed and the handshake does not complete, AutoControl died with - SIGSEGV instead of quietly using ydotool — the exact opposite of the - fail-closed promise. Measured one call at a time against the real library: - `ei_unref` is safe with no backend set up and safe after a *failed* setup, - and segfaults after a successful one. `ei_disconnect` crashes in the same - state, so it is not a refcounting mistake here; the header documents - `ei_unref` as correct for both outcomes, which makes this an upstream bug. - An opened backend is now abandoned rather than unreffed — a bounded leak of - one context per process, against a crash in a library that drives a desktop. - A sentinel in the verification re-checks the upstream state on every run and - says so when the workaround can go. -- **Every entry point the binding names is now resolved against the real - `libei.so`.** A misspelled symbol or a wrong `argtypes` sails past a mocked - symbol table and only surfaces on a user's machine; all 22 prototypes plus - the variadic `ei_seat_bind_capabilities` are checked for real. -- **The whole fail-closed chain runs end to end**: connect to a socket that - speaks no EI → handshake times out → `LibeiUnavailable` → `active_backend()` - returns None → `press_key` falls through to the ydotool CLI. -- **`liboeffis` is not packaged everywhere.** Arch and Fedora ship it; Debian - trixie does not. Without it there is no portal route, so `connect()` falls - back to the well-known EIS socket — which GNOME and KDE do not create. The - libei fast path is therefore unavailable on those systems, and says so - rather than looking mysteriously idle. - -### The Wayland Capture Path Now Meets a Real Compositor - -- **`screen.size()` reported one monitor while `grab_image()` returned the - whole layout.** The `wlr-randr` parser took the first `WxH` anywhere in the - document, which is the first output's current mode. On a two-monitor layout - that is half the screen — and the two are composed by the mss-shaped shim, - so the recorder, WebRTC and MCP monitor paths asked for a region half the - size of the screen and got it. `size()` is now the layout bounding box, from - a parser that reads every enabled output's mode *and* position. Found by - running against a real compositor; no mock had a second monitor. -- **`docker/Dockerfile.wayland` runs the backend under headless sway.** The - wlroots headless backend needs no GPU, no seat and no display, so a genuine - Wayland session fits in a container — and now in CI, as the - `wayland-verification` job. Two outputs are painted different solid colours, - because on a uniform screen a region grab cannot be caught reading the wrong - rectangle, and a red/blue swap cannot be caught at all. -- **21 checks that were previously mock-only now run against pixels the - compositor painted**: grim's argv and `-g` geometry, RGB channel order, - `wlr-randr`'s undocumented output format, `size()` / `grab_image()` / - `get_pixel()` / `screenshot()`, `je_auto_control.screenshot()`'s BGR output, - `grab_logical()` (the locator and OCR path), the mss shim, and `wtype`. -- **What the container cannot answer is stated rather than glossed over.** - ydotool needs `/dev/uinput` and headless sway consumes no libinput devices; - `xdg-desktop-portal-wlr` implements no RemoteDesktop, so there is no - `ConnectToEIS` to test. Both remain open in `Progress.md`. - -### libei Input, End to End - -- **The full portal handshake is implemented.** libei is not a - call-a-function-and-a-key-is-pressed library: a sender has to open an EIS - backend, bind a seat's capabilities, take a device *out of an event*, start - emulating on it, and follow every emission with `ei_device_frame` or nothing - is delivered. All of that now happens, so `press_key`, `set_position` and the - mouse buttons can emit without spawning a process per event. -- **The EIS socket comes from the desktop portal.** On GNOME and KDE it is not - a path on disk — it is a file descriptor handed over D-Bus by - `org.freedesktop.portal.RemoteDesktop.ConnectToEIS`, after a three-call - asynchronous session dance. No command-line tool can pass a file descriptor - into this process, so the `gdbus` route used for screenshots cannot work - here; `liboeffis` (which ships with libei for exactly this) does the dance. - Where liboeffis is absent, the well-known `$XDG_RUNTIME_DIR/eis-0` socket is - still tried. -- **Devices come from events, never from the context.** The previous binding - passed the `struct ei *` context to entry points that take a - `struct ei_device *` — pointer type confusion in a C library. That is now - impossible by construction. -- **A failed probe is paid once, not per keystroke.** The handshake involves a - portal round trip and possibly a consent dialog. The result is cached for the - process, so a host where libei is installed but unusable does not re-attempt - it on every key press. -- **Everything still falls back to ydotool.** Missing library, declined - consent, a partial capability grant, a paused device, a handshake that does - not complete — each raises `LibeiUnavailable`, which the keyboard and mouse - modules already treat as "use the CLI". Scroll deliberately stays on ydotool: - its direction convention is pinned by tests, and a wrong sign would fail - silently rather than loudly. -- **The ABI constants were checked against the upstream headers**, not - guessed. Three were wrong. `enum ei_device_capability` is a *bitmask*, so - `EI_DEVICE_CAP_KEYBOARD` is `1 << 2`, not 3; `OEFFIS_EVENT_CLOSED` comes - *before* `OEFFIS_EVENT_DISCONNECTED`; and `OEFFIS_DEVICE_ALL_DEVICES` is a - `= 0` sentinel rather than the OR of the device bits. The capability error - was the expensive one — no device would ever have reported the capability, - so every session would have timed out and silently used the CLI. The - verified values are now pinned by tests. The session also asks only for the - keyboard and pointer it actually drives, so the consent dialog does not - request a touchscreen grant nothing uses. - -### Wayland Capture Has a Floor Under It - -- **`xdg-desktop-portal` backs up the three CLI helpers.** None of `grim`, - `gnome-screenshot` or `spectacle` is guaranteed to be installed — GNOME has - not shipped `gnome-screenshot` by default since 42 — so - `org.freedesktop.portal.Screenshot` is tried last through `gdbus` rather than - giving up. It is awkward by nature: the portal returns a request handle and - answers later with a signal, so the listener starts before the call is made, - and the wait is bounded (30s) because a consent dialog can sit in front of it. -- **An operator can name their own capture command.** - `JE_AUTOCONTROL_WAYLAND_CAPTURE_COMMAND="mycap --png {output}"` wins over - every detected tool. `{output}` becomes a temporary PNG path, substituted per - argument after `shlex.split` and run without a shell, so a path with spaces - stays one argument. This is the escape hatch for a setup none of the built-in - tiers fit — including one where our argv guess for a helper turns out wrong. -- **libei refuses a connection it cannot emit through.** The binding only ever - holds an `ei` context, while every device entry point takes an `ei_device`, - and it runs none of libei's seat / device / `start_emulating` / `frame` - handshake — so it could not deliver input, but *could* pass the wrong pointer - into a C library on a host that opens `$XDG_RUNTIME_DIR/eis-0`. It now stops - at `connect()` with an explanation. Callers already treated that as "use the - ydotool CLI", which is what every real desktop was doing anyway. -- **The portal listener cannot wedge on shutdown.** Its pipe is closed only - after the reader thread lets go of it; closing a stream out from under a - blocked `read()` can hang on the buffer lock, which would have turned a - timed-out capture into the hang the timeout exists to prevent. - -### The Container Image Builds and Starts From a Windows Checkout - -- **Every container built on a Windows clone died on startup.** `.gitattributes` - said `* text=auto`, so `docker/entrypoint.sh` and `docker/entrypoint-xfce.sh` - were checked out with CRLF. The shebang then reads `#!/bin/sh`, the kernel - looks for an interpreter whose name ends in a carriage return, and the image - builds perfectly and then exits with `exec /usr/local/bin/autocontrol-entrypoint: - no such file or directory` — a message that names the file it just failed to - find. CI never saw it: a Linux runner checks the same file out with LF. - `*.sh text eol=lf` now pins it, and a test asserts no entrypoint carries CRLF. -- **`.dockerignore` was in `docker/`, where Docker does not look.** Docker reads - it from the build *context* root, and every documented build passes the - repository root (`docker build -f docker/Dockerfile .`), so the exclusions did - nothing: `.git`, `.venv`, `test/` and the caches were all being shipped to the - daemon on every build. Moved to the root, where it takes effect. -- **The `mss` shim test measured the host's monitor, not its own fake.** - `test_screen_grabber.py` patched `backend_grab_image` but left - `_backend_screen_size` reading the real `platform_wrapper.screen`, so - `monitors[0]` reported whatever display the developer had. It passed on a - 1920x1080 desktop and failed under a 1280x800 Xvfb, for reasons unrelated to - the code under test. The fake now owns both halves of the seam. - -## What's new (2026-08-17) - -### Wayland Sees the Screen - -- **Every capture path now goes through the platform backend.** `screenshot()`, - the image and anchor locators, OCR, smart waits, visual regression, screen - recording, the MCP monitor tools and remote desktop each reached for - `PIL.ImageGrab` or `mss` directly. Both read the X11 root window on Linux, - which under Wayland belongs to XWayland and does not composite native Wayland - windows. Pillow does fall back to `gnome-screenshot` / `grim` / `spectacle`, - but only inside `except OSError` around its X11 grab — so it fires when there - is no X display at all, and *not* while XWayland is up, which is the default - on GNOME, KDE and sway. `mss` has no fallback in any configuration. - `utils/cv2_utils/screen_grabber.py` is now the one place that decides how - pixels are read: a backend that publishes `grab_image` gets wrapped in - whichever library shape the caller already uses. Windows, macOS and Linux X11 - publish nothing and keep the real libraries, so their behaviour is - byte-for-byte unchanged. -- **Wayland capture covers three compositor families, not one.** `grim` only - speaks `wlr-screencopy`, which GNOME and KDE do not implement — so - `linux_wayland/capture.py` tries `grim` (sway, Hyprland, river), then - `gnome-screenshot`, then `spectacle`, and reports which one it used. Only - `grim` can take a region itself; the others capture the screen and the region - is cropped from it. -- **A missing capture tool fails loudly.** It raises with the install command - for each compositor rather than handing back an empty XWayland grab that - reads downstream as "template not found". The new `screen_capture` - diagnostics check names the tool in use before anything has to fail. -- **`screen.size()` and `get_pixel` work off GNOME/KDE too.** Resolution falls - back from `wlr-randr` to measuring a capture, and `get_pixel` crops a 1x1 - region from whichever capture path is available. - -### Which Program Owns That Window - -- **`foreground_window_process_id` / `window_process_id`** (`AC_foreground_window_pid`, - `AC_window_pid`, `ac_foreground_window_pid`, `ac_window_pid`, two Script Builder - specs; Windows backend `get_window_process_id`): a window title is whatever the - application decides to display, and unrelated programs share titles like - `Settings`, so "which program is the user in front of" had no reliable answer. - It does now, and callers can join it against a process list. Unavailable reads - as `None` — never a bare `0`, which would match the System Idle Process. -- **The two older window-input functions are deprecated and now work.** - `send_key_event_to_window` / `send_mouse_event_to_window` posted to the frame, - which reaches nothing in a window with child controls; they delegate to the - new pair, warn once, and keep their old argument types working. -- **`get_pixel` no longer risks a truncated device context.** The Windows screen - backend declared no prototypes, so an HDC — pointer-width — came back through - ctypes' default `c_int`, and the same truncated value was passed on to - `GetPixel` and `ReleaseDC`. Every prototype is declared and the module owns - private DLL handles, so the declarations cannot leak into other callers. -- **Act on a process's windows, not on a title** - (`windows_for_process_id`, `minimize_windows_for_process`): a browser names - its windows after the page they show and runs a dozen processes without any - window, so "minimise that application" was previously hand-rolled Win32 in - every caller — enumerate windows, ask each one which process owns it, filter, - minimise. That loop now lives here once. -- **Typing into a window without stealing focus actually works now** - (`post_key_to_window`, `post_click_to_window`, `AC_post_key_to_window`, - `AC_post_click_to_window`, `ac_post_key_to_window`, `ac_post_click_to_window`). - The existing `send_key_event_to_window` posted to the top-level frame, and - keyboard messages go to the control that *has focus* — so it silently did - nothing in any application with child controls while still reporting success. - Measured on Character Map: posting to the frame typed nothing, posting to the - focused edit typed the character; on Windows 11 Notepad the new path types - into a background window while the foreground window keeps focus. Clicks - resolve the deepest child under the point and convert to its client - coordinates. This is still best effort by nature — games and anything reading - raw input ignore posted messages — so both functions return whether the - messages were queued rather than claiming the application acted. -- **Half the clipboard was broken on 64-bit Windows, and nothing noticed.** - `set_clipboard_files`, `set_clipboard_html`, `set_clipboard_rtf` and - `set_clipboard_csv` raised `OverflowError` on every call — four writers that - had never once worked. Each module declared `restype` but not `argtypes`, so - a pointer-width memory handle went through ctypes' default `c_int`. The - pure-function tests could not see it (the byte packing was always right) and - nothing exercised the Win32 half. The prototypes now live once in - `utils/clipboard/win32_clipboard_api.py`, every clipboard module goes through - it, and a new test round-trips text, HTML, RTF, CSV and file lists through the - real clipboard — plus a static check that no module hand-rolls those calls - without declaring prototypes again. -- **`save_window_layout` no longer records windows it cannot restore.** Its - docstring promised titled windows; the default lister returned all of them, - while `restore_window_layout` addresses a window by title and skips blank ones. - On a real desktop that was 28 entries saved against 15 restorable — a caller - reporting the saved count was over-promising by a factor of two. The lister now - passes `titled_only=True`, and a round-trip test pins save and restore to the - same set. - -## What's new (2026-08-15) - -### Text Entry and On-Screen Location That Match What You See - -Three defects that made automation miss silently rather than fail loudly: text -that could not be typed, targets that could not be found, and coordinates that -were subtly wrong. All three produced "sometimes it works" behaviour, which is -harder to diagnose than a crash. - -- **Type anything, without the clipboard** (`type_unicode_keys`, - `type_unicode_text`, `AC_type_unicode_keys`, `AC_type_unicode_text`, - `ac_type_unicode_keys`, `ac_type_unicode_text`): `write` typed through the - 192-entry virtual-key table and *raised* on the first character outside it — - which on a US layout means `, . / : ? ! _ + @ %` as well as all CJK, so a URL - or a Chinese sentence failed as a whole string. The Windows backend gains - `press_unicode` / `release_unicode` / `type_unicode_unit` built on the - `KEYEVENTF_UNICODE` flag its `KeyboardInput` already understood, and `write` - now falls back to that route per character instead of raising. The existing - clipboard-paste `type_unicode` stays, but is no longer the only option: it - overwrites the user's clipboard and is refused by inputs that block paste, so - key injection is the default where a backend supports it. -- **Find text the engine split across boxes** (`find_spans`, `group_lines`): OCR - backends box one *word* at a time, so `Save As` and `另存新檔` had no single - box to compare against and `find_text_matches` reported nothing for text - plainly on screen. Matching now scans runs of consecutive boxes on a line — - grouped by vertical overlap, so it also works for backends that report no line - ids — and returns the shortest run that spells the target, with the union - rectangle and the weakest word's confidence. A one-box run is the old - behaviour, so nothing that matched before stops matching. -- **Capture in the coordinate space the mouse uses** (`grab_logical`, - `logical_virtual_rect`, `logical_scale`, `needs_rescale`): `find_image` and - `find_image_multi` captured through `ImageGrab.grab()`, which sees only the - primary monitor — a target on a second display could never be found. The - full-desktop capture has the opposite problem: it returns *physical* pixels - while a DPI-unaware process clicks in *logical* ones, so on a mixed-DPI - desktop (3840 physical against 3456 logical) a point read off the frame lands - up to ~116 px away. `monitor_layout.grab_logical` is now the single capture - primitive behind both OCR and template matching: it covers every monitor, - rescales into logical pixels, and reports the virtual-desktop origin to add to - a hit — which is negative whenever a monitor sits left of or above the - primary. Template matches are translated by that origin, so a located box is - directly clickable. `find_image` / `find_image_multi` also gained - `all_screens` and `screen_region`. -- **`visual_match` reports coordinates you can act on.** The scored matcher had - the same three problems plus one of its own: it captured through - `pil_screenshot` (primary monitor, physical pixels), and a hit found inside a - `region` was returned in *region-local* coordinates — so clicking a match was - wrong by the region's own offset. It now grabs through `grab_logical` and adds - the origin to every hit; a caller-supplied `haystack` is still its own - coordinate space, as it must be. Two more traps closed: a template that is - almost a single colour now raises `AutoControlFlatTemplateException` instead - of saturating the score map at 1.0 and "finding" the target at an arbitrary - position, and templates load through `imdecode` so a **non-ASCII path** no - longer reads as a corrupt file. Everything downstream of `_haystack_gray` - (`barcode`, `edge_lines`, `edge_match`) inherits the corrected capture. -- **Accessibility search that can actually find the control** (`window_title` - on `list_accessibility_elements` / `find_accessibility_element`, new - `find_accessibility_elements`, `contains`, `accessibility_status`, - `control_get_state`; commands `AC_a11y_find_all` / `AC_control_get_state`, - MCP `ac_a11y_find_all` / `ac_control_get_state`). Four things stood between - the API and a real target: the name had to match **exactly**, so a label - carrying an accelerator (`Save(&S)`) or trailing padding never matched; - `role="button"` never matched either, because the Windows backend reports the - raw `ControlType_50000` and nothing translated it; `max_results` truncated the - list *before* filtering, so an element past the cap could not be found however - specific the filter; and there was no way to search one window. Scoping is the - one that matters most for speed — measured on a busy desktop, walking - everything is 2,085 elements in **61 s** against 135 elements in **0.14 s** for - a single window. `contains` matching ranks an exact name first, so "OK" offers - the `OK` button before `OK and close`. `control_get_state` answers what pixels - cannot — a field's text scrolled out of view, a checkbox's true state, a - slider's exact number — in one call, with an absent key meaning "no such - state" rather than "empty"; password fields report only that they are password - fields, on both `get_value` and `get_state`, because UIA's masking is a - convention a custom-drawn control can ignore. Elements now also carry - `enabled`: a disabled control looks clickable and silently swallows the click. - Conversion cost is gone too — properties come back through one - `FindAllBuildCache` call instead of one cross-process read per property, and - the per-element `OpenProcess` for the app name is cached (converting 500 - elements: 0.02 s). - - **A desktop-wide search went from 61 s to about 2 s**, which took three - separate fixes because there were three separate causes: - - 1. *The root.* One `FindAll` from the desktop walks every window's subtree and - cannot be interrupted. An unscoped listing now takes **one top-level window - at a time in z-order**, so it can stop as soon as it has enough — and the - window the user is looking at is searched first. - 2. *The walk.* Even per window, `FindAll` is atomic: one 34,507-element window - took 10.3 s to answer a request for 200. The walk is now node by node - through `ControlViewWalker` with a cache request, so asking for 200 - elements costs 200 elements of work (0.036 s for 50, 0.114 s for 200, - 0.486 s for 1,000). - 3. *The provider.* UIA waits on the application itself. A full-screen game - that never answers made a single `ElementFromHandle` block for **60 s** — - and it was not detectable in advance: the window pumps messages, replies to - `WM_GETOBJECT`, and `IsHungAppWindow` says it is fine. The automation object - now comes from `CUIAutomation8` as `IUIAutomation2` with - `ConnectionTimeout` bounded, which brings that same call to 1.0 s. - - Measured end to end afterwards: 50 elements 0.20 s, 200 in 1.29 s, 1,000 in - 1.90 s, 3,000 in 3.87 s. Naming a window is still an order of magnitude - better (0.03 s) and remains the advice. - - `find_*` also separates `max_results` (how many matches to return) from - `scan_limit` (how many elements to examine) — one number cannot mean both, and - conflating them turns "up to 40 buttons" into "only look at the first 40 - elements on the desktop". - -### Telling You When Your Input Is Going Nowhere - -Sent input can be discarded before it reaches anything while the send call still -reports success — the caller is told "clicked (500, 300)", nothing happens, and -no error exists anywhere. `utils/input_reach` (`input_desktop_available`, -`input_reaches_system`, `AC_input_reachable`, `ac_input_reachable`) answers the -question directly. - -Two causes needing two checks. A locked workstation is detectable for free by -asking for the input desktop. Input *filtering* is not: measured on a machine -with an anti-cheat game in front, `SendInput` succeeds, `GetAsyncKeyState` never -sees the key, `OpenInputDesktop` reports everything fine, and the game's -integrity level is the same *Medium* as ours — so neither a privilege comparison -nor any cheap query can tell. The only honest test is to send a key and look, -which is why that probe is a diagnostic (it presses F13, which nothing binds) -rather than a gate in front of every action. - -### A Recording That Can Actually Be Replayed - -`record` captured presses and nothing else, which is not enough to reproduce a -session — and the gap was silent, because the recording looked fine until it was -played back. Three things were missing and one was leaking: - -- **Releases.** A press-only log cannot tell a drag from a click, and a modifier - held across several actions cannot be reconstructed. Verified before the - change: five press-and-release pairs produced five events. -- **The wheel.** `WM_MOUSEWHEEL` was not handled at all, so scrolling vanished. - Its `mouseData` high word is a *signed* notch count — read unsigned, a scroll - down becomes a scroll up by 65,534 notches. -- **Timing.** Without timestamps every step replays at once and no real - interface keeps up. `utils/input_macro` already had `replay_timeline` waiting - for `delta_ms` events that nothing produced. -- **A leaked thread per recording.** The listener pumped `GetMessage` once and - `stop_record` never woke it, so each record cycle left a thread blocked - forever. Verified: the thread was still alive after `stop_record` returned. - -`Win32InputHook` replaces both listeners with one hook that records press *and* -release, wheel deltas and a monotonic timestamp, pumps messages properly, and -exits on `WM_QUIT` when stopped. `stop_record_timeline` (`AC_stop_record_timeline`, -`ac_record_stop_timeline`) returns those events with `delta_ms`, ready for -`replay_timeline`. `stop_record` is untouched and still returns the historical -press-only queue, so existing callers keep working. - -New `utils/keyboard_layout` answers the other half: which character a key -produces. Punctuation differs per layout, so a hard-coded US table mislabels -every punctuation key on a German or Nordic keyboard. It asks the **foreground -window's** layout (the user types into what is in front, not into this process) -and only translates **after** recording — `ToUnicodeEx` mutates dead-key -composition state, so calling it mid-typing corrupts the character being -composed. - -### One Clipboard Image API Instead of Two - -`utils/clipboard` carried two `get_clipboard_image` / `set_clipboard_image` -pairs under identical names — one in `clipboard.py` taking PNG bytes, one in -`clipboard_image.py` taking a file path. Both had live callers, so importing -the wrong module failed at runtime, and only for whichever argument type you -passed. There is one pair now: `set_clipboard_image` accepts **either** PNG -bytes or a path, `clipboard_image.py` is gone, and both functions are exported -from the subpackage and the facade with `AC_clipboard_get_image` / -`AC_clipboard_set_image` commands — previously they were reachable only from -MCP and the GUI, never from `execute_action`. - -`windows/listener/` went with it: `Win32KeyboardListener` and -`Win32MouseListener` had no callers left once recording moved to -`win32_input_hook.py`. - -### Window Handles You Can Actually Use - -Window management listed windows but could not really operate on them. - -- **Handles are integers again.** The `EnumWindows` callback declared its hwnd - as `POINTER(c_int)`, so every handle came back as an `LP_c_long` object; - `int(hwnd)` on one raises `ValueError`. The list was readable and otherwise - useless — you could not focus, move or measure anything it returned, and the - `ac_list_windows` MCP tool raised outright because its handler called - `int(hwnd)`. Every Win32 prototype in `windows_window_manage` now declares - `argtypes` / `restype`, which also stops a 64-bit handle being truncated to - 32 bits. -- **`close_window_by_title` closes.** It used to minimise, because Win32's - `CloseWindow()` minimises despite its name and the wrapper passed that - through. It now posts `WM_CLOSE` — the same thing the window's own close - button does, so the application still gets to run its save prompts. - `minimize_window_by_title` keeps the old behaviour under an honest name. -- **New primitives**: `foreground_window` (what the user is working in), - `window_rect` (screen rectangle, negative coordinates and all), - `move_window_by_title` (omit width/height to reposition without resizing) and - `list_windows(titled_only=True)` — on this desktop that is 17 windows rather - than 34, the rest being shell and helper surfaces. -- **`focus_window` restores a minimised window** before raising it; focusing a - minimised window previously did nothing you could see. A maximised window - stays maximised. - -### URL Canonicalisation, Reachable From Every Surface - -`utils/url_canon` (RFC 3986 canonicalisation, normalisation and query helpers) -had a working headless core and passing tests, but none of its delivery -surfaces were connected, so it could only be reached by importing the submodule -directly. - -- **`canonicalize_url`, `normalize_url`, `urls_equal`, `build_query`, - `parse_query`** are now re-exported from the facade and listed in `__all__`. -- **`AC_canonicalize_url`, `AC_normalize_url`, `AC_urls_equal`** wire the same - functions into the executor, so they work from JSON action files, the socket - server, the scheduler and the Script Builder without Python glue; the - matching **`ac_canonicalize_url` / `ac_normalize_url` / `ac_urls_equal`** MCP - tools and three Script Builder `CommandSpec`s come with them. - -Comparing two URLs for "the same page" is the actual use: `urls_equal` ignores -query order and the fragment, so `?b=1&a=2` and `?a=2&b=1#top` match, which is -what a navigation assertion needs and what plain string comparison gets wrong. - -## What's new (2026-07-18) - -### Cross-Platform Reliability Hardening - -A full-project runtime audit swept every platform backend and utility for execution-time defects and unexpected behaviour, adding a headless regression test for each fix. There are **no API changes** — existing scripts keep working, they just fail less and behave correctly in more places. - -- **macOS (Retina / HiDPI)**: mouse-position reads and omitted-coordinate clicks now land at the correct point. The cursor y-flip used a *pixel*-based display height against a *point*-based cursor, offsetting every implicit click on a 2× display. -- **Remote-desktop relay**: fixed a hang on **Linux + CPython 3.14** where a paired pipe never exited after one peer disconnected — a cross-thread `shutdown()` no longer reliably wakes a blocked `recv()` there, so the pump now polls readability with `select()` and always re-checks its stop flag. -- **USB/IP server** now binds `127.0.0.1` by default (least-privilege); export the device to the LAN with an explicit `host="0.0.0.0"`. -- **Executor**: `AC_expect_poll` no longer crashes on a not-ready value (missing result key or a transiently-failing action) and keeps polling; `AC_parallel` branches are properly variable-scope-isolated, so a nested `AC_execute_action` can't race on the parent's scope; a malformed `run_suite` spec reports a clean error instead of aborting the run. -- **Windows Interception backend**: send-to-window click now performs a real press/release instead of silently no-opping on the button tuple. -- **Wayland**: a partial-coordinate `mouse_scroll` degrades gracefully instead of raising `NotImplementedError`. -- **Typed exceptions preserved at boundaries**: saving an action file with non-encodable text raises `AutoControlJsonActionException` (not a raw `UnicodeEncodeError`); a non-ASCII USB/IP busid no longer kills the client worker thread; SQLite data-source / query connections are always closed; USB ACL rule removal is case-insensitive to match the (case-insensitive) rule matching. -- Builds on the round-3 sweep that reparented the exception family under `AutoControlException` (assertions still propagate under `raise_on_error=False`) and hardened thread-safety across the socket server, scheduler, triggers, and MCP server. - -## What's new (2026-07-03) - -### Stable API, Failure Bundles, and Release Engineering - -A versioned entry point for new integrations, a portable failure-diagnostics format, and a hardened release pipeline. Full reference: [`docs/API_LIFECYCLE.md`](docs/API_LIFECYCLE.md) and [`docs/CAPABILITY_MATRIX.md`](docs/CAPABILITY_MATRIX.md). - -- **Stable `je_auto_control.api` façade**: small, lazy, typed namespace (`execute_action`, `execute_action_with_vars`, `generate_code`, `run_diagnostics`, failure bundles) so new consumers can import core automation without eagerly loading hundreds of optional integrations. Governed by a written lifecycle policy — stable removals need a deprecation warning plus two minor releases — with `utils/deprecation.deprecated` supplying consistent, metadata-carrying warnings. CI type-checks this surface with mypy and smoke-imports it on a Windows/Ubuntu/macOS × Python 3.10/3.14 matrix. -- **Failure bundles** (`create_failure_bundle` / `failure_bundle_on_error`, CLI `je_auto_control failure-bundle out.zip`): one atomic, self-contained `autocontrol.failure-bundle/v1` ZIP for diagnosing a failed run — manifest with runtime info, redacted error/context/events, redacted log tail, optional screenshot and diagnostics report, opt-in attachments. Collectors are best-effort: a broken screen grab or diagnostics probe is recorded in `collector_failures` instead of losing the bundle. `codegen --failure-bundle` wraps generated pytest flows so every generated test archives its own failure evidence. Secret redaction now also masks explicit `key=value` / `Authorization: Bearer` credential syntax regardless of entropy. -- **Release engineering**: publishing moves from push-to-main to immutable `v*` tags — the new `release.yml` verifies the tag matches the package version, builds, smoke-tests the wheel, attests build provenance, and publishes via PyPI Trusted Publishing. `quality.yml` gains dependency review, a coverage floor (fail-under 35, branch coverage), and a mypy gate on the stable API; a new platform-smoke workflow exercises the stable API on all three OSes. -- **Project docs**: new [`SECURITY.md`](SECURITY.md) (private-advisory reporting, response targets, operational defaults), [`CHANGELOG.md`](CHANGELOG.md) (Keep-a-Changelog compatibility record), API lifecycle policy, and a capability/platform support matrix. The Sphinx indexes catch up on v182–v223 feature docs in both languages. - -## What's new (2026-07-02) - -### Menu-Driven GUI: the Actions Menu Replaces In-Tab Buttons - -Every tab's commands now live in one predictable place. The window menu bar gains a dynamic **Actions** menu that rebuilds for the active tab; tabs keep only their inputs, tables, and result/status views instead of rows of buttons. Full reference: [`docs/source/Eng/doc/new_features/v223_features_doc.rst`](docs/source/Eng/doc/new_features/v223_features_doc.rst). - -- **Window-level Actions menu**: core tabs declare their commands at registration; feature tabs expose a `menu_actions()` hook returning `(label_key, handler)` pairs. 46 of 48 registered tabs now surface their commands this way — Script Builder and Remote Desktop intentionally keep their interactive panel layouts, and the menu shows a placeholder there. Buttons a window-level menu cannot replace stay in place (per-page browse buttons inside stacked trigger forms, the visibility-toggled data-source browse button, stateful auto-refresh checkboxes). A headless regression test guards the contract so no tab can silently lose its commands. - -## What's new (2026-06-26) - -### Trial and Force Action Modes (Playwright-style) - -Dry-run "is this control ready?" without clicking, or force a click past the gate. Full reference: [`docs/source/Eng/doc/new_features/v222_features_doc.rst`](docs/source/Eng/doc/new_features/v222_features_doc.rst). - -- **`act_with_mode`** (`AC_act_with_mode`): `actionability.act_when_ready` only waits-then-acts. Real flows need two more modes Playwright codified: **trial** (run every actionability check but *don't* act — a side-effect-free "would this work?" dry run) and **force** (skip the checks and act now — the escape hatch when the gate misjudges a control as occluded/disabled). `act_with_mode` adds both alongside the default `auto`, over the same injectable seams as the gate, returning `{mode, acted, actionable, reason, point, result}`. Reuses `actionability.wait_actionable`; fully testable without a screen. Completes the ROUND-15 input-fidelity lane (7/7). No `PySide6`. - -### Act In View — Scroll to a Target, Then Act When Actionable - -Click the row three pages down: scroll it into view, then gate on actionability before clicking. Full reference: [`docs/source/Eng/doc/new_features/v221_features_doc.rst`](docs/source/Eng/doc/new_features/v221_features_doc.rst). - -- **`act_in_view` / `ScrollPlan`** (`AC_act_in_view`): two reliability primitives stayed separate — `scroll_find.scroll_until_visible` brings an off-screen target on-screen, and `actionability.act_when_ready` waits for it to be visible/stable/enabled/unoccluded before acting. A real "click the off-screen row" step needs both. `act_in_view` composes them: scroll until the target is located, then run the actionability gate at its point and perform the action. `ScrollPlan` bundles the scroll search + its `locator`/`scroller` seams so the call stays within the argument limit; the actionability probes (`region_sampler`/`enabled_probe`/`hit_tester`) and gate `config` are injectable too, so the whole flow is testable without a screen. Closes the input-fidelity lane's composition gap. No `PySide6`. - -### Template-Free Element Proposal (Pixels to Elements) - -Get a clean numbered element list straight from the screen when there's no accessibility tree. Full reference: [`docs/source/Eng/doc/new_features/v220_features_doc.rst`](docs/source/Eng/doc/new_features/v220_features_doc.rst). - -- **`propose_elements` / `tag_kinds`** (`AC_propose_elements`, `AC_tag_kinds`): Set-of-Marks, `observation` and the grounding helpers all assume you already have element boxes — but a game, a custom-drawn app or a remote desktop has no accessibility tree. `propose_elements` builds that top-of-funnel list from pixels: detect widget boxes (closed-edge blobs via Canny + morphology + `connected_boxes`) and text boxes (`text_regions.find_text_regions`), fuse them — the `element_parse` `ocr > icon` priority *is* the "drop widget-that-is-really-text" cross-check — and return them in reading order, each tagged `text` or `widget`. `tag_kinds` is the pure labeller. cv2 imported lazily; the labeller is fully testable. Seventh and final feature of the ROUND-15 perception lane. No `PySide6`. - -### Classify a Widget from Its Pixel Shape - -Tell a checkbox from a radio button from a text field — from pixels, no model. Full reference: [`docs/source/Eng/doc/new_features/v219_features_doc.rst`](docs/source/Eng/doc/new_features/v219_features_doc.rst). - -- **`classify_widget` / `box_features` / `classify_icon`** (`AC_classify_widget`, `AC_classify_icon`): Set-of-Marks and element proposers return *boxes* but not *what each box is*; `form_fields.checkbox_state` reads a box already known to be a checkbox — the gap is the typing step before it. `box_features` extracts `{aspect, fill, edge_density, circularity}` for a box; `classify_widget` is the pure heuristic classifier (round→radio, wide-rounded→toggle, square-sparse→checkbox, wide-hollow→text_field, wide-filled→button, else icon); `classify_icon` composes them. The classifier is pure and fully testable; cv2/numpy imported lazily so the module stays importable. Sixth feature of the ROUND-15 perception lane. No `PySide6`. - -### Localize a Change to the Elements That Changed - -Turn a raw screen diff into "element 3 changed" by scoring a list of element boxes. Full reference: [`docs/source/Eng/doc/new_features/v218_features_doc.rst`](docs/source/Eng/doc/new_features/v218_features_doc.rst). - -- **`localize_changes` / `rank_changes`** (`AC_localize_changes`, `AC_rank_changes`): existing diffs answer *where* pixels changed (`motion_regions`, `perceptual_diff`, `ssim_changed_regions` → raw pixel regions) or which *accessibility* elements differ (`element_diff`, needs metadata) — but not "given a frame diff **and a list of element boxes**, which of *those* changed?". `localize_changes` diffs a reference against the current screen and scores each supplied element box by its mean per-pixel change; `rank_changes` is the pure ranker that flags `changed` (score ≥ `threshold`) and sorts most-changed first. Pairs with `set_of_marks`/accessibility boxes to give a per-element "what changed" feedback signal after a click. cv2/numpy imported lazily; ranking is pure and fully testable. Fifth feature of the ROUND-15 perception lane. No `PySide6`. - -### Theme-Invariant Matching (Light Template, Dark Mode) - -Find a button captured in light mode even after the app switches to dark mode. Full reference: [`docs/source/Eng/doc/new_features/v217_features_doc.rst`](docs/source/Eng/doc/new_features/v217_features_doc.rst). - -- **`normalize_theme` / `match_theme`** (`AC_match_theme`): `match_template` correlates raw pixel intensities, so a light-mode template scores terribly against the same control in dark mode — the polarity is inverted. The fix is to compare *structure*. `normalize_theme` maps an image to a polarity-invariant single channel (`sobel`/`laplacian` gradient magnitude — identical for an image and its colour inverse — or `zscore`); `match_theme` normalizes both the template and the screen, then locates the template via `visual_match.match_template`, finding it across a light/dark flip that defeats raw matching. cv2/numpy are imported lazily so the module stays importable everywhere. Fourth feature of the ROUND-15 perception lane. No `PySide6`. - -### Sample a Region's Text Contrast (WCAG) - -Grade the legibility of on-screen text when you only have a region, not the two colours. Full reference: [`docs/source/Eng/doc/new_features/v216_features_doc.rst`](docs/source/Eng/doc/new_features/v216_features_doc.rst). - -- **`grade_contrast` / `dominant_pair` / `region_contrast`** (`AC_grade_contrast`, `AC_dominant_pair`, `AC_region_contrast`): `a11y_audit.contrast_ratio` grades a foreground/background pair you already know — but a button or label on screen is a *patch of pixels*, not two known colours. `dominant_pair` splits sampled pixels at the mean luminance into the dominant foreground (minority, the text) and background (majority); `grade_contrast` grades a pair against the WCAG 2.x AA/AAA thresholds (normal + large text); `region_contrast` samples a screen region (through an injectable `sampler`) and grades it. The grading and split are pure and reuse `a11y_audit.contrast_ratio`, fully testable without a screen. Third feature of the ROUND-15 perception lane. No `PySide6`. - -### Set-of-Marks Label Layout (No Overlap, Readable Colour) - -Number every element without the labels piling up or vanishing into the background. Full reference: [`docs/source/Eng/doc/new_features/v215_features_doc.rst`](docs/source/Eng/doc/new_features/v215_features_doc.rst). - -- **`place_labels` / `label_color`** (`AC_place_labels`, `AC_label_color`): Set-of-Marks draws each numbered label at a fixed offset, so on dense UIs the numbers pile on top of each other and a dark label on a dark element vanishes. `place_labels` is greedy non-overlap placement — for each mark it tries a ring of candidate positions around its box (above/below/inside, left/right aligned) and takes the first that stays in bounds and clears every already-placed label; `label_color` picks black or white by whichever has the better WCAG contrast against the element background (reusing `a11y_audit.contrast_ratio`). Pure standard library, deterministic, fully testable without rendering. Second feature of the ROUND-15 perception lane. No `PySide6`. - -### Colour-Vision-Deficiency Simulation + Collision Check - -Check whether your red/green status colours are distinguishable to colour-blind users. Full reference: [`docs/source/Eng/doc/new_features/v214_features_doc.rst`](docs/source/Eng/doc/new_features/v214_features_doc.rst). - -- **`simulate_cvd` / `colors_collide` / `color_distance`** (`AC_simulate_cvd`, `AC_colors_collide`): status UIs lean on colour (green "ok" vs red "error"), but for the ~8% of men with a colour-vision deficiency those can be indistinguishable — and nothing in the framework could check it. `simulate_cvd` maps an RGB colour through a dichromat simulation matrix (protanopia/deuteranopia/tritanopia) at a given `severity`; `colors_collide` simulates two colours and reports whether they become confusable (a perceptual `redmean` distance below `threshold`); `color_distance` is the underlying metric. Pure standard library — no numpy/OpenCV, operating on plain RGB tuples, fully testable. First feature of the ROUND-15 perception lane. No `PySide6`. - -### Wait Until the App Is Idle - -Hold off the next click until the busy/wait cursor settles — don't act mid-churn. Full reference: [`docs/source/Eng/doc/new_features/v213_features_doc.rst`](docs/source/Eng/doc/new_features/v213_features_doc.rst). - -- **`wait_until_app_idle` / `idle_point`** (`AC_wait_until_app_idle`, `AC_idle_point`): a click fired while the app is still churning (busy cursor up, dialog mid-paint, long handler running) is dropped or mis-targeted. `smart_waits` watches *pixels* settle; this watches the app's *busy signal* settle, which is cheaper and survives animated-but-idle UI. It reuses `settle_detector.SettleTracker` — each poll feeds 1.0 when busy / 0.0 when idle, and returns once the app has read idle for `quiet_samples` polls in a row (a busy spike resets the run). `wait_until_app_idle` polls an injectable `busy_probe` (default = Windows busy/app-starting cursor) with injectable `clock`/`sleep`; `idle_point` is the pure analyser over a recorded busy/idle trace. Fully testable without an app. Fifth feature of the ROUND-15 input-fidelity lane. No `PySide6`. - -### Ensure a Control Is in the Desired State (Idempotent) - -Read-compare-act-verify instead of acting blind — don't double-toggle an already-checked box. Full reference: [`docs/source/Eng/doc/new_features/v212_features_doc.rst`](docs/source/Eng/doc/new_features/v212_features_doc.rst). - -- **`ensure_state` / `ensure_toggle`** (`AC_ensure_field_value`): automation that acts *unconditionally* double-toggles a box that was already checked or re-enters an already-correct field, and can't be safely re-run. The robust shape is read-compare-act-verify. `ensure_state` reads via an injectable `reader`, and only if it doesn't equal `desired` applies `setter` and re-reads (up to `attempts`); `ensure_toggle` is the boolean specialization that calls `toggle` only while the state differs. A control already in the desired state is left untouched (`changed=False`) — idempotent and safe to re-run, distinct from `idempotency` (a request-key replay cache) since this converges *device state*. The executor's `AC_ensure_field_value` idempotently sets a native control's value via the accessibility backend. Fourth feature of the ROUND-15 input-fidelity lane. No `PySide6`. - -### Adaptive Timeout from Observed Durations - -Stop guessing wait timeouts — learn them from how long the step actually takes. Full reference: [`docs/source/Eng/doc/new_features/v211_features_doc.rst`](docs/source/Eng/doc/new_features/v211_features_doc.rst). - -- **`recommend_timeout` / `timeout_stats`** (`AC_adaptive_timeout`, `AC_timeout_stats`): hard-coded waits are a perennial flakiness source — too short races a slow machine, too long makes every failure pay the full timeout. This learns the timeout from observed step durations: a high percentile (the slow-but-real case) scaled by a safety `factor`, clamped to a sane `[min_s, max_s]` band. `recommend_timeout` is the single number to feed a `wait_for_*` / actionability `GateConfig`; `timeout_stats` also exposes the percentiles and `floored`/`capped` flags for tuning. Both are pure and reuse `stats.percentile`; with no samples they fall back to `default_s`. Third feature of the ROUND-15 input-fidelity lane. No `PySide6`. - -### Verify a Field After Typing - -Read the field back and confirm the value actually landed — don't type and hope. Full reference: [`docs/source/Eng/doc/new_features/v210_features_doc.rst`](docs/source/Eng/doc/new_features/v210_features_doc.rst). - -- **`compare_field_value` / `verify_field_value` / `fill_and_verify`** (`AC_compare_field_value`, `AC_verify_field_value`): `field_entry` types into a control and *hopes* — a slow IME, focus steal, input mask or auto-format can silently mangle or drop characters, and nothing reads the field back. This is distinct from `action_effect` (did *anything* change near the target?) and `postcondition.text_present` (does the text appear *anywhere*?) — neither confirms *this* field equals *this* value. `compare_field_value` is the pure comparator (`exact`/`trim`/`ci`/`normalized` NFKC/`contains`); `verify_field_value` reads through an injectable `reader` (native accessibility value in the executor); `fill_and_verify` types via an injectable `filler`, reads back, and retries (optionally clearing first) until it matches or attempts run out. Every comparison and retry decision is pure and unit-tested without a real control. Second feature of the ROUND-15 input-fidelity lane. No `PySide6`. - -### Retry Budget — Deadline + Jitter - -Retry a flaky step bounded by a total time budget, with jittered backoff. Full reference: [`docs/source/Eng/doc/new_features/v209_features_doc.rst`](docs/source/Eng/doc/new_features/v209_features_doc.rst). - -- **`RetryBudget` / `run_with_budget` / `backoff_delay` / `jittered_delay`** (`AC_retry_delay`, `AC_plan_retry_delays`): `resilience.RetryPolicy` retries a fixed attempt count with plain exponential backoff — it can't express a *wall-clock deadline* ("give up after 30 s total, however many attempts that is") or *jitter* (randomized backoff so retrying workers don't resynchronize into a thundering herd). `RetryBudget` adds both: bounded by `max_attempts` *and/or* `deadline_s`, `run_with_budget` honours whichever is hit first and never sleeps past the deadline; delays use capped exponential backoff with a selectable `full`/`equal`/`none` jitter strategy. The randomness (`uniform`), clock and sleeper are all injectable, so every delay and giveup decision is deterministic in tests. First feature of the ROUND-15 input-fidelity lane. No `PySide6`. - -### Live IME State for Safe CJK Entry - -Wait for the input method to commit before reading a Japanese/Chinese/Korean field. Full reference: [`docs/source/Eng/doc/new_features/v208_features_doc.rst`](docs/source/Eng/doc/new_features/v208_features_doc.rst). - -- **`ime_state` / `is_composing` / `wait_for_composition_commit` / `decode_conversion_mode`** (`AC_ime_state`, `AC_is_composing`, `AC_wait_for_composition_commit`, `AC_decode_conversion_mode`): typing into a CJK field is unsafe while an IME is *composing* — the candidate text isn't committed, so reading the field back returns half-entered glyphs and the next keystroke edits the composition. `text_unicode` (`VK_PACKET`) is blind to this. `ime_state` exposes the focused window's live `{open, composing, composition, conversion}` (Windows IMM32, read-only) through an injectable `reader`; `is_composing` is the boolean gate; `wait_for_composition_commit` blocks until the IME commits (injectable `clock`/`sleep`/`reader`); `decode_conversion_mode` is the pure `IME_CMODE_*` bitmask decoder. All decode/wait logic is unit-tested without an IME. Sixth feature of the ROUND-15 cross-app OS lane. No `PySide6`. - -### Lock the Workstation + Wait for Unlock - -Lock the box at the end of a run, and block until a human unlocks it before resuming. Full reference: [`docs/source/Eng/doc/new_features/v207_features_doc.rst`](docs/source/Eng/doc/new_features/v207_features_doc.rst). - -- **`lock_session` / `plan_lock_session` / `wait_for_unlock` / `wait_for_lock` / `classify_lock_transitions`** (`AC_lock_session`, `AC_plan_lock_session`, `AC_wait_for_unlock`, `AC_classify_lock_transitions`): `session_guard` could *detect* a locked session and raise; this adds *acting* on it. `lock_session` locks the workstation now (`LockWorkStation` on Windows, `loginctl lock-session` / `CGSession -suspend` elsewhere) through an injectable `driver`; `wait_for_unlock` / `wait_for_lock` poll `session_guard.is_session_locked` (reusing its real Windows `OpenInputDesktop` probe) until the state flips or a timeout, with injectable `clock` / `sleep` / `probe`; `plan_lock_session` is the pure per-OS planner and `classify_lock_transitions` reduces a lock-state sample stream to `{event, locked}` lock/unlock events. `wait_for_unlock` is the blocking companion to `ensure_interactive_session`. Fifth feature of the ROUND-15 cross-app OS lane. No `PySide6`. - -### Read and Control the System Volume - -Set a known audio baseline before a run — mute, set 30%, or assert the level. Full reference: [`docs/source/Eng/doc/new_features/v206_features_doc.rst`](docs/source/Eng/doc/new_features/v206_features_doc.rst). - -- **`get_volume` / `set_volume` / `change_volume` / `is_muted` / `set_mute` / `mute` / `unmute` / `toggle_mute`** (`AC_get_volume`, `AC_set_volume`, `AC_change_volume`, `AC_set_mute`, `AC_toggle_mute`): the framework only had the blind media-key steps (`volume up` / `down` nudge by an unknown amount with no read-back). This adds absolute, read-backable control of the default output device — read or set the master level as an integer percent `0..100`, and read / set / toggle the mute flag. All logic (clamping, percent↔scalar conversion, toggle) is pure and runs through an injectable `VolumeDriver` seam, so it is fully unit-tested without an audio device; the default driver uses the Windows Core Audio `IAudioEndpointVolume` interface through the optional `pycaw` dependency (`pip install je_auto_control[audio]`), degrading with a clear error when absent. Fourth feature of the ROUND-15 cross-app OS lane. No `PySide6`. - -## What's new (2026-06-25) - -### Resolve the App Registered for a File Type - -Find out *which* app opens a file type — assert "PDFs open in Acrobat, not the browser". Full reference: [`docs/source/Eng/doc/new_features/v205_features_doc.rst`](docs/source/Eng/doc/new_features/v205_features_doc.rst). - -- **`normalize_ext` / `file_association`** (`AC_normalize_ext`, `AC_file_association`): `open_path` (`shell_open`) opens a file with whatever app is registered for it; this answers the inverse, read-only question — *which* app is that? Given `report.pdf` (or a bare `.pdf` / `pdf`) `file_association` returns the registered executable, friendly app name, open command line and MIME content type via the Windows `AssocQueryStringW` shell API. `normalize_ext` is the pure path/`.ext`/bare-`ext` → `.ext` helper. The assembly logic is unit-testable without Windows through an injectable `resolver` seam (the real shell API by default). The natural companion to `open_path`: one tells you what would open a file, the other opens it. Third feature of the ROUND-15 cross-app OS lane. No `PySide6`. - -### Idle Detection + Keep the Machine Awake - -Run only when the user has stepped away, and stop an overnight run from sleeping. Full reference: [`docs/source/Eng/doc/new_features/v204_features_doc.rst`](docs/source/Eng/doc/new_features/v204_features_doc.rst). - -- **`idle_seconds` / `is_idle` / `keep_awake` / `keep_awake_on` / `allow_sleep` / `plan_keep_awake`** (`AC_idle_seconds`, `AC_is_idle`, `AC_plan_keep_awake`, `AC_keep_awake_on`, `AC_allow_sleep`): long unattended runs get derailed two ways — the screensaver / power policy sleeps the box mid-run, or the run should hold while a human is actively using the machine. The framework had neither signal. `idle_seconds` / `is_idle` report time since the last keyboard / mouse input (`GetLastInputInfo` on Windows) through an injectable `probe`; `keep_awake` (scoped context manager) and `keep_awake_on` / `allow_sleep` (process-global on/off for JSON flows) stop the system and display sleeping, applied through an injectable `driver` (`SetThreadExecutionState` / `caffeinate` / `systemd-inhibit` by default) and restored on release. `plan_keep_awake` is the pure planner. All logic is unit-testable without touching the OS via the injected probe/driver. Second feature of the ROUND-15 cross-app OS lane. No `PySide6`. - -### Open Files / URLs with the Default App - -Hand a file to its default app, print it, or open a URL in the browser. Full reference: [`docs/source/Eng/doc/new_features/v203_features_doc.rst`](docs/source/Eng/doc/new_features/v203_features_doc.rst). - -- **`open_path` / `plan_open`** (`AC_open_path`, `AC_plan_open`): the framework could launch a literal `.exe`, but not the most common "hand off to another app" step — open `report.pdf` with its registered app, `print` a document, or open a URL in the default browser. This routes per-OS to `os.startfile` / `open` / `xdg-open` / `webbrowser`. `plan_open` is a pure planner that classifies the target (URL vs file path), validates it (URL scheme allow-list; `realpath` for files — a Windows drive `C:\` is correctly a path, not a scheme) and returns the dispatch descriptor; `open_path` runs it through an injectable `opener` (the real OS call by default), so the logic is unit-testable without launching anything. First feature of the ROUND-15 cross-app OS lane. No `PySide6`. - -### Reactive UIA Event Wait (focus change) - -Wait until focus lands on the dialog — a real, zero-latency UIA event, not polling. Full reference: [`docs/source/Eng/doc/new_features/v202_features_doc.rst`](docs/source/Eng/doc/new_features/v202_features_doc.rst). - -- **`wait_for_focus_change`** (`AC_wait_for_focus_change`): the accessibility recorder *polls* focus every ~250 ms, so it can miss a fast transition and reacts late. This blocks on the native `AddFocusChangedEventHandler` and returns the moment focus moves — the zero-latency, miss-free "wait until focus lands on the dialog" primitive, the accessibility-tree analogue of `wait_for_window` / `wait_for_image`. Returns the newly-focused element (or `None` on timeout). The real event subscription is registered/unregistered under a lock on the calling thread; dispatched through the injectable accessibility backend seam (headless-testable via a fake backend; real UIA in the Windows backend). No `PySide6`. - -### Container Selection + View Switching (Selection / MultipleView) - -Read what's selected in a listbox/grid, and switch Explorer-style views. Full reference: [`docs/source/Eng/doc/new_features/v201_features_doc.rst`](docs/source/Eng/doc/new_features/v201_features_doc.rst). - -- **`get_selection` / `list_views` / `set_view`** (`AC_get_selection`, `AC_list_views`, `AC_set_view`): `select_control_item` selects *one* item, but the container-level `SelectionPattern` answers "what is currently selected, and may it select multiple?" — the assertion target after selecting. `MultipleViewPattern` switches a control between its views (Explorer's list / details / tile / thumbnail), a precondition that otherwise needs fragile menu clicking. `get_selection` returns `{items, can_select_multiple, is_required}`, `list_views` returns `{current, views}`, and `set_view` switches by view name. Dispatched through the injectable accessibility backend seam (headless-testable via a fake backend; real UIA in the Windows backend). No `PySide6`. - -### Advanced TextPattern (find / select / read attributes) - -Search a control's text, select a match to replace it, and read font/colour formatting. Full reference: [`docs/source/Eng/doc/new_features/v200_features_doc.rst`](docs/source/Eng/doc/new_features/v200_features_doc.rst). - -- **`find_control_text` / `select_control_text` / `control_text_attributes`** (`AC_find_control_text`, `AC_select_control_text`, `AC_control_text_attributes`): `ax_text` shipped the three whole-range *reads*, but couldn't search for a substring, select a found range, or read text formatting — needed to assert "the error word is red and bold" or to place the selection at matched text before typing. This rounds out TextPattern: `find_control_text` searches the real content (not OCR) via `FindText`, `select_control_text` finds + selects a range so the next keystrokes replace it, and `control_text_attributes` reads `{font_name, font_size, bold, italic, foreground_color}`. Dispatched through the injectable accessibility backend seam (headless-testable via a fake backend; real UIA in the Windows backend). No `PySide6`. - -### MSAA Bridge for Legacy Controls (LegacyIAccessible) - -Automate the long tail of old Win32 controls that expose nothing via modern UIA. Full reference: [`docs/source/Eng/doc/new_features/v199_features_doc.rst`](docs/source/Eng/doc/new_features/v199_features_doc.rst). - -- **`legacy_info` / `legacy_default_action`** (`AC_legacy_info`, `AC_legacy_default_action`): many legacy Win32 / MFC / Delphi controls expose nothing useful via modern UIA patterns (`control_get_value` / `control_invoke` / `control_toggle` all return None), yet they're fully described through the MSAA `IAccessible` bridge — Name, Value, Description, Role, State and a **DefaultAction**. This reads that info and fires the default action via `LegacyIAccessiblePattern` — the last-resort fallback that makes old apps automatable. Dispatched through the injectable accessibility backend seam (headless-testable via a fake backend; real UIA in the Windows backend). No `PySide6`. - -### Move / Resize Elements + Window State (UIA Transform + Window) - -Move a floating panel, resize a control, and know if a window is modal-blocked. Full reference: [`docs/source/Eng/doc/new_features/v198_features_doc.rst`](docs/source/Eng/doc/new_features/v198_features_doc.rst). - -- **`move_element` / `resize_element` / `set_window_state` / `window_interaction_state`** (`AC_move_element`, `AC_resize_element`, `AC_set_window_state`, `AC_window_interaction_state`): this is UIA-**element-level**, not the HWND/title-level geometry in `window_layout`. `TransformPattern` moves/resizes a specific control or floating panel (dockable toolbars, MDI children, splitters) with no top-level window of its own; `WindowPattern` minimizes/maximizes a window and reports its **interaction state** (`ready` / `blocked_by_modal` / `not_responding`) — a reliable "is this window ready or modal-blocked?" signal pixel/title polling can't give. Dispatched through the injectable accessibility backend seam (headless-testable via a fake backend; real UIA in the Windows backend). No `PySide6`. - -### Table Headers + Cell Addressing (UIA TablePattern) - -Assert "the Status column of row 5 says Shipped" — by header, not by guessing indices. Full reference: [`docs/source/Eng/doc/new_features/v197_features_doc.rst`](docs/source/Eng/doc/new_features/v197_features_doc.rst). - -- **`table_headers` / `table_cell` / `cell_by_header`** (`AC_table_headers`, `AC_table_cell`, `AC_cell_by_header`): `read_control_table` (GridPattern) dumps a flat 2-D list of cell names with no header labels and no way to address one cell by (header, row) — you can dump a grid but not test one. This adds the missing half: `table_headers` reads the row/column header labels (TablePattern), `table_cell` reads the cell at `(row, column)` with its span (GridItemPattern), and `cell_by_header` resolves the column index from the headers so you can read the cell at `(row, "Status")` directly. Dispatched through the injectable accessibility backend seam (headless-testable via a fake backend; real UIA in the Windows backend). No `PySide6`. - -### Rich UIA Element Properties - -Know if a control is enabled / off-screen / has a tooltip before you act. Full reference: [`docs/source/Eng/doc/new_features/v196_features_doc.rst`](docs/source/Eng/doc/new_features/v196_features_doc.rst). - -- **`get_element_properties` / `is_element_enabled`** (`AC_get_element_properties`): the flat element list carries only name/role/bounds/app/id, but automation needs more before it acts — **is the control enabled** (don't click a disabled button), **is it off-screen**, its **item_status** (field validation/error), **help_text** (tooltip), and **accelerator_key** (drive via hotkey). This reads those high-value UIA properties (`enabled`/`offscreen`/`help_text`/`item_status`/`accelerator_key`/`access_key`/`orientation`); `is_element_enabled` is the common pre-action guard. Dispatched through the injectable accessibility backend seam (headless-testable via a fake backend; real UIA reads in the Windows backend). No `PySide6`. - -### Realize Off-Screen Items in Virtualized Lists / Grids - -Reach a row that isn't scrolled into view yet — the "element not found in a long list" fix. Full reference: [`docs/source/Eng/doc/new_features/v195_features_doc.rst`](docs/source/Eng/doc/new_features/v195_features_doc.rst). - -- **`realize_item`** (`AC_realize_item`): long lists / data grids / trees only materialize visible rows, so an off-screen row has no accessibility element at all — `list_accessibility_elements` / `read_control_table` / `select_control_item` can't see it, and `scroll_control_into_view` can't help because the element doesn't exist yet. This locates the item by property (UIA `ItemContainerPattern.FindItemByProperty`) and realizes it (`VirtualizedItemPattern.Realize`) so it becomes a real, clickable element. Match `by` name (default) or `automation_id`; locate the container by name/role/app. Dispatched through the injectable accessibility backend seam (headless-testable via a fake backend; real UIA in the Windows backend). No `PySide6`. - -### Per-Run Step Timeline (waterfall + bottleneck steps) - -Read why *this* run was slow — a step waterfall and its bottlenecks. Full reference: [`docs/source/Eng/doc/new_features/v194_features_doc.rst`](docs/source/Eng/doc/new_features/v194_features_doc.rst). - -- **`build_timeline` / `critical_steps`** (`AC_build_timeline`, `AC_critical_steps`): the action profiler aggregates timings by step *name* across runs — useless for "why was *this* run slow". This turns one run's ordered steps into a waterfall (each step's offset, duration, and `pct` share of the total) with the `bottleneck` step and a `parallelism` ratio (`> 1` when steps overlap via explicit `start` times); `critical_steps` ranks the dominant steps to optimise. A step is any `{name, duration, start?}` dict. Pure stdlib. No `PySide6`. - -### Flaky-Test Co-Failure Clustering - -Find the tests that flake *together* — and the shared root cause behind them. Full reference: [`docs/source/Eng/doc/new_features/v193_features_doc.rst`](docs/source/Eng/doc/new_features/v193_features_doc.rst). - -- **`cofailure_pairs` / `failure_clusters`** (`AC_cofailure_pairs`, `AC_failure_clusters`): flaky tests are rarely independent — a wobbly fixture or noisy dependency makes a *group* fail in the same runs (~75% of flaky tests cluster). Ranking tests one-by-one by flip rate misses that. This measures how often each pair of tests fails in the *same* runs (Jaccard over their failing-run sets) and groups tests above a threshold into connected clusters with a cohesion score — so you chase one root cause instead of N symptoms. Input is a list of runs, each the test names that failed in it. Pure stdlib. No `PySide6`. - -### Run-Trace Diff (what changed between two executions) - -See exactly what changed between a passing run and a failing one. Full reference: [`docs/source/Eng/doc/new_features/v192_features_doc.rst`](docs/source/Eng/doc/new_features/v192_features_doc.rst). - -- **`diff_runs` / `summarize_run_diff`** (`AC_diff_runs`): a run history says a run *failed* but not *what changed* from the run that passed. This aligns two step sequences with a longest-common-subsequence walk (so an inserted/removed step shifts the rest into place instead of mis-pairing everything) and classifies the differences: **added**/**removed** steps, **status_flips** (an aligned step that changed status — with the new failure's `failure_signature` when it carries an error), and **timing_regressions** (a step that got `regress_factor`× slower). `summarize_run_diff` renders a one-line summary. Pure stdlib over lists of `{name,status,duration,error}` step dicts. No `PySide6`. - -### Stable Failure Signatures - -Match the *same kind* of failure across runs, despite differing paths and ids. Full reference: [`docs/source/Eng/doc/new_features/v191_features_doc.rst`](docs/source/Eng/doc/new_features/v191_features_doc.rst). - -- **`normalize_error` / `failure_signature` / `group_failures`** (`AC_failure_signature`, `AC_group_failures`): two runs that failed the same way rarely have byte-identical error text — paths, line numbers, addresses, ids and timestamps differ every time — which defeats "is this the same failure?" and "which tests fail together?". This strips the variable parts of an error to a canonical form and hashes it (SHA-256), so the same kind of failure gets the same short signature across runs — the join key the rest of the test-robustness tools (run diffing, flake clustering) group on. `group_failures` buckets a list of errors by signature, most frequent first. Pure stdlib (`re` + `hashlib`). No `PySide6`. - -## What's new (2026-06-24) - -### Visual Saliency (where to look — spectral-residual) - -Find the region that stands out, with no template / colour / text. Full reference: [`docs/source/Eng/doc/new_features/v190_features_doc.rst`](docs/source/Eng/doc/new_features/v190_features_doc.rst). - -- **`saliency_map` / `salient_regions` / `most_salient`** (`AC_salient_regions`, `AC_most_salient`): when there's no template, colour or text to key on, an agent still needs a cue for *where to look*. This computes the spectral-residual saliency map (Hou & Zhang 2007 — log amplitude minus its local average, reconstructed through the phase) and turns it into ranked salient boxes in source pixel coordinates. The transform is a pure numpy FFT (`cv2.saliency` is in the forbidden opencv-contrib package, so it's re-implemented over base opencv); it reuses `visual_match`'s grayscale loader and `cv2_utils.blobs.connected_boxes`. Regions threshold at `mean + 2·std` by default. A coarse attention cue to *narrow* where a template / OCR pass then looks. No `PySide6`. - -### Display-Scale / Visual-DPI Detection - -Infer which display scale (DPI) a template renders at — and how confidently. Full reference: [`docs/source/Eng/doc/new_features/v189_features_doc.rst`](docs/source/Eng/doc/new_features/v189_features_doc.rst). - -- **`detect_scale` / `scale_sweep`** (`AC_detect_scale`, `AC_scale_sweep`): a template cropped at 100% scale won't match on a 150%-DPI machine, and `match_template` returns only the single best match — discarding the per-scale scores. This keeps the whole profile: `scale_sweep` scores the template at every scale, and `detect_scale` reports the winning scale as a DPI inference (`scale_percent`) with a confidence `margin` (how far it beats the runner-up). Reuses `visual_match._score_map` per scale; source is any ndarray / path / PIL image (or the live screen); scales default to the common Windows values. cv2/numpy lazily imported. No `PySide6`. - -### Image Quality Scoring (sharpness / contrast / brightness gate) - -Refuse to OCR a blurry or washed-out frame — score quality and gate before recognition. Full reference: [`docs/source/Eng/doc/new_features/v188_features_doc.rst`](docs/source/Eng/doc/new_features/v188_features_doc.rst). - -- **`image_quality` / `is_blurry` / `quality_gate`** (`AC_image_quality`, `AC_quality_gate`): OCR and template matching quietly fail on a blurry, washed-out or too-dark capture, and the caller can't tell a *missing* element from an *unreadable* one. This measures sharpness (variance of the Laplacian), contrast (grayscale stddev) and brightness (mean 0–255); `quality_gate` turns them into `{passed, issues}` flagging `blurry` / `low_contrast` / `too_dark` / `too_bright` so a script can pre-process or re-capture before OCR. Reuses `visual_match`'s grayscale loader (any ndarray / path / PIL image, or the live screen); cv2/numpy lazily imported. No `PySide6`. - -### Drop Files onto a Window (WM_DROPFILES) - -Complete a drag-and-drop programmatically — drop files onto a target window. Full reference: [`docs/source/Eng/doc/new_features/v187_features_doc.rst`](docs/source/Eng/doc/new_features/v187_features_doc.rst). - -- **`plan_file_drop` / `drop_files`** (`AC_plan_file_drop`, `AC_drop_files`): `clipboard_files` *stages* a file list on the clipboard for `Ctrl+V`; this actively **drops** files onto a target window by posting a `WM_DROPFILES` message. It reuses `clipboard_files.build_dropfiles` to pack the `DROPFILES` blob (shared byte layout, not re-implemented) and dispatches through an injectable driver seam, so the build-and-dispatch logic is unit-testable with a fake driver; the real `GlobalAlloc` + `PostMessage` lives in the default Win32 driver. `plan_file_drop` is a pure dry-run returning `{message, paths, point, wide, blob_size}`. No `PySide6`. - -### Clipboard Format Inspection (classify / diff available formats) - -See which formats are on the clipboard, and detect when its shape changes. Full reference: [`docs/source/Eng/doc/new_features/v186_features_doc.rst`](docs/source/Eng/doc/new_features/v186_features_doc.rst). - -- **`classify_format` / `classify_formats` / `diff_formats` / `list_clipboard_formats` / `clipboard_formats`** (`AC_clipboard_formats`, `AC_classify_formats`, `AC_diff_formats`): the clipboard usually holds the same content in several formats at once (a Word copy = text + HTML + RTF; a file copy = CF_HDROP; a screenshot = CF_DIB). This enumerates the live clipboard (`EnumClipboardFormats`) without consuming anything and classifies each format into a friendly category (text/image/files/html/rtf/csv/audio/…); `diff_formats` is a pure monitor primitive returning `{added, removed, changed}` between two snapshots. The classifier and diff are pure (registered names take priority over dynamic ids); only the live enumeration is Win32. No `PySide6`. - -### Rich Clipboard Formats (RTF and CSV/TSV) - -Put styled text and tables on the clipboard for cross-app paste into Word and Excel. Full reference: [`docs/source/Eng/doc/new_features/v185_features_doc.rst`](docs/source/Eng/doc/new_features/v185_features_doc.rst). - -- **`build_rtf` / `rtf_to_text` / `rows_to_csv` / `csv_to_rows` + `set_clipboard_rtf` / `get_clipboard_rtf` / `set_clipboard_csv` / `get_clipboard_csv`** (`AC_set_clipboard_rtf`, `AC_get_clipboard_rtf`, `AC_set_clipboard_csv`, `AC_get_clipboard_csv`): `rich_clipboard` added CF_HTML, but RTF (the format rich editors accept) and the `Csv` format Excel reads were still missing. This adds both: `build_rtf`/`rtf_to_text` build and strip RTF control words and `\uNNNN` / `\'XX` escapes in pure Python (fully unit-testable round-trip), and `rows_to_csv`/`csv_to_rows` wrap the stdlib `csv` module (delimiter-parametrised, so `\t` gives TSV). The codecs are platform-independent; the Win32 get/set share one generic byte-transfer helper, and the sets seed plain text so plain editors still paste. No `PySide6`. - -### Keyboard Focus Order (Tab sequence / WCAG audit / set-focus) - -Reason about keyboard navigation: the Tab order, a WCAG focus-order audit, and set-focus. Full reference: [`docs/source/Eng/doc/new_features/v184_features_doc.rst`](docs/source/Eng/doc/new_features/v184_features_doc.rst). - -- **`is_interactive_role` / `tab_order` / `audit_focus_order` / `focus_control`** (`AC_tab_order`, `AC_audit_focus_order`, `AC_focus_control`): nothing reasoned about *keyboard* navigation — only mouse coordinates and element values. This adds the keyboard layer: `tab_order` returns the focusable elements in the order Tab visits them (reading order), `audit_focus_order` is a WCAG 2.4.x report (the sequence + flagged problems like a focusable element with no visible area), and `focus_control` sets keyboard focus via UIA `SetFocus`. The first three are pure functions over `AccessibilityElement` lists — `tab_order` reuses `element_parse.reading_order` and `is_interactive_role` reuses `ax_tree_walk.humanize_role`, so no logic is duplicated; `focus_control` dispatches the injectable backend seam (real `SetFocus` in the Windows backend). No `PySide6`. - -### Readable, Addressable Accessibility Tree (role names + node paths) - -Turn a raw `ControlType_50000` tree dump into readable roles with a stable path per node. Full reference: [`docs/source/Eng/doc/new_features/v183_features_doc.rst`](docs/source/Eng/doc/new_features/v183_features_doc.rst). - -- **`control_type_name` / `humanize_role` / `humanize_tree` / `assign_node_paths` / `find_by_path`** (`AC_walk_tree`, `AC_humanize_role`): `dump_accessibility_tree` emits the platform's raw role (on Windows the bare UIA ControlType id, e.g. `ControlType_50000` for a button) and carries no stable per-node identity once serialised. This adds the pure post-processing it lacks: translate ControlType ids to friendly names, deep-copy a tree with every role humanised, stamp each node with a stable positional `path` (`"0.2.1"` — a pure stand-in for RuntimeId), and resolve a node back by path. `AC_walk_tree` is the readable counterpart to `AC_a11y_dump`. Pure-stdlib over `AXTreeNode`; unknown / non-UIA roles pass through unchanged. No `PySide6`. - -### Native Text Reading via the UIA TextPattern (document / selection / visible) - -Read the text in multiline editors and document controls where ValuePattern returns nothing. Full reference: [`docs/source/Eng/doc/new_features/v182_features_doc.rst`](docs/source/Eng/doc/new_features/v182_features_doc.rst). - -- **`get_control_text` / `get_selected_text` / `get_visible_text`** (`AC_get_control_text`, `AC_get_selected_text`, `AC_get_visible_text`): `control_get_value` reads through UIA ValuePattern, which returns an empty string on multiline edits, RichEdit / document controls and web text areas — exactly the controls whose text you most want. This reads through `TextPattern` instead: `get_control_text` returns the whole `DocumentRange`, `get_selected_text` the current `GetSelection`, `get_visible_text` only the on-screen `GetVisibleRanges`. Dispatched through the injectable `accessibility.backends.get_backend()` seam (headless-testable via a fake backend; real UIA calls in the Windows backend), returning `{text}` from the executor/MCP. No `PySide6`. - -### Extended UIA Control Patterns (Expand / Select / Range / Scroll) - -Drive tree nodes, list/combo items, sliders and scroll natively, not by pixel guessing. Full reference: [`docs/source/Eng/doc/new_features/v181_features_doc.rst`](docs/source/Eng/doc/new_features/v181_features_doc.rst). - -- **`expand_control` / `collapse_control` / `control_expand_state` / `select_control_item` / `control_range` / `set_control_range` / `scroll_control_into_view`** (`AC_expand_control`, `AC_select_control_item`, `AC_set_control_range`, …): the accessibility backend had only Value/Invoke/Toggle/Grid-read patterns, so treeviews, listboxes/combos, sliders and off-screen rows had no native call path. This adds ExpandCollapse / SelectionItem / RangeValue / ScrollItem patterns on top of the existing backend ABC, dispatched through the injectable `accessibility.backends.get_backend()` seam (headless-testable via a fake backend; real UIA calls in the Windows backend). No `PySide6`. - -### Pre-Match Settle Gating + Match Persistence - -Avoid matching mid-animation, and confirm a hit holds steady across frames. Full reference: [`docs/source/Eng/doc/new_features/v180_features_doc.rst`](docs/source/Eng/doc/new_features/v180_features_doc.rst). - -- **`region_stability` / `match_persistence`** (`AC_region_stability`, `AC_match_persistence`): `smart_waits.wait_until_screen_stable` gates a live loop with a boolean — it can't score stability on an injectable frame sequence or check whether a *match* held steady. `region_stability` scores consecutive-frame SSIM (`{stable, mean_ssim, min_ssim}`); `match_persistence` confirms a template is found in *every* frame with the centres agreeing within `agree_px` (`{persisted, n_hits, jitter}`). Reuses `ssim` + `visual_match` + `grounding_consensus`; injectable frames; no `PySide6`. - -### Colour-Aware Template Matching (HSV) - -Tell a red status dot from a green one of identical shape. Full reference: [`docs/source/Eng/doc/new_features/v179_features_doc.rst`](docs/source/Eng/doc/new_features/v179_features_doc.rst). - -- **`match_color` / `match_color_all`** (`AC_match_color`, `AC_match_color_all`): every `visual_match` matcher grayscales first, so red vs green of identical shape is indistinguishable; `color_region` finds known-colour blobs but can't template-match a multi-colour glyph. This matches on HSV hue/saturation with a colour-*distance* metric (`TM_SQDIFF_NORMED` — correlation would normalise away the absolute hue, scoring a red→green edge same as black→blue). Reuses `color_region`'s RGB loaders + `visual_match`'s resize/NMS/`Match`. `channels` default `("h","s")` (use `("h",)` for flat-saturation targets); for solid blobs use `find_color_region`. No `PySide6`. - -### Multi-Template Consensus Matching - -Vote several reference crops of one target into a single trustworthy location. Full reference: [`docs/source/Eng/doc/new_features/v178_features_doc.rst`](docs/source/Eng/doc/new_features/v178_features_doc.rst). - -- **`match_ensemble` / `vote_centers`** (`AC_match_ensemble`, `AC_vote_centers`): a button renders in several states (default/hover/pressed) but is one logical target; `ab_locator` picks one strategy and `match_template(scales=...)` sweeps one template — neither fuses multiple references. This matches each reference, clusters the hit centres, and accepts a target only when ≥ `min_votes` agree within `agree_px`, returning `{point, votes, n_candidates, spread}` — cutting false positives on themed/animated UI. Reuses `visual_match.match_template` + `grounding_consensus`; `vote_centers` is the pure voting core. No `PySide6`. - -### Per-Step Critic Features + Rule-Based Step Scorer - -Bundle the evidence to score an agent step, with a built-in rule-based scorer. Full reference: [`docs/source/Eng/doc/new_features/v177_features_doc.rst`](docs/source/Eng/doc/new_features/v177_features_doc.rst). - -- **`build_critic_record` / `score_step_rule_based` / `to_judge_prompt`** (`AC_build_critic_record`, `AC_score_step`): `trajectory_eval` scores a whole trajectory with no per-step evidence; `agent_trace` emits spans not quality; `agent_replay` stores steps but doesn't score. This composes `action_effect` + `observation_delta` + `postcondition` into one per-step record, then `score_step_rule_based` gives a deterministic `{outcome, process_score, reasons}` (no model needed) and `to_judge_prompt` renders it for an optional LLM-as-judge. Pure-stdlib aggregator; no `PySide6`. - -### Heading vs Body Classification + Document Outline - -Tell headings from body text by height and build a document outline. Full reference: [`docs/source/Eng/doc/new_features/v176_features_doc.rst`](docs/source/Eng/doc/new_features/v176_features_doc.rst). - -- **`classify_lines` / `outline`** (`AC_classify_lines`, `AC_outline`): nothing mapped line height to heading levels or built a section outline — `ocr/structure` / `element_parse` are positional and `text_blocks` doesn't rank. This applies the standard heuristic: a line taller than `heading_ratio` × the median line height is a heading, and distinct heading heights become levels (tallest = 1). `classify_lines` tags each line `{box, text, role, level}`; `outline` returns the headings in order as a table of contents. Pure-stdlib over line dicts; no `PySide6`. - -### Settle Detection Over a Churn Series - -Decide when the UI has gone quiet — as a pure, testable function over a change series. Full reference: [`docs/source/Eng/doc/new_features/v175_features_doc.rst`](docs/source/Eng/doc/new_features/v175_features_doc.rst). - -- **`settle_point` / `is_settled` / `SettleTracker`** (`AC_settle_point`): `smart_waits.wait_until_screen_stable` bakes the settle logic inside a `time.sleep` loop over live frames — you can't feed it a recorded series or unit-test the decision. This extracts it: given a stream of *churn* values (pixel delta / element-count delta / 0-1 digest-changed), it reports when churn stayed ≤ `max_churn` for `quiet_samples` in a row (a spike resets the run). `settle_point` returns the settle index, `SettleTracker` is the incremental form for a live loop. Pure-stdlib, no clock, no capture; no `PySide6`. - -### Paragraph & List Grouping of OCR Lines - -Group OCR lines into paragraphs and detect bulleted / numbered lists. Full reference: [`docs/source/Eng/doc/new_features/v174_features_doc.rst`](docs/source/Eng/doc/new_features/v174_features_doc.rst). - -- **`group_paragraphs` / `detect_lists`** (`AC_group_paragraphs`, `AC_detect_lists`): `text_regions` merges glyphs into lines but nothing grouped those lines into paragraphs or detected lists; `ocr/structure` stops at flat rows. `group_paragraphs` starts a new paragraph wherever the vertical gap exceeds `line_gap_factor` × the median line height; `detect_lists` recognises bullet (`•`/`-`/`*`) or ordinal (`1.`/`2)`/`a.`) items, returning `{text, marker, indent, box}`. Pure-stdlib over line dicts; reuses `table_grid_fill`'s box reader; no `PySide6`. - -### Column-Aware Reading Order (XY-Cut) - -Read multi-column layouts down each column instead of interleaving them. Full reference: [`docs/source/Eng/doc/new_features/v173_features_doc.rst`](docs/source/Eng/doc/new_features/v173_features_doc.rst). - -- **`flow_order` / `xy_cut` / `to_blocks`** (`AC_flow_order`, `AC_xy_cut`): `element_parse.reading_order` is a flat top-to-bottom sort that interleaves columns (reads A1, B1, A2, B2…). This recovers the correct order with recursive XY-cut — split at the widest whitespace valley (vertical → columns, horizontal → rows), so a two-column page reads A1, A2, B1, B2. `flow_order` returns the same `index`-tagged contract as `reading_order` (a drop-in column-aware upgrade, named to not shadow it); `xy_cut` exposes the region tree; `to_blocks` lists the leaf blocks. Pure-stdlib; no `PySide6`. - -### Grounding Self-Consistency (Consensus Over Proposals) - -Fuse several grounding proposals into one agreed target with an agreement score. Full reference: [`docs/source/Eng/doc/new_features/v172_features_doc.rst`](docs/source/Eng/doc/new_features/v172_features_doc.rst). - -- **`consensus_point` / `consensus_element` / `is_confident`** (`AC_consensus_point`, `AC_consensus_element`): a target can be grounded several ways at once (set-of-marks / OCR / template / a11y / N model samples) and they don't always agree. `ab_locator`/`element_scoring` rank *strategies* by history; `snap_to_element` snaps a *single* coordinate — neither fuses *simultaneous* proposals. This clusters candidate points (or votes candidate elements), returns the agreed `point` + an `agreement` fraction + `spread`, and `is_confident` flags low-agreement targets so the agent zooms / asks instead of clicking blind. Pure-stdlib; no `PySide6`. - -### Sub-Pixel Template-Match Refinement - -Refine a match's centre to a fraction of a pixel for drag / slider / high-DPI precision. Full reference: [`docs/source/Eng/doc/new_features/v171_features_doc.rst`](docs/source/Eng/doc/new_features/v171_features_doc.rst). - -- **`match_subpixel` / `refine_peak`** (`AC_match_subpixel`): every matcher returns *integer* coordinates from `cv2.minMaxLoc` — for a drag handle, fine slider or high-DPI display that rounding is the dominant click-placement error. This fits a parabola to the 3×3 score neighbourhood around the peak (independently on x/y, the standard NCC sub-pixel method) and returns a `SubPixelMatch` with float `cx`/`cy` + the applied `offset_x`/`offset_y`. Reuses `visual_match._score_map`; injectable `haystack`; no `PySide6`. - -### Repair-Tactic Policy for Failed / No-Effect Actions - -Pick the next repair tactic when an action does nothing — and drive the retry loop. Full reference: [`docs/source/Eng/doc/new_features/v170_features_doc.rst`](docs/source/Eng/doc/new_features/v170_features_doc.rst). - -- **`plan_repair` / `next_tactic` / `run_with_repair`** (`AC_plan_repair`): `self_healing`/`locator_repair` only fix a locator that *didn't resolve*; `loop_guard` only *detects* a stuck loop with no tactic selection. This consumes an effect verdict (e.g. from `action_effect`) and returns the ordered tactics to try — `wait_retry` / `relocate` / `nudge` / `scroll_into_view` / `escalate` — then `run_with_repair` drives a bounded retry loop with injected `act` / `verify` / `apply_tactic` / `verdict_for` / `sleep` seams, returning a `RepairOutcome`. Pure-stdlib state machine; no `PySide6`. Completes the self-correction trio with `action_effect` + `postcondition`. - -### Declarative Action Postconditions - -Assert an action's expected outcome as a JSON spec, diffed against the before-frame. Full reference: [`docs/source/Eng/doc/new_features/v169_features_doc.rst`](docs/source/Eng/doc/new_features/v169_features_doc.rst). - -- **`check_postcondition` / `compile_postcondition`** (`AC_check_postcondition`): `expect_poll`/`assert_eventually` poll a single condition with no action-bound spec and no before-baseline (so they can't express "a *new* dialog appeared"); `trajectory_eval` is whole-trajectory. This evaluates a small JSON spec of clauses — `appears`/`disappears` (diffed vs `before`), `enabled`/`disabled`, `text_present`/`text_absent`, `count` — against the after-observation, returning a per-clause `{ok, clauses, failed}` report. `compile_postcondition` turns a spec into an `after -> bool` predicate for `expect_poll`. Pure-stdlib; no `PySide6`. - -### Edge-Shape (Chamfer) Template Matching - -Locate flat icons by outline, robust to fill / theme / anti-aliasing. Full reference: [`docs/source/Eng/doc/new_features/v168_features_doc.rst`](docs/source/Eng/doc/new_features/v168_features_doc.rst). - -- **`edge_match` / `edge_match_all` / `chamfer_distance`** (`AC_edge_match`, `AC_edge_match_all`): intensity NCC (`visual_match`) drops when a control is re-filled / re-themed, and ORB (`feature_match`) needs corner texture flat-design glyphs lack. This matches by *edge shape*: Canny both images, distance-transform the scene edges, slide the template's edges over it and score by mean edge-to-edge distance (Chamfer). A perfect outline aligns at ~0 cost regardless of fill. Reuses `visual_match`'s loaders / resize / NMS / `Match` and `edge_lines`'s Canny default. Injectable `haystack`; no `PySide6`. - -### Action-Effect Classification (Did My Click Do Anything?) - -Tell an agent whether a click did anything — and whether it happened where it aimed. Full reference: [`docs/source/Eng/doc/new_features/v167_features_doc.rst`](docs/source/Eng/doc/new_features/v167_features_doc.rst). - -- **`classify_effect` / `effect_near_point` / `is_no_op`** (`AC_classify_effect`, `AC_effect_near_point`): `screen_state`/`element_diff` report what changed but never tie it to the action; `loop_guard` only flags a no-op after N repeats. This diffs the before/after observation and, given the action's target point, classifies the result on the *first* step as `no_op` / `changed_near_target` / `changed_elsewhere` (a surprise dialog) / `changed`, returning an `EffectVerdict` with the changed centres and a reason. Reuses `element_diff.match_elements` + `observation_delta`'s field-change check. Pure-stdlib; no `PySide6`. - -### Form Field Association (Multi-Direction) + Checkbox State - -Pair form labels with values even when the value is below or right-aligned, and read checkbox state. Full reference: [`docs/source/Eng/doc/new_features/v166_features_doc.rst`](docs/source/Eng/doc/new_features/v166_features_doc.rst). - -- **`associate_fields` / `match_labels_to_widgets` / `checkbox_state`** (`AC_associate_fields`, `AC_match_labels_to_widgets`): `ocr/structure` only pairs a `label:` with the *immediately next* cell — it can't handle label-above-value, two-column key/value, right-aligned values, or non-text widgets, and has no checkbox notion. This pairs each label with the nearest aligned value across *directions* (right / below) within `max_gap`, matches free-standing widgets (checkbox/radio/input) to their nearest label, and reads checkbox state from the box's dark-pixel fill ratio. Association is pure-stdlib; only `checkbox_state` touches pixels (behind the `visual_match` gray loader). No `PySide6`. - -### Whitespace-Projection Columns (Borderless Tables) - -Read borderless tables by inferring columns from the whitespace gaps. Full reference: [`docs/source/Eng/doc/new_features/v165_features_doc.rst`](docs/source/Eng/doc/new_features/v165_features_doc.rst). - -- **`detect_borderless_table` / `column_gutters` / `assign_columns` / `vertical_projection`** (`AC_detect_borderless_table`, `AC_column_gutters`): `ocr/structure` only detects a table when every row's cell-left-x matches — it fails on ragged / borderless / right-aligned columns; `edge_lines.find_grid` needs ruling lines a whitespace table doesn't have. This finds columns by the *gaps*: project OCR boxes onto the x-axis, read the persistent empty vertical bands as gutters, assign column indices, bucket rows by spacing, and emit `{n_rows, n_cols, rows, columns}`. Pure-stdlib difference-array projection (no numpy); reuses `table_grid_fill`'s box reader. No `PySide6`. - -### Auto-Thresholded Template Matching (Otsu on the Score Map) - -No more hand-tuned `min_score` — derive the match threshold from the score map. Full reference: [`docs/source/Eng/doc/new_features/v164_features_doc.rst`](docs/source/Eng/doc/new_features/v164_features_doc.rst). - -- **`match_auto` / `auto_threshold`** (`AC_match_auto`, `AC_auto_threshold`): every `match_template_all` call forces you to guess `min_score` (too low floods NMS, too high drops re-themed targets, and it differs per asset). This runs Otsu on the *correlation score histogram* to find the valley between background correlation and real matches, returns that cut-off plus a *separability* score (near 0 = unimodal, no clear match → don't trust it). `match_auto` returns one peak per above-threshold region (via `connected_boxes`, avoiding duplicate hits on a wide peak), clamped by a `floor`. Reuses the new `visual_match._score_map`; injectable `haystack`; no `PySide6`. - -### Token-Budgeted Observation Delta (What Changed) - -Tell an agent *what changed* since the last step, not the whole screen again. Full reference: [`docs/source/Eng/doc/new_features/v163_features_doc.rst`](docs/source/Eng/doc/new_features/v163_features_doc.rst). - -- **`delta_observation` / `delta_index` / `summarize_delta`** (`AC_delta_observation`): `serialize_observation` renders one full frame (blows the token budget every turn); `element_diff` gives the stable-ID correspondence but stops at matched/added/removed element pairs. This is the missing serializer — it diffs two frames, classifies matched elements as changed (role/name/enabled/value/moved) or stable, and renders only the churn as `+ [i] role "name"` / `~ [i] … (fields)` / `- …` lines (added & changed first, stable dropped, capped at `max_lines`). Reuses `element_diff.match_elements` + `observation.observation_index`. Pure-stdlib; no `PySide6`. - -### Fill a Ruling-Line Grid With OCR Text (Addressable Tables) - -Turn a bordered table's lines + OCR words into an addressable `R x C` table. Full reference: [`docs/source/Eng/doc/new_features/v162_features_doc.rst`](docs/source/Eng/doc/new_features/v162_features_doc.rst). - -- **`populate_table` / `assign_text_to_grid` / `table_to_records` / `table_to_csv`** (`AC_populate_table`): `edge_lines.find_grid` recovers a table's ruling-line geometry but the cells come back *empty*; OCR gives the text but no structure — nothing joined them. This drops OCR boxes into the grid (assigned by cell-centre, gated by an overlap fraction so a box straddling a thin rule isn't double-counted), concatenates each cell's text in reading order, flags merged-cell spans, and converts straight to records / CSV. Pure-stdlib over plain dicts — no image, no OCR engine, no device. No `PySide6`. - -### Trust-Scored Template Matching (Ambiguity / PSR) - -Know when a template match is strong but *ambiguous* before clicking it. Full reference: [`docs/source/Eng/doc/new_features/v161_features_doc.rst`](docs/source/Eng/doc/new_features/v161_features_doc.rst). - -- **`match_with_trust` / `score_peaks`** (`AC_match_with_trust`): `match_template` returns only the top score and clicks it — but a button repeated in a toolbar or a near-identical sibling correlates ~0.95 in two places, so a high score is not an *unambiguous* match. This adds a Lowe-style ratio test *for pixel templates* (ORB got one via `feature_match`; `match_template` never did): it inspects the whole correlation surface, compares the global peak to the next-best peak outside an exclusion window, computes the peak-to-sidelobe ratio (PSR), and returns a `TrustedMatch` with `second_score` / `peak_ratio` / `psr` / `is_ambiguous`. Reuses a new `visual_match._score_map` (the full `matchTemplate` surface the public matchers discard) — no matching code duplicated. Injectable `haystack`; no `PySide6`. - -## What's new (2026-06-23) - -### Clipboard File-Drop List (CF_HDROP) - -Put a list of files on the clipboard, ready to paste into Explorer. Full reference: [`docs/source/Eng/doc/new_features/v160_features_doc.rst`](docs/source/Eng/doc/new_features/v160_features_doc.rst). - -- **`build_dropfiles` / `parse_dropfiles` / `set_clipboard_files` / `get_clipboard_files`** (`AC_set_clipboard_files`, `AC_get_clipboard_files`): the clipboard carried text, images and (via `rich_clipboard`) HTML, but never a *file list* — the `CF_HDROP` payload Explorer reads to paste files as a real copy. Building it is fiddly (20-byte `DROPFILES` header + double-null-terminated UTF-16 path list + `pFiles` offset). This isolates the packing into pure, fully testable `build_dropfiles` / `parse_dropfiles` byte functions, with thin Windows-only `set`/`get_clipboard_files` wrappers on top — the same split `rich_clipboard` uses for `CF_HTML`. No `PySide6`. - -### Coarse Labelled Screen Grid (VLM Grounding) - -Refer to screen regions as grid cells ("click C3") instead of raw pixels. Full reference: [`docs/source/Eng/doc/new_features/v159_features_doc.rst`](docs/source/Eng/doc/new_features/v159_features_doc.rst). - -- **`grid_cells` / `cell_for_point` / `point_for_cell`** (`AC_grid_cells`, `AC_cell_for_point`, `AC_point_for_cell`): VLM grounding is far more reliable when a model names a coarse cell than when it emits hallucinated pixel coordinates. This lays an `rows`x`cols` grid over the screen (or a `region`), labels each cell spreadsheet-style (`A1` top-left, past `Z` → `AA`), and maps both ways — point → containing cell, named cell → centre point (ready to click). Pure-stdlib geometry; the only device-bound path is the default that reads the live screen size, so every function is headless-testable with an explicit `region`. No `PySide6`. - -### Rotation- & Scale-Tolerant Template Matching - -Find templates that are rotated or skewed, not just scaled. Full reference: [`docs/source/Eng/doc/new_features/v158_features_doc.rst`](docs/source/Eng/doc/new_features/v158_features_doc.rst). - -- **`match_rotated` / `match_rotated_all` / `scale_space`** (`AC_match_rotated`, `AC_match_rotated_all`): `match_template` sweeps *scales* but assumes axis-aligned — OpenCV's `matchTemplate` isn't rotation-invariant, so a skewed control, a rotated icon or a dial at a different angle is missed. This sweeps `angles` (each warped with `cv2.warpAffine`) crossed with a `np.linspace` scale-space, returns the best-correlating `RotatedMatch` carrying the recovered `scale` + `angle` (the `*_all` form NMS-dedupes neighbouring angles/scales). Reuses `visual_match`'s loaders / resize / method table / NMS — no matching or geometry code duplicated. Injectable `haystack`; headless-testable; no `PySide6`. - -### Barcode Decoding (1-D) - -Read EAN / UPC / Code-128 barcodes off the screen or an image. Full reference: [`docs/source/Eng/doc/new_features/v157_features_doc.rst`](docs/source/Eng/doc/new_features/v157_features_doc.rst). - -- **`read_barcodes`** (`AC_read_barcodes`): the framework decoded QR codes (`read_qr`) but had no reader for the *1-D* barcodes (EAN-13/8, UPC-A, Code-128) that label physical goods, inventory tickets and shipping labels. This decodes them via OpenCV's `cv2.barcode.BarcodeDetector`, returning `{text, type, points}` per code. The decode step is an injectable seam (default calls OpenCV; tests pass their own `decoder`), so it's fully headless-testable and degrades gracefully — an OpenCV build without the `barcode` module returns `[]` instead of raising. Reuses the shared `visual_match` haystack loader; no `PySide6`. - -### Weighted Candidate Scoring - -Rank ambiguous element candidates by a confidence score. Full reference: [`docs/source/Eng/doc/new_features/v156_features_doc.rst`](docs/source/Eng/doc/new_features/v156_features_doc.rst). - -- **`score_candidates` / `best_candidate`** (`AC_score_candidates`, `AC_best_candidate`): `anchor_locator` is a single relation + distance sort and `ab_locator` races whole strategies by elapsed time — neither ranks ambiguous candidates by a *weighted* mix of role match + fuzzy name similarity + anchor proximity + enabled-state. This returns `ScoredCandidate`s best-first with a `matched_on` breakdown; the name similarity is injectable (default `fuzzy_ratio`, reused — no new string-distance code). Pure-stdlib over element dicts; powers self-heal / grounding when several boxes could be the target. Headless-testable. - -### Geometry-Aware Element Diff & Stable IDs - -Track elements across frames by overlap, with stable IDs. Full reference: [`docs/source/Eng/doc/new_features/v155_features_doc.rst`](docs/source/Eng/doc/new_features/v155_features_doc.rst). - -- **`match_elements` / `assign_stable_ids`** (`AC_match_elements`, `AC_assign_stable_ids`): `diff_snapshots` keys identity on `(role, name)` — it can't match a renamed-but-stationary control or a moved one, nor give persistent IDs across frames. This matches element boxes by IoU (reusing `element_parse.iou`): `match_elements` returns `{matched, added, removed}`; `assign_stable_ids` carries each element's `id` from a `prior` frame (a moved button keeps its id, a new one gets a fresh id) — so an agent can reliably refer to "element 7" turn-over-turn. Pure-stdlib, headless-testable. - -### Portable Agent-Trajectory Trace (Record & Replay) - -Log an agent's observation→action steps and replay them. Full reference: [`docs/source/Eng/doc/new_features/v154_features_doc.rst`](docs/source/Eng/doc/new_features/v154_features_doc.rst). - -- **`record_step` / `to_jsonl` / `from_jsonl` / `replay_trace`** (`AC_replay_trace`): `agent_trace` records OTel spans (observability), `trajectory_eval` only scores, `semantic_recording` replays human macros — none is a replayable obs→action transcript. This is the OmniTool-style `{step, observation, action, result}` JSONL with a deterministic replay driver (injectable `runner`, no live model). The executor command replays each step's AC action through the executor. Pure-stdlib, headless-testable; build regression / training datasets from agent runs. - -### Pre-Action Grounding Guard - -Reject out-of-bounds clicks; snap near-misses onto the real element. Full reference: [`docs/source/Eng/doc/new_features/v153_features_doc.rst`](docs/source/Eng/doc/new_features/v153_features_doc.rst). - -- **`validate_action` / `snap_to_element` / `in_bounds`** (`AC_validate_action`): `guardrail` scans text and `loop_guard` detects loops — neither validates a coordinate action before dispatch, so a hallucinated `(9999,-5)` click fires into nothing and a 5px-off click misses. This rejects off-screen coordinates and, given `targets`, snaps a near-miss onto the nearest element's centre, returning `{ok, reason, snapped}`. Pure-stdlib geometry over element dicts; the executor `screen` defaults to the live screen. Headless-testable; plugs in front of an agent loop's dispatch. - -### Token-Budgeted A11y Text Observation - -Turn the a11y tree into an indexed text block a VLM can act on. Full reference: [`docs/source/Eng/doc/new_features/v152_features_doc.rst`](docs/source/Eng/doc/new_features/v152_features_doc.rst). - -- **`serialize_observation` / `observation_index` / `flatten_tree`** (`AC_serialize_observation`, `AC_observation_index`): `describe_screen` gives role *counts* + a flat label list — no stable index, no `[12] button "Submit" @(x,y)` lines, no viewport clip, no token budget. This flattens a (nested) element tree to interactive-only, clips to the viewport, orders reading-style, caps at `max_elements`, assigns a stable `index`, and renders the lines a model acts on ("click [12]"). Pure-stdlib over element dicts; pairs with `fuse_elements`/`set_of_marks`. Headless-testable. - -### Canonical Computer-Use Action Schema - -Bridge Anthropic / OpenAI agent actions to AutoControl commands. Full reference: [`docs/source/Eng/doc/new_features/v151_features_doc.rst`](docs/source/Eng/doc/new_features/v151_features_doc.rst). - -- **`from_anthropic` / `from_openai_cua` / `to_ac_command` / `canonical_action`** (`AC_cua_command`): `tool_use_schema` exports AC_* signatures and `coordinate_space` rescales — neither *normalizes an inbound action payload*. Anthropic emits `{action:"left_click", coordinate:[x,y]}`, OpenAI CUA emits `{type:"click", x, y, button}`; these adapters map both to a canonical action and then to a runnable `[AC_*, params]` (with optional coordinate-space `scale`). Pure-stdlib, headless-testable; the executor command returns `{canonical, command}` for any source. - -### Window Client-Area Geometry - -Click *inside* a window regardless of its title bar / borders. Full reference: [`docs/source/Eng/doc/new_features/v150_features_doc.rst`](docs/source/Eng/doc/new_features/v150_features_doc.rst). - -- **`get_client_rect` / `client_point` / `frame_insets` / `client_to_screen`** (`AC_get_client_rect`, `AC_client_point`): `get_window_geometry` returns only the *outer* bbox — there was no client-area rect, frame-inset math, or client→screen mapping. `client_point("App", x, y)` maps a content-relative point to the screen so a click lands inside the window regardless of chrome; `frame_insets` reports border/title-bar thickness. `frame_insets`/`client_to_screen` are pure geometry (headless-testable); `get_client_rect` uses an injectable Win32 reader (`GetClientRect`+`ClientToScreen`). - -### Perceptual (YIQ) Image Diff with Anti-Alias Suppression - -Visual-regression diffing that ignores anti-aliased edges. Full reference: [`docs/source/Eng/doc/new_features/v149_features_doc.rst`](docs/source/Eng/doc/new_features/v149_features_doc.rst). - -- **`perceptual_diff` / `assert_perceptual`** (`AC_perceptual_diff`): `image_difference` counts raw per-channel deltas and `ssim_compare` is a global score — neither uses a perceptual metric or ignores anti-aliasing, the #1 source of false-positive visual-diff failures. This compares in YIQ space (pixelmatch's colour metric) and, by default, removes thin 1px anti-aliased edge diffs via a morphological open so only solid changes count (`include_aa=True` keeps them). Returns `{diff_pixels, diff_ratio, regions}`; `assert_perceptual` / `max_diff_ratio` gate a regression test. Injectable image pair → headless-testable (a 1px fringe → 0, a solid block → counted). - -### Soft Assertions (Aggregate Failures) - -Verify many things, report every failure at once. Full reference: [`docs/source/Eng/doc/new_features/v148_features_doc.rst`](docs/source/Eng/doc/new_features/v148_features_doc.rst). - -- **`SoftAssertions`** (`AC_soft_assert`): `assert_all` takes a pre-built spec list up front — there was no scoped accumulator you sprinkle `check()` calls into that raises everything on block exit (JUnit5 `assertAll` / Playwright `expect.soft`). `with SoftAssertions() as soft: soft.check(...)` records pass/fail (never raising mid-block, returns the bool to branch on), then raises once on exit listing every failure — and never masks an exception already propagating. The executor command aggregates a JSON `checks` list (eq/ne/gt/lt/contains/truthy). Pure-stdlib, headless-testable. - -### Window Z-Order (Always-On-Top / Front / Back) - -Pin a window on top, raise it, or push it behind. Full reference: [`docs/source/Eng/doc/new_features/v147_features_doc.rst`](docs/source/Eng/doc/new_features/v147_features_doc.rst). - -- **`set_topmost` / `bring_to_front` / `send_to_back` / `plan_zorder`** (`AC_set_topmost`, `AC_bring_to_front`, `AC_send_to_back`): the raw `set_window_position` existed but wasn't in the facade, had no title wrapper and no topmost semantics — the standard RPA "always-on-top" was missing. `plan_zorder` is a pure action→`SetWindowPos` constant lookup (headless-testable); the title-based setters apply it through an injectable driver (the `snap_window` seam pattern), Win32 by default. - -### Localized Motion / Activity Detection - -Find which sub-regions are animating between two frames. Full reference: [`docs/source/Eng/doc/new_features/v146_features_doc.rst`](docs/source/Eng/doc/new_features/v146_features_doc.rst). - -- **`changed_regions` / `has_motion` / `activity_score`** (`AC_changed_regions`, `AC_has_motion`): `wait_until_screen_stable` is a boolean poll, `ssim_changed_regions` is structural (ignores fast motion), `diff_screenshots` isn't activity blobs. This is the cheap absdiff path — threshold the per-pixel difference, dilate, and return the moved-region boxes (largest first), a boolean, and the fraction of pixels that moved. Pick a quiet area or locate a spinner. Two injectable frames → headless-testable; reuses the shared connected-components helper; `after` defaults to a live screen grab in the executor. - -### Colour-Histogram Fingerprint & Change Detection - -Tell whether the view is "the same" despite lighting / scale. Full reference: [`docs/source/Eng/doc/new_features/v145_features_doc.rst`](docs/source/Eng/doc/new_features/v145_features_doc.rst). - -- **`image_histogram` / `compare_histograms` / `histogram_changed`** (`AC_image_histogram`, `AC_histogram_changed`): `image_dedup`'s perceptual hash is spatial (brittle to colour/theme) and `color_stats` is one colour. A normalized colour histogram is the illumination/scale-robust "same view, or palette shifted?" signal (theme switch, reload, rotated banner). `image_histogram` returns a per-channel histogram (`hsv`/`rgb`/`gray`); `compare_histograms` does correlation/chisqr/intersection/bhattacharyya; `histogram_changed` compares a reference vs the live screen. Injectable image → headless-testable; base OpenCV (`cv2.calcHist`/`compareHist`). - -### Rich Clipboard (HTML / CF_HTML) - -Copy and paste *formatted* HTML into Word / Outlook. Full reference: [`docs/source/Eng/doc/new_features/v144_features_doc.rst`](docs/source/Eng/doc/new_features/v144_features_doc.rst). - -- **`build_cf_html` / `parse_cf_html` / `set_clipboard_html` / `get_clipboard_html`** (`AC_set_clipboard_html`, `AC_get_clipboard_html`): the base clipboard handles plain text + image only — rich paste needs `CF_HTML`, whose byte-offset header (`StartHTML`/`EndHTML`/`StartFragment`/`EndFragment`) is famously error-prone. `build_cf_html`/`parse_cf_html` compute and recover it in pure Python (round-trip tested, correct across multi-byte UTF-8); `set/get_clipboard_html` wrap them over the Win32 clipboard (with a plain-text fallback). Byte-offset math is headless-testable; only the I/O is Windows. - -### Composable / Filtered Candidate Locators - -Refine located elements with a chain: `.within(panel).filter(has_text="Delete").nth(1)`. Full reference: [`docs/source/Eng/doc/new_features/v143_features_doc.rst`](docs/source/Eng/doc/new_features/v143_features_doc.rst). - -- **`from_boxes` / `Candidates`** (`AC_locate_chain`): `anchor_locator` is a single relation and `grid_locator` is cells — neither supports composable refinement of a candidate set (the Selenium-4 / Playwright chained-locator idiom). This is a pure post-filter over boxes from *any* source (template / OCR / a11y / `fuse_elements`): `within` (region clip), `filter` (`has_text` / `near` / area / predicate), `sort_reading`, `nth` / `first` / `last`, `resolve()` / `center()`. Every method returns a new `Candidates` (no mutation) → fully headless-testable. The executor command applies a JSON `ops` list. - -### Retrying Value Assertions (expect.poll) - -Retry *any* value until it matches, not just the built-in checks. Full reference: [`docs/source/Eng/doc/new_features/v142_features_doc.rst`](docs/source/Eng/doc/new_features/v142_features_doc.rst). - -- **`expect_poll` / `assert_poll` + matchers** (`AC_expect_poll`): `assert_eventually` only polls the fixed dict-spec checks (text/image/pixel/…). This polls any zero-arg `getter` against any `matcher` (`to_equal` / `to_contain` / `to_be_greater_than` / `to_match_regex` / `to_be_truthy` / `to_be_stable`) until it passes or times out — an OCR'd total, a row count stabilising, a custom predicate. Injectable `clock`/`sleep` → deterministic, mirrors Playwright's `expect.poll`. The executor command re-runs a nested action until a key of its result matches. - -### Line / Grid / Separator Detection (Hough) - -Find table grid lines and UI dividers from raw pixels. Full reference: [`docs/source/Eng/doc/new_features/v141_features_doc.rst`](docs/source/Eng/doc/new_features/v141_features_doc.rst). - -- **`find_lines` / `find_grid` / `find_separators`** (`AC_find_lines`, `AC_find_grid`, `AC_find_separators`): `grid_locator` clusters *already-found* boxes and `shape_locator` finds closed rectangles — neither finds a table's ruling lines or a divider from pixels. Canny + probabilistic Hough detects straight segments (classified horizontal/vertical/diagonal), `find_grid` recovers `{rows, cols, cells}` so you can address "row 3, col 2", and `find_separators` returns the coordinates of long dividers. Injectable haystack → headless-testable; base OpenCV (`cv2.HoughLinesP`). - -### Model-Free Text-Region Detection (MSER) - -Find where text is on screen without running OCR. Full reference: [`docs/source/Eng/doc/new_features/v140_features_doc.rst`](docs/source/Eng/doc/new_features/v140_features_doc.rst). - -- **`find_text_regions` / `find_text_lines`** (`AC_find_text_regions`, `AC_find_text_lines`): `shape_locator` finds rectangles (not text) and `locate_text` needs an OCR engine *and* the exact string — neither answers "where is *any* text?". MSER finds the glyph/word/line blobs, so a script can crop candidate boxes to feed OCR (faster + more accurate than full-frame) or detect a label appeared with no OCR dependency. `merge` unions MSER's nested per-glyph regions; `find_text_lines` groups glyphs into per-line boxes; a blank screen returns `[]`. Base OpenCV (`cv2.MSER_create`), injectable haystack → headless-testable. - -### HSV Colour-Space Segmentation - -Find "any shade of red" regardless of lighting. Full reference: [`docs/source/Eng/doc/new_features/v139_features_doc.rst`](docs/source/Eng/doc/new_features/v139_features_doc.rst). - -- **`dominant_hue_regions` / `segment_hsv` / `color_mask`** (`AC_dominant_hue_regions`, `AC_segment_hsv`): `find_color_region` masks in RGB with a per-channel ± box — it can't match "the same colour at a different brightness" (status lights, highlights, theme tints). HSV separates hue from brightness, so a hue band + saturation/value floor catches every shade across lighting. `dominant_hue_regions(hue=…)` handles red's 0/180 wrap automatically; `segment_hsv` takes an explicit band; both return `{x,y,width,height,area,center}` blobs reusing the shared connected-components helper. Injectable haystack → headless-testable. - -### Fuse & Order On-Screen Element Boxes - -Turn raw OCR + icon + a11y boxes into one clean, numbered element list. Full reference: [`docs/source/Eng/doc/new_features/v138_features_doc.rst`](docs/source/Eng/doc/new_features/v138_features_doc.rst). - -- **`iou` / `merge_boxes` / `fuse_elements` / `reading_order`** (`AC_fuse_elements`, `AC_reading_order`): `set_of_marks` numbers a clean element list but nothing *produced* it — a real screen parse yields three overlapping sources with duplicates and no order. These supply the missing step: drop near-duplicate boxes by IoU, union OCR/icon/a11y keeping the most trustworthy source on overlap (`source_priority` a11y > ocr > icon), and sort top-to-bottom/left-to-right with a stable `index`. Plain `dict` boxes → pure-stdlib, fully headless-testable; pairs directly with `set_of_marks`. - -### Actionability Gate (Wait Until Ready Before Acting) - -Don't click until the target is genuinely ready. Full reference: [`docs/source/Eng/doc/new_features/v137_features_doc.rst`](docs/source/Eng/doc/new_features/v137_features_doc.rst). - -- **`wait_actionable` / `act_when_ready`** (`AC_wait_actionable`): Playwright/Cypress run an actionability check before every click — present + stopped moving + enabled + not covered — but AutoControl had none (`self_heal_click` clicks immediately; `wait_until_screen_stable` watches the whole frame). This composes the four checks into one gate and returns an `ActionabilityReport` (per-check booleans, target `point`, `reason` = first failing check). Every signal is an injectable callable (`bbox_provider` / `region_sampler` / `enabled_probe` / `hit_tester`) plus an injectable `clock`/`sleep`, so it's fully deterministic and headless-testable. The executor command gates on a template image. - -### Multi-Monitor / Virtual-Desktop Geometry - -Place windows and points correctly across several displays. Full reference: [`docs/source/Eng/doc/new_features/v136_features_doc.rst`](docs/source/Eng/doc/new_features/v136_features_doc.rst). - -- **`enumerate_monitors` + `Monitor` / `virtual_bounds` / `monitor_at_point` / `monitor_for_window` / `to_local` / `to_virtual` / `remap_point`** (`AC_enumerate_monitors`, `AC_monitor_at_point`): `snap_window` / `arrange_grid` / the layout planner all assumed a single primary `(width, height)` — monitor-blind, unable to tile on a second display or handle a negative-origin virtual desktop. This adds the physical layer: union virtual bounds, which-monitor-owns-this-point/window, virtual↔monitor-local conversion, and equivalent-spot remapping across resolutions/DPI. Pure geometry over `Monitor` dataclasses → fully headless-testable; `enumerate_monitors` has an injectable provider (default `mss`). - -### Image Pre-processing for OCR / Template Matching - -Clean up the screen before reading or matching it. Full reference: [`docs/source/Eng/doc/new_features/v135_features_doc.rst`](docs/source/Eng/doc/new_features/v135_features_doc.rst). - -- **`preprocess_image` + `to_grayscale` / `binarize` / `upscale` / `denoise` / `deskew` / `enhance_contrast`** (`AC_preprocess_image`): `locate_text` and `match_template` fed the *raw* capture to OCR / the matcher — small text, dark themes, low contrast and skew wrecked both, with no preprocessing seam anywhere. This adds the standard pipeline (grayscale → upscale → binarize → deskew → denoise → CLAHE) that multiplies their accuracy. Injectable haystack → ndarray; `detect_skew_angle` measures text rotation; `binarize` does otsu / adaptive. The executor command writes the cleaned image to a path. Headless-testable on synthetic arrays. - -### Arrange Multiple Windows (Grid / Cascade) - -Lay out a whole set of windows in one call. Full reference: [`docs/source/Eng/doc/new_features/v134_features_doc.rst`](docs/source/Eng/doc/new_features/v134_features_doc.rst). - -- **`arrange_grid` / `arrange_cascade`** (`AC_arrange_grid`, `AC_arrange_cascade`): `snap_window` moves *one* window and the layout planner only *computes* rectangles — these close the loop, taking a list of window titles and actually moving every match into a grid (auto near-square shape, or explicit `rows`/`cols` + `gap`) or a diagonal cascade. Build on the layout planner and reuse `snap_window`'s injectable `mover`/`screen_size` seams, so they are fully headless-testable; return the count moved. - -### Window Tiling / Layout Geometry Planner - -Compute where to place application windows — halves, grids, cascades. Full reference: [`docs/source/Eng/doc/new_features/v133_features_doc.rst`](docs/source/Eng/doc/new_features/v133_features_doc.rst). - -- **`tile_rect` / `grid_rects` / `cascade_rects`** (`AC_tile_rect`, `AC_grid_rects`, `AC_cascade_rects`): `save/restore_window_layout` replay *exact* saved positions and `snap_window` moves *one* window — nothing *computes* a fresh multi-window layout. This pure-geometry planner returns the target rectangles for halves, quadrants, thirds, an R×C grid and a staggered cascade given a screen work area, so a script can lay out windows deterministically. Returns `WindowRect` (`.as_tuple()` / `.to_dict()`); `gap` insets tiles; cross-platform and fully headless-testable; composes with any window-move backend. - -### Locate UI Elements by Edge / Contour (No Template) - -Find the clickable boxes on a screen you have never seen. Full reference: [`docs/source/Eng/doc/new_features/v132_features_doc.rst`](docs/source/Eng/doc/new_features/v132_features_doc.rst). - -- **`find_shapes` / `find_rectangles`** (`AC_find_shapes`, `AC_find_rectangles`): every other locator needs something to look *for* — a template, a colour, some text. These need nothing: Canny edge detection + contour extraction returns the bounding boxes (`{x,y,width,height,area,center,aspect}`, largest first) of the distinct shapes, so a script can enumerate cards / buttons / input fields structurally and click the Nth one. `find_rectangles` keeps only convex quads and adds an `aspect_range=(min,max)` w/h filter (`(1.5,8)` wide buttons). Injectable haystack → headless-testable. - -### ORB Feature Matching (Rotation / Scale / Theme Robust) - -Find a target even when it is rotated, rescaled or re-themed. Full reference: [`docs/source/Eng/doc/new_features/v131_features_doc.rst`](docs/source/Eng/doc/new_features/v131_features_doc.rst). - -- **`feature_match`** (`AC_feature_match`): pixel template matching (`match_template` / `match_masked`) correlates pixels, so it breaks the moment the target is rotated, scaled by an unlisted factor, or re-coloured (light/dark theme, hover). This matches ORB *keypoints* and fits a RANSAC homography, returning the four projected `corners`, the `center`, the `inliers` count and an inlier-fraction `score`. ORB border/patch sizes auto-scale down for icon-sized templates (OpenCV's defaults reject them). Core OpenCV only (no contrib); injectable haystack → headless-testable. - -### Structural-Similarity (SSIM) Comparison - -Perceptual screen comparison that tells you *what* changed. Full reference: [`docs/source/Eng/doc/new_features/v130_features_doc.rst`](docs/source/Eng/doc/new_features/v130_features_doc.rst). - -- **`ssim_compare` / `ssim_changed_regions`** (`AC_ssim_compare`, `AC_ssim_changed_regions`): pixel diff (`diff_screenshots`) fires on a one-pixel shift; a histogram (`detect_drift`) is blind to layout. SSIM is the standard visual-regression metric — tolerant of small illumination changes, sensitive to structural change. `ssim_compare` returns a 0..1 score (1.0 = identical); `ssim_changed_regions` returns boxes of what moved. `ignore=[[x,y,w,h]]` masks live clocks / cursors. Pure NumPy + OpenCV (no scikit-image); injectable image pair → headless-testable. - -### Masked Template Matching - -Match icons regardless of their background. Full reference: [`docs/source/Eng/doc/new_features/v129_features_doc.rst`](docs/source/Eng/doc/new_features/v129_features_doc.rst). - -- **`match_masked` / `match_masked_all`** (`AC_match_masked`, `AC_match_masked_all`): plain template matching scores *every* pixel, so an icon clipped from one background fails over a different one. These count only the pixels you mark relevant — an explicit grayscale `mask`, or an RGBA template's alpha channel — so transparent / "don't care" pixels stop dragging the score down. Returns the same `Match` (score/center) as scored template matching; OpenCV masked `TM_CCORR_NORMED`, NaNs zeroed. Injectable haystack → headless-testable. - -### Locate On-Screen Regions by Colour - -Find the green status pill / red banner by colour. Full reference: [`docs/source/Eng/doc/new_features/v128_features_doc.rst`](docs/source/Eng/doc/new_features/v128_features_doc.rst). - -- **`find_color_region` / `find_color_regions`** (`AC_find_color_region`): `color_stats` only describes a region's colour and `assert_pixel` checks one point — neither *locates* a coloured region. This masks pixels within `tolerance` of a target RGB and returns the connected blobs' boxes (`{x,y,width,height,area,center}`, largest first) — for status lights, progress fills, error banners where a template is brittle. Injectable haystack → headless-testable; OpenCV/NumPy via `je_open_cv`. - -### Confidence-Returning Template Matching - -Template matching that returns the score, searches multiple scales, and finds all occurrences. Full reference: [`docs/source/Eng/doc/new_features/v127_features_doc.rst`](docs/source/Eng/doc/new_features/v127_features_doc.rst). - -- **`match_template` / `match_template_all` / `best_matches` / `TemplateMatch`** (`AC_match_template`, `AC_match_template_all`): the existing matcher (`find_object`) is single-scale and *discards the score*. This returns a `Match` with `score`/`scale`/`center`, searches `scales` for DPI/zoom tolerance, and enumerates every occurrence with non-maximum suppression. Injectable `haystack` (ndarray/path/PIL) → headless-testable on synthetic arrays; OpenCV/NumPy via the `je_open_cv` dependency. - -### Wait for Window Title (Regex) - -Block until a window title matches a regex (or vanishes). Full reference: [`docs/source/Eng/doc/new_features/v126_features_doc.rst`](docs/source/Eng/doc/new_features/v126_features_doc.rst). - -- **`wait_until_window_title`** (`AC_wait_window_title`): `wait_for_window` matches a title substring and only waits for *appear*; `wait_until_window_closed` is substring vanish. This matches a regular expression by default (`regex=False` for substring) and can wait for the title to vanish (`present=False`) — e.g. wait for a tab to navigate to `r".*— Checkout$"`. Injectable title source, headless-testable. - -### Grid / Table Cell Addressing - -Address a table cell by (row, column) from cell bounding boxes. Full reference: [`docs/source/Eng/doc/new_features/v125_features_doc.rst`](docs/source/Eng/doc/new_features/v125_features_doc.rst). - -- **`cluster_grid` / `locate_cell`** (`AC_grid_cell`): `anchor_locator` does pairwise relations but nothing addresses a 2-D grid. Given the cell bounding boxes (from `locate_all_image` / `find_text_matches`), this clusters them into rows (by centre-y within `row_tolerance`) and columns (by centre-x) and returns the centre of the 0-based `(row, col)` cell — ready to click. Pure clustering, fully headless-testable. - -### Anchor Ordinal & Locate-All - -Pick the Nth anchor-relative match, or enumerate them all. Full reference: [`docs/source/Eng/doc/new_features/v124_features_doc.rst`](docs/source/Eng/doc/new_features/v124_features_doc.rst). - -- **`anchor_locate(..., ordinal=N)` / `anchor_locate_all`** (`AC_anchor_locate` ordinal, `AC_anchor_locate_all`): `anchor_locate` always returned the single nearest match — no way to grab "the 2nd row below the header" or list every row. Adds a 1-based `ordinal` selector (backward-compatible; `ordinal=1` = nearest) and `anchor_locate_all` returning every match sorted by distance — the building block for table/list-row selection. Pure ranking core, deterministic. - -### Held Modifiers Across an Action Group - -Hold ctrl/shift down across several actions, released even on error. Full reference: [`docs/source/Eng/doc/new_features/v123_features_doc.rst`](docs/source/Eng/doc/new_features/v123_features_doc.rst). - -- **`hold_modifiers` / `plan_with_modifiers`** (`AC_with_modifiers`): `hotkey` releases its keys immediately — there was no way to hold a modifier down across several independent actions (shift-click range select, ctrl-click multi-select) with a guaranteed release. `hold_modifiers` is a context manager that presses on enter and releases in reverse on exit (in a `finally`, so nothing leaks); `plan_with_modifiers` is the pure plan. Injectable sink, deterministic. - -### Unicode Text Entry (Emoji / CJK) - -Type any Unicode (emoji / CJK / accented) that `write` can't. Full reference: [`docs/source/Eng/doc/new_features/v122_features_doc.rst`](docs/source/Eng/doc/new_features/v122_features_doc.rst). - -- **`type_unicode` / `plan_paste` / `unicode_code_units`** (`AC_type_unicode`): `write` types through the virtual-key table and *raises* on emoji/CJK/many accented chars. `type_unicode` enters any text reliably by setting the clipboard and pasting (`modifier` ctrl/command). `unicode_code_units` splits text into UTF-16 code units (surrogate pairs) for KEYEVENTF_UNICODE backends. Pure-planning + injectable sink, deterministic. - -### Wait for Region Colour - -Block until a colour fills (or leaves) a screen region. Full reference: [`docs/source/Eng/doc/new_features/v121_features_doc.rst`](docs/source/Eng/doc/new_features/v121_features_doc.rst). - -- **`wait_until_color`** (`AC_wait_color`): `wait_for_pixel` matches one point exactly and `wait_until_pixel_changes` detects any change at one point — neither waits for "the status light turns green" / "the progress bar fills" / "the red banner is gone". This counts pixels within `tolerance` of `target_rgb` over a region and succeeds when that fraction crosses `min_fraction` (or drops below it, `present=False`). Injectable sampler, headless-testable. Pure-stdlib. - -### Relative Mouse Movement - -Nudge the pointer by a delta from where it is. Full reference: [`docs/source/Eng/doc/new_features/v120_features_doc.rst`](docs/source/Eng/doc/new_features/v120_features_doc.rst). - -- **`move_mouse_relative` / `relative_target`** (`AC_move_mouse_relative`): the mouse wrapper only had absolute `set_mouse_position` — no `moveRel(dx, dy)` for relative-pointer / canvas / FPS apps and incremental drags. Reads the live position and moves by the delta; `relative_target` is the pure arithmetic, and the getter/setter are injectable for headless tests. Pure-stdlib, deterministic. - -### Hold Key / Auto-Repeat - -Hold a key for a duration, or auto-repeat it at a fixed rate. Full reference: [`docs/source/Eng/doc/new_features/v119_features_doc.rst`](docs/source/Eng/doc/new_features/v119_features_doc.rst). - -- **`hold_key` / `plan_key_hold`** (`AC_hold_key`): `type_keyboard` is an instant down+up — there was no "hold this key for N seconds" (game movement, hold-to-scroll) or "send it at R presses/second" (auto-repeat). `plan_key_hold` builds the deterministic op-plan (press/wait/release, or N spaced key events for `rate_hz`); `hold_key` routes waits to an injectable `sleep` and keys to an injectable `sink`. Pure-planning, deterministic. - -### Wait Until Gone (Blocking Vanish Waits) - -Block until a spinner / toast / dialog disappears. Full reference: [`docs/source/Eng/doc/new_features/v118_features_doc.rst`](docs/source/Eng/doc/new_features/v118_features_doc.rst). - -- **`wait_until_gone` / `wait_until_image_gone` / `wait_until_text_gone`** (`AC_wait_image_gone`, `AC_wait_text_gone`): `wait_for_image`/`wait_for_text` only block until something *appears*, and `observer` fires async callbacks on vanish — there was no *blocking* "wait until this image/text disappears then continue" call. The generic `wait_until_gone` takes any predicate (headless-testable); the image/text helpers build it from the locate functions. `gone_for_s` debounces flicker. Returns a `WaitOutcome`. Pure-stdlib. - -### Clear-Then-Type Field Entry - -Reliably set a text field's value (the Playwright `fill` idiom). Full reference: [`docs/source/Eng/doc/new_features/v117_features_doc.rst`](docs/source/Eng/doc/new_features/v117_features_doc.rst). - -- **`set_field_text` / `plan_field_set`** (`AC_set_field_text`): there was no single "focus → clear → set value" primitive, and `write` raises on emoji/CJK. This clears the field (select-all + delete) then enters the text — optionally via the clipboard (`paste=True`) which is the Unicode-safe path `write` can't do. `modifier` is the platform command key (`ctrl`/`command`). Pure-planning + injectable sink, deterministic. - -## What's new (2026-06-22) - -### Multi-Waypoint Mouse Gestures - -Move or drag the pointer through a polyline of waypoints. Full reference: [`docs/source/Eng/doc/new_features/v116_features_doc.rst`](docs/source/Eng/doc/new_features/v116_features_doc.rst). - -- **`plan_path` / `move_along_path` / `drag_path` / `path_easings`** (`AC_move_along_path`, `AC_drag_path`): `humanize` and `tween_drag` only interpolate a single start→end hop — there was no way to drive an arbitrary chain of waypoints (signatures, marquee selects, multi-stop drags) with the button held across the whole path. `plan_path` is pure eased point math (reusing `tween_drag`'s easings, junctions de-duplicated); the move/drag dispatch through an injectable sink for headless testing. Pure-stdlib, deterministic. - -### Check-Digit Algorithms - -Compute / verify Luhn, Verhoeff, Damm and ISO 7064 MOD 97-10 check digits. Full reference: [`docs/source/Eng/doc/new_features/v115_features_doc.rst`](docs/source/Eng/doc/new_features/v115_features_doc.rst). - -- **`luhn_validate` / `luhn_check_digit` / `verhoeff_*` / `damm_*` / `mod97_10_*`** (`AC_checksum_validate`, `AC_checksum_digit`): `pii_text` detects card/IBAN shapes by regex and `data_quality` does regex validation, but nothing computed or verified a *check digit*. This adds the four schemes behind most identifiers (cards/IMEI, national IDs, IBAN) — the shared engine `identifier_validate` builds on. Pure-stdlib, deterministic. - -### GNU gettext Catalog I/O (.po / .mo) - -Read/compile the de-facto translation format. Full reference: [`docs/source/Eng/doc/new_features/v114_features_doc.rst`](docs/source/Eng/doc/new_features/v114_features_doc.rst). - -- **`parse_po` / `read_mo` / `GettextCatalog` / `parse_po_file` / `read_mo_file`** (`AC_gettext_translate`, `AC_gettext_ngettext`): the repo pseudo-localises and renders ICU messages but couldn't read GNU gettext `.po`/`.mo`. This parses `.po` (contexts, plurals, the `Plural-Forms` header via `gettext.c2py`), compiles a standards-compliant `.mo` that Python's own `gettext.GNUTranslations` loads, and exposes `gettext`/`ngettext`/`pgettext`. Pure-stdlib, deterministic. - -### ICU-lite MessageFormat (Plural / Select) - -Render count-aware localised messages. Full reference: [`docs/source/Eng/doc/new_features/v113_features_doc.rst`](docs/source/Eng/doc/new_features/v113_features_doc.rst). - -- **`format_message` / `plural_category` / `ordinal_category`** (`AC_format_message`): `i18n_test.check_catalog` only compares placeholder sets and `interpolate` is flat `${var}` — neither renders `"{count, plural, one {# item} other {# items}}"`. This implements the ICU MessageFormat subset most apps use: `select`, `plural`, `selectordinal` with CLDR categories, exact `=N` selectors, the `#` count, `offset:`, nesting and apostrophe quoting. Injectable plural rules. Pure-stdlib, deterministic. - -### Locale-Aware List Formatting - -Join items the way a language expects ("A, B, and C"). Full reference: [`docs/source/Eng/doc/new_features/v112_features_doc.rst`](docs/source/Eng/doc/new_features/v112_features_doc.rst). - -- **`format_list`** (`AC_format_list`): a naive `", ".join` gives "A, B, C" with no "and"/"or" and no localisation. This implements the CLDR list-pattern composition with conjunction / disjunction / unit styles and per-locale conjunction words + serial-comma rule (`en`/`es`/`fr`/`de`/`pt`) — `format_list(["a","b","c"])` → "a, b, and c", `locale="es"` → "a, b y c". Pure-stdlib, deterministic. - -### Bidirectional-Text QA (Trojan-Source Scan) - -Catch invisible Unicode directional formatting (RTL QA + Trojan-source). Full reference: [`docs/source/Eng/doc/new_features/v111_features_doc.rst`](docs/source/Eng/doc/new_features/v111_features_doc.rst). - -- **`detect_bidi_issues` / `bidi_controls` / `is_bidi_balanced` / `base_direction` / `is_trojan_source` / `strip_bidi_controls` / `has_bidi_controls`** (`AC_bidi_check`, `AC_bidi_strip`): `confusables` catches lookalike characters, but bidi controls (LRO/RLO/PDF, isolates, marks) can silently reorder rendered text — an RTL-QA gap and the "Trojan Source" attack (CVE-2021-42574). This lists the controls, checks nesting balance, infers base direction, and flags reordering formatting. Pure-stdlib (`unicodedata`), deterministic. - -### Readability Scoring - -Score how hard text is to read; gate generated copy on a reading grade. Full reference: [`docs/source/Eng/doc/new_features/v110_features_doc.rst`](docs/source/Eng/doc/new_features/v110_features_doc.rst). - -- **`flesch_reading_ease` / `flesch_kincaid_grade` / `gunning_fog` / `smog_index` / `automated_readability_index` / `readability_report` / `readability_stats` / `count_syllables`** (`AC_readability_report`): the text utilities canonicalise, match and rank text but never scored *difficulty*. This adds the classic English readability formulae over a deterministic tokeniser and syllable heuristic, so a test can assert an on-screen message or label stays within a target reading grade. Pure-stdlib (`re`/`math`), deterministic. - -### Confusable / Homoglyph Detection - -Catch Unicode visual spoofing (IDN-homograph phishing, lookalike labels). Full reference: [`docs/source/Eng/doc/new_features/v109_features_doc.rst`](docs/source/Eng/doc/new_features/v109_features_doc.rst). - -- **`confusable_skeleton` / `is_confusable` / `detect_homoglyphs` / `is_mixed_script` / `scripts_of`** (`AC_confusable_scan`, `AC_confusable_compare`): a Cyrillic `"а"` is pixel-for-pixel a Latin `"a"`, so `"pаypal"` reads as `"paypal"` yet compares unequal. Following Unicode TR39, this folds confusables to a prototype skeleton (strings match when skeletons match) and flags mixed-script tokens. Pure-stdlib (`unicodedata`), deterministic. - -### Locale-Aware String Collation - -Sort strings the way a reader of the language expects. Full reference: [`docs/source/Eng/doc/new_features/v108_features_doc.rst`](docs/source/Eng/doc/new_features/v108_features_doc.rst). - -- **`sort_strings` / `collation_compare` / `collation_key`** (`AC_collation_sort`, `AC_collation_compare`): Python's default `sorted` is codepoint order, so `"Z" < "a"` and `"ä"` lands far from `"a"`. This Unicode-Collation-lite key orders by base letter, then accent (secondary), then case (tertiary), with an optional `tailoring` alphabet so Swedish puts `å ä ö` after `z`. Pure-stdlib (`unicodedata`), deterministic across platforms — unlike `locale.strxfrm`. - -### Transactional Outbox - -Durably buffer events and drain them at-least-once. Full reference: [`docs/source/Eng/doc/new_features/v107_features_doc.rst`](docs/source/Eng/doc/new_features/v107_features_doc.rst). - -- **`Outbox`** (`AC_outbox_enqueue`, `AC_outbox_pending`): `events.cloud_events` posts synchronously with no durability — a crash or network blip loses the event. The outbox persists each event first, then `drain`s pending entries through an injected sink with at-least-once delivery: a sink failure leaves the entry pending for retry until `max_attempts`, after which it is dead-lettered. `save` / `load` keep events across restarts. Pure-stdlib, deterministic. - -### Optimistic-Concurrency Versioned Store - -Update only if the version is unchanged (compare-and-swap / If-Match). Full reference: [`docs/source/Eng/doc/new_features/v106_features_doc.rst`](docs/source/Eng/doc/new_features/v106_features_doc.rst). - -- **`VersionedStore` / `VersionConflict` / `if_match_header` / `check_if_match`** (`AC_cas_put`, `AC_cas_get`): `http_conditional` used ETag for read caching but never for write concurrency. This local compare-and-swap store `put`s only when `expected_version` matches (raising `VersionConflict` on a stale write), bumps a monotonic version, and bridges to HTTP `If-Match` — the write side of the ETag story. Pure-stdlib, deterministic. - -### Per-Stream Sequence-Gap Detection - -Detect missing / out-of-order / duplicate messages by sequence number. Full reference: [`docs/source/Eng/doc/new_features/v105_features_doc.rst`](docs/source/Eng/doc/new_features/v105_features_doc.rst). - -- **`SequenceTracker`** (`AC_sequence_observe`): nothing tracked per-stream monotonic sequence numbers. `observe(stream, seq)` classifies each as `ok` / `duplicate` / `gap` (with the `missing` numbers) / `reorder` (late arrivals fill gaps), and exposes `gaps` and `high_water`. Complements `dedup_window`. Pure-stdlib, deterministic. - -### Time-Windowed Deduplication - -Drop duplicate/redelivered messages within a TTL window. Full reference: [`docs/source/Eng/doc/new_features/v104_features_doc.rst`](docs/source/Eng/doc/new_features/v104_features_doc.rst). - -- **`DedupWindow`** (`AC_dedup_check`): `work_queue` dedups only in-flight references, so a completed reference re-enqueues and redelivered webhooks reprocess. This sliding-window inbox `check_and_mark`s a message id — `True` the first time, `False` for a duplicate within `ttl_s` — converting at-least-once delivery to exactly-once-in-window. Injectable clock, bounded size. Pure-stdlib, deterministic. - -### Idempotency-Key Store - -Run a side effect once, replay its response on retries. Full reference: [`docs/source/Eng/doc/new_features/v103_features_doc.rst`](docs/source/Eng/doc/new_features/v103_features_doc.rst). - -- **`IdempotencyStore` / `request_fingerprint` / `IdempotencyConflict`** (`AC_idempotency_begin`, `AC_idempotency_complete`): `RetryPolicy` re-executes and `work_queue` dedups only in-flight refs — nothing cached the first result. This Stripe-style store returns `new`/`in_progress`/`completed` for a key, replays the stored response, raises on a fingerprint conflict, and supports injectable-clock TTL + JSON persistence. Pure-stdlib, deterministic. - -### Moving-Average Smoothing - -Smooth a noisy value series. Full reference: [`docs/source/Eng/doc/new_features/v102_features_doc.rst`](docs/source/Eng/doc/new_features/v102_features_doc.rst). - -- **`sma` / `wma` / `ewma` / `rolling`** (`AC_sma`, `AC_ewma`): `stats.describe` summarizes a whole sample and `timeseries` rolls counters into rates, but nothing smoothed a noisy signal. This adds trailing simple/weighted/exponentially-weighted moving averages and a generic rolling reducer, all returning a same-length list aligned to the input timeline. Pure-stdlib, deterministic. - -### Single-Series Anomaly Detection - -Flag the spike in one live metric series. Full reference: [`docs/source/Eng/doc/new_features/v101_features_doc.rst`](docs/source/Eng/doc/new_features/v101_features_doc.rst). - -- **`detect_anomalies` / `mad_anomalies` / `zscore_anomalies` / `ewma_control`** (`AC_detect_anomalies`): `data_drift` is two-batch distribution shift and `slo.burn_alerts` only thresholds budget burn — neither points at *which* value in one series is anomalous. This flags outliers via robust MAD (modified z-score), plain z-score, and an EWMA control chart (with an optional in-control baseline) — `{index, value, score, is_anomaly}` records. Pure-stdlib, deterministic. - -### Near-Duplicate Text Detection (SimHash / MinHash) - -Fingerprint text to find near-dups at scale. Full reference: [`docs/source/Eng/doc/new_features/v100_features_doc.rst`](docs/source/Eng/doc/new_features/v100_features_doc.rst). - -- **`simhash` / `near_duplicates` / `minhash_signature` / `minhash_similarity`** (`AC_simhash`, `AC_near_duplicates`): `fuzzy_dedupe` is O(n²) pairwise with no stable fingerprint and `image_dedup` only hashes pixels. This adds the text analog — SimHash (Hamming-distance near-dup clustering) and MinHash (estimated Jaccard) using a fixed `blake2b` hash for deterministic fingerprints. Pairs with `normalize_text`. Pure-stdlib. - -### String-Distance Similarity Metrics - -Match typos and reordered tokens. Full reference: [`docs/source/Eng/doc/new_features/v99_features_doc.rst`](docs/source/Eng/doc/new_features/v99_features_doc.rst). - -- **`levenshtein` / `damerau_levenshtein` / `jaro` / `jaro_winkler` / `jaccard` / `dice` / `similarity`** (`AC_text_similarity`): `fuzzy` exposed only difflib's gestalt ratio. This adds the edit-distance and token-set metrics it lacks — Jaro-Winkler (standard for short labels), Damerau (transposition-aware), and char-n-gram Jaccard/Dice — plus a unified `similarity()` that normalizes every metric to `[0, 1]`. Pairs with `normalize_text`. Pure-stdlib, deterministic. - -### Time-Series Transforms - -Turn counters into rates; downsample and resample. Full reference: [`docs/source/Eng/doc/new_features/v98_features_doc.rst`](docs/source/Eng/doc/new_features/v98_features_doc.rst). - -- **`ts_rate` / `ts_irate` / `ts_increase` / `ts_delta` / `ts_downsample` / `ts_resample`** (`AC_ts_rate`, `AC_ts_downsample`): `observability` counters store only the current value (no counter→rate anywhere) and `cost_telemetry` only buckets by day. This adds Prometheus-style reset-aware rate/increase/delta over `(timestamp, value)` series, tumbling-bucket downsampling (avg/sum/min/max/first/last/count), and grid resampling (last/linear/none). No wall clock — deterministic. Pure-stdlib. - -### Unicode Text Normalisation & Slugify - -Canonicalize text before fuzzy/search/OCR matching. Full reference: [`docs/source/Eng/doc/new_features/v97_features_doc.rst`](docs/source/Eng/doc/new_features/v97_features_doc.rst). - -- **`normalize_text` / `deaccent` / `slugify` / `normalize_quotes` / `fold_whitespace`** (`AC_normalize_text`, `AC_slugify`): `fuzzy` and `search_index.tokenize` only lowercase and OCR matching only `.lower()`+substring, so `"Café"` (NFC) vs `"Café"` (NFD) vs `"cafe"` compare unequal. This adds the missing canonicalization layer (NFKC + casefold + whitespace fold, accent stripping, smart-quote mapping, ASCII slugs). Pure-stdlib (`unicodedata`), deterministic. - -### JSON-Schema Compatibility Checking - -Classify schema changes as backward/forward/full. Full reference: [`docs/source/Eng/doc/new_features/v96_features_doc.rst`](docs/source/Eng/doc/new_features/v96_features_doc.rst). - -- **`check_compatibility` / `diff_schemas` / `is_backward_compatible` / `is_forward_compatible` / `is_full_compatible`** (`AC_check_compatibility`): we could validate against and generate JSON Schemas but couldn't answer "will an old consumer still read new data?". This classifies changes (added-required field, removed field, narrowed/widened type, enum add/remove) under Confluent/Avro backward/forward/full rules over the object subset. Pure-stdlib, deterministic. - -### Typed Configuration Schema - -Validate config into a typed object. Full reference: [`docs/source/Eng/doc/new_features/v95_features_doc.rst`](docs/source/Eng/doc/new_features/v95_features_doc.rst). - -- **`ConfigSchema` / `ConfigField` / `validate_config` / `coerce`** (`AC_validate_config`): `assets._coerce` coerces one value and `json_schema` validates structure, but nothing bound a resolved config dict into a typed object with required-field enforcement and choice constraints. This coerces types (`str`/`int`/`float`/`bool`), applies defaults, enforces required/choices, and returns `{ok, config, errors}` — a stdlib pydantic-settings analog. Pure-stdlib, deterministic. - -### OTLP/JSON Span Export - -Export spans the way a collector ingests them. Full reference: [`docs/source/Eng/doc/new_features/v94_features_doc.rst`](docs/source/Eng/doc/new_features/v94_features_doc.rst). - -- **`spans_to_otlp` / `attributes_to_otlp` / `write_otlp`** (`AC_spans_to_otlp`): `agent_trace.to_otel` returned flat dicts that aren't valid OTLP/JSON (no resourceSpans/scopeSpans nesting, times not as uint64 strings). This wraps spans in the proper envelope with hex IDs, uint64-string times, and OTLP `KeyValue` attribute encoding — what an OpenTelemetry collector's file exporter reads. Pairs with `trace_context`. Pure-stdlib, deterministic. - -### Canonical Log Lines & Structured Logging - -One wide event per run, with trace correlation. Full reference: [`docs/source/Eng/doc/new_features/v93_features_doc.rst`](docs/source/Eng/doc/new_features/v93_features_doc.rst). - -- **`CanonicalLogLine` / `JSONLogFormatter` / `bind_trace_context`** (`AC_canonical_log`): `logging_instance` emits a fixed pipe-delimited string with no JSON and no trace/span fields. This adds a Stripe-style canonical log line (field accumulator + `timer` with injectable clock) and a JSON `logging.Formatter` that carries `trace_id`/`span_id` — the log-trace correlation counterpart to `trace_context`. Pure-stdlib, deterministic. - -### Conditional HTTP Requests & Cache Validators - -Skip re-downloading unchanged resources (ETag / 304). Full reference: [`docs/source/Eng/doc/new_features/v92_features_doc.rst`](docs/source/Eng/doc/new_features/v92_features_doc.rst). - -- **`store_validators` / `conditioned_call` / `is_fresh` / `parse_cache_control` / `is_not_modified`** (`AC_parse_cache_control`, `AC_store_validators`): `http_request` never sent `If-None-Match`/`If-Modified-Since` nor read `Cache-Control`, so every poll re-downloaded. This extracts validators, parses `Cache-Control` (max-age/no-store/…), decides freshness by an explicit age, conditions the next request, and detects `304 Not Modified`. Pure-stdlib, deterministic. - -### Cookie Jar (HTTP Session Carry) - -Carry a session across HTTP calls. Full reference: [`docs/source/Eng/doc/new_features/v91_features_doc.rst`](docs/source/Eng/doc/new_features/v91_features_doc.rst). - -- **`CookieJar` / `parse_set_cookie`** (`AC_cookie_header`, `AC_parse_set_cookie`): `http_request` is stateless — no session cookies persisted across calls, so a login-then-call flow couldn't carry a session headlessly. This parses `Set-Cookie` headers into a jar, builds the `Cookie` request header, and saves/loads the jar as JSON (cookies cleared on `Max-Age<=0`/empty). Pure-stdlib, deterministic. - -### HTTP Content Negotiation & Decompression - -Build `Accept` headers and decode gzip/deflate. Full reference: [`docs/source/Eng/doc/new_features/v90_features_doc.rst`](docs/source/Eng/doc/new_features/v90_features_doc.rst). - -- **`build_accept` / `build_accept_encoding` / `parse_quality_values` / `decode_body` / `negotiated_call`** (`AC_decode_body`, `AC_parse_quality_values`): `urllib`/`http_request` never set `Accept-Encoding` nor decoded `Content-Encoding`, so compressed bodies arrived raw. This adds `Accept`/`Accept-Encoding` builders, a q-value parser (sorted by quality), and gzip/deflate (incl. raw deflate) decoding. Brotli excluded (not stdlib). Pure-stdlib, deterministic. - -### multipart/form-data Build & Parse - -Build file-upload bodies. Full reference: [`docs/source/Eng/doc/new_features/v89_features_doc.rst`](docs/source/Eng/doc/new_features/v89_features_doc.rst). - -- **`build_multipart` / `parse_multipart` / `MultipartFile`** (`AC_build_multipart`, `AC_parse_multipart`): `http_request` sent only JSON/raw — there was no file upload, and stdlib `cgi` (which parsed multipart) was removed in 3.13. This assembles a `multipart/form-data` body from text fields and files with an injectable boundary (byte-stable), and parses one back into `{fields, files}`. Pure-stdlib, deterministic. - -### Secret Redaction for Config & Logs - -Mask secrets before logging or exporting. Full reference: [`docs/source/Eng/doc/new_features/v88_features_doc.rst`](docs/source/Eng/doc/new_features/v88_features_doc.rst). - -- **`redact_config` / `redact_secret_text`** (`AC_redact_config`, `AC_redact_secret_text`): `utils/redaction` only blurs screenshots and `secrets_scan` only *detects* — neither returned a masked copy. This reuses the `secrets_scan` detector (key-name patterns, AWS/bearer formats, high-entropy) to return a redacted deep copy of a config structure, and to mask secret-looking tokens in a free-text log line (preserving surrounding words). Vault refs (`${secrets.*}`) are left intact. Pure-stdlib, deterministic. - -### RFC 8288 Link Header & Pagination - -Parse `Link` headers and follow `rel="next"`. Full reference: [`docs/source/Eng/doc/new_features/v87_features_doc.rst`](docs/source/Eng/doc/new_features/v87_features_doc.rst). - -- **`parse_link_header` / `next_url` / `links_by_rel` / `paginate`** (`AC_parse_link_header`, `AC_next_url`): paginated REST APIs return `Link: <...>; rel="next"` but nothing parsed it. This parses the header (quoted values with commas, multiple links), indexes by relation, and `paginate` walks `rel="next"` over an injected `fetch` (transport/cassette) up to `max_pages`. Pure-stdlib, deterministic. - -### Referential Integrity Checks - -Foreign-key, unique, accepted-values and row-count checks across tables. Full reference: [`docs/source/Eng/doc/new_features/v86_features_doc.rst`](docs/source/Eng/doc/new_features/v86_features_doc.rst). - -- **`check_foreign_key` / `check_unique_key` / `check_accepted_values` / `check_row_count`** (`AC_check_foreign_key`, `AC_check_unique_key`, `AC_check_accepted_values`, `AC_check_row_count`): `validate_rows` is intra-row, single-table (its `unique` only dedupes within one batch). This adds dbt-style generic checks — parent/child foreign keys across two tables, single/composite key uniqueness, accepted-values, and row-count bounds — over rows from `load_rows`/`query_sqlite`. Pure-stdlib, deterministic. - -### URI-Scheme Value References - -Store pointers, not secrets, in config. Full reference: [`docs/source/Eng/doc/new_features/v85_features_doc.rst`](docs/source/Eng/doc/new_features/v85_features_doc.rst). - -- **`resolve_ref` / `resolve_refs_in` / `is_ref` / `RefResolver`** (`AC_resolve_ref`, `AC_resolve_refs`): `interpolate` hardcoded only `${secrets.NAME}` and `AssetStore` refs were vault-name-only — there was no general read-time indirection. This resolves `env://VAR`, `file://path` (with an optional `base_dir` traversal guard), and `secret://name` (injectable resolver or the governance broker), and walks nested structures resolving every reference. Env reader / secret resolver / base dir are injectable. Pure-stdlib, deterministic. - -## What's new (2026-06-21) - -### W3C Baggage Propagation - -Carry cross-cutting key-value context across HTTP. Full reference: [`docs/source/Eng/doc/new_features/v84_features_doc.rst`](docs/source/Eng/doc/new_features/v84_features_doc.rst). - -- **`Baggage` / `parse_baggage` / `format_baggage` / `inject_baggage` / `extract_baggage`** (`AC_baggage_parse`, `AC_baggage_format`): `trace_context` carried trace/span identity but nothing propagated cross-cutting context (`run_id`/`tenant`/`experiment`). This implements the W3C Baggage header — a percent-encoded `key=value` list — with an immutable `Baggage` (set/remove return new instances) and case-insensitive inject/extract over a headers dict. Pairs with `trace_context`. Pure-stdlib, deterministic. - -### Dataset Diff (Row-Set Change Report) - -Diff two tabular extracts by key. Full reference: [`docs/source/Eng/doc/new_features/v83_features_doc.rst`](docs/source/Eng/doc/new_features/v83_features_doc.rst). - -- **`diff_rows` / `cell_changes` / `summarize_diff`** (`AC_diff_rows`, `AC_cell_changes`): the framework diffed screens/snapshots but had nothing to diff two **tabular** row-sets by key. This keys both sides and reports `{added, removed, changed, unchanged}` (changed carries `{key, old, new}`), expands per-cell `{key, column, old, new}` changes, and counts each bucket. Supports composite keys; last-write-wins on duplicates. Pure-stdlib, deterministic. - -### Distribution Drift Detection - -Check whether today's data is shaped like the baseline. Full reference: [`docs/source/Eng/doc/new_features/v82_features_doc.rst`](docs/source/Eng/doc/new_features/v82_features_doc.rst). - -- **`psi` / `ks_two_sample` / `categorical_drift` / `detect_drift`** (`AC_detect_drift`, `AC_categorical_drift`): `stats` had A/B experiment tests but no Population Stability Index and no KS two-sample test for reference-vs-current distributions. This adds PSI (quantile-binned log-ratio), the KS statistic with a Kolmogorov p-value, and a categorical chi-square + total-variation summary — pairing with `data_profile`. `detect_drift` gives a one-call `{psi, drifted, ks}` verdict. Pure-stdlib, deterministic. - -### Layered Configuration Resolver - -Compose config with `defaults < file < env < CLI` precedence. Full reference: [`docs/source/Eng/doc/new_features/v81_features_doc.rst`](docs/source/Eng/doc/new_features/v81_features_doc.rst). - -- **`LayeredConfig` / `deep_merge` / `SourceTrace`** (`AC_resolve_config`, `AC_explain_config`): `json_patch.merge_patch` merges two docs, `config_sync` is last-write-wins, `AssetStore` is flat-per-env — none compose an ordered precedence stack with deep merge or report which layer won each key. `add_layer(name, mapping, priority)` then `resolve()` deep-merges (nested dicts recursively, scalars/lists replaced); `explain("db.host")` names the winning layer. Layers are caller-supplied (env passed in, never `os.environ` implicitly). Pure-stdlib, deterministic. - -### Server-Sent Events (SSE) Client Parser - -Consume `text/event-stream` responses. Full reference: [`docs/source/Eng/doc/new_features/v80_features_doc.rst`](docs/source/Eng/doc/new_features/v80_features_doc.rst). - -- **`parse_event_stream` / `SSEParser` / `SSEEvent`** (`AC_parse_sse`): the MCP HTTP transport emits SSE, but nothing consumed it — a streaming LLM/agent/chatops endpoint left `http_request` with a raw blob. This implements the WHATWG event-stream parsing algorithm (`event`/`data`/`id`/`retry`, comments, the leading-space rule, blank-line dispatch) with an incremental `feed` for chunks and a one-shot `parse_event_stream`. Pure-stdlib, fully deterministic. - -### Dotenv (.env) Parsing - -Read 12-factor `.env` files into config. Full reference: [`docs/source/Eng/doc/new_features/v79_features_doc.rst`](docs/source/Eng/doc/new_features/v79_features_doc.rst). - -- **`parse_dotenv` / `load_dotenv` / `dotenv_values` / `dump_dotenv`** (`AC_parse_dotenv`, `AC_load_dotenv`): `load_vars_from_json` ingested flat JSON but nothing read the de-facto `.env` file. This parses `KEY=VALUE` lines (`export` prefixes, single/double quoting, `\n`/`\t` escapes, inline comments) into a plain dict — no `python-dotenv` dependency. The loader merges into a caller-supplied mapping rather than mutating `os.environ`, so it stays safe and deterministic. Pure-stdlib. - -### RFC 9457 Problem Details Parsing - -Read standardized API errors out of HTTP responses. Full reference: [`docs/source/Eng/doc/new_features/v78_features_doc.rst`](docs/source/Eng/doc/new_features/v78_features_doc.rst). - -- **`parse_problem` / `is_problem` / `raise_for_problem` / `ProblemDetails`** (`AC_parse_problem`): `http_request` returned a non-2xx body unparsed, so flows and `assert_http` had no structured way to read a standardized API error. This parses the RFC 9457 `application/problem+json` document — registered `type`/`title`/`status`/`detail`/`instance` members plus vendor extensions — returning `None` for non-problem responses or raising `HttpProblemError`. Pure-stdlib, fully deterministic. - -### Data Profiling & Schema Inference - -Survey a row-set and propose a validation schema. Full reference: [`docs/source/Eng/doc/new_features/v77_features_doc.rst`](docs/source/Eng/doc/new_features/v77_features_doc.rst). - -- **`profile_rows` / `infer_schema`** (`AC_profile_rows`, `AC_infer_schema`): `validate_rows` consumes a hand-written schema and `stats.describe` summarizes one numeric list — nothing surveyed a whole row-set. This profiles each column (null fraction, cardinality, inferred type, top values, numeric min/max/mean) and infers a `validate_rows`-compatible schema (required where non-null, unique where distinct, numeric bounds) — the profiler step that feeds the existing validator. Pure-stdlib, fully deterministic. - -### W3C Trace Context Propagation - -Correlate spans and logs across HTTP boundaries. Full reference: [`docs/source/Eng/doc/new_features/v76_features_doc.rst`](docs/source/Eng/doc/new_features/v76_features_doc.rst). - -- **`SpanContext` / `new_root_context` / `child_context` / `inject_context` / `extract_context`** (`AC_trace_inject`, `AC_trace_extract`): the existing tracer and `agent_trace` spans carried no IDs, so a span on one side of an HTTP call couldn't be correlated with the work it triggered on the other. This implements the W3C Trace Context standard — generate/parse/propagate `traceparent` + `tracestate` headers (version-`00`, rejects malformed/all-zero IDs), with an injectable RNG for deterministic IDs in tests. Pure-stdlib. - -### HTTP Record & Replay Cassette - -Re-run API flows in CI with no live server. Full reference: [`docs/source/Eng/doc/new_features/v75_features_doc.rst`](docs/source/Eng/doc/new_features/v75_features_doc.rst). - -- **`Cassette` / `CassetteMissError`** (`AC_http_replay`): the HTTP client hardcoded its `urllib` transport, so a flow driving a real API couldn't be re-run offline. The client now exposes a `build_call` / `urllib_transport` seam, and this adds a VCR-style cassette — `replay` returns a recorded response for a matching request (pure, no network — the CI-valuable half), `recording_transport` is a thin pass-through over the live transport. Match on `method`/`url` (optionally `body`); `save`/`load` JSON cassettes. Pure-stdlib. - -### Bulkhead & Rate-Limit Headers - -Cap concurrency, honor server back-off. Full reference: [`docs/source/Eng/doc/new_features/v74_features_doc.rst`](docs/source/Eng/doc/new_features/v74_features_doc.rst). - -- **`Bulkhead` / `next_delay` / `parse_retry_after` / `parse_ratelimit`** (`AC_bulkhead_run`, `AC_retry_after`): `resilience` recovers and `rate_limit` paces, but nothing capped *simultaneous* in-flight calls (a slow dependency could exhaust every worker) and the HTTP client ignored `Retry-After`/`RateLimit-*`. This adds a bulkhead (bounded-concurrency permit that sheds load with `BulkheadFullError` when full) and parsers for the server's advised delay (delta-seconds or HTTP-date). Non-blocking permit counting → deterministic, no threads in tests. Pure-stdlib. - -### Streaming Latency Percentiles - -Mergeable p99 for load/soak runs. Full reference: [`docs/source/Eng/doc/new_features/v73_features_doc.rst`](docs/source/Eng/doc/new_features/v73_features_doc.rst). - -- **`LatencyDigest` / `exact_percentiles`** (`AC_percentiles`): `stats.percentile` needs the full sorted list; this adds a HdrHistogram-style digest with O(1) `record`, bounded memory (significant-figure buckets), and `merge` for cross-shard aggregation — the property you need for a correct aggregate p99 from per-worker results. `exact_percentiles` covers the small-set case (arbitrary quantiles). Pure-stdlib `math`. - -### Service-Level Objectives (SLO) - -SLI, error budget and burn-rate alerts. Full reference: [`docs/source/Eng/doc/new_features/v72_features_doc.rst`](docs/source/Eng/doc/new_features/v72_features_doc.rst). - -- **`evaluate_slo` / `burn_rate` / `burn_alerts` / `default_burn_rules`** (`AC_evaluate_slo`, `AC_burn_alerts`): the framework emitted raw signals but had no SLO layer. This computes the SLI over outcome records (`[{timestamp, ok}]`), the error budget against a target, and the **multi-window multi-burn-rate** alerts from the Google SRE workbook (page 14.4×@1h, 6×@6h; ticket 1×@3d — firing only when both windows exceed the threshold). Records are plain data, clock injectable, fully deterministic. Pure-stdlib. - -### Chaos Experiments - -Inject faults, verify the system holds. Full reference: [`docs/source/Eng/doc/new_features/v71_features_doc.rst`](docs/source/Eng/doc/new_features/v71_features_doc.rst). - -- **`ChaosExperiment` / `run_experiment` / `Probe` / `latency_fault` / `exception_fault`** (`AC_run_chaos`): `resilience` *recovers* from failures; this *causes* them and checks a steady-state hypothesis still holds (Chaos Toolkit lifecycle — verify before, inject faults, verify after, roll back LIFO). Probes/faults/rollbacks are callables; the clock/RNG/sleep are injectable so experiments run **deterministically** in tests with no real failures or sleeping. `AC_run_chaos` drives an action-list spec. Pure-stdlib. - -### JSON Contract & Snapshot Matching - -Match, diff and snapshot JSON payloads. Full reference: [`docs/source/Eng/doc/new_features/v70_features_doc.rst`](docs/source/Eng/doc/new_features/v70_features_doc.rst). - -- **`match_json` / `diff_json` / `normalize_json` / `snapshot_json`** (`AC_match_json`, `AC_diff_json`): `json_schema` validates against an authored schema and `jsonpath` extracts, but nothing matched two payloads with relaxed rules or diffed them path-by-path. This adds contract/snapshot matching — `partial` (subset), `match_type` (Pact-style `like`), `ignore` volatile paths — returning `{path, kind}` mismatches (`missing`/`extra`/`changed`), plus golden-master `snapshot_json`. Composes with `json_schema` + `json_patch`; pure-stdlib. - -### SLSA Build Provenance - -Attest what was built. Full reference: [`docs/source/Eng/doc/new_features/v69_features_doc.rst`](docs/source/Eng/doc/new_features/v69_features_doc.rst). - -- **`build_provenance` / `subject_for` / `verify_provenance` / `write_provenance`** (`AC_build_provenance`, `AC_verify_provenance`): the framework signs action files and inventories deps (SBOM) but couldn't attest *what was produced by which build*. This adds an in-toto v1 Statement with a SLSA v1 provenance predicate over file `sha256` digests, and a verifier that re-hashes the artifacts (tamper → mismatch). Complements `action_signing` + `sbom`; pure-stdlib `hashlib`+`json`, fully offline. - -### Feature Flags - -Toggle behavior with targeting & rollout. Full reference: [`docs/source/Eng/doc/new_features/v68_features_doc.rst`](docs/source/Eng/doc/new_features/v68_features_doc.rst). - -- **`FlagStore` / `evaluate_flag` / `is_enabled` / `assign_variant`** (`AC_evaluate_flag`, `AC_flag_enabled`): `decision_table` is one-shot DMN and `ab_locator` is locator A/B — neither is a product flag store with sticky % rollout. This adds an OpenFeature-shaped engine: targeting rules (`eq`/`in`/`semver_*`…), weighted variants, kill switch, and consistent-hash bucketing (`sha256(key.salt.context_key)`) so a subject is **sticky**. Returns `{value, variant, reason}` (`TARGETING_MATCH`/`SPLIT`/`DISABLED`/`ERROR`). Pure-stdlib, deterministic. - -### Text Diff, Patch & Three-Way Merge - -Apply and merge text diffs. Full reference: [`docs/source/Eng/doc/new_features/v67_features_doc.rst`](docs/source/Eng/doc/new_features/v67_features_doc.rst). - -- **`unified_diff` / `apply_unified` / `three_way_merge`** (`AC_unified_diff`, `AC_apply_unified`, `AC_three_way_merge`): `difflib` *generates* a unified diff but the stdlib can't *apply* one, and there was no three-way merge. This adds the missing applier (walks `@@` hunks, verifies context, raises on mismatch) and a line-based three-way merge (non-overlapping edits combine cleanly; overlapping ones emit `<<<<<<<` conflict markers). Complements `json_patch` (structured JSON); pure-stdlib `difflib`. - -### Calendar Recurrence Rules (RRULE) - -Schedule "every 2nd Tuesday". Full reference: [`docs/source/Eng/doc/new_features/v66_features_doc.rst`](docs/source/Eng/doc/new_features/v66_features_doc.rst). - -- **`parse_rrule` / `occurrences` / `next_occurrence`** (`AC_rrule_occurrences`, `AC_rrule_next`): the scheduler's cron is 5-field interval-only — it can't express "every 2nd Tuesday", "the last weekday of the month", or "every weekday for 10 occurrences". This adds an RFC 5545 (iCalendar) RRULE parser + occurrence expander supporting `FREQ`/`INTERVAL`/`COUNT`/`UNTIL`/`BYDAY` (with ordinals like `2MO`/`-1FR`)/`BYMONTHDAY`/`BYMONTH`/`BYSETPOS`/`WKST`. Pure-stdlib `datetime`+`calendar`, injectable clock for deterministic `next_occurrence`. - -### Statistics & A/B Significance - -Decide whether a difference is real. Full reference: [`docs/source/Eng/doc/new_features/v65_features_doc.rst`](docs/source/Eng/doc/new_features/v65_features_doc.rst). - -- **`describe` / `percentile` / `two_proportion_z_test` / `welch_t_test` / `cohens_d` / `chi_square_2x2`** (`AC_describe_stats`, `AC_ab_significance`): `ab_locator` ranks by raw success rate and `run_history` stores durations, but nothing computed percentiles or significance. This adds the analysis layer — summary stats + p50/p90/p95/p99, a two-proportion z-test (with CI), Welch's t-test (exact t-distribution p-value via the incomplete beta — no SciPy), Cohen's d, and a 2×2 chi-square. The normal CDF is exact via `math.erf`; validated against textbook values (incl. the chi²=z² identity). Pure-stdlib `math`+`statistics`. - -### Full-Text Search (BM25) - -Rank a document corpus by relevance. Full reference: [`docs/source/Eng/doc/new_features/v64_features_doc.rst`](docs/source/Eng/doc/new_features/v64_features_doc.rst). - -- **`SearchIndex` / `search_documents` / `tokenize`** (`AC_search_documents`, `ac_search_documents`): `fuzzy` is pairwise and `skill_library` matches substrings alphabetically — neither ranks a corpus by relevance. This adds an inverted-index search ranked with Okapi BM25 (`k1=1.5`, `b=0.75`, `IDF = ln(1+(N−df+0.5)/(df+0.5))`) or TF-IDF, so a rare term out-ranks a common one, term frequency saturates, and long docs are normalized down. Incremental `add`/`remove`, optional stop-words, deterministic ranking. Pure-stdlib `math`+`collections`+`re` — no database. - -### JSON Pointer, Patch & Merge Patch - -Address, diff and patch JSON. Full reference: [`docs/source/Eng/doc/new_features/v63_features_doc.rst`](docs/source/Eng/doc/new_features/v63_features_doc.rst). - -- **`resolve_pointer` / `make_patch` / `apply_patch` / `merge_patch` / `make_merge_patch`** (`AC_resolve_pointer`, `AC_apply_json_patch`, `AC_make_json_patch`, `AC_merge_patch`): `jsonpath` is read-only and `approval` compares whole artifacts — nothing could address one location, compute a structured delta, or apply a partial update. This adds the three IETF primitives — JSON Pointer (RFC 6901), JSON Patch (RFC 6902, all six ops, **atomic** apply), and JSON Merge Patch (RFC 7386, `null` deletes) — for config-drift detection, partial updates, HTTP PATCH bodies, and golden-master deltas. Pure-stdlib `json`+`copy`, validated against the RFC test vectors. - -### Client-Side Rate Limiting - -Stay under API quotas. Full reference: [`docs/source/Eng/doc/new_features/v62_features_doc.rst`](docs/source/Eng/doc/new_features/v62_features_doc.rst). - -- **`TokenBucket` / `SlidingWindowLimiter` / `throttle`** (`AC_rate_limit`, `ac_rate_limit`): `RetryPolicy`/`CircuitBreaker` recover from failures but nothing shaped the *rate* of calls. This adds a token bucket (smooth rate + burst), a sliding-window limiter (Cloudflare's O(1) weighted counter), and a leading-edge throttle decorator. Every limiter takes an injectable `clock` (and `acquire` a `sleep`) so it's fully deterministic in CI with no real delays. `AC_rate_limit` gates an action against a named bucket, returning `{acquired, tokens, wait}`. - -### JSON Web Tokens (JWT) - -Mint and verify bearer tokens for the APIs you automate. Full reference: [`docs/source/Eng/doc/new_features/v61_features_doc.rst`](docs/source/Eng/doc/new_features/v61_features_doc.rst). - -- **`encode_jwt` / `decode_jwt` / `ClaimsPolicy`** (`AC_jwt_encode`, `AC_jwt_decode`): the framework had HMAC *file* signing and an ACME-bound RS256 JWS, but nothing to mint/verify a compact bearer JWT. This adds a pure-stdlib HS256/384/512 codec with full claim validation (`exp`/`nbf`/`aud`/`iss`, injectable clock) that drops straight into `http_request`'s bearer auth. Safe by default: rejects `alg:none`, enforces an algorithm allowlist (anti-confusion), and compares signatures with `hmac.compare_digest`. `AC_jwt_decode` returns `{ok, claims}` so flows can branch without raising. - -### License Policy Gate - -Flag disallowed dependency licenses. Full reference: [`docs/source/Eng/doc/new_features/v60_features_doc.rst`](docs/source/Eng/doc/new_features/v60_features_doc.rst). - -- **`evaluate_sbom` / `evaluate_license` / `normalize_spdx` / `license_findings_to_sarif`** (`AC_check_licenses`, `ac_check_licenses`): the SBOM recorded each dependency's license name but never *judged* it. This normalizes license strings to SPDX ids and evaluates them against an allowlist/denylist (with a built-in `DEFAULT_COPYLEFT` set), understanding SPDX expressions (`OR` = choice, `AND` = all), then bridges violations into SARIF (`denied`→error, `unknown`→warning). Pure-stdlib, fully offline — the license-compliance lane beside the OSV vulnerability lane. - -### OpenVEX Vulnerability Triage - -Suppress the vulns that don't affect you. Full reference: [`docs/source/Eng/doc/new_features/v59_features_doc.rst`](docs/source/Eng/doc/new_features/v59_features_doc.rst). - -- **`vex_statement` / `build_vex` / `apply_vex`** (`AC_apply_vex`, `ac_apply_vex`): the OSV scanner surfaces every known CVE forever — there was no way to record "we checked, this one doesn't affect us". This authors [OpenVEX](https://openvex.dev) 0.2.0 statements and applies them to the scanner's findings: `not_affected`/`fixed` **suppress** a finding, `affected`/`under_investigation` **annotate** it. Statements join on the vuln id *or* an alias, optionally product-scoped; `not_affected` requires a justification or impact statement. Pure-stdlib; chains directly after `AC_scan_vulns`. - -### Dependency Vulnerability Scanning (OSV) - -Match the SBOM against known CVEs. Full reference: [`docs/source/Eng/doc/new_features/v58_features_doc.rst`](docs/source/Eng/doc/new_features/v58_features_doc.rst). - -- **`scan_components` / `match_package` / `is_affected` / `findings_to_sarif`** (`AC_scan_vulns`, `ac_scan_vulns`): `build_sbom` only *inventoried* dependencies and `to_sarif` only *exported* findings — nothing ever **produced** a vulnerability finding. This matches the SBOM's `(ecosystem, name, version)` components against an [OSV](https://osv.dev) advisory database (sweeping `introduced`/`fixed`/`last_affected` ranges, PEP-503 name normalization, severity→SARIF level) and bridges results into the existing SARIF exporter for GitHub/Azure DevOps code scanning. The advisory DB is **injected as data** (offline, deterministic); the live `osv.dev` query is an optional `fetcher` seam. Pure-stdlib `re`. - -### JSON Schema Validation - -Validate nested JSON against a real schema. Full reference: [`docs/source/Eng/doc/new_features/v57_features_doc.rst`](docs/source/Eng/doc/new_features/v57_features_doc.rst). - -- **`validate_json` / `is_valid` / `assert_schema`** (`AC_validate_json`, `ac_validate_json`): the framework only *generated* JSON Schema and `data_quality` is a flat per-column checker — neither could validate a nested API request/response body. This adds the consumer: a JSON Schema (Draft 2020-12 subset) validator that reports **every** violation as `{path, keyword, message}` (e.g. `$.age maximum`). Covers `type` (incl. integral-float `integer`), `enum`/`const`, numeric/string bounds, array & object keywords, `allOf`/`anyOf`/`oneOf`/`not`, boolean schemas and local `$ref`. Pure-stdlib `re`; pairs with `json_query` and the `http_request` helper. - -## What's new (2026-06-20) - -### SARIF 2.1.0 Findings Export - -Unify scanner findings for GitHub code scanning. Full reference: [`docs/source/Eng/doc/new_features/v56_features_doc.rst`](docs/source/Eng/doc/new_features/v56_features_doc.rst). - -- **`to_sarif` / `write_sarif` / `make_finding` / `from_lint_issues` / `from_audit_findings`** (`AC_export_sarif`, `ac_export_sarif`): the framework's findings producers (action-lint, secrets scan, WCAG audit, guardrail) had no common export. This builds a SARIF 2.1.0 document — with auto rule catalog and stable `partialFingerprints` for cross-run dedupe — that GitHub/Azure DevOps code scanning ingests as line-anchored alerts. Pure-stdlib `json`+`hashlib`; adapters normalize the existing lint/audit shapes. - -### Text PII Detection & Redaction - -Mask PII in text before it leaks. Full reference: [`docs/source/Eng/doc/new_features/v55_features_doc.rst`](docs/source/Eng/doc/new_features/v55_features_doc.rst). - -- **`detect_pii` / `redact_pii_text`** (`AC_detect_pii` / `AC_redact_pii`, `ac_*`): image redaction existed but text (OCR, clipboard, LLM I/O, logs) had no string-level PII handling. This detects emails / phones / SSNs / credit cards / IPv4 / IBANs over plain text and redacts with `label` / `mask` / `partial` / `hash`. Overlapping spans dedupe (a card isn't also a phone); patterns are backtracking-safe. Pure-stdlib `re`+`hashlib`. - -### Self-Healing Locator Write-Back - -Persist corrected locators so heals aren't forgotten. Full reference: [`docs/source/Eng/doc/new_features/v54_features_doc.rst`](docs/source/Eng/doc/new_features/v54_features_doc.rst). - -- **`RepairStore` / `repair_from_heal`** (`AC_repair_record` / `AC_repair_resolved` / `AC_repair_pending` / `AC_repair_approve`, `ac_*`): runtime self-healing previously **threw away** the corrected location, so every run re-healed. This records the corrected locator (coords/VLM description/method) from a heal, **auto-applies** it when `confidence >= auto_threshold` (default 0.9) or queues a reviewable suggestion, and `resolved(key)` returns the learned fix for reuse. Closes the heal→durable-fix loop; pure-stdlib, fully testable. - -### DMN-Style Decision Tables - -Externalize branching into reviewable rule tables. Full reference: [`docs/source/Eng/doc/new_features/v53_features_doc.rst`](docs/source/Eng/doc/new_features/v53_features_doc.rst). - -- **`evaluate_table` / `DecisionTable`** (`AC_decision_table`, `ac_decision_table`): replaces nested `AC_if_var` chains with rows of `conditions -> outputs` and a hit policy (`UNIQUE`/`FIRST`/`PRIORITY`/`COLLECT`). Cell conditions are wildcard / literal / `{op, value}` using the executor's standard comparators (reused, not duplicated). Pure-stdlib, fully testable; the DMN way to keep business rules data-driven. - -### Saga / Compensating Rollback - -Undo completed steps when a later one fails. Full reference: [`docs/source/Eng/doc/new_features/v52_features_doc.rst`](docs/source/Eng/doc/new_features/v52_features_doc.rst). - -- **`Saga` / `run_saga`** (`AC_run_saga`, `ac_run_saga`): records a compensating action per step; on any failure runs the completed steps' compensations in **LIFO** order — the durable-transaction primitive `AC_try` (single-block) couldn't provide. Forward actions/compensations are callables (or JSON action lists), so it's fully unit-tested with no side effects; compensation is best-effort (a failing undo is logged, rollback continues). Returns `{ok, completed, compensated, failed_step, error}`. - -### JSONPath Querying - -Query API/DB JSON with wildcards, recursion, filters. Full reference: [`docs/source/Eng/doc/new_features/v51_features_doc.rst`](docs/source/Eng/doc/new_features/v51_features_doc.rst). - -- **`json_query` / `json_query_one` / `json_extract`** (`AC_json_query` / `AC_json_extract`, `ac_*`): the executor's path walker only split on `.` and indexed — this adds a JSONPath subset (`$`, `.key`, `[n]`/`[-n]`, `*`/`[*]`, `..` recursive descent, `[?(@.k op v)]` filters) over parsed JSON, so array-bearing API/DB responses are easy to extract from. `json_extract` runs a `{key: path}` mapping into a flat dict. Pure-stdlib `re`; the path engine `AC_http_to_var` and DB-row flows were missing. - -### Multi-Channel Webhook Notifications - -Alert Teams/Discord/Slack/webhook. Full reference: [`docs/source/Eng/doc/new_features/v50_features_doc.rst`](docs/source/Eng/doc/new_features/v50_features_doc.rst). - -- **`notify_webhook` / `WebhookChannel`** (`AC_notify_webhook`, `ac_notify_webhook`): `notify` was desktop-toast only and ChatOps shipped Slack only — this sends to **Slack / Discord / Microsoft Teams / raw** webhooks, building the transport-shaped payload (Slack & Teams MessageCard use `text`, Discord uses `content`) and POSTing via the egress-guarded HTTP client. The `poster` transport is injectable (or `set_default_poster`), so sending is unit-tested with no network. - -### Outbound CloudEvents Emitter - -Emit run/automation events as CloudEvents. Full reference: [`docs/source/Eng/doc/new_features/v49_features_doc.rst`](docs/source/Eng/doc/new_features/v49_features_doc.rst). - -- **`to_cloudevent` / `EventEmitter` / `post_cloudevent`** (`AC_emit_event`, `ac_emit_event`): the repo could receive webhooks but not **emit** events — this wraps run-lifecycle/assertion/failure data in a CloudEvents 1.0 (CNCF) envelope and optionally POSTs it over the egress-guarded HTTP client (interop with Knative, Azure Event Grid, iPaaS, generic webhooks). The `sink`/`poster` transport is injectable, so emission is unit-tested with no network. - -### Environment-Scoped Typed Asset Store - -Per-environment typed config + credential refs. Full reference: [`docs/source/Eng/doc/new_features/v48_features_doc.rst`](docs/source/Eng/doc/new_features/v48_features_doc.rst). - -- **`AssetStore` / `active_environment`** (`AC_set_asset` / `AC_get_asset` / `AC_list_assets`, `ac_*`): the orchestrator "Assets/lockers" pillar — centrally-managed config values that differ by environment (dev/staging/prod) and carry a type (`text`/`int`/`bool`/`credential`). `get` coerces to the declared type and falls back to the default env; `credential` assets hold a secret *reference* that `resolve` turns into the real value via an injected resolver (Python-only, so secrets never enter `get`/executor records). Fills the gap the secret vault (secret-only) and config-sync (whole-blob) left. - -### Task / Process Mining (Automation-Candidate Discovery) - -Discover what to automate from recorded action logs. Full reference: [`docs/source/Eng/doc/new_features/v47_features_doc.rst`](docs/source/Eng/doc/new_features/v47_features_doc.rst). - -- **`mine_action_log` / `find_repeated_sequences` / `directly_follows` / `rank_automation_candidates`** (`AC_mine_actions`, `ac_mine_actions`): mines a recorded action log for frequent, repeatable command n-grams, builds a directly-follows graph, and ranks automation candidates by `count × length` — the RPA "task mining" pillar AutoControl recorded data for but never analysed. Pure-stdlib; operates on the existing action-list shape; a candidate that recurs and spans several steps is a strong "extract into a skill" signal. - -### Stuck-Loop Guard (Agent Loop Progress Detection) - -Catch agents stuck in no-progress loops. Full reference: [`docs/source/Eng/doc/new_features/v46_features_doc.rst`](docs/source/Eng/doc/new_features/v46_features_doc.rst). - -- **`LoopGuard` / `digest_result`** (`AC_loop_guard_observe` / `AC_loop_guard_reset`, `ac_*`): the top computer-use failure mode is an agent repeating an action with no effect — and the model can't see its own loop. `LoopGuard` watches the `(tool, args, result)` stream and flags `repeat` (same call N times), `ping_pong` (A-B-A-B), and `no_op` (observation digest unchanged), escalating `ok`→`warn`→`critical` by run length. Complements the step/time budget and offline trajectory eval; pure-stdlib, deterministic. - -### Coordinate-Space Mapping (Model Grid ⇄ Physical Pixels) - -Translate computer-use model clicks to real pixels. Full reference: [`docs/source/Eng/doc/new_features/v45_features_doc.rst`](docs/source/Eng/doc/new_features/v45_features_doc.rst). - -- **`CoordinateSpace` / `xga_space` / `normalized_space` / `downscale_png`** (`AC_to_physical` / `AC_to_model`, `ac_*`): computer-use/VLA models click in a fixed grid (Anthropic downscales to XGA; Gemini returns a 1000×1000 grid), not physical pixels. This maps both ways (round + clamp), `xga_space` aspect-preserves without upscaling, and `downscale_png` resizes a screenshot to the model's input size (Pillow, already core). Pure-arithmetic mapping — unit-tested without a model/GPU. - -### Voice-Command Router - -Trigger flows hands-free from recognized speech. Full reference: [`docs/source/Eng/doc/new_features/v44_features_doc.rst`](docs/source/Eng/doc/new_features/v44_features_doc.rst). - -- **`VoiceRouter`** (`AC_voice_register` / `AC_voice_dispatch` / `AC_voice_list` / `AC_voice_clear`, `ac_*`): map spoken trigger phrases to `AC_*` action lists; feed it recognized text and it runs the closest registered command (phrase matching reuses the fuzzy matcher, so "save the file" fires "save file"). **Speech-to-text is out of scope and injectable** — the router takes text and a `recognizer`/`runner` callable, so routing is fully unit-tested without audio or any speech dependency (a real Vosk/mic recogniser plugs into `listen_once`). - -### Locale-Aware Number, Currency & Date Parsing - -Parse localized numbers/currency/dates. Full reference: [`docs/source/Eng/doc/new_features/v43_features_doc.rst`](docs/source/Eng/doc/new_features/v43_features_doc.rst). - -- **`parse_decimal` / `parse_number` / `format_decimal` / `format_currency` / `format_date`** (`AC_parse_decimal` / `AC_parse_number` / `AC_format_decimal` / `AC_format_currency` / `AC_format_date`, `ac_*`): OCR/UI text like `"1.234,56"` (de_DE) parses correctly to `1234.56` via **Babel**'s CLDR data, and values format back per-locale. `babel` is an optional `[locale]` extra, imported lazily; functional tests run under `importorskip` (wiring/facade always verified). - -### Perceptual-Hash Image Dedupe - -Collapse near-identical screenshots. Full reference: [`docs/source/Eng/doc/new_features/v42_features_doc.rst`](docs/source/Eng/doc/new_features/v42_features_doc.rst). - -- **`average_hash` / `dhash` / `hamming_distance` / `images_similar` / `dedupe_images`** (`AC_image_hash` / `AC_dedupe_images`, `ac_*`): perceptual hashing maps visually similar images to close fingerprints, so near-duplicate frames in a recording or step report cluster by Hamming distance and collapse to one representative. Uses **Pillow** (already core — no extra dep); the dedupe/compare logic is pure Python with an injectable `hasher`, so clustering is unit-tested without any image and the real Pillow path under `importorskip`. - -### S3-Compatible Artifact Store - -Push run artifacts to object storage. Full reference: [`docs/source/Eng/doc/new_features/v41_features_doc.rst`](docs/source/Eng/doc/new_features/v41_features_doc.rst). - -- **`S3ArtifactStore`** (`AC_s3_upload` / `AC_s3_download` / `AC_s3_list` / `AC_s3_delete`, `ac_*`): upload/download/list/delete reports, screenshots, and recordings against any S3-compatible bucket (AWS S3, MinIO, R2). `boto3` is an **optional** `[s3]` extra and the client is **injectable**, so the store's logic — and the executor path — are fully unit-tested with a fake client (no boto3/network); the live AWS path is honestly noted as CI-unverifiable. The whole API is relative to the store `prefix`. A module-level default store backs the commands. - -### Fuzzy String Matching & Dedupe - -Match noisy OCR/UI text robustly. Full reference: [`docs/source/Eng/doc/new_features/v40_features_doc.rst`](docs/source/Eng/doc/new_features/v40_features_doc.rst). - -- **`fuzzy_ratio` / `fuzzy_best_match` / `fuzzy_matches` / `fuzzy_dedupe`** (`AC_fuzzy_ratio` / `AC_fuzzy_best_match` / `AC_fuzzy_dedupe`, `ac_*`): score similarity (0..1), pick the closest candidate from a list, or collapse near-duplicates — so a flow can act on "the button that *looks like* Submit" rather than an exact label. The default backend is stdlib `difflib` (**zero extra deps**); the optional `[fuzzy]` extra adds `rapidfuzz` for speed, with scores normalised either way. `ignore_case` and `score_cutoff` supported. - -## What's new (2026-06-19) - -### Video Step-Overlay Report - -Caption screenshots into a walkthrough video. Full reference: [`docs/source/Eng/doc/new_features/v39_features_doc.rst`](docs/source/Eng/doc/new_features/v39_features_doc.rst). - -- **`write_step_video`** (`AC_write_step_video`, `ac_write_step_video`): turns per-step screenshots into a shareable video where each frame is held for a few seconds with its caption and a pass/fail colour banner burned in. The assembly logic (`build_overlay_plan` / `render_overlay_frame`) is separated from OpenCV via injectable `loader`/`drawer`/`writer_factory` hooks — unit-testable with fakes and no `cv2`/`numpy` dependency; the real path lazily imports `cv2` only when those hooks are absent. The visual companion to the HTML/JSON reports. - -### Agent Observability (GenAI OpenTelemetry Spans) - -OTel GenAI-convention spans for LLM runs. Full reference: [`docs/source/Eng/doc/new_features/v38_features_doc.rst`](docs/source/Eng/doc/new_features/v38_features_doc.rst). - -- **`AgentTrace`** (`AC_trace_record` / `AC_trace_summary` / `AC_trace_export` / `AC_trace_reset`, `ac_*`): records spans whose attributes follow the OpenTelemetry **GenAI semantic conventions** (`gen_ai.operation.name`, `gen_ai.system`, `gen_ai.request.model`, `gen_ai.usage.input_tokens`/`output_tokens`, `gen_ai.tool.name`) and the `"{operation} {model}"` span name. `to_otel()` drops into an OTLP exporter; `summary()` rolls up token cost and latency; an `operation()` context manager times live blocks and marks errors. Pure-stdlib (no `opentelemetry` dep), injectable clock; pairs with trajectory evaluation (record here, score there). - -### Compliance Control Report (SOC2 / ISO 27001) - -Map governance evidence to named controls. Full reference: [`docs/source/Eng/doc/new_features/v37_features_doc.rst`](docs/source/Eng/doc/new_features/v37_features_doc.rst). - -- **`build_compliance_report`** (`AC_compliance_report`, `ac_compliance_report`): the framework already ships the controls an auditor cares about — egress allowlist, JIT credential leases, maker-checker approval, secrets scanner, audit logging, CycloneDX SBOM. This maps a flat `evidence` mapping to SOC2 (CC6.1/CC6.3/CC6.8/CC7.3/CC8.1) and ISO 27001 (A.5.23/A.8.16/A.8.30) controls, each marked `satisfied`/`gap`/`not_assessed`, and renders JSON or a standalone HTML table. The capstone of the governance set — a reporting aid, not a certification. - -### Agent Trajectory Evaluation - -Score an agent run against a rubric. Full reference: [`docs/source/Eng/doc/new_features/v36_features_doc.rst`](docs/source/Eng/doc/new_features/v36_features_doc.rst). - -- **`evaluate_trajectory`** (`AC_evaluate_trajectory`, `ac_evaluate_trajectory`): scores a recorded trajectory (ordered `{action, args, observation}` steps) against a declarative rubric — `required_actions` (+`ordered`), `forbidden_actions`, `max_steps`, `success_contains`. Returns `{passed, score, steps, checks}` where `score` is the fraction of applicable checks passed and each `check` pinpoints a violated expectation. A deterministic, dependency-free signal for agent regression testing; the rubric is plain data so it lives in JSON action files and travels over MCP. - -### Approval Testing (Golden-Master Baselines) - -Lock outputs against a human-approved baseline. Full reference: [`docs/source/Eng/doc/new_features/v35_features_doc.rst`](docs/source/Eng/doc/new_features/v35_features_doc.rst). - -- **`verify_artifact` / `approve_artifact`** (`AC_verify_artifact` / `AC_approve_artifact` / `AC_pending_artifacts`, `ac_*`): golden-master / snapshot testing for *any* artifact (text, JSON, OCR output, screenshot bytes). `verify_artifact` compares produced content to `.approved.`; a mismatch or missing baseline writes `.received.` for review and fails, and `approve_artifact` promotes a reviewed received file to the baseline. Complements pixel diffing with a review-gated baseline you commit alongside the test; names are path-traversal-checked. - -### Network Egress Allowlist Guard - -Pin which hosts automation may reach. Full reference: [`docs/source/Eng/doc/new_features/v34_features_doc.rst`](docs/source/Eng/doc/new_features/v34_features_doc.rst). - -- **`EgressPolicy` / `set_egress_policy`** (`AC_egress_allow` / `AC_egress_check` / `AC_egress_reset`, `ac_*`): an allow list (default-deny) and/or deny list of `fnmatch` host globs (`*.example.com`) consulted by **every** `http_request` (so `AC_http` and all features built on it are covered at once). Blocked hosts raise `EgressBlocked` *before* a socket opens. Starts in allow-all mode — no behavior change until an operator locks egress down. Closes the exfiltration surface for unattended automation. - -### Just-In-Time Credential Leases - -Zero standing privilege for secrets. Full reference: [`docs/source/Eng/doc/new_features/v33_features_doc.rst`](docs/source/Eng/doc/new_features/v33_features_doc.rst). - -- **`CredentialBroker`** (`AC_lease_secret` / `AC_lease_valid` / `AC_revoke_lease` / `AC_lease_active`, `ac_*`): a consumer takes a short-lived *lease* (token bound to a secret name + expiry); the real value is fetched only at `redeem` time, only while valid, through a pluggable resolver (an unlocked `SecretManager`, env, vault). Secret values never enter executor/MCP records — the executor/MCP/Builder surfaces manage the lease lifecycle only; `redeem` is a deliberate Python-API-only escape hatch. Clock and resolver injectable. - -### Maker-Checker Approval Gate - -Segregation of duties for high-risk steps. Full reference: [`docs/source/Eng/doc/new_features/v32_features_doc.rst`](docs/source/Eng/doc/new_features/v32_features_doc.rst). - -- **`ApprovalGate`** (`AC_approval_request` / `AC_approval_approve` / `AC_approval_reject` / `AC_approval_status`, `ac_*`): a *maker* files a high-risk action and gets a token; a *checker* — required to be a **different** principal — approves or rejects it; the action proceeds only once `is_approved` is true. State is an optional shared JSON file so the dispatcher and the human approver can run as separate processes. Pure-stdlib, SOC2-style four-eyes control. - -### Plugin SDK - -Third-party `AC_*` commands via entry points. Full reference: [`docs/source/Eng/doc/new_features/v31_features_doc.rst`](docs/source/Eng/doc/new_features/v31_features_doc.rst). - -- **`discover_plugins` / `load_plugins`** (`AC_list_plugins` / `AC_load_plugins`, `ac_*`): a pip package registers new executor commands declaratively in the `je_auto_control.commands` entry-point group; AutoControl discovers and registers them at runtime (immediately usable from JSON flows, socket server, scheduler, MCP). Broken plugins are skipped; the declarative, namespaced complement to the runtime path loader. - -### MCP Structured Output - -MCP 2025-06-18 structured tool output. Full reference: [`docs/source/Eng/doc/new_features/v30_features_doc.rst`](docs/source/Eng/doc/new_features/v30_features_doc.rst). - -- **`MCPTool(output_schema=...)`** — a tool may declare an `outputSchema`; its dict result is returned as `structuredContent` in the `tools/call` response so clients/LLMs consume a typed, schema-validated object instead of re-parsing text. `to_descriptor()` advertises it in `tools/list`; non-dict results and schema-less tools are unchanged. `ac_validate_rows` is the first built-in to adopt it. - -### Tweened Drag - -Deterministic eased drags. Full reference: [`docs/source/Eng/doc/new_features/v29_features_doc.rst`](docs/source/Eng/doc/new_features/v29_features_doc.rst). - -- **`tween_points` / `tween_drag` / `easing_names`** (`AC_tween_drag`, `ac_tween_drag`): drag from `start` to `end` along an eased curve (linear / ease_in_out_quad / ease_out_cubic / ease_in_cubic) — deterministic, pure-math path, injectable sink for tests; complements the humanized jitter. - -### Process-Doc (SOP) Generator - -Turn an action list into a step-by-step SOP. Full reference: [`docs/source/Eng/doc/new_features/v28_features_doc.rst`](docs/source/Eng/doc/new_features/v28_features_doc.rst). - -- **`generate_sop` / `write_sop`** (`AC_generate_sop`, `ac_generate_sop`): map a recorded/authored action list to numbered, human-readable steps + an HTML document (UiPath Task-Capture deliverable); content HTML-escaped, unknown commands degrade gracefully. - -### Heal Analytics & Secret Scan - -Two pure-stdlib audit/analysis tools. Full reference: [`docs/source/Eng/doc/new_features/v27_features_doc.rst`](docs/source/Eng/doc/new_features/v27_features_doc.rst). - -- **Self-heal analytics** — `analyze_heal_log` / `heal_stats` (`AC_heal_stats`, `ac_heal_stats`): aggregate the self-heal log into heal-rate, strategy mix, fallback-rate, avg latency and the most-brittle locators — catch decaying selectors before they fail. -- **Secret scan** — `scan_secrets(data)` (`AC_scan_secrets`, `ac_scan_secrets`): flag hardcoded secrets in action JSON (by key name, value pattern, or high entropy) that should use `${secrets.*}`; vault refs ignored, previews masked. - -### CI Annotations & Clipboard History - -Two pure-stdlib utilities. Full reference: [`docs/source/Eng/doc/new_features/v26_features_doc.rst`](docs/source/Eng/doc/new_features/v26_features_doc.rst). - -- **CI annotations** — `emit_annotations(results)` (`AC_ci_annotations`, `ac_ci_annotations`): turn result dicts into GitHub Actions workflow commands (`::error file=...,line=...::msg`) so failures show inline in a PR, no reporter action needed. -- **Clipboard history** — `ClipboardHistory` / `default_clipboard_history` (`AC_clip_history_capture`/`list`/`search`/`start`/`stop`, `ac_clip_history_*`): a capped, searchable, newest-first ring buffer of copied text with an optional background poller. - -### Resilience Primitives - -Reusable retry + circuit-breaker primitives. Full reference: [`docs/source/Eng/doc/new_features/v25_features_doc.rst`](docs/source/Eng/doc/new_features/v25_features_doc.rst). - -- **RetryPolicy** — `RetryPolicy(...).run(fn)` / `retry_call(fn)`: retry on configured exceptions with exponential backoff (injectable sleep). (The existing `AC_retry` flow command already retries an action body; this is the reusable callable wrapper.) -- **CircuitBreaker** — `CircuitBreaker` / `CircuitOpenError` (`AC_circuit_call`, `ac_circuit_call`): open after N consecutive failures, short-circuit until a reset timeout, then half-open — stops a retry storm hammering a downed dependency. Injectable clock; `AC_circuit_call` runs an action list through a named breaker. - -### Timed Input Macros - -Replay input with timing fidelity + a press-hold-release DSL, full stack. Full reference: [`docs/source/Eng/doc/new_features/v24_features_doc.rst`](docs/source/Eng/doc/new_features/v24_features_doc.rst). - -- **Timed timeline replay** — `replay_timeline(events, speed=...)` (`AC_replay_timeline`, `ac_replay_timeline`): replay events honoring each `delta_ms` gap, scaled by `speed` and clampable; ops = move/click/scroll/press/release/key. -- **Input-sequence DSL** — `run_sequence(steps)` (`AC_input_sequence`, `ac_input_sequence`): declarative press/hold/release chords + `repeat`/`wait`. Both inject sink+sleep for deterministic tests. - -### Semantic Screen State - -The semantic companion to the pixel diff, full stack. Full reference: [`docs/source/Eng/doc/new_features/v23_features_doc.rst`](docs/source/Eng/doc/new_features/v23_features_doc.rst). - -- **Snapshot & diff** — `snapshot` / `diff_snapshots` / `snapshot_screen` / `screen_changed` (`AC_screen_snapshot` / `AC_screen_diff` / `AC_screen_changed`, `ac_*`): normalize the a11y tree to `{role, name, bbox}` and report what **appeared / vanished / moved** with a human-readable summary — the feedback signal an agent needs to verify a step ("Save dialog appeared"). -- **Describe the screen** — `describe_screen` (`AC_describe_screen`, `ac_describe_screen`): a compact "where am I" — role counts + interactive control labels. - -### Set-of-Marks Overlay - -The standard VLM-grounding format, full stack. Full reference: [`docs/source/Eng/doc/new_features/v22_features_doc.rst`](docs/source/Eng/doc/new_features/v22_features_doc.rst). - -- **Number elements** — `mark_elements` / `render_marks` / `resolve_mark` (pure + Pillow): assign `1..N` to interactable elements (with centre/role/text), draw numbered red boxes on a screenshot, and map a chosen number back to its element — so a VLM picks a *number* instead of guessing pixels (directly strengthens the existing VLM locator). -- **Mark-then-click loop** — `mark_screen(render_path=...)` / `mark_click(n)` (`AC_mark_screen` / `AC_mark_click`, `ac_*`): number the live a11y tree (+ optional overlay screenshot), feed marks+image to a model, then click mark `n`. - -### Checkpoint & Resume - -Durable execution for long flows + a `py.typed` marker, full stack. Full reference: [`docs/source/Eng/doc/new_features/v21_features_doc.rst`](docs/source/Eng/doc/new_features/v21_features_doc.rst). - -- **Flow checkpoint & resume** — `run_resumable(actions, run_id=..., store=...)` / `CheckpointStore` (`AC_run_resumable` / `AC_checkpoint_status` / `AC_checkpoint_clear`, `ac_*`): persist step-index + variables after each step; on re-run with the same `run_id`, fast-forward past completed steps and rehydrate variables — a flow that crashes at step 400 resumes at 400, not 0. Pluggable (SQLite default), cleared on completion. -- **`py.typed` marker** — ships the PEP 561 marker so Mypy/Pyright/Pylance honor AutoControl's inline type hints in downstream code (the repo's typed API was previously invisible to type checkers). - -### i18n / l10n Testing - -Three pure-stdlib internationalization/localization testing helpers that compound, full stack. Full reference: [`docs/source/Eng/doc/new_features/v20_features_doc.rst`](docs/source/Eng/doc/new_features/v20_features_doc.rst). - -- **Pseudo-localization** — `pseudo_localize` / `pseudo_localize_catalog` (`AC_pseudo_localize`, `ac_pseudo_localize`): accent + pad UI strings (placeholders preserved, `⟦…⟧` wrapped) to flush out hardcoded text and pre-stress layout before real translation. -- **Text-overflow detection** — `check_overflow(elements)` (`AC_check_overflow`, `ac_check_overflow`): flag text whose estimated width exceeds its widget bounds (the #1 l10n bug), computed from the a11y bounds AutoControl already reads. -- **Catalog completeness** — `check_catalog(base, target)` (`AC_check_catalog`, `ac_check_catalog`): diff a translation catalog for missing / orphaned / empty keys and placeholder mismatches — a CI gate against blank UI. - -### Data Quality - -Three pure-stdlib data-quality helpers (the gate between `load_rows`/OCR and downstream entry), full stack. Full reference: [`docs/source/Eng/doc/new_features/v19_features_doc.rst`](docs/source/Eng/doc/new_features/v19_features_doc.rst). - -- **Row schema validation** — `validate_rows(rows, schema)` (`AC_validate_rows`, `ac_validate_rows`): declarative per-field rules (type/required/regex/min/max/min_len/max_len/allowed/unique); returns `{ok, valid, invalid, errors}` so bad scraped/OCR data is caught before it corrupts an ERP/form. -- **Field extraction** — `extract_fields(text, fields, patterns)` (`AC_extract_fields`, `ac_extract_fields`): named regex presets (email/url/ipv4/phone/date_iso/amount/hashtag) + custom patterns over free text / OCR blobs. -- **Row masking** — `mask_rows(rows, rules)` (`AC_mask_rows`, `ac_mask_rows`): mask columns before export — `redact` / `hash` (SHA-256) / `partial` (keep last 4); complements the screenshot-only redaction. - -### SBOM & Suite Sharding - -Two pure-stdlib ops tools (security + scale research angles), full stack. Full reference: [`docs/source/Eng/doc/new_features/v18_features_doc.rst`](docs/source/Eng/doc/new_features/v18_features_doc.rst). - -- **CycloneDX SBOM** — `build_sbom` / `write_sbom` (`AC_generate_sbom`, `ac_generate_sbom`): emit a CycloneDX 1.6 dependency SBOM (name/version/purl/license) for supply-chain compliance (EU CRA / EO 14028); `root` limits to a package's closure, `extra_components` inventories action files. No third-party dependency. -- **Duration-aware suite sharding** — `shard_flows` / `merge_results` (`AC_shard_suite` / `AC_merge_results`): bin-pack flows into N shards balanced by historical per-flow duration (so the slowest worker, not test count, defines runtime), then merge per-shard reports into one rollup. - -### Reactive Observer - -A non-blocking screen observer (SikuliX `observe` model), full stack (facade, `AC_*`, MCP, Script Builder). Full reference: [`docs/source/Eng/doc/new_features/v17_features_doc.rst`](docs/source/Eng/doc/new_features/v17_features_doc.rst). - -- **`ScreenObserver`** (`AC_observe_add` / `AC_observe_remove` / `AC_observe_list` / `AC_observe_poll` / `AC_observe_start` / `AC_observe_stop`, `ac_observe_*`): register watches that fire on **appear** / **vanish** / **change** of an image/text/pixel and run a callback or action list — react to dialogs/progress/status while the main flow continues. -- **Testable by design** — detection is an injectable `predicate`; transition logic is unit-tested via `poll_once()` with synthetic values. Built-in `image_predicate` / `text_predicate` / `pixel_predicate` wrap the existing locate/OCR/pixel helpers. - -### WCAG 2.2 Audit - -The accessibility audit gains a WCAG 2.2 / EN 301 549 success-criterion layer, full stack (facade, `AC_*`, MCP, Script Builder). Full reference: [`docs/source/Eng/doc/new_features/v16_features_doc.rst`](docs/source/Eng/doc/new_features/v16_features_doc.rst). - -- **WCAG-tagged conformance audit** — `wcag_audit(level="AA")` (`AC_wcag_audit`, `ac_wcag_audit`): tags every defect with its WCAG success-criterion id/level/impact (4.1.2, 1.4.3, 1.4.10) and returns a conformance report with `by_criterion`/`by_impact` counts, filtered to A/AA/AAA — mappable to EN 301 549 for EAA compliance evidence. -- **Target Size (SC 2.5.8)** — `audit_target_size(elements, min_px=24)`: new WCAG 2.2 rule flagging interactive targets smaller than 24×24 px, computed from element bounds; `tag_issue` adds SC tagging to any existing audit issue. - -### Memory & Determinism - -Two pure-stdlib tools from the agent/QA research round, full stack (facade, `AC_*`, MCP, Script Builder). Full reference: [`docs/source/Eng/doc/new_features/v15_features_doc.rst`](docs/source/Eng/doc/new_features/v15_features_doc.rst). - -- **Agent episodic memory** — `AgentMemory` (`AC_memory_remember` / `AC_memory_recall` / `AC_memory_recent` / `AC_memory_forget` / `AC_memory_stats`, `ac_memory_*`): SQLite store of `(goal → trajectory → outcome)` episodes with keyword recall to inject past experience into the planner's context — cross-run learning, no embedding dependency. -- **Deterministic run** — `DeterministicRun` / `seed_everything` (`AC_seed_everything`, `ac_seed_everything`): pin the RNG seed and freeze `time.time` for a `with` block (recording the choices for replay) to kill time/randomness flakiness; `time.monotonic` left intact so timeouts still work. - -### Office I/O - -Headless read/write for Excel/Word/PowerPoint, full stack (facade, `AC_*`, MCP, Script Builder). Optional extra: `pip install je_auto_control[office]`. Full reference: [`docs/source/Eng/doc/new_features/v14_features_doc.rst`](docs/source/Eng/doc/new_features/v14_features_doc.rst). - -- **Excel** — `read_workbook` / `write_workbook` (`AC_read_workbook` / `AC_write_workbook`, `ac_read_workbook` / `ac_write_workbook`): read an `.xlsx` worksheet into row dicts (first row = keys) and write rows back, no GUI. -- **Word** — `read_document` / `write_document` (`AC_read_document` / `AC_write_document`): read/write `.docx` paragraphs. -- **PowerPoint** — `read_presentation` / `write_presentation` (`AC_read_presentation` / `AC_write_presentation`): read per-slide text; write slides as `{title, body:[...]}`. - -The backing libraries (`openpyxl`/`python-docx`/`python-pptx`) are optional — each call raises a clear error if missing, and `import je_auto_control` pulls none of them. - -### Agent Toolkit - -Three pure-stdlib tools for LLM/agent-driven automation, full stack (facade, `AC_*`, MCP, Script Builder). Full reference: [`docs/source/Eng/doc/new_features/v13_features_doc.rst`](docs/source/Eng/doc/new_features/v13_features_doc.rst). - -- **Skill / playbook library** — `SkillLibrary` (`AC_skill_save` / `AC_skill_run` / `AC_skill_list` / `AC_skill_remove` / `AC_skill_search`, `ac_skill_*`): store named, reusable action sequences on disk, search them by name/description/tags, and replay across runs — the durable counterpart to in-memory macros. -- **Prompt-injection guardrail** — `assess_text` / `scan_text` / `redact_text` (`AC_guard_text`, `ac_guard_text`): scan untrusted screen/OCR text for injection patterns (instruction-override, system-prompt exfiltration, jailbreak/chat-template markers …) before feeding it to an LLM; returns `{suspicious, score, findings, redacted}`. -- **A2A agent card** — `build_agent_card` / `write_agent_card` (`AC_agent_card`, `ac_agent_card`): publish an A2A agent card so other agents can discover and call AutoControl as a GUI-automation peer. - -### Authoring & Debugging - -Two pure-stdlib authoring-time tools, full stack (facade, `AC_*`, MCP, Script Builder). Full reference: [`docs/source/Eng/doc/new_features/v12_features_doc.rst`](docs/source/Eng/doc/new_features/v12_features_doc.rst). - -- **Element repository** — `ElementRepository` (`AC_element_save` / `AC_element_find` / `AC_element_click` / `AC_element_remove` / `AC_element_list`, `ac_element_*`): save native-UI locators under friendly names (object repository) and reuse them — `repo.click("login.submit")` instead of repeating name/role everywhere; a UI change is fixed in one place. -- **Step debugger / tracer** — `FlowDebugger` (breakpoints, `step`/`continue_`/`run_to_end`, live `variables()`) and `trace_actions` (`AC_debug_trace`, `ac_debug_trace`): step through an action list one command at a time with variables persisting across steps, or get a per-step `{index, command, result}` trace (with `dry_run` to plan without running). - -### Test & Tooling Batch - -Three pure-stdlib quality-of-life tools, full stack (facade, `AC_*`, MCP, Script Builder). Full reference: [`docs/source/Eng/doc/new_features/v11_features_doc.rst`](docs/source/Eng/doc/new_features/v11_features_doc.rst). - -- **Synthetic test data** — `generate_rows(schema, count, seed=...)` / `write_dataset` (`AC_generate_data`, `ac_generate_data`): deterministic fake rows (name/email/phone/int/choice/date…) to drive data-driven runs without real PII; no Faker. -- **MCP registry manifest** — `write_server_manifest("server.json", include_tools=True)` (`AC_mcp_manifest`, `ac_mcp_manifest`): publish a registry-valid `server.json` so MCP agents/IDEs can discover this server. -- **Risk-based test selection** — `rank_flows` / `select_flows` (`AC_rank_tests` / `AC_select_tests`): rank flows by recent failures, flakiness, staleness and never-run from run history; run the riskiest first or only the top-k. - -### Transactional Queue - -Turn AutoControl from "run a script" into "run a robot." A SQLite-backed work queue implements the production-RPA dispatcher/performer pattern: enqueue items, process one at a time with per-item status, dedup and retry, so a run of thousands is **resumable after a crash** and parallelizable. Pure stdlib, full stack. Full reference: [`docs/source/Eng/doc/new_features/v10_features_doc.rst`](docs/source/Eng/doc/new_features/v10_features_doc.rst). - -- **Dispatcher/performer** — `WorkQueue.add()` enqueues (dedupes by reference); `get_next()` atomically claims the oldest item; `complete()` / `fail()` record the outcome. `AC_queue_add` / `AC_queue_next` / `AC_queue_complete` / `AC_queue_fail` / `AC_queue_stats`. -- **Failure semantics** — application errors retry up to `max_retries`; **business** errors (`BusinessError` / `kind="business"`) never retry. `stats()` gives per-status counts for dashboards. - -### Unattended Reliability - -Three practitioner-pain fixes for unattended / login automation, all headless and full-stack. Full reference: [`docs/source/Eng/doc/new_features/v9_features_doc.rst`](docs/source/Eng/doc/new_features/v9_features_doc.rst). - -- **OTP / TOTP for 2FA** — `generate_totp` / `verify_totp` (`AC_otp_to_var`, `ac_generate_otp`): mint the current 6-digit code from a base32 secret to type into a login form (reuses the remote-desktop TOTP engine). -- **Native file dialogs** — `handle_file_dialog` (`AC_handle_file_dialog`): wait for the OS Open/Save/folder dialog, type the path, confirm — in one call, with an injectable driver. -- **Locked-session guard** — `ensure_interactive_session` / `is_session_locked` (`AC_assert_session_active`): fail clearly when the workstation is locked / disconnected instead of emitting phantom clicks. - -### Popup Watchdog - -The #1 cause of unattended-automation failure is an unexpected dialog the script never coded for (UAC, "session expiring", Windows Update, a modal). The popup watchdog runs a concurrent guard thread that watches for registered patterns and dismisses them independently of the main flow. Surfaced by the practitioner pain-point research as the top unattended failure cause; full stack (facade, `AC_*`, MCP, Script Builder), fully headless. Full reference: [`docs/source/Eng/doc/new_features/v8_features_doc.rst`](docs/source/Eng/doc/new_features/v8_features_doc.rst). - -- **Auto-dismiss popups** — `default_popup_watchdog.add_window_rule(title, action="close")` then `.start()` (`AC_watchdog_add` / `AC_watchdog_start` / `AC_watchdog_stop` / `AC_watchdog_list`): closes a matching window or presses a key (`enter`/`esc`) when it appears. -- **Custom rules** — `PopupWatchdog` / `WatchdogRule` pair any detector (image/a11y/text) with a dismisser; a failing rule is logged and skipped, never killing the guard loop. - -### Native UI Control - -Object-level desktop automation: read and drive native controls through the OS accessibility API (by name / role / app / **AutomationId**) instead of clicking pixels or OCR-ing text — far more reliable for native apps. The accessibility layer previously only listed/found/clicked; it now also acts. Ships through the full stack (facade, `AC_*`, MCP, Script Builder) with a Windows UIAutomation backend; unsupported backends raise a clear error. Full reference: [`docs/source/Eng/doc/new_features/v7_features_doc.rst`](docs/source/Eng/doc/new_features/v7_features_doc.rst). - -- **Read / set value** — `control_get_value` / `control_set_value` (`AC_control_get_value` / `AC_control_set_value`): read a textbox/combo value (no OCR) and set it in one call (no per-key typing). -- **Invoke / toggle** — `control_invoke` / `control_toggle` (`AC_control_invoke` / `AC_control_toggle`): press a button or flip a checkbox via its control pattern. -- **Read a table/grid** — `read_control_table` (`AC_read_table`): scrape a grid/list/table control into rows of cell strings — desktop data extraction without OCR. -- Targets a control by `name` / `role` / `app_name` / `automation_id` (the stable Windows identifier), so it survives layout/localization changes. - -### Additional updates - -Two headless cores that shipped without the rest of their stack are now -first-class. Both gain a facade re-export, an `AC_*` executor command, an -MCP tool, and a Script Builder entry, with headless tests. Full reference: -[`docs/source/Eng/doc/new_features/v6_features_doc.rst`](docs/source/Eng/doc/new_features/v6_features_doc.rst). - -- **Visual regression (golden images)** — `take_golden` / `compare_to_golden` (`AC_take_golden` / `AC_assert_visual`): capture a baseline screenshot and fail when the screen drifts beyond a pixel tolerance, with a highlighted diff image and mask regions. `AC_assert_visual` auto-creates the baseline on first run. PIL-only. -- **Finite-state machine** — `run_state_machine` (`AC_run_state_machine`): drive a script as a declarative `{initial, states}` spec whose `on_enter` actions run through the executor and whose transitions fire on `after` / `if_var_eq` / predicate guards, bounded by `max_steps` / `global_timeout_s`. - -## What's new (2026-06-18) - -Eight headless capabilities that round out scripting, integration, and CI -use: a real command-line interface, recording-to-code generation, and -first-class HTTP / SQL / email / PDF / wait steps. Each ships a headless -Python API, an `AC_*` executor command, an MCP tool, and a visual Script -Builder entry, and is covered by headless tests (network / SMTP / PDF -backends are injected, so nothing touches the outside world). Full -reference page: -[`docs/source/Eng/doc/new_features/v5_features_doc.rst`](docs/source/Eng/doc/new_features/v5_features_doc.rst). - -**Command-line interface** -- **`je_auto_control` console script** — run and inspect action files from a shell / CI: `run` (with `--var`, `--dry-run`), `validate` (alias `lint`), `list-commands`, `fmt`, `record`, `codegen`, `version`. - -**Code generation** -- **Recording → code** — `generate_code` / `generate_code_file` (`AC_generate_code`, `je_auto_control codegen`) turn a recording or action file into a pytest test, standalone Python, or Robot suite. The default `calls` style emits readable `ac.(...)` statements, falling back to `ac.execute_action([...])` for flow control. - -**Integrations** -- **HTTP / API** — `http_request` (`AC_http_request`): method, headers, JSON or raw body, basic / bearer auth, explicit timeout; non-2xx responses are returned (not raised) so you can assert on status. `AC_http_to_var` now shares the client and can POST bodies. -- **SQL** — `query_sqlite` (`AC_sql_to_var` / `AC_assert_db`): read-only, parameter-bound SQLite queries into a variable, or a scalar assertion (e.g. `SELECT COUNT(*) ... == 0`). -- **Email (SMTP)** — `send_email` (`AC_send_email`): stdlib SMTP with TLS on by default (STARTTLS or implicit SSL over a verified context), attachments, and multiple recipients. -- **PDF** — `extract_pdf_text` / `pdf_metadata` / `assert_pdf_text` (`AC_pdf_to_var` / `AC_assert_pdf_text`): text extraction and content assertions, backed by the optional `pypdf` extra (`pip install je_auto_control[pdf]`). - -**Smart waits** -- **Wait for a file** — `wait_until_file` (`AC_wait_for_file`) blocks until a file exists and its size stops growing (a download finished writing). -- **Wait for a TCP port** — `wait_until_port` (`AC_wait_for_port`) blocks until `host:port` accepts connections (pairs with `launch_process`). -- **Wait for a process** — `wait_until_process` (`AC_wait_for_process`) blocks until a process appears or exits — the companion to `launch_process` / `kill_process` (requires psutil). - -**Security** — HTTP / SMTP enforce http/https or TLS with verified certificates and explicit timeouts; SQL is read-only and parameter-bound; file paths are resolved before I/O. - -## What's new (2026-06-17) - -Thirty-plus automation primitives across input realism, vision, flow -control, triggers, window management, and file security — plus recoverable -deletion and an editor undo. Each ships with a headless API, an `AC_*` -executor command, and a visual Script Builder entry; vision and window -features keep their geometry / IO operations injectable so the logic is -fully unit-tested. Full reference page: -[`docs/source/Eng/doc/new_features/v4_features_doc.rst`](docs/source/Eng/doc/new_features/v4_features_doc.rst). - -**Human-like input** -- **Human-like mouse motion** — `move_mouse_humanized` walks an eased, bowed cubic-Bezier path with optional overshoot + jitter, deterministic by `seed` (`AC_human_move`). -- **Human-like typing** — `type_text_humanized` types character by character with a jittered per-key delay and optional "thinking" pauses, seedable (`AC_human_type`). - -**Vision** -- **VLM natural-language assertion** — `assert_by_description` asks a vision-language model whether the screen matches a description; the `verify()` companion to `locate_by_description` (`AC_assert_vlm`). -- **Scroll-to-find** — `scroll_until_visible` scrolls a direction until a template image or OCR text appears, or the budget runs out (`AC_scroll_to_find`). -- **Region colour stats** — `region_color_stats` reports a region's average + dominant colour and that colour's pixel fraction (`AC_region_color_stats`). -- **QR reading** — `read_qr_codes` decodes QR codes in a screen region via OpenCV's `QRCodeDetector` (no new dependency) (`AC_read_qr`). - -**Flow control & variables** -- **Reusable macros** — `AC_define_macro` / `AC_call_macro`: define a named, parameterised action sub-routine once and call it with `${arg}` bindings. -- **In-process parallel** — `AC_parallel` runs branch action lists concurrently, each on an isolated executor so branches never race on shared variables. -- **Performance-budget assertion** — `assert_duration` / `AC_assert_duration` fails a block that takes longer than a millisecond budget. -- **Read into a variable** — `AC_ocr_to_var`, `AC_shell_to_var`, `AC_read_file_to_var`, `AC_http_to_var` (body or dotted JSON path), `AC_now_to_var` (strftime), `AC_random_to_var` (seeded int / float / choice). -- **Transform a variable** — `AC_transform_var`: upper / lower / strip / title / replace / regex-extract / slice, in place or into a new variable. -- **Assert a variable** — `assert_variable` / `AC_assert_var`: eq / ne / lt / gt / contains / regex through the assertion DSL. - -**Triggers & smart waits** -- **Composite triggers** — `AllOfTrigger` / `AnyOfTrigger` / `SequenceTrigger` combine any existing trigger by boolean AND / OR / ordered sequence. -- **Cron trigger** — `CronTrigger` fires on a five-field cron expression, composing with the boolean triggers (e.g. "at 09:00 *and* only if the image is on screen"). -- **More smart waits** — `wait_until_clipboard_changes` (`AC_wait_clipboard_change`) and `wait_until_window_closed` (`AC_wait_window_closed`). - -**Window management** -- **Per-window capture** — `capture_window` screenshots exactly a window's bounds by title (`AC_capture_window`). -- **Layout save / restore** — `save_window_layout` / `restore_window_layout` snapshot every window's position to JSON and move them all back later (`AC_save_window_layout` / `AC_restore_window_layout`). -- **Snap / tile** — `snap_window` moves a window to a screen half, quarter, or maximize (`AC_snap_window`). - -**File security & safety** -- **Action-file signing** — `sign_action_file` / `verify_action_file` (HMAC-SHA256 sidecar); `execute_files` can require signatures via `JE_AUTOCONTROL_REQUIRE_SIGNED_ACTIONS` (`AC_sign_action_file` / `AC_verify_action_file`). -- **Action-file encryption** — `encrypt_action_file` / `decrypt_action_file` (Fernet, AES-128-CBC + HMAC) (`AC_encrypt_action_file` / `AC_decrypt_action_file`). -- **Recoverable deletion** — `move_to_trash` sends a file to the OS recycle bin (Win32 `SHFileOperation` undo flag / macOS Trash / Linux XDG trash, preferring `send2trash`) (`AC_move_to_trash`). - -**Reporting & notifications** -- **Screenshot annotation** — `annotate_screenshot` draws labelled boxes / translucent highlights / arrows / text onto a capture (`AC_annotate_screenshot`). -- **Desktop notifications** — `notify` shows a cross-platform toast (notify-send / osascript / PowerShell), injection-safe (`AC_notify`). - -**GUI** -- **Recording Editor undo** — every edit is snapshotted; **Ctrl+Z** (and an Undo button) restore the prior state. -- **Triggers tab** — "Combine selected" wraps chosen triggers into a composite; new **Cron** trigger type. -- **Assertions tab** — new **VLM** ("screen matches description") assertion kind. -- Every new `AC_*` command appears in the visual **Script Builder**. - -**Fixes** — repaired the USB-passthrough approval-prompt crash on PySide6 6.11.1 (`Q_ARG(object)` → a Qt signal), eight stale / broken GUI + USB tests, two lost exception chains, and brought thirteen functions back under the cyclomatic-complexity gate. - -## What's new (2026-06) - -Nine additions that turn the automation primitives into a full **QA / test -framework**: assert screen state, drive scripts from data, detect and -quarantine flaky tests, run a scored suite, emit CI-native reports, audit -accessibility / i18n, fan a script across a device matrix, and assert on -audio / video. Each ships with a headless API, an `AC_*` executor command, -an `ac_*` MCP tool, and a Qt GUI tab. Full reference page: -[`docs/source/Eng/doc/new_features/v3_features_doc.rst`](docs/source/Eng/doc/new_features/v3_features_doc.rst). - -**Assertions** -- **Assertion DSL** — verify screen state instead of only driving it: `assert_text` (OCR, `regex` + `present=False` for absence), `assert_image`, `assert_pixel`, `assert_window`, `assert_clipboard` (`equals` / `contains` / `regex`, `present=False` to confirm a secret was cleared), `assert_process` (a named process is / isn't running, via psutil). Returns an `AssertionResult`; raises `AutoControlAssertionException` on mismatch with optional failure screenshot (`AC_assert_text / _image / _pixel / _window / _clipboard / _process`). -- **Off-screen assertions** — `assert_file` (existence / substring / SHA-256 / minimum size — verify a download or export) and `assert_http` (an http/https endpoint returns a status + optional body text, always with an explicit timeout). Both extend the DSL beyond the screen and plug into the combinators below (`AC_assert_file / AC_assert_http`). -- **Assertion combinators** — `assert_all([...specs])` runs a batch as *soft assertions* (every spec is checked, all failures collected before raising) and returns a `GroupAssertionResult`; `assert_any([...specs])` is the OR-complement (passes when at least one spec passes, short-circuiting — e.g. *either* a success dialog *or* a redirect confirms a login); `assert_eventually(spec, timeout, interval)` retries one declarative assertion spec until it passes or times out (e.g. poll a health endpoint until it returns 200, or wait for a download file to appear). Both are spec-driven (`{"kind": "text", "text": "Saved"}`, `{"kind": "http", "url": "..."}`) so they work identically from Python, JSON, and MCP across every assertion kind — text/image/pixel/window/clipboard/process/file/http (`AC_assert_all / AC_assert_eventually`). -- **Media assertions** — `assert_audio_activity` (record + RMS threshold for sound vs silence) and `assert_video_changes` (mean frame-to-frame diff over a segment for motion vs static); pure numeric cores, lazy `sounddevice` / OpenCV (`AC_assert_audio / AC_assert_video_changes`). - -**Data-driven execution** -- **Data sources** — `load_rows` connectors for CSV / JSON / SQLite / Excel / inline; the `AC_for_each_row` block command runs a body once per row with `${row.column}` access. SQLite is single read-only `SELECT`/`WITH` only; paths are `realpath`-validated. `${var}` interpolation now resolves dotted dict-key / list-index paths while preserving types (`AC_load_data`). - -**Flaky detection & quarantine** -- **Flaky report** — score intermittent failures from run history by pass↔fail flip rate, grouped by script / source (`AC_flaky_report`). -- **Quarantine** — a persistent (mode 0600) skip-list the suite runner honours; `auto_quarantine_from_flakiness` auto-populates it above a flip-rate threshold (`AC_quarantine_add / _remove / _list / _clear / _auto`). - -**Suite runner + CI reports** -- **QA suite orchestration** — `run_suite` turns action lists into scored cases with setup / teardown, tags, and data-driven expansion; assertion failures → failed, other exceptions → error, quarantined → skipped (`AC_run_suite`). -- **JUnit / Allure reports** — `write_junit_xml` + `write_allure_results` (or `junit_path` / `allure_dir` on `AC_run_suite`) emit reports Jenkins / GitHub Actions / GitLab CI / Allure parse natively. - -**Audit, matrix, media** -- **Accessibility / i18n audit** — reuse the a11y tree + OCR to find missing accessible names, WCAG contrast-ratio failures, and ellipsis-truncated strings (`AC_audit_accessibility / AC_audit_contrast`). -- **Mobile device matrix** — fan one action list across many Android / iOS devices in parallel, each on an isolated executor, targeting the current device via `${device.*}`; per-device pass/fail, failures isolated (`AC_run_device_matrix`). - -## What's new (2026-05) - -Twenty-seven additions covering smarter locators, deeper IDE / ops -tooling, four new platforms (Wayland, Wayland-libei, Android -widget-tree, iOS), screenshot PII redaction, and a generic -plan-execute-verify agent loop. Each ships with a headless API, an -`AC_*` executor command, an `ac_*` MCP tool, and (where it makes -sense) a Qt GUI tab. Full reference page: -[`docs/source/Eng/doc/new_features/v2_features_doc.rst`](docs/source/Eng/doc/new_features/v2_features_doc.rst). - -**Locator + selector intelligence** -- **Self-healing locator** — `image_template → VLM` fallback with a JSON-lines audit log (`AC_self_heal_locate / _click`). -- **Anchor-based locator** — find element B by spatial relation (`above`, `below`, `left_of`, `right_of`, `near`) to anchor A; anchor and target can use different backends (image / OCR / VLM / a11y). -- **OCR with structured output** — cluster raw OCR matches into rows, tables, and `label:value` form fields (`AC_ocr_read_structure`). -- **Smart waits** — `wait_until_screen_stable`, `wait_until_pixel_changes`, `wait_until_region_idle`: frame-diff replacements for `time.sleep`. -- **A/B locator framework** — race N strategies for the same target; recommend the historically best one from a persisted ledger. - -**Operations + observability** -- **LLM cost telemetry** — per-call token + USD log with day / model / provider rollup (`record_llm_call`, `summarise_llm_costs`). -- **Trace replay UI** — scrubbable timeline over the existing time-travel recordings with per-step action list. -- **Failure → ticket automation** — fan a failure report out to Jira / Linear / GitHub Issues when a scheduled / triggered / REST run fails. -- **Container CI templates** — GitHub Actions + GitLab CI workflows that build the image, run the headless pytest suite under Xvfb, and smoke-test the REST entrypoint; XFCE+x11vnc Dockerfile variant for flows that need a real WM. -- **Cross-host DAG orchestrator** — parallel execution with skip-on-failure cascade across local + admin-console-registered hosts (`run_dag`, `AC_run_dag`). -- **Multi-viewer presence** — roster + controller/observer roles for the remote desktop, with a thread-safe Python `PresenceRegistry` independent of aiortc. - -**Agent + integrations** -- **Computer-use high-level API** — `run_computer_use(goal, ...)` wraps `ComputerUseAgentBackend` + `AgentLoop`; auto-detects display size; bounded by `max_steps` / `wall_seconds`. -- **Generic agent loop JSON + MCP** — `AC_run_agent` / `ac_run_agent` expose the closed-loop `AgentLoop` (plan → act → verify → retry) with pluggable Anthropic / OpenAI backends; the Anthropic-only Computer-Use raw path remains via `AC_computer_use`. -- **WebRunner convenience commands** — `web_open` / `web_quit` / `web_screenshot` / `web_current_url` on top of the existing `je_web_runner` bridge; same surface exposed as `AC_web_*` and `ac_web_*`. -- **Chat-ops bot** — transport-agnostic `CommandRouter` + polling Slack adapter. Built-in commands: `/help`, `/scripts`, `/run`, `/screenshot`, `/status`. RBAC via `required_role`. - -**Privacy + safety** -- **Screenshot PII redaction** — `RedactionEngine` with built-in detectors for email / credit card / SSN / phone (regex against caller-supplied OCR tokens) plus accessibility-tree secure-text-field detection. Forced regions for sticky overlays. Env-var-driven default policy `JE_AUTOCONTROL_REDACTION=off|moderate|strict`. Wired through `AC_redact_screenshot` + `ac_redact_screenshot`. - -**Platform coverage** -- **Wayland CLI backend** — `wtype` / `ydotool` / `grim` with `XDG_SESSION_TYPE` auto-detect and X11 (XWayland) fallback; override via `JE_AUTOCONTROL_LINUX_DISPLAY_SERVER=x11|wayland|auto`. -- **Wayland libei native** — ctypes binding to `libei.so.*` for microsecond-latency input; opt-in via `JE_AUTOCONTROL_WAYLAND_INPUT_BACKEND=libei|cli|auto`. Defaults to libei when loadable. -- **macOS Accessibility deep-dive** — recursive `dump_accessibility_tree()` plus a polling `AccessibilityRecorder` for focus / bounds events. -- **Android — adb shell primitives** — `AC_android_tap/swipe/key/text/screenshot` route through `adb` for any phone over USB / Wi-Fi adb. No daemon required. -- **Android — uiautomator2 widget tree** — `AC_android_find_element/click_element/dump_hierarchy` add selector-based widget lookup (`text` / `resource_id` / `description` / `class_name`) and live XML hierarchy dump on top of the adb path. -- **iOS — XCUITest via WebDriverAgent** — new `je_auto_control.ios.*` namespace: `tap`, `swipe`, `long_press`, `type_text`, `press_key`, `screenshot`, `screen_size`, `find_element` / `click_element` (XCUITest selectors: `name`, `class_name`, `predicate`), `dump_source`. Seven new `AC_ios_*` executor commands and matching `ac_ios_*` MCP tools. `facebook-wda` is an optional pip dep; loads lazily so non-Mac hosts still import the package. - -**Developer experience** -- **autocontrol-lsp completion** — the language server now tracks `didOpen` / `didChange` / `didClose`, publishes diagnostics for invalid JSON and unknown `AC_*` commands, and provides signature help generated from the live executor table. -- **`.pyi` stub generator** — `python -m je_auto_control.utils.stubs.generator je_auto_control/actions.pyi` emits an IDE-facing stub so every `AC_*` command autocompletes with parameter hints. -- **VS Code extension** — bundled extension now ships `AutoControl: Run / Screenshot / Preview` commands that hit the local REST API. -- **Browser extension recorder** — Manifest V3 extension under `browser-extension/`: capture clicks, typing, navigation, form submissions in a tab and export them as `AC_web_*` / `WR_*` JSON. -- **pytest plugin + Gherkin BDD** — `pytest11` entry point auto-loads; `@pytest.mark.autocontrol` arms screenshot-on-failure; `bdd_steps.register_pytest_bdd_steps(pytest_bdd)` wires `Given/When/Then` onto every `AC_*` verb. -- **Visual flow editor** — node-based view that round-trips to the same JSON action format the list-based Script Builder uses. - ---- +**This log has moved to [`docs/updates/`](docs/updates/README.md).** Every former +`## What's new (…)` section of this file is now one `#release` entry there, in a +monthly batch file (`docs/updates/YYYY-MM.md`), next to every other piece of +finished work. Each section's text was carried over unchanged except that its +headings moved down one level. This file stays so that existing links still land +somewhere. + +- **Index, newest first, with the query commands**: [docs/updates/README.md](docs/updates/README.md) +- **Every entry, one line each** (from the repository root): `rg -n "^## U-2" docs/updates` +- **Release notes only**: `rg -n "^## U-2.*#release" docs/updates` +- **One month**: `rg -n "^## U-202608" docs/updates` +- **The full text of one entry**: `rg -n -A 60 "^## U-20260824-01" docs/updates` + +Without `rg`: `git grep -n "^## U-2" -- docs/updates`. + +**Finding an old section**: `What's new (YYYY-MM-DD)` became entry `U-YYYYMMDD-01`. +The two sections dated only by month got the day their last content was added: +`What's new (2026-06)` is `U-20260605-01` and `What's new (2026-05)` is `U-20260525-01`. + +Compatibility changes are still recorded in [CHANGELOG.md](CHANGELOG.md); open work +is in [Progress.md](Progress.md). The older translated notes (through 2026-08-20) are +[README/WHATS_NEW_zh-TW.md](README/WHATS_NEW_zh-TW.md) and +[README/WHATS_NEW_zh-CN.md](README/WHATS_NEW_zh-CN.md). diff --git a/architecture.md b/architecture.md new file mode 100644 index 00000000..b97e7e46 --- /dev/null +++ b/architecture.md @@ -0,0 +1,198 @@ +# AutoControl Architecture + +> Short overview for people and agents. Per-module detail lives in [`architecture_explore.md`](architecture_explore.md). +> Last verified: 2026-09-22 against `0e8e25b` on `feat/coverage-to-80`. + +## 1. Purpose + +AutoControl (`je_auto_control`) is a cross-platform GUI automation framework: mouse and keyboard control, +screen capture, image recognition, OCR, accessibility-tree lookup, action scripting and report generation +behind one headless Python API. Every feature is also reachable from JSON action files (`AC_*` commands), +a CLI, TCP / REST / MCP servers, a pytest plugin and an optional PySide6 GUI. Backends cover Windows, +macOS, Linux X11, Linux Wayland, Android and iOS. + +## 2. Layers and directories + +Each layer calls only the one below it: +entry points → execution core (`utils/executor/`) → headless capabilities (`utils/`) → `wrapper/` → one OS backend. + +| Path | Responsibility | +| --- | --- | +| `je_auto_control/__init__.py` | Facade: re-exports the public API and lists it in `__all__`. Must import without PySide6. | +| `je_auto_control/api/` | Small versioned headless facade (`core.py`); the supported entry for new integrations per `docs/API_LIFECYCLE.md`. | +| `je_auto_control/cli.py`, `__main__.py` | Main CLI and the legacy argparse entry point. | +| `je_auto_control/utils/executor/` | Execution core: `Executor.event_dict` (`AC_*` name → callable) in `action_executor.py`, block commands in `flow_control.py`, validation in `action_schema.py`. | +| `je_auto_control/utils/` | Headless capability layer, one subpackage per feature, zero Qt imports. Grouped by theme in `architecture_explore.md` §5.4. | +| `je_auto_control/utils/{socket_server,rest_api,mcp_server,pytest_plugin}/` | Server and integration surfaces (§3). | +| `je_auto_control/utils/{remote_desktop,usb,usbip}/` | Remote desktop (TCP / WebSocket / WebRTC) and USB passthrough. | +| `je_auto_control/utils/webrunner_bridge/` | Optional bridge that runs WebRunner `WR_*` commands (§6). | +| `je_auto_control/wrapper/` | Platform-neutral API (`auto_control_mouse/keyboard/screen/image/record/window.py`); `platform_wrapper.py` picks the backend; `backend_contract.py` types the seam; `window_backends/`. | +| `je_auto_control/{windows,osx,linux_with_x11,linux_wayland}/` | Desktop OS backends; only the running OS's backend is imported. | +| `je_auto_control/{android,ios}/` | Mobile device control (adb / uiautomator2, WebDriverAgent). | +| `je_auto_control/gui/` | Optional PySide6 GUI (`[gui]` extra): `main_window.py`, tab registry `main_widget.py`, `script_builder/`, `remote_desktop/`, `language_wrapper/`. | +| `autocontrol-lsp/` | Separate distribution: language server for `AC_*` action JSON, plus a `vscode/` client. | +| `test/` | `unit_test/headless/` (CI gate), `unit_test/flow_control/`, `integrated_test/`, `gui_test/`, `manual_test/`, `verify/`. | +| `docs/` | Sphinx docs, `API_LIFECYCLE.md`, `CAPABILITY_MATRIX.md`. | +| `examples/`, `benchmarks/` | Runnable example scripts; latency smoke benchmark. | +| `docker/`, `k8s/helm/`, `ci_templates/` | Container images and backend verification harnesses, Helm chart, GitLab CI template. | +| `browser-extension/`, `AutoControl/`, `exe/`, `autocontrol_driver/` | Manifest v3 companion extension, project-template sample, packaged GUI launcher, driver build script. | + +## 3. Entry points and public interfaces + +| Surface | Exact name | Notes | +| --- | --- | --- | +| Python facade | `import je_auto_control` | Broad historical surface (`__all__`). | +| Stable API | `je_auto_control.api` → `api/core.py` | `execute_action`, `execute_action_with_vars`, `generate_code`, `run_diagnostics`, `create_failure_bundle`, `failure_bundle_on_error`, `FailureBundleOptions`. | +| Main CLI | `je_auto_control` → `je_auto_control.cli:main` | Subcommands `run` (`--var`, `--dry-run`), `validate` / `lint`, `fmt`, `list-commands`, `record`, `codegen`, `failure-bundle`, `list-jobs`, `start-server`, `start-rest`, `version`. | +| Legacy CLI | `python -m je_auto_control` (`__main__.py`) | `-e/--execute_file FILE`, `-d/--execute_dir DIR`, `-c/--create_project PATH`, `--execute_str JSON`. `--execute_str` also accepts a double-encoded JSON string. Any error exits 1 with a log line rather than a traceback, and a `-d` path that is not a directory is an error. | +| MCP server | `je_auto_control_mcp` → `utils/mcp_server/__main__.py:main` | stdio; `start_mcp_stdio_server()`; HTTP transport via the `AC_start_mcp_http_server` command. | +| REST API | `je_auto_control start-rest`, `python -m je_auto_control.utils.rest_api`, `start_rest_api_server()` | Default `127.0.0.1:9939`, bearer token + rate limit. | +| TCP server | `je_auto_control start-server`, `start_autocontrol_socket_server()` | `utils/socket_server/auto_control_socket_server.py`, default `127.0.0.1:9938`, JSON action lists. | +| pytest plugin | `pytest11` entry point `je_auto_control.utils.pytest_plugin.plugin` | Loaded automatically once the package is installed (see coverage rule in §7). | +| LSP | `autocontrol-lsp` → `autocontrol_lsp.server.server:run`; `python -m autocontrol_lsp.server` | Command list is read from the live executor. | +| GUI | `start_autocontrol_gui()` in `gui/__init__.py`; `exe/start_autocontrol_gui.py` | Needs `pip install je_auto_control[gui]`; PySide6 is imported only under `gui/`. | +| Action lint | `python -m je_auto_control.utils.action_lint` | Used by `.github/workflows/action-json-lint.yml`. | + +## 4. Main flows + +**A. JSON action script (primary path)** + +``` +action.json → utils/json/json_file.read_action_json + → Executor.execute_action (utils/executor/action_executor.py) + → action_schema.validate_actions (unknown AC_* names rejected before anything runs) + → _execute_event → flow_control.BLOCK_COMMANDS (AC_loop / AC_if_* / AC_try / AC_retry …) + | event_dict[name](**args) (${var} / ${secrets.*} interpolated via utils/script_vars) + → wrapper/auto_control_*.py → wrapper/platform_wrapper.py → windows/ | osx/ | linux_with_x11/ | linux_wayland/ + → record dict {"execute: [...]": result or repr(error)} +``` + +Errors of the `AutoControlException` family are recorded, not raised (unless `raise_on_error=True`); +`AutoControlAssertionException` always propagates. Every path that runs an action file from disk (`execute_files`, +the CLI, the scheduler, triggers, hotkeys, webhooks, the MCP run tool, the GUI) loads it with +`read_executable_action_json`, which reads it once and verifies those bytes against the `.sig` sidecar when +`JE_AUTOCONTROL_REQUIRE_SIGNED_ACTIONS` is set (`utils/action_signing/`). + +**B. Remote and external drivers** all feed the single global `executor`: + +``` +TCP socket_server | REST rest_api | MCP mcp_server | utils/scheduler | utils/triggers | utils/chatops + → execute_action → same Executor instance → flow A +AC_web_run / AC_web_run_actions → utils/webrunner_bridge/bridge.py + → je_web_runner.utils.executor.action_executor.executor.event_dict["WR_*"] +``` + +**C. Record → edit → generate code** + +``` +wrapper/auto_control_record.record → OS listener (e.g. windows/record/win32_input_hook.py) + → stop_record / record_to_json → action list + → utils/recording_edit (trim / filter / rescale) | utils/semantic_recording (anchors for cross-machine replay) + → utils/codegen (pytest / python / robot) +``` + +## 5. Extension points + +**New feature or `AC_*` command**, in this order (CLAUDE.md › Feature Delivery Rules): + +1. Headless implementation in `je_auto_control/utils//` (or `wrapper/`); no PySide6; optional deps imported lazily. +2. Re-export public names in `je_auto_control/__init__.py` and its `__all__`. +3. Register the `AC_*` name in `Executor.event_dict` (`utils/executor/action_executor.py`); commands with nested + action bodies go in `BLOCK_COMMANDS` (`utils/executor/flow_control.py`). +4. Describe its parameters in `gui/script_builder/command_schema.py` (Script Builder form). +5. Optional MCP tool: factory in `utils/mcp_server/tools/_factories.py`, adapter in the themed handler module — + `_handlers_input.py`, `_handlers_screen.py`, `_handlers_system.py`, `_handlers_runs.py`, + `_handlers_scheduling.py`, `_handlers_remote.py`, `_handlers_locators.py`, `_handlers_operations.py`, + `_handlers_qa.py`, `_handlers_executor_bridge.py` (a three-line delegation to an executor function), or + `_handlers.py` for data, text and the WebRunner bridge. +6. GUI: thin widget in `gui/`, registered in `gui/main_widget.py` (`_add_tab`) with commands exposed through + `menu_actions()`; strings in every `gui/language_wrapper/*.py` catalogue. +7. Headless test in `test/unit_test/headless/`. +8. Update `architecture_explore.md` (and `README.md` + `README/` translations if a quoted count changes), then run + `python test/unit_test/headless/test_doc_line_counts.py --fix`. Regenerate the typed stub with + `python -m je_auto_control.utils.stubs.generator je_auto_control/actions.pyi`. + +**Other seams** + +- Runtime commands without touching core: `add_command_to_executor({"AC_x": fn})`, `utils/plugin_loader/` + (directory scan) or `utils/plugin_sdk/` (package entry points). +- New OS backend: package `je_auto_control//` + assembly module `wrapper/_platform_.py` satisfying + `wrapper/backend_contract.py` + one branch in `wrapper/platform_wrapper.py`; window management in `wrapper/window_backends/`. +- New accessibility / OCR / vision / llm / agent / hotkey backend: implement the base class in that subpackage's + `backends/` directory; a null fallback keeps imports dependency-free. +- New report format: add a generator beside `utils/generate_report/generate_{html,json,xml}_report.py`. + +## 6. Cross-project boundaries + +| Consumer | How it uses this repo | What it relies on | +| --- | --- | --- | +| Jeffrey_RPA | Editable install of **this working tree**: uncommitted changes here reach it immediately. Single facade `JeffreyRPA/_gui_control.py`. | Top-level names (e.g. `click_mouse`, `hotkey`, `write`, `screen_size`, `get_pixel`, `post_click_to_window`) and internal paths `je_auto_control.wrapper.auto_control_window`, `je_auto_control.wrapper.auto_control_keyboard.WRITE_CONTROL_KEYS`, `je_auto_control.utils.monitor_layout` (`logical_virtual_rect`, `enumerate_monitors`), and `wrapper.platform_wrapper.keyboard_keys_table` / `mouse_keys_table` — it validates every key name a user types against the keyboard table and reverse-looks-up recorded virtual keys through it, so a name removed there becomes a rejected hotkey over in that repo. | +| PyBreeze | Subprocess `python -m je_auto_control --execute_str ` / `--execute_file `; on Windows the JSON string arrives double-encoded. | Legacy CLI flags; also embeds `je_auto_control.gui.main_widget.AutoControlGUIWidget` and calls `record` / `stop_record` in-process. | +| TestPioneer | Optional extra `gui = ["je_auto_control"]`; `parallel_run` starts `python -m je_auto_control --execute_file `. | `execute_action`, `execute_files`, `RecordingThread`; the `--execute_file` flag. | + +**Guarded by** `test/unit_test/headless/test_cross_project_contracts.py`: every legacy CLI flag (short and long, run as a +real child process, including PyBreeze's double-encoded `--execute_str`), the facade names in the three rows above +(Jeffrey_RPA's list is every `ac.` in `_gui_control.py`), the `auto_control_window` functions Jeffrey_RPA calls, +its three internal imports, the two key tables (shape everywhere, Windows key names on Windows), and +`AutoControlGUIWidget`. The test only knows what this table knows: when a consumer starts relying on something +new, add it to both. + +**Outbound (optional):** `utils/webrunner_bridge/bridge.py` imports WebRunner's *internal* +`je_web_runner.utils.executor.action_executor.executor` lazily, for `AC_web_*` commands and `gui/webrunner_tab.py`. +`je_web_runner` is not a declared dependency; when it is missing the bridge raises `WebRunnerBridgeError`. +Moving that WebRunner module breaks the bridge. + +**Import-time contracts** + +- `import je_auto_control` must not load PySide6; the GUI window is imported only inside `start_autocontrol_gui()`. + `test/unit_test/headless/test_facade_import_is_light.py` also keeps `cv2`, `numpy`, `PIL`, `cryptography`, + `je_open_cv` and `mss` off the import path. +- `utils/logging/logging_instance.py` attaches a file handler at import and sets the level of its own logger only + (never the root logger — that would push every third-party library's DEBUG records into the host's handlers). The + handler opens its file on the first record, and nothing logs during the import: importing writes no file at all. + The file is `$JE_AUTOCONTROL_LOG_FILE` as read when the file is opened (so a `conftest.py` can still set it after + the `pytest11` plugin imported the package; a relative path resolves against the cwd then), else + `~/.je_auto_control/logs/AutoControlGUI.log`, shared by every process: appended to, rotated to `.1` past 10 MB + only when a process opens it, and swapped for `os.devnull` with one `RuntimeWarning` when it cannot be opened. + Consumers that must keep the log out of a shared file (a test suite) set the variable before importing; + changing the working directory no longer redirects it. +- No module reads the home directory at import; every `~/.je_auto_control/` path is resolved when used, so a + consumer can redirect `HOME` / `USERPROFILE` after the import (this repo's `test/conftest.py` gives each test run + a temporary home). `test/unit_test/headless/test_state_paths_follow_home.py` scans the package for violations. + +**De-facto public:** `docs/API_LIFECYCLE.md` calls `je_auto_control.utils.*` internal, but the internal paths in the +table above are used by sibling repos; treat renaming or removing them as a breaking change and check the consumers +first. `AC_*` command names and the legacy CLI flags are public too (action files live outside this repo). + +## 7. Design constraints + +- Every feature ships a headless API, a facade export, an `AC_*` command and a thin GUI tab whose commands live in the + Actions menu (enforced by `test_actions_menu_gui.py`). → CLAUDE.md › Feature Delivery Rules › Every feature ships both a headless API and a GUI surface +- The top-level package stays Qt-free. → same section +- `architecture_explore.md` changes in the same commit as the code; counts are measured, never estimated; + `test_doc_counts.py` and `test_doc_line_counts.py` fail CI on drift. → CLAUDE.md › Feature Delivery Rules › `architecture_explore.md` is updated with every change +- Agreed-but-unfinished work is recorded in `Progress.md` (open items only). → CLAUDE.md › Feature Delivery Rules › Outstanding work goes in `Progress.md` +- Flat exception hierarchy: every framework error derives from `AutoControlException`; assertion failures keep + propagating. → CLAUDE.md › Coding Standards › Project-specific rules +- Validate at boundaries and reject unknown command names; servers bind `127.0.0.1` unless explicitly opted in. → same +- No `print()` or runtime `assert` in library code; lazy imports for optional and platform deps; release platform + resources in `finally` / `with`; guard shared state with locks or queues; pin dependency versions. → same +- Size limits (cyclomatic ≤ 10, cognitive ≤ 15, function ≤ 75 lines, file ≤ 750 lines, line ≤ 120) are a review + standard; grandfathered over-limit files are listed in `Progress.md`. → CLAUDE.md › Coding Standards › Size and complexity limits +- Run ruff, pylint, bandit and radon before committing; every suppression carries an inline reason. → CLAUDE.md › Coding Standards › Automated verification +- Measure coverage with `python -m coverage run -m pytest`, never `pytest --cov` (the pytest11 plugin imports the + facade first), with the `[webrtc]` extra installed; never loosen `python_files = ["test_*.py"]`. → CLAUDE.md › Development Commands +- Tests cover the headless path, avoid sleeps over 1 s, are order-independent, and keep the Qt `deleteLater()` + flush fixture in `test/unit_test/headless/conftest.py`. → CLAUDE.md › Testing +- Commit messages are imperative and explain why; attribution rules apply. → CLAUDE.md › Commit Conventions + +## 8. When to update this file + +- A top-level package or directory in §2 is added, removed or renamed, or an OS backend is added or dropped. +- An entry point changes: console script, `python -m` module, server surface, pytest or LSP plugin, legacy CLI flag. +- The executor contract changes (action shape, validation, error containment, signing) or the layer order in §2. +- A new extension mechanism appears or the ordered steps in §5 change. +- A sibling repo starts or stops depending on this one, starts using another internal path, the WebRunner bridge + target moves, or the log file name or location changes. +- A hard rule in CLAUDE.md is added or changed. +- On every edit, refresh the "Last verified" line. Module-level changes belong in `architecture_explore.md`, not here. diff --git a/architecture_explore.md b/architecture_explore.md index b3438b8a..72e9e86d 100644 --- a/architecture_explore.md +++ b/architecture_explore.md @@ -6,7 +6,7 @@ > 擷取每個模組的 docstring 與頂層公開名稱；統計數字取自實際檔案，非估算。 > 指令數與公開 API 數以 `executor.known_commands()` 與 `je_auto_control.__all__` 在工作樹上實測取得。 > -> **掃描時間**：2026-08-21　**版本**：`pyproject.toml` version `0.0.220`　**分支**：`feat/typing-contract-and-coverage-ratchet` +> **掃描時間**：2026-09-22　**版本**：`pyproject.toml` version `0.0.221`　**分支**：`feat/coverage-to-80` --- @@ -19,11 +19,11 @@ iOS（WebDriverAgent）。核心能力是滑鼠／鍵盤控制、影像辨識、 | 指標 | 數值 | | --- | ---: | -| Python 模組總數（含周邊子專案） | 1,032 | -| 程式碼總行數 | 141,316 | +| Python 模組總數（含周邊子專案） | 1,049 | +| 程式碼總行數 | 146,823 | | `je_auto_control/utils/` 子套件數 | 310 | | `AC_*` 動作指令數（`known_commands()` 實測） | 773 | -| 套件門面 `__all__` 公開名稱數 | 1,238 | +| 套件門面 `__all__` 公開名稱數 | 1,241 | | GUI 分頁數（`main_widget` 註冊） | 48 | | MCP 工具數（`build_default_tool_registry()` 實測） | 676 | | `test_*.py` 測試檔／測試函式 | 478 / 4,654 | @@ -119,7 +119,9 @@ action.json ─► utils/json/json_file.read_action_json 錯誤處理原則：`AutoControlException` 家族在此被「收納」成紀錄而非中止整份腳本； 但 `AutoControlAssertionException`（`AC_assert_*` 失敗）即使在 `raise_on_error=False` 下仍會往上拋， -確保斷言不會被靜默吃掉。`execute_files` 會先呼叫 `require_signed_actions` 驗簽。 +確保斷言不會被靜默吃掉。從磁碟執行動作檔的路徑（`execute_files`、CLI、排程器、觸發器、熱鍵、 +webhook、MCP 執行工具、GUI）都經 `read_executable_action_json` 載入：檔案只讀一次， +設定 `JE_AUTOCONTROL_REQUIRE_SIGNED_ACTIONS` 時就以這份位元組對 `.sig` 驗簽，再解析同一份位元組。 **B. 錄製 → 重播 → 產碼** @@ -153,13 +155,14 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 | 模組 | 行數 | 職責 | | --- | ---: | --- | | `je_auto_control/__init__.py` | 1,970 | **套件門面**。集中匯入並再匯出 1,200 個公開名稱，以功能區塊註解分段（callback／exception／executor／a11y／vision／clipboard…）。 | -| `je_auto_control/__main__.py` | 74 | 舊版 argparse 進入點：`-e` 執行單檔、`-d` 執行整個目錄、`--execute_str` 執行 JSON 字串、`-c` 建立專案。 | -| `je_auto_control/cli.py` | 323 | **主 CLI**（`je_auto_control` console script）。子命令：`run`（含 `--var`／`--dry-run`）、`validate`／`lint`、`list-commands`、`fmt`、`record`、`codegen`、`failure-bundle`、`list-jobs`、`start-server`、`start-rest`、`version`。所有子命令延遲匯入，確保不碰 Qt。 | +| `je_auto_control/__main__.py` | 87 | 舊版 argparse 進入點：`-e` 執行單檔、`-d` 執行整個目錄、`--execute_str` 執行 JSON 字串、`-c` 建立專案。 | +| `je_auto_control/cli.py` | 338 | **主 CLI**（`je_auto_control` console script）。子命令：`run`（含 `--var`／`--dry-run`）、`validate`／`lint`、`list-commands`、`fmt`、`record`、`codegen`、`failure-bundle`、`list-jobs`、`start-server`、`start-rest`、`version`。所有子命令延遲匯入，確保不碰 Qt。 | | `je_auto_control/api/__init__.py` | 22 | 版本化整合進入點。 | | `je_auto_control/api/core.py` | 19 | **穩定無頭 API 門面**：只暴露 `execute_action`、`execute_action_with_vars`、`generate_code`、`run_diagnostics`、`create_failure_bundle`、`failure_bundle_on_error`、`FailureBundleOptions`。mypy 型別契約以此為起點，現已擴到整包（見「設定基線」）。 | | `je_auto_control/utils/deprecation.py` | 35 | 公開 API 的一致性棄用警告。 | -| `je_auto_control/utils/http_headers.py` | 45 | 入站 HTTP 標頭的共用防禦式解析。 | -| `je_auto_control/utils/sqlite_support.py` | 70 | 選用標準函式庫 `sqlite3` 的取用點：`require_sqlite3()`／`sqlite3_available()`／`SQLITE_ERRORS`。十個以 SQLite 存放狀態的子系統都經由這裡，所以 FreeBSD 這種把 `sqlite3` 另外包成 `databases/py-sqlite3` 的 Python 仍然 import 得起門面。 | +| `je_auto_control/utils/http_headers.py` | 115 | 入站 HTTP 標頭與 chunked 內文的共用防禦式解析。 | +| `je_auto_control/utils/sqlite_support.py` | 112 | 選用標準函式庫 `sqlite3` 的取用點：`require_sqlite3()`／`sqlite3_available()`／`SQLITE_ERRORS`。十個以 SQLite 存放狀態的子系統都經由這裡，所以 FreeBSD 這種把 `sqlite3` 另外包成 `databases/py-sqlite3` 的 Python 仍然 import 得起門面。 | +| `je_auto_control/utils/timeouts.py` | 33 | 把使用者給的逾時換成截止時間：`deadline_after()` 拒絕 NaN（`json` 接受它，而 `clock() >= NaN` 永遠不成立，輪詢迴圈會永遠跑下去），負值與無限大維持原意；`clamp_poll_interval()` 把背景迴圈的輪詢間隔夾在 0.05 秒到 1 小時之間（`Event.wait(inf)` 在 Windows 會丟 `OverflowError`）。 | ### 5.2 wrapper 抽象層 @@ -169,47 +172,47 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 | --- | ---: | --- | | `wrapper/platform_wrapper.py` | 116 | **Strategy 樞紐**。依 `sys.platform` 匯入唯一後端並匯出 `keyboard`、`keyboard_check`、`keyboard_keys_table`、`mouse`、`mouse_keys_table`、`special_mouse_keys_table`、`screen`、`recorder`；八個名稱都帶著 `backend_contract` 的型別出去，其中 `keyboard`／`mouse` 因為四個分支綁的是三種互不相容的形狀，先落在私有的 `_keyboard`／`_mouse`（`Any`）上再標合約；載入失敗直接拋 `AutoControlException`（fail fast）。 | | `wrapper/backend_contract.py` | 238 | 平台縫的型別合約：`ScreenBackend`／`KeyboardCheckBackend`／`RecorderBackend` 三個跨平台 Protocol，加上 `keyboard`／`mouse` 各自的三份——`Win32*`（SendInput 與 Interception）、`Darwin*`（Quartz）、`X11Unix*`（XTest／uinput／Wayland／BSD），因為這兩個名稱的呼叫形狀真的因平台而異；`KeyboardBackend`／`MouseBackend` 依 `sys.platform` 別名到其中一組，所以呼叫端被檢查的是它真的會走到的簽章。四個 `_platform_*` 組裝模組各自標注自己綁的是什麼，少一個成員就在該後端自己的檔案裡紅掉，而不是在三層之上的呼叫點。 | -| `wrapper/_platform_windows.py` | 334 | Windows 後端組裝：Win32 ctypes 模組 + 虛擬鍵表 + 選用 Interception 驅動。 | -| `wrapper/_platform_osx.py` | 160 | macOS 後端組裝（Quartz 事件 + osx 虛擬鍵表）。 | -| `wrapper/_platform_linux.py` | 278 | X11 後端組裝（python-Xlib + 選用 uinput）。 | -| `wrapper/_platform_wayland.py` | 61 | Wayland 後端組裝（libei／ydotool／grim）。 | -| `wrapper/auto_control_mouse.py` | 451 | 滑鼠 API：位置讀寫、按下／放開／點擊、捲動、座標前處理、送訊息給指定視窗。 | -| `wrapper/auto_control_keyboard.py` | 304 | 鍵盤 API：鍵表查詢、按下／放開／敲擊、`write` 字串、`hotkey` 組合鍵、按鍵狀態偵測。 | +| `wrapper/_platform_windows.py` | 318 | Windows 後端組裝：Win32 ctypes 模組 + 虛擬鍵表 + 選用 Interception 驅動。 | +| `wrapper/_platform_osx.py` | 157 | macOS 後端組裝（Quartz 事件 + osx 虛擬鍵表）。 | +| `wrapper/_platform_linux.py` | 275 | X11 後端組裝（python-Xlib + 選用 uinput）。 | +| `wrapper/_platform_wayland.py` | 58 | Wayland 後端組裝（libei／ydotool／grim）。 | +| `wrapper/auto_control_mouse.py` | 491 | 滑鼠 API：位置讀寫、按下／放開／點擊、捲動、座標前處理、送訊息給指定視窗。 | +| `wrapper/auto_control_keyboard.py` | 368 | 鍵盤 API：鍵表查詢、按下／放開／敲擊、`write` 字串、`hotkey` 組合鍵、按鍵狀態偵測。**`type_keyboard` 與 `hotkey` 的放開走 `finally`**（見下）。 | | `wrapper/auto_control_screen.py` | 111 | 螢幕 API：`screen_size`、`screenshot`（可指定區域）、`get_pixel`。 | | `wrapper/auto_control_image.py` | 83 | 影像 API：`locate_all_image`、`locate_image_center`、`locate_and_click`。 | -| `wrapper/auto_control_record.py` | 114 | 錄製 API：`record`／`stop_record`／`record_to_json`（支援 stop event 與逾時）。 | -| `wrapper/auto_control_window.py` | 278 | 視窗管理門面：列舉、尋找、聚焦、等待、關閉、顯示狀態、幾何、所屬行程 PID、依行程列舉／最小化視窗、不搶焦點的投遞式輸入（目前僅 Windows 實作）。 | -| `wrapper/window_backends/` | 985 | 視窗管理的平台縫（`base` / `windows_backend` / `x11_backend` / `macos_backend` / `null_backend`）。放在 `wrapper/` 而不是 `utils/`，因為它必須 import `windows/`、`linux_with_x11/`、`osx/`，而 `utils/` 在分層上在那三者之上。 | +| `wrapper/auto_control_record.py` | 124 | 錄製 API：`record`／`stop_record`／`record_to_json`（支援 stop event 與逾時）。 | +| `wrapper/auto_control_window.py` | 287 | 視窗管理門面：列舉、尋找、聚焦、等待、關閉、顯示狀態、幾何、所屬行程 PID、依行程列舉／最小化視窗、不搶焦點的投遞式輸入（目前僅 Windows 實作）。 | +| `wrapper/window_backends/` | 988 | 視窗管理的平台縫（`base` / `windows_backend` / `x11_backend` / `macos_backend` / `null_backend`）。放在 `wrapper/` 而不是 `utils/`，因為它必須 import `windows/`、`linux_with_x11/`、`osx/`，而 `utils/` 在分層上在那三者之上。 | ### 5.3 平台後端 -#### Windows（`windows/`，23 檔／1,906 行） +#### Windows（`windows/`，23 檔／1,957 行） | 模組 | 行數 | 職責 | | --- | ---: | --- | | `core/utils/win32_ctype_input.py` | 73 | `SendInput` 的 ctypes 結構定義與送出。 | | `core/utils/win32_vk.py` | 188 | Windows 虛擬鍵碼對照表。 | -| `core/utils/win32_keypress_check.py` | 21 | `GetAsyncKeyState` 按鍵狀態查詢。 | +| `core/utils/win32_keypress_check.py` | 22 | `GetAsyncKeyState` 按鍵狀態查詢。 | | `mouse/win32_ctype_mouse_control.py` | 220 | 滑鼠事件產生（含多螢幕絕對座標換算）。 | -| `keyboard/win32_ctype_keyboard_control.py` | 55 | 鍵盤事件產生。 | -| `record/win32_input_hook.py` | 207 | 單一一組低階鍵鼠 hook（`WH_KEYBOARD_LL`／`WH_MOUSE_LL`）＋訊息迴圈，產生帶時間戳的事件時間軸；停止時以 `PostThreadMessageW(WM_QUIT)` 收掉執行緒，不會每錄一次就漏一條。 | +| `keyboard/win32_ctype_keyboard_control.py` | 98 | 鍵盤事件產生。 | +| `record/win32_input_hook.py` | 253 | 單一一組低階鍵鼠 hook（`WH_KEYBOARD_LL`／`WH_MOUSE_LL`）＋訊息迴圈，產生帶時間戳的事件時間軸；停止時以 `PostThreadMessageW(WM_QUIT)` 收掉執行緒，不會每錄一次就漏一條。 | | `record/win32_record.py` | 41 | 把 `win32_input_hook` 的時間軸轉成 action list（含按鍵放開、滾輪與間隔）；整形本體與 macOS 共用 `utils/input_macro/recorder_base.py`。 | -| `screen/win32_screen.py` | 89 | 螢幕尺寸與像素讀取。**每支 Win32 函式都明寫 argtypes/restype**（HDC 是指標寬度，走預設的 c_int 會截斷，錯誤會沉默地擴散到 GetPixel／ReleaseDC），並持有自己的 user32／gdi32 handle。import 時呼叫 `SetProcessDPIAware()`——**行程層級且不可還原**，實體↔邏輯座標換算請走 `utils/monitor_layout`。 | -| `window/windows_window_manage.py` | 366 | 視窗列舉／聚焦／關閉／最小化／幾何／所屬行程 PID／投遞式輸入（`auto_control_window` 的實作）。**每支 Win32 函式都明寫 argtypes/restype**，並持有自己的 user32 handle，避免把原型外溢到別的模組；hwnd 一律是 int。 | +| `screen/win32_screen.py` | 95 | 螢幕尺寸與像素讀取。**每支 Win32 函式都明寫 argtypes/restype**（HDC 是指標寬度，走預設的 c_int 會截斷，錯誤會沉默地擴散到 GetPixel／ReleaseDC），並持有自己的 user32／gdi32 handle。import 時呼叫 `SetProcessDPIAware()`——**行程層級且不可還原**，實體↔邏輯座標換算請走 `utils/monitor_layout`。 | +| `window/windows_window_manage.py` | 374 | 視窗列舉／聚焦／關閉／最小化／幾何／所屬行程 PID／投遞式輸入（`auto_control_window` 的實作）。**每支 Win32 函式都明寫 argtypes/restype**，並持有自己的 user32 handle，避免把原型外溢到別的模組；hwnd 一律是 int。 | | `message/window_message.py` | 97 | 直接對視窗送 `WM_*` 訊息（背景輸入）。 | -| `interception/_dll.py` | 231 | `interception.dll` 的延遲 ctypes 載入與結構定義。 | -| `interception/keyboard.py` | 71 | 經 Interception 驅動的鍵盤輸入（繞過部分反自動化偵測）。 | -| `interception/mouse.py` | 161 | 經 Interception 驅動的滑鼠輸入。 | +| `interception/_dll.py` | 230 | `interception.dll` 的延遲 ctypes 載入與結構定義。 | +| `interception/keyboard.py` | 70 | 經 Interception 驅動的鍵盤輸入（繞過部分反自動化偵測）。 | +| `interception/mouse.py` | 160 | 經 Interception 驅動的滑鼠輸入。 | -#### macOS（`osx/`，17 檔／915 行） +#### macOS（`osx/`，17 檔／919 行） | 模組 | 行數 | 職責 | | --- | ---: | --- | -| `core/utils/osx_vk.py` | 114 | macOS 虛擬鍵碼表。 | +| `core/utils/osx_vk.py` | 113 | macOS 虛擬鍵碼表。 | | `mouse/osx_mouse.py` | 137 | Quartz `CGEvent` 滑鼠事件。 | -| `keyboard/osx_keyboard.py` | 129 | Quartz 鍵盤事件。 | +| `keyboard/osx_keyboard.py` | 137 | Quartz 鍵盤事件。 | | `keyboard/osx_keyboard_check.py` | 24 | 按鍵狀態查詢。 | -| `listener/osx_listener.py` | 253 | 專屬執行緒上的 listen-only `CGEventTap`＋自己的 `CFRunLoopRunInMode` 切片；不在 import 時建 `NSApplication`，也不用會卡住呼叫緒的 `AppHelper.runEventLoop()`。修飾鍵由 `flagsChanged` 的旗標還原成 press／release，座標取 `CGEventGetLocation`（左上原點，與重播送出的座標同一空間）。 | +| `listener/osx_listener.py` | 257 | 專屬執行緒上的 listen-only `CGEventTap`＋自己的 `CFRunLoopRunInMode` 切片；不在 import 時建 `NSApplication`，也不用會卡住呼叫緒的 `AppHelper.runEventLoop()`。修飾鍵由 `flagsChanged` 的旗標還原成 press／release，座標取 `CGEventGetLocation`（左上原點，與重播送出的座標同一空間）。 | | `record/osx_record.py` | 41 | 錄製。捕捉後的整形（舊版按下事件 Queue、時間軸、只錄滑鼠／只錄鍵盤）走共用的 `utils/input_macro/recorder_base.py`。 | | `screen/osx_screen.py` | 143 | 螢幕擷取與尺寸（含 Retina 座標處理）。 | | `pid/pid_control.py` | 64 | 以 PID 操作應用程式。 | @@ -218,16 +221,16 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 | 模組 | 行數 | 職責 | | --- | ---: | --- | -| `core/utils/x11_linux_display.py` | 14 | 共用 `Xlib.display.Display` 實例。 | -| `core/utils/x11_linux_vk.py` | 197 | X11 keysym 對照表。 | -| `mouse/x11_linux_mouse_control.py` | 133 | XTest 滑鼠事件。 | -| `keyboard/x11_linux_keyboard_control.py` | 85 | XTest 鍵盤事件。 | -| `listener/x11_linux_listener.py` | 195 | XRecord 監聽。 | -| `record/x11_linux_record.py` | 73 | 錄製。 | -| `screen/x11_linux_screen.py` | 62 | 螢幕尺寸與擷取。 | -| `uinput/_device.py` | 234 | `/dev/uinput` 封裝（核心層輸入，選用）。 | -| `uinput/keyboard.py` | 33 | uinput 鍵盤後端，介面與 X11 版一致。 | -| `uinput/mouse.py` | 116 | uinput 滑鼠後端。 | +| `core/utils/x11_linux_display.py` | 16 | 共用 `Xlib.display.Display` 實例。 | +| `core/utils/x11_linux_vk.py` | 199 | X11 keysym 對照表。 | +| `mouse/x11_linux_mouse_control.py` | 155 | XTest 滑鼠事件。 | +| `keyboard/x11_linux_keyboard_control.py` | 88 | XTest 鍵盤事件。 | +| `listener/x11_linux_listener.py` | 208 | XRecord 監聽。 | +| `record/x11_linux_record.py` | 76 | 錄製。 | +| `screen/x11_linux_screen.py` | 65 | 螢幕尺寸與擷取。 | +| `uinput/_device.py` | 244 | `/dev/uinput` 封裝（核心層輸入，選用）。 | +| `uinput/keyboard.py` | 32 | uinput 鍵盤後端，介面與 X11 版一致。 | +| `uinput/mouse.py` | 115 | uinput 滑鼠後端。 | #### Linux Wayland（`linux_wayland/`，17 檔／2,870 行） @@ -253,13 +256,13 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 | 模組 | 行數 | 職責 | | --- | ---: | --- | -| `android/adb_client.py` | 183 | `adb` CLI 的薄封裝。 | -| `android/client.py` | 91 | `uiautomator2.Device` 的延遲封裝。 | -| `android/find.py` | 104 | uiautomator2 widget 樹的元素查詢。 | -| `ios/client.py` | 94 | `facebook-wda`（WebDriverAgent）封裝。 | -| `ios/find.py` | 86 | XCUITest 無障礙查詢。 | -| `ios/input.py` | 46 | iOS 觸控與按鍵原語。 | -| `ios/screen.py` | 32 | iOS 裝置螢幕擷取與尺寸。 | +| `android/adb_client.py` | 197 | `adb` CLI 的薄封裝。 | +| `android/client.py` | 127 | `uiautomator2.Device` 的延遲封裝。 | +| `android/find.py` | 107 | uiautomator2 widget 樹的元素查詢。 | +| `ios/client.py` | 122 | `facebook-wda`（WebDriverAgent）封裝。 | +| `ios/find.py` | 93 | XCUITest 無障礙查詢。 | +| `ios/input.py` | 51 | iOS 觸控與按鍵原語。 | +| `ios/screen.py` | 34 | iOS 裝置螢幕擷取與尺寸。 | ### 5.4 能力層 `utils/`（310 個子套件） @@ -268,77 +271,77 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 ### 5.4.1 執行引擎與腳本資產 -> 24 個套件、約 12,926 行。 +> 24 個套件、約 14,072 行。 | 模組 | 行數 | 職責 | | --- | ---: | --- | | `utils/action_lint/` | 328 | action 檔 linter 與 JSON Schema 產生器（CI 用 `python -m` 進入點） | -| `utils/action_signing/` | 248 | action 檔 HMAC-SHA256 簽章與 Fernet 加密，`execute_files` 會強制驗簽 | +| `utils/action_signing/` | 362 | action 檔 HMAC-SHA256 簽章與 Fernet 加密，`execute_files` 會強制驗簽 | | `utils/checkpoint/` | 120 | 流程檢查點與續跑，讓長 action list 具持久性 | -| `utils/codegen/` | 158 | 由 action list 產生可執行的 pytest / python / robot 測試碼 | -| `utils/dag/` | 478 | 跨主機 DAG 編排器（圖模型 + runner） | +| `utils/codegen/` | 255 | 由 action list 產生可執行的 pytest / python / robot 測試碼 | +| `utils/dag/` | 492 | 跨主機 DAG 編排器（圖模型 + runner） | | `utils/decision_table/` | 103 | DMN 風格決策表：規則 + 命中策略，把分支外部化 | | `utils/deterministic/` | 98 | 決定性執行控制：固定亂數種子 + 凍結時鐘 | -| `utils/executor/` | 9,081 | **核心**。`Executor` 指令分派表（773 個 `AC_*`）、參數插值、乾跑、逐步 callback；`flow_control` 提供 34 個區塊指令（迴圈／分支／try／巨集／變數） | +| `utils/executor/` | 9,402 | **核心**。`Executor` 指令分派表（773 個 `AC_*`）、參數插值、乾跑、逐步 callback；`flow_control` 提供 34 個區塊指令（迴圈／分支／try／巨集／變數） | | `utils/flow_debugger/` | 142 | action list 的單步除錯器與追蹤器 | -| `utils/input_macro/` | 342 | 定時輸入事件：錄製結果的整形（`timeline`／`InputRecorder`，Windows 與 macOS 共用）、重播與宣告式輸入序列 DSL | -| `utils/json/` | 74 | action JSON 檔讀寫與正規化格式化（`fmt --check` 的後端） | -| `utils/json_store/` | 61 | JSON 字典檔持久化的共用小工具（內部管線） | -| `utils/loop_guard/` | 140 | 機械式卡死迴圈偵測（agent loop 用） | -| `utils/plugin_loader/` | 85 | 掃描外部 Python 外掛目錄並註冊其 `AC_` callable | -| `utils/plugin_sdk/` | 68 | 外掛 SDK：透過 entry points 發佈／載入第三方 `AC_*` 指令 | +| `utils/input_macro/` | 451 | 定時輸入事件：錄製結果的整形（`timeline`／`InputRecorder`，Windows 與 macOS 共用）、重播與宣告式輸入序列 DSL | +| `utils/json/` | 99 | action JSON 檔讀寫與正規化格式化（`fmt --check` 的後端） | +| `utils/json_store/` | 232 | JSON 字典檔持久化的共用小工具（內部管線） | +| `utils/loop_guard/` | 154 | 機械式卡死迴圈偵測（agent loop 用） | +| `utils/plugin_loader/` | 128 | 掃描外部 Python 外掛目錄並註冊其 `AC_` callable | +| `utils/plugin_sdk/` | 80 | 外掛 SDK：透過 entry points 發佈／載入第三方 `AC_*` 指令 | | `utils/project/` | 186 | 專案腳手架：建立目錄結構與範本 action 檔 | | `utils/recording_edit/` | 150 | 不重錄的前提下裁切／過濾／縮放已錄製的 action list | -| `utils/saga/` | 93 | Saga 協調器：失敗時以 LIFO 補償動作回滾 | +| `utils/saga/` | 100 | Saga 協調器：失敗時以 LIFO 補償動作回滾 | | `utils/script_vars/` | 190 | 執行期變數作用域與 `${var}` / `${secrets.*}` 插值 | -| `utils/skill_library/` | 116 | 具名可重用 action 序列（skill）的持久化倉庫 | -| `utils/state_machine/` | 181 | 宣告式有限狀態機驅動 action JSON | -| `utils/stubs/` | 236 | 為 `AC_*` 指令面產生型別 stub | -| `utils/test_record/` | 66 | 全域測試紀錄單例，記錄每個動作的參數與例外 | -| `utils/work_queue/` | 182 | 交易式工作佇列（dispatcher／performer），支撐大量批次執行 | +| `utils/skill_library/` | 115 | 具名可重用 action 序列（skill）的持久化倉庫 | +| `utils/state_machine/` | 260 | 宣告式有限狀態機驅動 action JSON | +| `utils/stubs/` | 287 | 為 `AC_*` 指令面產生型別 stub | +| `utils/test_record/` | 70 | 全域測試紀錄單例，記錄每個動作的參數與例外 | +| `utils/work_queue/` | 268 | 交易式工作佇列（dispatcher／performer），支撐大量批次執行 | ### 5.4.2 框架基礎設施 -> 14 個套件、約 2,654 行。 +> 14 個套件、約 2,836 行。 | 模組 | 行數 | 職責 | | --- | ---: | --- | -| `utils/callback/` | 200 | Observer 模式：`callback_executor` 以字串名觸發功能，執行後呼叫回呼 | -| `utils/config_bundle/` | 400 | 使用者設定的單檔匯出／匯入 | +| `utils/callback/` | 204 | Observer 模式：`callback_executor` 以字串名觸發功能，執行後呼叫回呼 | +| `utils/config_bundle/` | 424 | 使用者設定的單檔匯出／匯入 | | `utils/critical_exit/` | 98 | 監看緊急停止鍵的守護執行緒，用於中止失控腳本 | | `utils/diagnostics/` | 322 | 跨子系統的「一切正常嗎」健檢，附 `python -m` 進入點 | | `utils/dbus_client/` | 683 | 只用標準函式庫的 D-Bus session bus 客戶端。原本在 `linux_wayland/` 為 portal 交握而寫，AT-SPI 無障礙後端成為第二個使用者後搬到這裡（`utils/` 在分層上在各 OS 套件之上） | -| `utils/exception/` | 210 | **例外階層根**。所有錯誤繼承 `AutoControlException`，加上集中式錯誤訊息字串（`exception_tags`） | -| `utils/failure_bundle/` | 187 | 可攜、已遮蔽的失敗診斷 ZIP（截圖 + 診斷 + log 尾段） | -| `utils/file_process/` | 26 | 目錄檔案列舉（`execute_dir` 的後端） | -| `utils/logging/` | 71 | `autocontrol_logger` 單例 + 輪替檔案 handler | -| `utils/package_manager/` | 98 | 動態載入套件並把 executor 注入其中 | +| `utils/exception/` | 212 | **例外階層根**。所有錯誤繼承 `AutoControlException`，加上集中式錯誤訊息字串（`exception_tags`） | +| `utils/failure_bundle/` | 219 | 可攜、已遮蔽的失敗診斷 ZIP（截圖 + 診斷 + log 尾段） | +| `utils/file_process/` | 40 | 目錄檔案列舉（`execute_dir` 的後端） | +| `utils/logging/` | 161 | `autocontrol_logger` 單例 + 家目錄共用記錄檔 handler（`JE_AUTOCONTROL_LOG_FILE` 可改） | +| `utils/package_manager/` | 101 | 動態載入套件並把 executor 注入其中 | | `utils/path_guard/` | 99 | 命令列傳入路徑的正規化與邊界檢查（防路徑穿越） | | `utils/platform_id/` | 62 | 作業系統家族的單一判定點。`sys.platform` 原本在一百多處跟字面清單比對，而那些清單都沒有 BSD；`is_x11_unix()` 問的是「這是不是 X11 unix」，這才是守衛一直想問的問題 | -| `utils/shell_process/` | 159 | `ShellManager`：以 argv list 執行外部命令（禁用 `shell=True`） | +| `utils/shell_process/` | 172 | `ShellManager`：以 argv list 執行外部命令（禁用 `shell=True`） | | `utils/start_exe/` | 39 | 啟動另一個執行檔行程 | ### 5.4.3 排程、觸發與背景監看 -> 11 個套件、約 3,554 行。 +> 11 個套件、約 3,910 行。 | 模組 | 行數 | 職責 | | --- | ---: | --- | -| `utils/hotkey/` | 727 | 全域熱鍵守護行程，把 OS 層熱鍵綁到 action 檔（Win/macOS/X11 三後端） | -| `utils/idle_keepawake/` | 212 | 偵測使用者閒置時間並在無人值守執行期間阻止系統睡眠 | -| `utils/lock_session/` | 163 | 鎖定工作站、等待解鎖並分類鎖定狀態轉換 | -| `utils/observer/` | 220 | 反應式畫面觀察者，在出現／消失／變化時觸發 | -| `utils/recurrence/` | 324 | RFC 5545 重複規則解析與發生時間展開 | -| `utils/scheduler/` | 352 | 間隔式與 cron 式的 action JSON 排程器 | +| `utils/hotkey/` | 837 | 全域熱鍵守護行程，把 OS 層熱鍵綁到 action 檔（Win/macOS/X11 三後端） | +| `utils/idle_keepawake/` | 216 | 偵測使用者閒置時間並在無人值守執行期間阻止系統睡眠 | +| `utils/lock_session/` | 164 | 鎖定工作站、等待解鎖並分類鎖定狀態轉換 | +| `utils/observer/` | 229 | 反應式畫面觀察者，在出現／消失／變化時觸發 | +| `utils/recurrence/` | 373 | RFC 5545 重複規則解析與發生時間展開 | +| `utils/scheduler/` | 439 | 間隔式與 cron 式的 action JSON 排程器 | | `utils/session_guard/` | 62 | 驅動輸入前先偵測工作階段是否已鎖定／非互動 | -| `utils/triggers/` | 1,152 | 事件驅動觸發引擎：影像／視窗／像素／檔案／webhook／IMAP 郵件 | +| `utils/triggers/` | 1,241 | 事件驅動觸發引擎：影像／視窗／像素／檔案／webhook／IMAP 郵件 | | `utils/voice/` | 87 | 語音指令路由：把辨識到的語句對應到 `AC_*` action list | -| `utils/watchdog/` | 173 | 背景彈窗／中斷看門狗，供無人值守自動化 | +| `utils/watchdog/` | 180 | 背景彈窗／中斷看門狗，供無人值守自動化 | | `utils/watcher/` | 82 | 無頭輪詢原語：滑鼠位置、像素顏色、log tail | ### 5.4.4 輸入模擬與動作品質 -> 22 個套件、約 2,643 行。 +> 22 個套件、約 2,665 行。 | 模組 | 行數 | 職責 | | --- | ---: | --- | @@ -351,67 +354,67 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 | `utils/field_entry/` | 76 | 清空再輸入的欄位填寫慣用法（Playwright `fill`） | | `utils/gamepad/` | 311 | 虛擬遊戲手把後端（Windows ViGEmBus 驅動） | | `utils/humanize/` | 190 | 擬人輸入：貝茲曲線滑鼠路徑 + 抖動打字節奏 | -| `utils/ime_state/` | 144 | 讀取即時 IME 組字／轉換狀態，確保 CJK 輸入安全 | -| `utils/key_hold/` | 107 | 按住按鍵一段時間，或以固定頻率自動重複 | +| `utils/ime_state/` | 146 | 讀取即時 IME 組字／轉換狀態，確保 CJK 輸入安全 | +| `utils/key_hold/` | 109 | 按住按鍵一段時間，或以固定頻率自動重複 | | `utils/modifier_state/` | 76 | 跨一組動作按住修飾鍵，並保證安全釋放 | -| `utils/mouse_path/` | 94 | 多路徑點滑鼠手勢（沿折線移動或拖曳） | +| `utils/mouse_path/` | 106 | 多路徑點滑鼠手勢（沿折線移動或拖曳） | | `utils/mouse_relative/` | 59 | 相對位移滑鼠移動 | | `utils/postcondition/` | 138 | 宣告式的動作預期結果規格，對照畫面驗證 | -| `utils/step_repair/` | 114 | 失敗／無效動作的修復策略（自我修正迴圈） | +| `utils/step_repair/` | 117 | 失敗／無效動作的修復策略（自我修正迴圈） | | `utils/table_grid_fill/` | 143 | 以 OCR 文字填滿格線表格，取得可定址的表格 | | `utils/input_reach/` | 111 | 送出去的輸入到不到得了：桌面鎖定查詢（免費）＋ 實際送一個 F13 確認沒有被過濾（有副作用，只給診斷用） | | `utils/keyboard_layout/` | 148 | 向系統問「這個鍵盤配置下每個鍵印出什麼字」（`ToUnicodeEx`），問不到退回 US 對照表 | | `utils/text_unicode/` | 151 | 輸入任意 Unicode（emoji／CJK／重音字）：優先送字元按鍵事件，不支援時退回剪貼簿貼上 | -| `utils/tween_drag/` | 95 | 沿曲線的緩動插值拖曳 | +| `utils/tween_drag/` | 98 | 沿曲線的緩動插值拖曳 | | `utils/verify_field/` | 112 | 打字後讀回欄位，確認內容確實落地 | ### 5.4.5 影像辨識與畫面分析 -> 37 個套件、約 5,105 行。 +> 37 個套件、約 5,403 行。 | 模組 | 行數 | 職責 | | --- | ---: | --- | | `utils/annotate/` | 115 | 截圖標註：畫框、highlight、箭頭、標籤 | | `utils/barcode/` | 53 | 一維條碼（EAN／UPC）解碼，解碼器可注入 | -| `utils/color_match/` | 105 | 在 HSV 通道上做顏色感知的樣板比對 | -| `utils/color_region/` | 79 | 以顏色定位畫面區域（遮罩 + 連通元件） | -| `utils/color_stats/` | 96 | 區域顏色統計：平均色與主色 | +| `utils/color_match/` | 127 | 在 HSV 通道上做顏色感知的樣板比對 | +| `utils/color_region/` | 78 | 以顏色定位畫面區域（遮罩 + 連通元件） | +| `utils/color_stats/` | 98 | 區域顏色統計：平均色與主色 | | `utils/coordinate_space/` | 84 | 模型網格座標與實體像素之間的座標空間對映 | -| `utils/cv2_utils/` | 637 | OpenCV 基礎層：擷取後端選擇（`screen_grabber`，Pillow／mss 或平台後端）、截圖、樣板比對（走 `grab_logical`，涵蓋所有螢幕）、螢幕錄影、影片錄製、連通元件、影像堆疊的取用口（`optional`，Windows arm64 沒有 wheel 時語意報錯） | +| `utils/cv2_utils/` | 798 | OpenCV 基礎層：擷取後端選擇（`screen_grabber`，Pillow／mss 或平台後端）、截圖、樣板比對（走 `grab_logical`，涵蓋所有螢幕）、螢幕錄影、影片錄製（兩者都經 `frame_clock` 依 fps 配速）、連通元件、影像堆疊的取用口（`optional`，Windows arm64 沒有 wheel 時語意報錯）、非 ASCII 路徑也讀寫得到的影像檔存取（`image_file`） | | `utils/edge_lines/` | 120 | 以 Hough 轉換偵測線條／格線／分隔線 | | `utils/edge_match/` | 112 | 邊緣形狀（Chamfer／距離轉換）樣板比對 | | `utils/feature_match/` | 130 | ORB 特徵比對：在旋轉／縮放／主題變更下定位樣板 | | `utils/hsv_segment/` | 91 | HSV 色彩空間分割（抗光照的顏色遮罩 + blob 框） | | `utils/icon_classify/` | 113 | 從像素形狀判斷一個框是哪一類元件 | -| `utils/image_dedup/` | 83 | 感知雜湊影像去重（Pillow aHash/dHash） | +| `utils/image_dedup/` | 90 | 感知雜湊影像去重（Pillow aHash/dHash） | | `utils/image_quality/` | 77 | 在 OCR／比對前評分影像品質（銳利度／對比／亮度） | | `utils/img_histogram/` | 99 | 顏色直方圖指紋與變化偵測（抗光照） | | `utils/marks_layout/` | 124 | Set-of-Marks 標籤的不重疊排版與可讀配色 | -| `utils/match_autothresh/` | 105 | Otsu 自動門檻，免去手動調 `min_score` | +| `utils/match_autothresh/` | 108 | Otsu 自動門檻，免去手動調 `min_score` | | `utils/match_ensemble/` | 63 | 多樣板共識比對（多張參考圖投票到同一位置） | | `utils/match_stability/` | 68 | 比對前的靜止閘門與跨影格的比對持續性 | | `utils/match_trust/` | 136 | 樣板比對可信度評分（次峰比 + peak-to-sidelobe） | | `utils/monitor_layout/` | 317 | 多螢幕／虛擬桌面幾何（在哪個螢幕、位置、重映射）＋ `logical_frame` 以滑鼠座標空間擷取畫面 | | `utils/motion_regions/` | 73 | 兩影格間的局部變化／活動偵測（absdiff） | | `utils/perceptual_diff/` | 100 | 感知式（YIQ）影像差異，抑制反鋸齒邊緣誤報 | -| `utils/preprocess/` | 185 | OCR／比對前的影像前處理（灰階、二值化、去傾斜…） | +| `utils/preprocess/` | 219 | OCR／比對前的影像前處理（灰階、二值化、去傾斜…） | | `utils/qr/` | 60 | 從影像或螢幕區域解碼 QR code（OpenCV） | | `utils/rotated_match/` | 145 | 容忍旋轉與縮放的樣板比對（尺度空間 × 角度掃描） | | `utils/saliency/` | 107 | 頻譜殘差視覺顯著性：顯著圖與排序後的顯著區域 | | `utils/scale_detect/` | 84 | 偵測樣板實際渲染的顯示縮放／視覺 DPI | | `utils/screen_grid/` | 143 | 供 VLM 接地用的粗粒度標號網格（點 ↔ 格對映） | -| `utils/set_of_marks/` | 150 | Set-of-Marks 疊圖：為畫面元素編號供 VLM 指認 | +| `utils/set_of_marks/` | 154 | Set-of-Marks 疊圖：為畫面元素編號供 VLM 指認 | | `utils/shape_locator/` | 105 | 以邊緣／輪廓偵測定位元件（矩形／形狀，免樣板） | -| `utils/ssim/` | 140 | 結構相似度比較：感知分數 + 變化區域 | -| `utils/subpixel_match/` | 101 | 以二次曲面擬合做次像素級比對精修 | +| `utils/ssim/` | 141 | 結構相似度比較：感知分數 + 變化區域 | +| `utils/subpixel_match/` | 103 | 以二次曲面擬合做次像素級比對精修 | | `utils/theme_normalize/` | 92 | 主題無關的影像正規化，讓亮色樣板能配對深色模式 | -| `utils/video_report/` | 133 | 影片步驟疊圖報告：把截圖加字幕串成操作導覽影片 | -| `utils/visual_match/` | 454 | 會回傳信心值的樣板比對（分數、多尺度、find-all + NMS）；擷取走 `grab_logical`，命中座標已加回虛擬桌面原點，單色樣板直接拒收 | -| `utils/visual_regression/` | 226 | 桌面 GUI 的視覺回歸測試（黃金圖比對） | +| `utils/video_report/` | 164 | 影片步驟疊圖報告：把截圖加字幕串成操作導覽影片 | +| `utils/visual_match/` | 475 | 會回傳信心值的樣板比對（分數、多尺度、find-all + NMS）；擷取走 `grab_logical`，命中座標已加回虛擬桌面原點，單色樣板直接拒收 | +| `utils/visual_regression/` | 237 | 桌面 GUI 的視覺回歸測試（黃金圖比對） | ### 5.4.6 OCR 與文字理解 -> 19 個套件、約 3,196 行。 +> 19 個套件、約 3,252 行。 | 模組 | 行數 | 職責 | | --- | ---: | --- | @@ -419,30 +422,30 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 | `utils/column_layout/` | 150 | 從垂直空白推斷欄位，處理無框線表格 | | `utils/confusables/` | 112 | 易混淆／同形字偵測（Unicode 欺騙骨架） | | `utils/form_fields/` | 128 | 多方向關聯表單標籤與值，並讀取核取方塊狀態 | -| `utils/fuzzy/` | 94 | 模糊字串比對與去重（預設 difflib，有 rapidfuzz 則優先） | +| `utils/fuzzy/` | 96 | 模糊字串比對與去重（預設 difflib，有 rapidfuzz 則優先） | | `utils/grid_locator/` | 71 | 以 (row, column) 從邊界框定址表格／網格儲存格 | | `utils/guardrail/` | 108 | 針對畫面／OCR 文字的啟發式 prompt-injection 防護 | | `utils/heading_segment/` | 69 | 判定 OCR 行是標題或內文，建出文件大綱 | | `utils/near_dup/` | 105 | 近似重複文字偵測（SimHash／MinHash） | -| `utils/ocr/` | 1,113 | OCR 引擎門面 + 三個後端（Tesseract／EasyOCR／PaddleOCR）、版面結構化與跨詞比對（`text_span`） | +| `utils/ocr/` | 1,126 | OCR 引擎門面 + 三個後端（Tesseract／EasyOCR／PaddleOCR）、版面結構化與跨詞比對（`text_span`） | | `utils/pii_text/` | 98 | 自由文字中的 PII 偵測與遮蔽（email／電話／SSN／卡號／IP／IBAN） | | `utils/readability/` | 137 | 可讀性評分（Flesch、Flesch-Kincaid、Gunning Fog、SMOG、ARI） | | `utils/reading_flow/` | 119 | 以遞迴 XY-cut 推導欄位感知的閱讀順序 | | `utils/search_index/` | 142 | 記憶體內 BM25／TF-IDF 全文檢索 | | `utils/text_blocks/` | 88 | 把 OCR 行組成段落與項目符號／編號清單 | -| `utils/text_diff/` | 148 | unified diff 產生、套用與三方合併 | -| `utils/text_normalize/` | 72 | Unicode 正規化與 slug 產生 | +| `utils/text_diff/` | 187 | unified diff 產生、套用與三方合併 | +| `utils/text_normalize/` | 74 | Unicode 正規化與 slug 產生 | | `utils/text_regions/` | 161 | 免模型的畫面文字區域偵測（MSER）：區域與行 | | `utils/text_similarity/` | 165 | 字串距離度量（文字比對用） | ### 5.4.7 無障礙樹與原生控制項 -> 16 個套件、約 4,313 行。 +> 16 個套件、約 4,359 行。 | 模組 | 行數 | 職責 | | --- | ---: | --- | | `utils/a11y_audit/` | 355 | 以無障礙樹 + OCR 進行無障礙與 i18n 稽核 | -| `utils/accessibility/` | 2,835 | 跨平台無障礙樹定位與錄製；Windows UIA／macOS AX／null 三後端。支援限定視窗（換搜尋起點，不是過濾）、逐節點可中斷走訪、`IUIAutomation2` 連線逾時、名稱子字串比對與排序、`control_get_state` 一次讀完值／勾選／選取／數值（密碼欄位不回內容） | +| `utils/accessibility/` | 2,890 | 跨平台無障礙樹定位與錄製；Windows UIA／macOS AX／null 三後端。支援限定視窗（換搜尋起點，不是過濾）、逐節點可中斷走訪、`IUIAutomation2` 連線逾時、名稱子字串比對與排序、`control_get_state` 一次讀完值／勾選／選取／數值（密碼欄位不回內容） | | `utils/ax_events/` | 29 | 反應式 UIA 事件等待（focus-changed） | | `utils/ax_props/` | 44 | 讀取豐富 UIA 屬性（enabled／offscreen／help／status／快捷鍵） | | `utils/ax_text/` | 102 | 透過 UIA TextPattern 取得原生文字（讀取／尋找／選取／屬性） | @@ -450,7 +453,7 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 | `utils/contrast_map/` | 120 | 取樣實際顏色以評定畫面文字的可讀性（WCAG） | | `utils/control_patterns/` | 88 | 延伸 UIA 控制項模式動作（Expand／Select／Range／Scroll） | | `utils/cvd_simulate/` | 125 | 模擬色覺缺陷並標示在該狀況下會撞色的顏色 | -| `utils/element_repository/` | 122 | 原生 UI 元素的具名定位器倉庫（object repository） | +| `utils/element_repository/` | 113 | 原生 UI 元素的具名定位器倉庫（object repository） | | `utils/focus_order/` | 95 | 鍵盤焦點順序：預期 Tab 序列、WCAG 稽核與設定焦點 | | `utils/legacy_accessible/` | 45 | MSAA 橋接，處理 UIA 無法建模的舊控制項 | | `utils/selection_view/` | 57 | 容器選取狀態與檢視切換（Selection／MultipleView 模式） | @@ -460,142 +463,142 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 ### 5.4.8 元素定位、自我修復與智慧等待 -> 23 個套件、約 4,014 行。 +> 23 個套件、約 4,083 行。 | 模組 | 行數 | 職責 | | --- | ---: | --- | -| `utils/ab_locator/` | 336 | A/B 定位器框架：同時競速 N 種策略並記錄各自勝率 | +| `utils/ab_locator/` | 351 | A/B 定位器框架：同時競速 N 種策略並記錄各自勝率 | | `utils/adaptive_timeout/` | 84 | 由觀測到的步驟耗時推導等待逾時，而非硬猜 | | `utils/anchor_locator/` | 457 | 錨點定位器：以空間關係組合 影像／OCR／VLM／a11y 四種來源 | -| `utils/app_idle/` | 108 | 等應用程式不再忙碌，再驅動下一步 | +| `utils/app_idle/` | 109 | 等應用程式不再忙碌，再驅動下一步 | | `utils/change_localize/` | 80 | 把畫面變化歸因到實際改變的元素框 | | `utils/critic_features/` | 85 | 每步的 critic 特徵集合與規則式步驟評分 | | `utils/element_diff/` | 88 | 跨影格的幾何感知元素比對（穩定 ID、移動追蹤） | | `utils/element_parse/` | 106 | 融合並排序畫面元素框（IoU、合併、多來源融合、閱讀順序） | | `utils/element_proposal/` | 86 | 免樣板、免模型地從原始像素提出乾淨元素清單 | | `utils/element_scoring/` | 105 | 加權候選評分（角色 + 名稱相似度 + 鄰近度 + 啟用狀態） | -| `utils/expect_poll/` | 137 | 反覆取值直到符合條件（Playwright `expect.poll` 風格） | +| `utils/expect_poll/` | 138 | 反覆取值直到符合條件（Playwright `expect.poll` 風格） | | `utils/grounding_consensus/` | 127 | 對同一目標的多個接地提案做自我一致性投票 | | `utils/heal_analytics/` | 77 | 自癒事件記錄的分析（治癒率、脆弱定位器） | | `utils/locator_chain/` | 112 | 可組合／可過濾的候選定位器（chained-locator 慣用法） | | `utils/locator_repair/` | 117 | 自癒回寫：把修正後的定位器持久化 | | `utils/observation/` | 92 | 供 VLM／agent 接地用的 token 預算內、帶索引的 a11y 文字觀察 | | `utils/observation_delta/` | 103 | token 預算內的觀察差異：兩個 UI 影格之間變了什麼 | -| `utils/screen_state/` | 143 | 語義畫面狀態：快照／差異與結構化畫面描述 | +| `utils/screen_state/` | 182 | 語義畫面狀態：快照／差異與結構化畫面描述 | | `utils/scroll_find/` | 84 | 捲動直到目標影像／文字可見 | -| `utils/self_healing/` | 342 | 自癒定位器：先影像樣板、失敗改用 VLM，並留稽核記錄 | +| `utils/self_healing/` | 352 | 自癒定位器：先影像樣板、失敗改用 VLM，並留稽核記錄 | | `utils/semantic_recording/` | 423 | 為錄製內容加上語義錨點，支援換機重播與自癒重播 | | `utils/settle_detector/` | 76 | 以純函式介面判定 UI 是否已靜止 | -| `utils/smart_waits/` | 646 | 智慧等待：以影格差異取代 `time.sleep` | +| `utils/smart_waits/` | 649 | 智慧等待：以影格差異取代 `time.sleep` | ### 5.4.9 AI / Agent / LLM -> 13 個套件、約 20,643 行。 +> 13 個套件、約 21,265 行。 | 模組 | 行數 | 職責 | | --- | ---: | --- | | `utils/a2a/` | 92 | A2A（agent-to-agent）agent card 產生 | -| `utils/agent/` | 1,250 | 閉環 Computer-Use Agent 主迴圈 + Anthropic／OpenAI／Computer-Use 三後端 | -| `utils/agent_memory/` | 153 | agent 的持久化情節記憶（goal → trajectory → outcome） | +| `utils/agent/` | 1,438 | 閉環 Computer-Use Agent 主迴圈 + Anthropic／OpenAI／Computer-Use 三後端 | +| `utils/agent_memory/` | 152 | agent 的持久化情節記憶（goal → trajectory → outcome） | | `utils/agent_replay/` | 63 | 可攜的 agent 軌跡追蹤（記錄 observation→action 並重播） | -| `utils/agent_trace/` | 129 | agent 可觀測性：OpenTelemetry GenAI 慣例的 LLM span | -| `utils/cost_telemetry/` | 292 | 每次呼叫的 LLM 成本遙測：token 數 + 估算美金 | +| `utils/agent_trace/` | 153 | agent 可觀測性：OpenTelemetry GenAI 慣例的 LLM span | +| `utils/cost_telemetry/` | 307 | 每次呼叫的 LLM 成本遙測：token 數 + 估算美金 | | `utils/cua_action/` | 127 | 標準化 computer-use 動作結構（Anthropic／OpenAI → `AC_*`） | -| `utils/llm/` | 357 | 自然語言 → action list 規劃器 + Anthropic／null 後端 | +| `utils/llm/` | 365 | 自然語言 → action list 規劃器 + Anthropic／null 後端 | | `utils/mcp_registry/` | 92 | MCP registry `server.json` 資訊清單產生（可被發現） | -| `utils/mcp_server/` | 17,354 | **無頭 MCP 伺服器**（16K LOC，預設註冊 676 個工具＝657 個 `ac_*` + 19 個別名）：stdio + HTTP 傳輸、工具工廠與處理器、資源、prompt、稽核、限流、外掛熱重載 | -| `utils/tool_use_schema/` | 180 | 把 `AC_*` 指令匯出成 Claude／OpenAI 的 tool-use schema | -| `utils/trajectory_eval/` | 106 | agent 軌跡評估：依評分規準為一次執行打分 | -| `utils/vision/` | 448 | VLM 元素定位器（依描述找元素）+ Anthropic／OpenAI／null 後端 | +| `utils/mcp_server/` | 17,656 | **無頭 MCP 伺服器**（16K LOC，預設註冊 676 個工具＝657 個 `ac_*` + 19 個別名）：stdio + HTTP 傳輸、工具工廠與處理器、資源、prompt、稽核、限流、外掛熱重載 | +| `utils/tool_use_schema/` | 189 | 把 `AC_*` 指令匯出成 Claude／OpenAI 的 tool-use schema | +| `utils/trajectory_eval/` | 113 | agent 軌跡評估：依評分規準為一次執行打分 | +| `utils/vision/` | 518 | VLM 元素定位器（依描述找元素）+ Anthropic／OpenAI／null 後端 | ### 5.4.10 遠端桌面與 USB -> 6 個套件、約 17,903 行。 +> 6 個套件、約 18,791 行。 | 模組 | 行數 | 職責 | | --- | ---: | --- | -| `utils/admin/` | 328 | 多主機管理主控台：平行輪詢 N 個 AutoControl REST 端點 | -| `utils/config_sync/` | 246 | 透過訊令伺服器做跨機器設定同步 | +| `utils/admin/` | 398 | 多主機管理主控台：平行輪詢 N 個 AutoControl REST 端點 | +| `utils/config_sync/` | 279 | 透過訊令伺服器做跨機器設定同步 | | `utils/device_matrix/` | 138 | 行動裝置矩陣：同一 action list 於多台裝置平行執行 | -| `utils/remote_desktop/` | 11,990 | **遠端桌面子系統**（56 檔／11.7K LOC）：TCP／WebSocket／WebRTC 三條傳輸路徑、主機與檢視端、訊令伺服器、TURN／中繼、多檢視者、錄影、信任清單、TOTP、稽核鏈 | -| `utils/usb/` | 4,281 | 跨平台 USB 列舉／熱插拔／裝置直通（WinUSB、IOKit、libusb 後端 + ACL + WebRTC DataChannel 通道） | -| `utils/usbip/` | 920 | USB/IP 線路協定主機端（協定封包、TCP 伺服器、libusb URB 後端） | +| `utils/remote_desktop/` | 12,559 | **遠端桌面子系統**（56 檔／11.7K LOC）：TCP／WebSocket／WebRTC 三條傳輸路徑、主機與檢視端、訊令伺服器、TURN／中繼、多檢視者、錄影、信任清單、TOTP、稽核鏈 | +| `utils/usb/` | 4,472 | 跨平台 USB 列舉／熱插拔／裝置直通（WinUSB、IOKit、libusb 後端 + ACL + WebRTC DataChannel 通道） | +| `utils/usbip/` | 945 | USB/IP 線路協定主機端（協定封包、TCP 伺服器、libusb URB 後端） | ### 5.4.11 伺服器、網路協定與外部整合 -> 24 個套件、約 5,923 行。 +> 24 個套件、約 6,250 行。 | 模組 | 行數 | 職責 | | --- | ---: | --- | -| `utils/acme_v2/` | 598 | 完整 ACME v2 用戶端（RFC 8555），不依賴 certbot | -| `utils/chatops/` | 633 | Chat-ops bot：接收 Slack／Discord／webhook 的 slash 指令並路由到動作 | -| `utils/cookie_jar/` | 103 | RFC 6265 cookie jar | +| `utils/acme_v2/` | 610 | 完整 ACME v2 用戶端（RFC 8555），不依賴 certbot | +| `utils/chatops/` | 649 | Chat-ops bot：接收 Slack／Discord／webhook 的 slash 指令並路由到動作 | +| `utils/cookie_jar/` | 121 | RFC 6265 cookie jar | | `utils/email_send/` | 116 | SMTP 寄信（email 觸發器的發送端搭檔） | | `utils/events/` | 82 | 對外 CloudEvents 發送（執行生命週期事件） | -| `utils/http_cassette/` | 110 | 錄製／重播 HTTP 互動，做離線決定性 API 測試 | -| `utils/http_client/` | 135 | 零依賴 HTTP(S) 用戶端，供 action 步驟呼叫 API | -| `utils/http_conditional/` | 87 | 條件式 HTTP 請求與快取驗證器 | +| `utils/http_cassette/` | 153 | 錄製／重播 HTTP 互動，做離線決定性 API 測試 | +| `utils/http_client/` | 193 | 零依賴 HTTP(S) 用戶端，供 action 步驟呼叫 API | +| `utils/http_conditional/` | 107 | 條件式 HTTP 請求與快取驗證器 | | `utils/http_content/` | 103 | HTTP 內容協商與回應解壓縮 | | `utils/http_problem/` | 116 | RFC 9457 problem+json 解析 | -| `utils/jwt/` | 172 | JWT（HMAC 家族）編碼、解碼與 claim 驗證 | -| `utils/link_header/` | 112 | RFC 8288 Link header 解析與分頁 | +| `utils/jwt/` | 219 | JWT（HMAC 家族）編碼、解碼與 claim 驗證 | +| `utils/link_header/` | 115 | RFC 8288 Link header 解析與分頁 | | `utils/multipart/` | 139 | multipart/form-data 建構與解析 | | `utils/notify/` | 95 | 跨平台桌面通知 | | `utils/notify_channels/` | 100 | 對外聊天／webhook 通知（Slack／Discord／Teams／raw） | | `utils/otp/` | 37 | TOTP 一次性密碼產生（自動化 2FA 登入） | -| `utils/outbox/` | 92 | 交易式 outbox，保證至少一次的事件投遞 | -| `utils/pytest_plugin/` | 373 | pytest 外掛 + BDD step library（`pytest11` entry point） | -| `utils/rest_api/` | 1,751 | 純標準庫 REST 前端：路由、Bearer 驗證、限流、Prometheus 指標、OpenAPI 3.1 產生 | -| `utils/socket_server/` | 131 | 執行 action JSON 的執行緒式 TCP 指令伺服器（預設綁 127.0.0.1） | -| `utils/sse_client/` | 112 | Server-Sent Events 用戶端解析 | -| `utils/tls_acme/` | 448 | TLS 自動化：HTTP-01 挑戰伺服器、金鑰／CSR、自動續期 | +| `utils/outbox/` | 107 | 交易式 outbox，保證至少一次的事件投遞 | +| `utils/pytest_plugin/` | 380 | pytest 外掛 + BDD step library（`pytest11` entry point） | +| `utils/rest_api/` | 1,793 | 純標準庫 REST 前端：路由、Bearer 驗證、限流、Prometheus 指標、OpenAPI 3.1 產生 | +| `utils/socket_server/` | 156 | 執行 action JSON 的執行緒式 TCP 指令伺服器（預設綁 127.0.0.1） | +| `utils/sse_client/` | 126 | Server-Sent Events 用戶端解析 | +| `utils/tls_acme/` | 455 | TLS 自動化：HTTP-01 挑戰伺服器、金鑰／CSR、自動續期 | | `utils/url_canon/` | 117 | RFC 3986 URL 正規化與查詢字串工具 | | `utils/webrunner_bridge/` | 161 | 把 action JSON 橋接到 WebRunner（`je_web_runner`） | ### 5.4.12 報表、可觀測性與測試治理 -> 34 個套件、約 6,903 行。 +> 34 個套件、約 7,128 行。 | 模組 | 行數 | 職責 | | --- | ---: | --- | | `utils/anomaly/` | 107 | 單一序列異常偵測 | -| `utils/approval/` | 102 | Approval testing：以核可基準線驗證產出物 | -| `utils/assertion/` | 863 | 斷言 DSL：畫面狀態驗證 + 組合子 | +| `utils/approval/` | 118 | Approval testing：以核可基準線驗證產出物 | +| `utils/assertion/` | 881 | 斷言 DSL：畫面狀態驗證 + 組合子 | | `utils/baggage/` | 111 | W3C Baggage 傳遞 | | `utils/canonical_log/` | 90 | canonical log line 與結構化 JSON 日誌 | | `utils/ci_annotations/` | 62 | 由執行結果輸出 CI 工作流程註記（GitHub Actions） | | `utils/compliance/` | 136 | 合規：把治理證據對應到 SOC2／ISO 27001 控制項 | -| `utils/failure_hooks/` | 396 | 失敗 → 工單自動化：開 Jira／Linear／GitHub issue | +| `utils/failure_hooks/` | 395 | 失敗 → 工單自動化：開 Jira／Linear／GitHub issue | | `utils/failure_signature/` | 74 | 把錯誤訊息正規化成穩定的 SHA-256 失敗簽章並分群 | | `utils/flake_cluster/` | 103 | 以共同失敗 Jaccard 相似度為易碎測試分群 | | `utils/flakiness/` | 150 | 以執行歷史分析不穩定測試 | -| `utils/generate_report/` | 308 | HTML／JSON／XML 三種報表產生器（Template Method） | +| `utils/generate_report/` | 293 | HTML／JSON／XML 三種報表產生器（Template Method） | | `utils/media_assert/` | 233 | 媒體斷言：音訊活動與影片動態檢查 | -| `utils/observability/` | 668 | Prometheus 格式指標 + OpenTelemetry 相容 trace + `/metrics` 匯出伺服器 | +| `utils/observability/` | 696 | Prometheus 格式指標 + OpenTelemetry 相容 trace + `/metrics` 匯出伺服器 | | `utils/otlp_export/` | 81 | OTLP/JSON span 匯出 | | `utils/percentiles/` | 103 | 可合併的串流延遲摘要與精確百分位數 | | `utils/process_doc/` | 85 | 由錄製的 action list 產生逐步 SOP 文件 | | `utils/process_mining/` | 110 | 流程探勘：從動作日誌挖掘可自動化的候選 | -| `utils/profiler/` | 424 | 逐動作效能剖析器 + 資源剖析器 | -| `utils/quarantine/` | 190 | 易碎測試隔離區，讓套件執行器跳過已知不穩定案例 | +| `utils/profiler/` | 426 | 逐動作效能剖析器 + 資源剖析器 | +| `utils/quarantine/` | 200 | 易碎測試隔離區，讓套件執行器跳過已知不穩定案例 | | `utils/run_diff/` | 123 | 兩次執行軌跡的差異（LCS 對齊：新增／移除／狀態翻轉／退化） | -| `utils/run_history/` | 377 | 執行歷史儲存與產出物管理 | -| `utils/sarif/` | 134 | 以 SARIF 2.1.0 匯出發現項，供 GitHub／Azure code scanning | +| `utils/run_history/` | 405 | 執行歷史儲存與產出物管理 | +| `utils/sarif/` | 163 | 以 SARIF 2.1.0 匯出發現項，供 GitHub／Azure code scanning | | `utils/slo/` | 112 | SLO 評估：SLI、錯誤預算與多視窗燃燒率告警 | | `utils/smoothing/` | 67 | 數列移動平均平滑 | -| `utils/soft_assert/` | 62 | 軟斷言：累積檢查並在區塊結束時一次拋出 | -| `utils/stats/` | 213 | 描述統計與 A/B 顯著性檢定（純標準庫） | +| `utils/soft_assert/` | 74 | 軟斷言：累積檢查並在區塊結束時一次拋出 | +| `utils/stats/` | 220 | 描述統計與 A/B 顯著性檢定（純標準庫） | | `utils/step_timeline/` | 81 | 每次執行的步驟瀑布圖與瓶頸（關鍵路徑）步驟排名 | | `utils/test_select/` | 123 | 以執行歷史做風險導向的測試選取 | | `utils/test_shard/` | 87 | 以耗時為權重的套件切分與分片結果合併 | -| `utils/test_suite/` | 442 | QA 套件編排：把扁平 action list 評分為測試案例 + CI 報表 | +| `utils/test_suite/` | 527 | QA 套件編排：把扁平 action list 評分為測試案例 + CI 報表 | | `utils/time_travel/` | 381 | 錄製 session 的時光回溯除錯（控制器 + 播放器） | | `utils/timeseries/` | 143 | 時間序列轉換（rate／降採樣／重採樣） | -| `utils/trace_context/` | 162 | W3C Trace Context 傳遞 | +| `utils/trace_context/` | 168 | W3C Trace Context 傳遞 | ### 5.4.13 資料來源、結構驗證與 i18n -> 24 個套件、約 3,895 行。 +> 24 個套件、約 4,124 行。 | 模組 | 行數 | 職責 | | --- | ---: | --- | @@ -604,78 +607,78 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 | `utils/data_drift/` | 125 | 分布漂移偵測 | | `utils/data_profile/` | 121 | 資料剖析與結構推斷 | | `utils/data_quality/` | 186 | 資料品質：列結構驗證、欄位擷取、遮蔽 | -| `utils/data_source/` | 182 | 資料驅動執行：從 CSV／JSON／SQLite／Excel 載入資料列 | +| `utils/data_source/` | 192 | 資料驅動執行：從 CSV／JSON／SQLite／Excel 載入資料列 | | `utils/dataset_diff/` | 89 | 表格資料列差異比對（CDC 風格） | | `utils/gettext_catalog/` | 296 | GNU gettext 目錄 I/O（解析 .po、編譯／讀取 .mo、訊息查詢） | -| `utils/i18n_test/` | 130 | 國際化／在地化測試輔助 | +| `utils/i18n_test/` | 196 | 國際化／在地化測試輔助 | | `utils/json_contract/` | 135 | JSON 契約／快照比對：`match_json`、`diff_json`、`snapshot_json` | -| `utils/json_patch/` | 312 | JSON Pointer（6901）、JSON Patch（6902）與 Merge Patch（7386） | -| `utils/json_schema/` | 374 | JSON Schema（Draft 2020-12 子集）驗證 | -| `utils/jsonpath/` | 179 | 精簡 JSONPath 查詢 | +| `utils/json_patch/` | 322 | JSON Pointer（6901）、JSON Patch（6902）與 Merge Patch（7386） | +| `utils/json_schema/` | 419 | JSON Schema（Draft 2020-12 子集）驗證 | +| `utils/jsonpath/` | 225 | 精簡 JSONPath 查詢 | | `utils/list_format/` | 72 | 地區感知清單格式化（CLDR 風格的「A、B 和 C」） | | `utils/locale_collation/` | 128 | 地區感知字串排序（決定性多層排序鍵） | | `utils/locale_parse/` | 68 | 地區感知數字／貨幣／日期解析與格式化（選用 babel） | | `utils/message_format/` | 236 | ICU-lite MessageFormat（plural／select／selectordinal） | | `utils/office/` | 162 | Office 文件無頭讀寫（Excel／Word／PowerPoint） | -| `utils/pdf/` | 87 | PDF 讀取與斷言（選用 pypdf 後端） | +| `utils/pdf/` | 115 | PDF 讀取與斷言（選用 pypdf 後端） | | `utils/referential/` | 75 | 跨資料集的參照完整性檢查 | | `utils/schema_compat/` | 162 | JSON Schema 相容性分級 | -| `utils/sql/` | 78 | 對 SQLite 的臨時唯讀 SQL 查詢 | +| `utils/sql/` | 84 | 對 SQLite 的臨時唯讀 SQL 查詢 | | `utils/test_data/` | 205 | 帶種子的合成測試資料產生（純標準庫） | -| `utils/xml/` | 252 | XML 檔讀寫與結構變更（`defusedxml`） | +| `utils/xml/` | 270 | XML 檔讀寫與結構變更（`defusedxml`） | ### 5.4.14 安全、機密與合規 -> 13 個套件、約 2,294 行。 +> 13 個套件、約 2,545 行。 | 模組 | 行數 | 職責 | | --- | ---: | --- | -| `utils/config_redaction/` | 75 | 設定結構與 log 字串的機密遮蔽 | +| `utils/config_redaction/` | 79 | 設定結構與 log 字串的機密遮蔽 | | `utils/egress/` | 114 | 無頭 HTTP 用戶端的網路外連允許清單守衛 | -| `utils/governance/` | 199 | 治理：maker-checker 核准閘門與即時憑證租約 | +| `utils/governance/` | 231 | 治理：maker-checker 核准閘門與即時憑證租約 | | `utils/license_policy/` | 139 | 以 SBOM 元件評估 SPDX 授權允許／拒絕政策 | | `utils/provenance/` | 104 | SLSA 建置來源證明（in-toto v1） | -| `utils/rbac/` | 272 | 角色型存取控制與逐使用者稽核歸因 | -| `utils/redaction/` | 467 | 截圖遮蔽層：規則偵測 + 政策 + 協調器（上傳 VLM 前先遮） | -| `utils/sbom/` | 110 | SBOM（CycloneDX）產生 | +| `utils/rbac/` | 299 | 角色型存取控制：使用者、角色與權杖驗證（尚未接到 REST／MCP） | +| `utils/redaction/` | 499 | 截圖遮蔽層：規則偵測 + 政策 + 協調器（上傳 VLM 前先遮） | +| `utils/sbom/` | 118 | SBOM（CycloneDX）產生 | | `utils/secret_ref/` | 126 | URI scheme 形式的值參照解析 | -| `utils/secrets/` | 272 | 加密機密儲存庫，供 `${secrets.NAME}` 解析 | -| `utils/secrets_scan/` | 98 | 掃描 action JSON／資料中應入庫卻硬編碼的機密 | -| `utils/vex/` | 130 | OpenVEX 陳述撰寫與漏洞分類處置 | -| `utils/vuln_scan/` | 188 | 以 OSV 比對 SBOM 元件的漏洞（純標準庫） | +| `utils/secrets/` | 340 | 加密機密儲存庫，供 `${secrets.NAME}` 解析 | +| `utils/secrets_scan/` | 130 | 掃描 action JSON／資料中應入庫卻硬編碼的機密 | +| `utils/vex/` | 143 | OpenVEX 陳述撰寫與漏洞分類處置 | +| `utils/vuln_scan/` | 223 | 以 OSV 比對 SBOM 元件的漏洞（純標準庫） | ### 5.4.15 韌性、流量控制與設定 -> 14 個套件、約 1,706 行。 +> 14 個套件、約 1,952 行。 | 模組 | 行數 | 職責 | | --- | ---: | --- | -| `utils/artifact_store/` | 114 | S3 相容產出物儲存（報表／截圖／錄影） | -| `utils/assets/` | 155 | 環境範圍的型別化資產／設定儲存（UiPath Assets 風格） | -| `utils/bulkhead/` | 134 | Bulkhead 併發隔離 + 伺服器限流標頭解析 | +| `utils/artifact_store/` | 128 | S3 相容產出物儲存（報表／截圖／錄影） | +| `utils/assets/` | 178 | 環境範圍的型別化資產／設定儲存（UiPath Assets 風格） | +| `utils/bulkhead/` | 141 | Bulkhead 併發隔離 + 伺服器限流標頭解析 | | `utils/chaos/` | 153 | 決定性混沌實驗（穩態假說 + 故障注入） | -| `utils/dedup_window/` | 63 | 時間視窗內的訊息去重 | -| `utils/dotenv/` | 101 | `.env` 檔解析與序列化 | +| `utils/dedup_window/` | 72 | 時間視窗內的訊息去重 | +| `utils/dotenv/` | 149 | `.env` 檔解析與序列化 | | `utils/feature_flags/` | 173 | 功能旗標評估，含目標規則與決定性灰度 | -| `utils/idempotency/` | 114 | 冪等鍵儲存與已存回應重放 | +| `utils/idempotency/` | 142 | 冪等鍵儲存與已存回應重放 | | `utils/layered_config/` | 110 | 分層設定解析 | -| `utils/optimistic/` | 105 | 樂觀併發的版本化儲存 | -| `utils/rate_limit/` | 162 | 用戶端限流：token bucket、滑動視窗、throttle | -| `utils/resilience/` | 110 | 韌性原語：退避重試與斷路器 | -| `utils/retry_budget/` | 147 | 重試預算：以牆鐘期限與 full jitter 約束重試 | +| `utils/optimistic/` | 135 | 樂觀併發的版本化儲存 | +| `utils/rate_limit/` | 204 | 用戶端限流：token bucket、滑動視窗、throttle | +| `utils/resilience/` | 144 | 韌性原語：退避重試與斷路器 | +| `utils/retry_budget/` | 158 | 重試預算：以牆鐘期限與 full jitter 約束重試 | | `utils/sequence_gap/` | 65 | 逐串流的序號缺口偵測 | ### 5.4.16 系統、視窗與剪貼簿 -> 16 個套件、約 2,415 行。 +> 16 個套件、約 2,439 行。 | 模組 | 行數 | 職責 | | --- | ---: | --- | -| `utils/clipboard/` | 496 | 跨平台無頭剪貼簿存取（文字 + 影像）＋ `win32_clipboard_api.py`：**所有剪貼簿格式共用的 Win32 原型與 open/alloc/lock 流程**（`open_clipboard()` 會等過短暫被別的行程佔住的剪貼簿——Win32 一次只允許一個行程開啟，別人正在複製就必然失敗）（`argtypes` 只宣告一半曾讓四支 writer 在 64 位元上必然丟 `OverflowError`，見 CHANGELOG）。`set_clipboard_image` 同時接受 PNG 位元組與檔案路徑——先前這個名字在本子套件裡有**兩份不同簽章的實作**（`clipboard.py` 吃 bytes、`clipboard_image.py` 吃路徑），匯錯來源只會在執行期才炸，已合併成一支 | -| `utils/clipboard_files/` | 96 | 剪貼簿檔案清單（CF_HDROP）：純 DROPFILES 封裝 + Win32 存取 | +| `utils/clipboard/` | 446 | 跨平台無頭剪貼簿存取（文字 + 影像）＋ `win32_clipboard_api.py`：**所有剪貼簿格式共用的 Win32 原型與 open/alloc/lock 流程**（`open_clipboard()` 會等過短暫被別的行程佔住的剪貼簿——Win32 一次只允許一個行程開啟，別人正在複製就必然失敗）（`argtypes` 只宣告一半曾讓四支 writer 在 64 位元上必然丟 `OverflowError`，見 CHANGELOG）。`set_clipboard_image` 同時接受 PNG 位元組與檔案路徑——先前這個名字在本子套件裡有**兩份不同簽章的實作**（`clipboard.py` 吃 bytes、`clipboard_image.py` 吃路徑），匯錯來源只會在執行期才炸，已合併成一支 | +| `utils/clipboard_files/` | 106 | 剪貼簿檔案清單（CF_HDROP）：純 DROPFILES 封裝 + Win32 存取 | | `utils/clipboard_formats/` | 151 | 檢視與分類剪貼簿可用格式（純分類／差異 + Win32 列舉） | -| `utils/clipboard_history/` | 109 | 剪貼簿歷史：環形緩衝 + 背景輪詢器 | -| `utils/clipboard_rich_formats/` | 254 | 豐富剪貼簿格式 — RTF 與 CSV/TSV 編解碼 + Windows 存取 | +| `utils/clipboard_history/` | 111 | 剪貼簿歷史：環形緩衝 + 背景輪詢器 | +| `utils/clipboard_rich_formats/` | 282 | 豐富剪貼簿格式 — RTF 與 CSV/TSV 編解碼 + Windows 存取 | | `utils/file_assoc/` | 92 | 解析哪個應用程式被註冊來開啟某副檔名 | | `utils/file_dialog/` | 60 | 驅動原生檔案 開啟／儲存／資料夾選擇 對話框 | | `utils/file_drop/` | 96 | 以 WM_DROPFILES 把檔案拖放到視窗 | @@ -683,7 +686,7 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 | `utils/shell_open/` | 97 | 以預設應用開啟檔案，或以預設瀏覽器開啟 URL | | `utils/system_volume/` | 194 | 讀取與控制系統主音量與靜音狀態 | | `utils/trash/` | 90 | 把檔案移到系統資源回收筒（可復原刪除） | -| `utils/window_capture/` | 258 | 逐視窗截圖、視窗版面儲存／還原、貼齊與排列 | +| `utils/window_capture/` | 292 | 逐視窗截圖、視窗版面儲存／還原、貼齊與排列 | | `utils/window_geometry/` | 81 | 視窗客戶區幾何（外框內縮、client→screen 對映） | | `utils/window_layout/` | 134 | 視窗拼貼／版面規劃器（左右半、四象限、網格、層疊） | | `utils/window_zorder/` | 76 | 視窗 z 序控制（最上層／移到最前／送到最後） | @@ -692,122 +695,133 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 上表以子套件為單位；以下把行數最大的幾個子系統展開到檔案層。 -#### `utils/executor/`（9,081 行）— 執行核心 +#### `utils/executor/`（9,402 行）— 執行核心 | 檔案 | 行數 | 職責 | | --- | ---: | --- | -| `action_executor.py` | 8,131 | `Executor` 類別與 `event_dict` 分派表（773 個指令），另含數百個把 utils 能力接成指令的 adapter 函式；全域單例 `executor` 與 `add_command_to_executor()` 擴充點。 | -| `flow_control.py` | 530 | 真正的流程控制：`AC_loop`／`AC_for_each`／`AC_while_*`／`AC_if_*`／`AC_try`／`AC_retry`／`AC_parallel`／`AC_define_macro`／`AC_call_macro`／變數指令（`AC_set_var`／`AC_get_var`／`AC_inc_var`）。`LoopBreak`／`LoopContinue` 以例外實作。34 個區塊指令的分派表 `BLOCK_COMMANDS` 也在這裡，含下一列匯入的資料來源指令。 | -| `flow_data_commands.py` | 253 | `AC_*_to_var` 資料來源與轉換指令：shell、時鐘、亂數、PDF、TOTP、SQL、檔案、HTTP、OCR，加上 `AC_assert_var`／`AC_assert_db`／`AC_assert_duration`／`AC_transform_var`。都不執行巢狀 action list，所以沒有迴圈／分支語意。 | +| `action_executor.py` | 8,279 | `Executor` 類別與 `event_dict` 分派表（773 個指令），另含數百個把 utils 能力接成指令的 adapter 函式；全域單例 `executor` 與 `add_command_to_executor()` 擴充點。 | +| `flow_control.py` | 622 | 真正的流程控制：`AC_loop`／`AC_for_each`／`AC_while_*`／`AC_if_*`／`AC_try`／`AC_retry`／`AC_parallel`／`AC_define_macro`／`AC_call_macro`／變數指令（`AC_set_var`／`AC_get_var`／`AC_inc_var`）。`LoopBreak`／`LoopContinue` 以例外實作。34 個區塊指令的分派表 `BLOCK_COMMANDS` 也在這裡，含下一列匯入的資料來源指令。 | +| `flow_data_commands.py` | 262 | `AC_*_to_var` 資料來源與轉換指令：shell、時鐘、亂數、PDF、TOTP、SQL、檔案、HTTP、OCR，加上 `AC_assert_var`／`AC_assert_db`／`AC_assert_duration`／`AC_transform_var`。都不執行巢狀 action list，所以沒有迴圈／分支語意。 | | `action_schema.py` | 128 | action list 的結構驗證：形狀、參數型別、未知指令拒絕。單一走訪同時支援兩種消費方式：`validate_actions()` 遇到第一個問題就拋、`unknown_command_names()` 收齊全部不認得的名字（REST `/execute` 用它回 400）。 | +| `action_redaction.py` | 72 | 記錄與紀錄鍵用的遮蔽：`AC_secret_*` 的參數（金庫通行碼、機密值）在寫進 log、當成結果紀錄的鍵之前換成 `***`，巢狀在區塊指令裡的也一樣。 | | `mouse_aliases.py` | 39 | 單鍵點擊別名（`AC_click_left` 等），executor 與 callback executor 共用。 | -#### `utils/mcp_server/`（17,354 行，676 個工具）— 最大子系統 +#### `utils/mcp_server/`（17,656 行，676 個工具）— 最大子系統 | 檔案 | 行數 | 職責 | | --- | ---: | --- | -| `tools/_factories.py` | 8,739 | 工具工廠：每個函式回傳一個領域的 `MCPTool` 清單（把 `AC_*` 能力包成 MCP 工具）。 | -| `tools/_handlers.py` | 4,651 | 把 MCP 工具呼叫橋接到 AutoControl 無頭 API 的 adapter。 | -| `server.py` | 717 | JSON-RPC 2.0 over stdio 的最小 MCP 伺服器：連線範圍狀態、行內／併發分派、工具與 resource／prompt 處理器。 | -| `http_transport.py` | 521 | MCP 的 HTTP 傳輸。 | -| `http_sessions.py` | 234 | MCP 的 HTTP 傳輸用的 session 身分:`Mcp-Session-Id` 註冊表,以及每個 session 那條常駐的 server→client SSE 串流。 | -| `_client_requests.py` | 232 | 伺服器主動送出的請求：`roots/list`／`elicitation/create`／`sampling/createMessage`,對應表與回應路由,以及破壞性工具的確認交握。 | +| `tools/_factories.py` | 8,992 | 工具工廠：每個函式回傳一個領域的 `MCPTool` 清單（把 `AC_*` 能力包成 MCP 工具）。 | +| `tools/_handlers.py` | 545 | 把 MCP 工具呼叫橋接到 AutoControl 無頭 API 的 adapter；主題模組拆完之後這裡留的是資料／文字／HTTP 那一類與 WebRunner 橋接。 | +| `tools/_handlers_qa.py` | 414 | 同一種 adapter，QA 主題：斷言 DSL、資料驅動、SQL／PDF／郵件／HTTP 步驟、codegen、視覺回歸、狀態機、flaky 偵測與隔離、suite runner、無障礙稽核、裝置矩陣、媒體斷言。從 `_handlers.py` 依主題拆出的第一塊（750 行上限）；兩者互不引用。 | +| `tools/_handlers_input.py` | 212 | 同一種 adapter，輸入主題：滑鼠、鍵盤、虛擬手把（ViGEm）。 | +| `tools/_handlers_screen.py` | 305 | 同一種 adapter，螢幕主題：擷取、像素、影像與文字搜尋、螢幕錄影。 | +| `tools/_handlers_system.py` | 566 | 同一種 adapter，桌面工作階段：視窗、行程與 shell、開檔、閒置與睡眠、音量、鎖定、輸入法狀態、欄位驗證與重試、色彩對比、變更排序、元件分類、剪貼簿。 | +| `tools/_handlers_runs.py` | 110 | 同一種 adapter，執行主題：executor、執行歷史、錄製、動作檔。 | +| `tools/_handlers_scheduling.py` | 200 | 同一種 adapter，排程主題：排程器、觸發器、熱鍵常駐。 | +| `tools/_handlers_remote.py` | 66 | 同一種 adapter，遠端桌面的 host 與 viewer。 | +| `tools/_handlers_executor_bridge.py` | 1,448 | 252 個純委派（中位數 3 行，最長的 16 行全是參數簽章）：每個都是 `from action_executor import _x` 再 `return _x(...)`，沒有分支邏輯。超過 750 行,理由記在 `Progress.md` 的豁免表（再切只能照 MCP 工廠領域分,會把同一種委派散進十幾個沒有語意邊界的檔）。 | +| `tools/_handlers_locators.py` | 417 | 同一種 adapter，定位主題：無障礙樹、智慧等待、自我修復、螢幕觀察、座標空間、視覺與 OCR、影像去重、元件倉庫、A/B 定位。 | +| `tools/_handlers_operations.py` | 647 | 同一種 adapter，營運主題：agent 與其記憶／追蹤、治理與合規、成本與遙測、失敗掛鉤、看門狗、速率限制、檢查點、核可、產物與資產、測試選擇與分片、佇列與 saga。 | +| `server.py` | 718 | JSON-RPC 2.0 over stdio 的最小 MCP 伺服器：連線範圍狀態、行內／併發分派、工具與 resource／prompt 處理器。 | +| `http_transport.py` | 585 | MCP 的 HTTP 傳輸。 | +| `http_sessions.py` | 247 | MCP 的 HTTP 傳輸用的 session 身分:`Mcp-Session-Id` 註冊表,以及每個 session 那條常駐的 server→client SSE 串流。 | +| `_client_requests.py` | 249 | 伺服器主動送出的請求：`roots/list`／`elicitation/create`／`sampling/createMessage`,對應表與回應路由,以及破壞性工具的確認交握。 | | `_protocol.py` | 167 | JSON-RPC 線路格式：版本與識別常數、`_MCPError`、決定失敗工具行為的錯誤 tuple、envelope 產生器、工具回傳值轉 `content` 區塊。不碰伺服器狀態。 | -| `resources.py` | 303 | MCP resource 提供者。 | +| `resources.py` | 307 | MCP resource 提供者。 | | `prompts.py` | 220 | MCP prompt 目錄。 | | `fake_backend.py` | 184 | CI／無頭測試用的記憶體內假後端。 | -| `plugin_watcher.py` | 149 | 檔案變更時熱重載外掛工具的背景 watcher。 | -| `tools/_base.py` | 147 | 工具註冊表的共用型別與輔助。 | -| `tools/_validation.py` | 107 | MCP 工具用到的 JSON Schema 子集驗證器。 | -| `tools/plugin_tools.py` | 90 | 把外掛載入的 `AC_*` callable 包成 `MCPTool`。 | +| `plugin_watcher.py` | 168 | 檔案變更時熱重載外掛工具的背景 watcher。 | +| `tools/_base.py` | 146 | 工具註冊表的共用型別與輔助。 | +| `tools/_validation.py` | 122 | MCP 工具用到的 JSON Schema 子集驗證器。 | +| `tools/plugin_tools.py` | 89 | 把外掛載入的 `AC_*` callable 包成 `MCPTool`。 | | `log_bridge.py` | 90 | 把 Python logging 記錄橋接成 MCP `notifications/message`。 | -| `audit.py` | 78 | MCP 工具呼叫稽核記錄。 | +| `audit.py` | 87 | MCP 工具呼叫稽核記錄。 | | `context.py` | 71 | 傳給 opt-in 工具處理器的每次呼叫上下文。 | | `rate_limit.py` | 48 | 工具呼叫的 token bucket 限流。 | | `__main__.py` | 88 | `je_auto_control_mcp` console script 進入點。 | -#### `utils/remote_desktop/`（11,990 行／56 檔） +#### `utils/remote_desktop/`（12,559 行／56 檔） 三條傳輸路徑並存：**TCP**（JPEG 影格）、**WebSocket**（同協定換傳輸）、**WebRTC**（aiortc 視訊 + DataChannel）。 | 檔案 | 行數 | 職責 | | --- | ---: | --- | -| `webrtc_host.py` | 702 | WebRTC 主機：串流螢幕視訊並接受檢視端輸入;session 生命週期、DataChannel 接線、檔案收發。 | -| `webrtc_viewer.py` | 662 | WebRTC 檢視端：接收視訊並送出輸入。 | -| `host.py` | 625 | TCP 主機：接受迴圈、TLS 包裝、連線／認證握手、音訊與剪貼簿廣播、檔案推送、單次 token。 | -| `viewer.py` | 623 | TCP 檢視端。 | -| `host_service.py` | 542 | 無頭 WebRTC 主機執行器 + 多平台服務安裝器。 | -| `host_client.py` | 406 | TCP 主機的每連線處理器：一個檢視端一個實例,擁有它的認證交換、sender／audio／receiver 三條執行緒,以及入站訊息的路由表。 | +| `webrtc_host.py` | 716 | WebRTC 主機：串流螢幕視訊並接受檢視端輸入;session 生命週期、DataChannel 接線、檔案收發。 | +| `webrtc_viewer.py` | 672 | WebRTC 檢視端：接收視訊並送出輸入。 | +| `host.py` | 669 | TCP 主機：接受迴圈、TLS 包裝、連線／認證握手、音訊與剪貼簿廣播、檔案推送、單次 token。 | +| `viewer.py` | 634 | TCP 檢視端。 | +| `host_service.py` | 558 | 無頭 WebRTC 主機執行器 + 多平台服務安裝器。 | +| `host_client.py` | 453 | TCP 主機的每連線處理器：一個檢視端一個實例,擁有它的認證交換、sender／audio／receiver 三條執行緒,以及入站訊息的路由表。 | | `registry.py` | 370 | `AC_remote_*` 指令使用的行程級單例。 | | `webrtc_transport.py` | 369 | 共用 WebRTC 管線：asyncio 橋接執行緒、螢幕視訊軌、設定。 | -| `multi_viewer.py` | 314 | 每個連入檢視端各跑一個 `WebRTCDesktopHost` 的協調器。 | -| `signaling_server.py` | 297 | 獨立的 WebRTC SDP 交換 rendezvous 服務。 | -| `audit_log.py` | 288 | SQLite 雜湊鏈稽核記錄。 | +| `multi_viewer.py` | 339 | 每個連入檢視端各跑一個 `WebRTCDesktopHost` 的協調器。 | +| `signaling_server.py` | 427 | 獨立的 WebRTC SDP 交換 rendezvous 服務。 | +| `audit_log.py` | 355 | SQLite 雜湊鏈稽核記錄。 | | `host_capture.py` | 297 | TCP 主機的影格與游標產生：螢幕列舉、監視器索引轉擷取區域、預設 JPEG／游標 provider,以及 `FrameProductionMixin`（游標輪詢、擷取迴圈、上線編碼）。 | -| `ws_protocol.py` | 277 | 最小 RFC 6455 WebSocket 框架與握手。 | -| `file_transfer.py` | 273 | 分塊檔案傳輸。 | -| `relay.py` | 270 | NAT 穿透失敗時的 TCP 中繼。 | -| `fingerprint.py` | 250 | TOFU 主機指紋驗證。 | +| `ws_protocol.py` | 284 | 最小 RFC 6455 WebSocket 框架與握手。 | +| `file_transfer.py` | 339 | 分塊檔案傳輸。 | +| `relay.py` | 314 | NAT 穿透失敗時的 TCP 中繼。 | +| `fingerprint.py` | 245 | TOFU 主機指紋驗證。 | | `turn_config.py` | 234 | coturn 設定產生器。 | | `presence.py` | 221 | 多檢視者的執行緒安全在場註冊表。 | | `jpeg_recorder_encrypted.py` | 223 | AES-GCM 加密版 session 錄影。 | -| `address_book.py` | 209 | 檢視端的主機通訊錄。 | -| `audio.py` / `webrtc_audio.py` / `webrtc_mic.py` | 206 / 190 / 152 | 音訊擷取播放、音訊軌、麥克風上行。 | -| `webrtc_files.py` | 205 | 專屬 DataChannel 的分塊檔案傳輸。 | -| `webrtc_host_auth.py` | 222 | 檢視端認證與核准：token 檢查、信任清單／IP 白名單自動放行、手動接受／拒絕、SAS、逾時關閉。 | +| `address_book.py` | 213 | 檢視端的主機通訊錄。 | +| `audio.py` / `webrtc_audio.py` / `webrtc_mic.py` | 205 / 189 / 151 | 音訊擷取播放、音訊軌、麥克風上行。 | +| `webrtc_files.py` | 249 | 專屬 DataChannel 的分塊檔案傳輸。 | +| `webrtc_host_auth.py` | 237 | 檢視端認證與核准：token 檢查、信任清單／IP 白名單自動放行、手動接受／拒絕、SAS、逾時關閉。 | | `lan_discovery.py` | 189 | mDNS／Zeroconf 區網探索。 | | `video_codec.py` | 181 | TCP／WS 路徑的可插拔視訊編解碼。 | | `webrtc_host_media.py` | 194 | 重新協商與 recvonly 軌管理。aiortc 沒有 `removeTransceiver`,所以開／關不對稱——開是加軌重新 offer,關只能設 inactive 並停掉 receiver。 | | `hw_codec.py` | 169 | 硬體 H.264 編碼偵測與啟用。 | -| `webrtc_stats.py` | 163 | 把 aiortc 的 `RTCStats` 報告輪詢成精簡 dict。 | +| `webrtc_stats.py` | 167 | 把 aiortc 的 `RTCStats` 報告輪詢成精簡 dict。 | | `connect_coordinator.py` | 149 | 由使用者輸入的目標決定該用哪條傳輸。 | | `adaptive_bitrate.py` | 148 | 依統計調整主機擷取 FPS。 | -| `signaling_client.py` | 145 | 純標準庫的訊令用戶端。 | -| `trust_list.py` | 144 | 自動接受的檢視端信任清單。 | +| `signaling_client.py` | 151 | 純標準庫的訊令用戶端。 | +| `trust_list.py` | 139 | 自動接受的檢視端信任清單。 | | `webrtc_inspector.py` | 138 | 行程級的 `StatsSnapshot` 滾動視窗。 | -| `input_dispatch.py` | 133 | 在主機端套用輸入訊息。 | +| `input_dispatch.py` | 139 | 在主機端套用輸入訊息。 | | `session_recorder.py` | 134 | 以 PyAV 把 WebRTC 影格錄成 mp4。 | -| `totp.py` | 130 | RFC 6238 TOTP（零外部相依）。 | -| `file_sync.py` | 139 | 輪詢式資料夾鏡像。 | +| `totp.py` | 140 | RFC 6238 TOTP（零外部相依）。 | +| `file_sync.py` | 141 | 輪詢式資料夾鏡像。 | | `transport.py` | 123 | 可插拔的型別化訊息傳輸。 | -| `host_access.py` | 105 | TCP 主機的檢視端核准與存取控制：`PendingViewer`、權限字串、分享碼的 TOTP 候選值、IP 白名單。`host` 與 `host_client` 共用,所以獨立成模組。 | +| `host_access.py` | 112 | TCP 主機的檢視端核准與存取控制：`PendingViewer`、權限字串、分享碼的 TOTP 候選值、IP 白名單。`host` 與 `host_client` 共用,所以獨立成模組。 | | `protocol.py` | 96 | 長度前綴的 TCP 框架。 | -| `resume_tokens.py` / `session_quality_cache.py` / `rate_limit.py` | 95 / 86 / 85 | 快速重連 token、每 session 品質快取、檢視端限流。 | -| `host_id.py` / `viewer_id.py` | 82 / 78 | 主機與檢視端的持久身分。 | -| `permissions.py` / `clipboard_sync.py` / `wake_on_lan.py` / `session_actions.py` / `auth.py` | 65 / 73 / 57 / 41 / 29 | 逐 session 權限、剪貼簿同步、WOL、SAS 注入與螢幕遮蔽、HMAC 挑戰回應。 | -| `ws_host.py` / `ws_viewer.py` / `jpeg_recorder.py` | 41 / 30 / 139 | WebSocket 傳輸變體與 TCP 路徑錄影。 | +| `resume_tokens.py` / `session_quality_cache.py` / `rate_limit.py` | 94 / 85 / 84 | 快速重連 token、每 session 品質快取、檢視端限流。 | +| `host_id.py` / `viewer_id.py` | 81 / 77 | 主機與檢視端的持久身分。 | +| `permissions.py` / `clipboard_sync.py` / `wake_on_lan.py` / `session_actions.py` / `auth.py` | 64 / 72 / 56 / 40 / 28 | 逐 session 權限、剪貼簿同步、WOL、SAS 注入與螢幕遮蔽、HMAC 挑戰回應。 | +| `ws_host.py` / `ws_viewer.py` / `jpeg_recorder.py` | 40 / 29 / 146 | WebSocket 傳輸變體與 TCP 路徑錄影。 | -#### `utils/usb/`（4,281 行）與 `utils/usbip/`（920 行） +#### `utils/usb/`（4,472 行）與 `utils/usbip/`（945 行） | 檔案 | 行數 | 職責 | | --- | ---: | --- | -| `usb/passthrough/session.py` | 595 | 逐 peer 的 USB 直通 session。 | -| `usb/passthrough/viewer_client.py` | 561 | 檢視端的直通協定用戶端。 | -| `usb/passthrough/backend.py` | 464 | 後端 ABC + libusb 實作。 | -| `usb/passthrough/winusb_backend.py` | 458 | Windows WinUSB 後端（ctypes）。 | -| `usb/passthrough/acl.py` | 433 | 逐裝置 ACL。 | -| `usb/passthrough/iokit_backend.py` | 222 | macOS IOKit 後端。 | -| `usb/passthrough/webrtc_channel.py` | 168 | 把直通協定橋到 WebRTC `usb` DataChannel。 | -| `usb/passthrough/loopback.py` | 158 | 行程內 loopback 傳輸（測試用）。 | +| `usb/passthrough/session.py` | 642 | 逐 peer 的 USB 直通 session。 | +| `usb/passthrough/viewer_client.py` | 575 | 檢視端的直通協定用戶端。 | +| `usb/passthrough/backend.py` | 463 | 後端 ABC + libusb 實作。 | +| `usb/passthrough/winusb_backend.py` | 488 | Windows WinUSB 後端（ctypes）。 | +| `usb/passthrough/acl.py` | 495 | 逐裝置 ACL。 | +| `usb/passthrough/iokit_backend.py` | 221 | macOS IOKit 後端。 | +| `usb/passthrough/webrtc_channel.py` | 180 | 把直通協定橋到 WebRTC `usb` DataChannel。 | +| `usb/passthrough/loopback.py` | 159 | 行程內 loopback 傳輸（測試用）。 | | `usb/passthrough/protocol.py` | 133 | 線路框格式。 | -| `usb/passthrough/descriptor.py` | 133 | USB 標準裝置描述元解析。 | -| `usb/passthrough/key_provider.py` | 123 | ACL 的可插拔 HMAC 金鑰來源。 | -| `usb/passthrough/commands.py` | 151 | 無頭直通指令（單一真實來源）。 | -| `usb/usb_devices.py` | 286 | 跨平台 USB 裝置列舉。 | -| `usb/usb_watcher.py` | 214 | 輪詢式 USB 熱插拔監看。 | -| `usbip/protocol.py` | 331 | USB/IP 線路格式封裝／解析。 | -| `usbip/server.py` | 236 | USB/IP 主機端 TCP 伺服器。 | -| `usbip/libusb_backend.py` | 209 | 以 PyUSB／libusb 執行 URB 的正式後端。 | -| `usbip/backend.py` | 88 | 可插拔 URB 執行後端。 | - -#### `utils/rest_api/`（1,751 行） +| `usb/passthrough/descriptor.py` | 132 | USB 標準裝置描述元解析。 | +| `usb/passthrough/key_provider.py` | 125 | ACL 的可插拔 HMAC 金鑰來源。 | +| `usb/passthrough/commands.py` | 150 | 無頭直通指令（單一真實來源）。 | +| `usb/usb_devices.py` | 296 | 跨平台 USB 裝置列舉。 | +| `usb/usb_watcher.py` | 260 | 輪詢式 USB 熱插拔監看。 | +| `usbip/protocol.py` | 330 | USB/IP 線路格式封裝／解析。 | +| `usbip/server.py` | 256 | USB/IP 主機端 TCP 伺服器。 | +| `usbip/libusb_backend.py` | 212 | 以 PyUSB／libusb 執行 URB 的正式後端。 | +| `usbip/backend.py` | 87 | 可插拔 URB 執行後端。 | + +#### `utils/rest_api/`（1,793 行） | 檔案 | 行數 | 職責 | | --- | ---: | --- | -| `rest_server.py` | 480 | HTTP 前端主體。 | +| `rest_server.py` | 508 | HTTP 前端主體。 | | `rest_handlers.py` | 486 | 端點實作。 | | `rest_openapi.py` | 422 | 走訪路由表產生 OpenAPI 3.1 規格。 | -| `rest_auth.py` | 143 | Bearer token 驗證 + 逐 client 限流閘門。 | +| `rest_auth.py` | 157 | Bearer token 驗證 + 逐 client 限流閘門。 | | `rest_metrics.py` | 75 | Prometheus 曝露端點。 | | `rest_registry.py` | 75 | 保存執行中 REST 伺服器的行程級單例。 | | `__main__.py` | 56 | `python -m je_auto_control.utils.rest_api` 進入點。 | @@ -816,7 +830,7 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 | 子套件 | 檔案組成 | | --- | --- | -| `accessibility/` | `accessibility_api.py`（公開 API）、`element.py`（dataclass）、`tree.py`（遞迴樹傾印）、`recorder.py`（輪詢式事件錄製）、`backends/`：`base.py` 330 行抽象、`windows_backend.py` 915 行（comtypes UIA）、`windows_query.py` 170 行（UIA 搜尋起點、可中斷走訪、快取請求、NULL COM 指標判定與 `UIA_ERRORS`）、`windows_state.py` 98 行（控制項狀態讀取與密碼欄位判定）、`macos_backend.py` 125 行（pyobjc AX）、`null_backend.py` fallback | +| `accessibility/` | `accessibility_api.py`（公開 API）、`element.py`（dataclass）、`tree.py`（遞迴樹傾印）、`recorder.py`（輪詢式事件錄製）、`backends/`：`base.py` 330 行抽象、`windows_backend.py` 801 行（comtypes UIA）、`windows_reads.py` 142 行（pattern／文字範圍／表頭／元素屬性的純讀取與 `UIA_READ_ERRORS`）、`windows_query.py` 176 行（UIA 搜尋起點、可中斷走訪、快取請求、NULL COM 指標判定與 `UIA_ERRORS`）、`windows_state.py` 98 行（控制項狀態讀取與密碼欄位判定）、`macos_backend.py` 125 行（pyobjc AX）、`null_backend.py` fallback | | `agent/` | `agent_loop.py`、`computer_use.py`、`backends/`：`anthropic.py`、`anthropic_computer_use.py`（435 行）、`openai.py`、`base.py` | | `ocr/` | `ocr_engine.py`（門面）、`structure.py`（版面）、`backends/`：`tesseract_backend.py`、`easyocr_backend.py`、`paddleocr_backend.py`、`base.py` | | `vision/` | `vlm_api.py`、`backends/`：`anthropic_backend.py`、`openai_backend.py`、`null_backend.py`、`_parse.py`、`base.py` | @@ -831,7 +845,7 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 | `semantic_recording/` | `enrich.py`（加錨點）、`replay.py`（換機重播）、`self_healing.py`（自癒重播） | | `tls_acme/` | `challenge.py`、`keys.py`、`renewal.py` | | `pytest_plugin/` | `plugin.py`（pytest11 進入點）、`keywords.py`、`bdd_steps.py`（Gherkin） | -| `cv2_utils/` | `screen_grabber.py`、`screenshot.py`、`template_detection.py`、`screen_record.py`、`video_recording.py`、`blobs.py`、`optional.py` | +| `cv2_utils/` | `screen_grabber.py`、`screenshot.py`、`template_detection.py`、`screen_record.py`、`video_recording.py`、`frame_clock.py`、`blobs.py`、`optional.py`、`image_file.py` | | `action_lint/` | `linter.py`、`schema.py`、`__main__.py`（CI 使用） | | `time_travel/` | `controller.py`、`player.py` | | `dag/` | `graph.py`、`runner.py` | @@ -861,17 +875,17 @@ GUI 是**選用 extra**（`pip install je_auto_control[gui]`，PySide6 + qt-mate | 模組 | 行數 | 職責 | | --- | ---: | --- | | `gui/__init__.py` | 23 | `start_autocontrol_gui()`：**唯一**會延遲匯入 PySide6 的地方，維持頂層套件 Qt-free。 | -| `main_window.py` | 290 | `QMainWindow`：選單列（File／Actions／View／…）、可關閉分頁、即時語言切換、字級預設、qt-material 主題。分頁分為 core／editing／detection／automation／system 五類。 | +| `main_window.py` | 289 | `QMainWindow`：選單列（File／Actions／View／…）、可關閉分頁、即時語言切換、字級預設、qt-material 主題。分頁分為 core／editing／detection／automation／system 五類。 | | `main_widget.py` | 423 | 擁有 `QTabWidget`，註冊 48 個分頁，並暴露 show/hide/list API 給選單列。核心分頁在註冊時直接宣告 `(label_key, handler)` 動作對；分頁本體都在下列 mixin。 | -| `_auto_click_tab.py` | 270 | 自動點擊分頁的 mixin 建構器。 | -| `_screenshot_tab.py` | 127 | 截圖／取像素分頁 mixin。 | -| `_image_detect_tab.py` | 106 | 影像偵測分頁 mixin。 | -| `_script_tab.py` | 105 | 腳本執行分頁 mixin。 | -| `_record_tab.py` | 101 | 錄製／回放分頁 mixin。 | -| `_report_tab.py` | 81 | 報表分頁 mixin。 | -| `_i18n_helpers.py` | 67 | 需要即時語言切換的分頁共用的翻譯註冊 mixin。 | -| `language_wrapper/` | 4,977 | 四語系字典（英／日／簡中／繁中）+ `multi_language_wrapper` 執行期切換器與監聽註冊表。 | -| `selector/` | 183 | 拖曳選取螢幕區域的半透明全螢幕覆蓋層與樣板裁切工具（互動式，但都有對應的程式化 API）。 | +| `_auto_click_tab.py` | 286 | 自動點擊分頁的 mixin 建構器。 | +| `_screenshot_tab.py` | 136 | 截圖／取像素分頁 mixin。 | +| `_image_detect_tab.py` | 114 | 影像偵測分頁 mixin。 | +| `_script_tab.py` | 115 | 腳本執行分頁 mixin。 | +| `_record_tab.py` | 110 | 錄製／回放分頁 mixin。 | +| `_report_tab.py` | 88 | 報表分頁 mixin。 | +| `_i18n_helpers.py` | 66 | 需要即時語言切換的分頁共用的翻譯註冊 mixin。 | +| `language_wrapper/` | 4,999 | 四語系字典（英／日／簡中／繁中）+ `multi_language_wrapper` 執行期切換器與監聽註冊表。 | +| `selector/` | 179 | 拖曳選取螢幕區域的半透明全螢幕覆蓋層與樣板裁切工具（互動式，但都有對應的程式化 API）。 | > **分頁指令一律走 Actions 選單**：分頁本身只放輸入、表格與結果檢視，指令由視窗層選單暴露。 > 核心分頁在 `main_widget.py` 註冊時宣告動作；功能分頁實作 `menu_actions()`（目前 40 個檔案有此 hook）。 @@ -930,16 +944,17 @@ GUI 是**選用 extra**（`pip install je_auto_control[gui]`，PySide6 + qt-mate | diagnostics | `diagnostics_tab.py` | 91 | 執行子系統檢查並顯示結果。 | | report | `_report_tab.py` | 81 | 產生 HTML／JSON／XML 報表。 | -#### 遠端桌面 GUI（`gui/remote_desktop/`，18 檔／6,336 行） +#### 遠端桌面 GUI（`gui/remote_desktop/`，19 檔／6,393 行） | 模組 | 行數 | 職責 | | --- | ---: | --- | -| `webrtc_panel.py` | 2,545 | WebRTC 子分頁主體。 | +| `webrtc_panel.py` | 2,530 | WebRTC 子分頁主體。 | | `webrtc_dialogs.py` | 493 | WebRTC GUI 用的自訂對話框與清單元件（待審檢視者、信任清單、通訊錄、遠端檔案表、稽核記錄、LAN 瀏覽）。 | | `advanced_group.py` | 92 | 兩個 WebRTC 面板共用的 Advanced STUN／TURN（含選用硬體編碼器）群組，含它寫回面板的 Protocol。 | +| `trusted_group.py` | 70 | WebRTC host 面板的信任 viewer 清單群組（移除／清空／匯入／匯出），含它寫回面板的 Protocol。 | | `connection_screen.py` | 672 | Quick Connect —— AnyDesk 風格單畫面入口。 | | `viewer_panel.py` | 542 | 「控制另一台機器」子分頁。 | -| `webrtc_known_hosts.py` | 340 | TOFU 釘選庫瀏覽器：`KnownHostsDialog` 與帶外釘選用的小表單。由 `webrtc_dialogs` 再匯出。 | +| `webrtc_known_hosts.py` | 342 | TOFU 釘選庫瀏覽器：`KnownHostsDialog` 與帶外釘選用的小表單。由 `webrtc_dialogs` 再匯出。 | | `host_panel.py` | 334 | 「分享這台機器」子分頁。 | | `frame_display.py` | 228 | 繪製 JPEG 影格並發出遠端輸入事件的元件。 | | `webrtc_workers.py` | 195 | 訊令流程的背景 `QThread` worker。 | @@ -982,7 +997,7 @@ GUI 是**選用 extra**（`pip install je_auto_control[gui]`，PySide6 + qt-mate | **新 GUI 分頁** | 在 `gui/` 新增 widget（只做 UI 翻譯）→ 在 `main_widget.py` `_add_tab` 註冊 → 提供 `menu_actions()` | 主視窗選單建構邏輯 | | **新 OCR／VLM／LLM／a11y 後端** | 在對應 `backends/` 實作 base 協定 | 呼叫端 | | **新報表格式** | 仿 `generate_report/` 既有三者的骨架新增產生器 | 執行紀錄收集 | -| **新 MCP 工具** | 在 `mcp_server/tools/_factories.py` 加工廠、`_handlers.py` 加 adapter | 傳輸層 | +| **新 MCP 工具** | 在 `mcp_server/tools/_factories.py` 加工廠、`_handlers.py` 加 adapter（QA 主題加在 `_handlers_qa.py`） | 傳輸層 | --- @@ -992,10 +1007,9 @@ GUI 是**選用 extra**（`pip install je_auto_control[gui]`，PySide6 + qt-mate | 檔案 | 用途 | | --- | --- | -| `dev.yml` | 開發分支測試。 | -| `stable.yml` | 合併到 main 後版本遞增並上傳 PyPI（使用 `PYPI_API_TOKEN`）。 | +| `stable.yml` | 每次 push／PR 到 `main` 與每日排程跑 Windows 五版本的示範腳本；合併到 main 後版本遞增並上傳 PyPI（使用 `PYPI_API_TOKEN`）。 | | `release.yml` | 發佈流程（上傳步驟目前關閉）。 | -| `quality.yml` | 靜態分析與型別檢查。 | +| `quality.yml` | ruff、bandit、dependency review、九格矩陣的 headless pytest（含 coverage 地板）與 mypy。 | | `platform-smoke.yml` | 跨平台煙霧測試。 | | `docker.yml` | 容器映像建置。 | | `action-json-lint.yml` | 用 `python -m je_auto_control.utils.action_lint` 檢查 action JSON。 | @@ -1003,8 +1017,8 @@ GUI 是**選用 extra**（`pip install je_auto_control[gui]`，PySide6 + qt-mate **設定基線**（`pyproject.toml`）： - **pytest**：`testpaths` 限定 `test/unit_test/headless` 與 `test/unit_test/flow_control`；`--strict-markers --strict-config`。 -- **coverage**：`fail_under = 50`（棘輪：實測九宮格矩陣最低的一格是 50.26%，取整數當地板），排除 `gui/` 與 `language_wrapper/`。 -- **mypy**：對**整包**把關，尚未過關的模組列在 `test/verify/typing_contract_exempt.txt`（155 個，只准變少）； +- **coverage**：`fail_under = 81`（棘輪：九格矩陣實測 81.40%（ubuntu-22.04／3.14）到 82.73%（windows-2022／3.12），取最低一格的整數當地板；維護者 2026-08-23 訂的 80 已達成），排除 `gui/` 與 `language_wrapper/`。量法一律是 `coverage run -m pytest`，不是 `pytest --cov`（見 CLAUDE.md）。 +- **mypy**：對**整包**把關，尚未過關的模組列在 `test/verify/typing_contract_exempt.txt`（只准變少，目前 **0** 個——整包都已過關）； `test/verify/typing_contract_verify.py` 會分別以 `win32`／`linux`／`darwin` 三個目標平台各跑一次並取聯集， 所以 Windows 與 macOS 後端在 Ubuntu runner 上也被檢查。非基礎相依的第三方模組一律以 `follow_imports = "skip"` + `follow_imports_for_stubs` 壓成 `Any`，閘門才不會因為裝了哪個 extra 而改變判定。 @@ -1019,32 +1033,53 @@ socket 預設綁 `127.0.0.1`；資源一律用 `with`。 讓 executor、背景輪詢迴圈、請求處理器與 GUI slot 這四種收納邊界能用單一 `except` 攔住整個家族。 **不可**新增直接繼承 `Exception` 的兄弟類別——那會靜默逃出每一道邊界。 +**「按下 → 放開」之間的失敗必須走 `finally`，不能靠 `except`**（2026-09-09）。 +`auto_control_keyboard` 的 `hotkey` 與 `type_keyboard` 原本是「按下，然後放開」而中間 +沒有任何保護：任何一步拋例外，已經按下去的鍵就**留在按下狀態**——那是使用者真實的 +鍵盤，一個卡住的 `Ctrl` 或 `Alt` 會讓之後每一次點選與按鍵都變成別的意思，而畫面上沒有 +任何跡象。`hotkey(["ctrl", "shift", "esc"])` 在第三個鍵上失敗是這個缺陷最貴的形態。 + +**為什麼是 `finally` 而不是把型別加進 `except`**：這兩支收的是 +`(OSError, RuntimeError, AttributeError, TypeError, ValueError)`，而 +`press_keyboard_key` / `release_keyboard_key` 丟的是 `AutoControlKeyboardException` +——它屬於 `AutoControlException` 家族，**不在那五個的任何一個底下**（實測確認）。也就是說**最可能發生 +的失敗**（鍵名不在對照表裡、平台不支援、後端出錯）根本走不到那個 `except`。這和上面 +那條「不可新增直接繼承 `Exception` 的兄弟類別」是同一個問題的另一面：那條防的是例外逃出 +家族，這裡是 `except` 只列了內建型別、沒列家族本身，於是漏接了它們**自己**的例外。`finally` 是唯一每條離開路徑都會跑到的地方。 + +收尾由 `_release_still_held(still_held, is_shift)` 負責：只放開**真的按下去而且還沒放開** +的鍵（按成功才記，所以不會去放開一個從沒按下的鍵——那會取消使用者自己正按著的鍵）， +倒著放，而且**絕不往外拋**（清理路徑再丟例外只會把原因蓋掉）。 +守門：`test/unit_test/headless/test_keyboard_release_on_failure.py`（11 支，變異驗證 5/5）， +其中一支專門釘住「`AutoControlKeyboardException` 不在那五個型別底下」這個**前提**——整個 +設計靠它成立，前提哪天變了，這批註解也要跟著改。 + --- ## 8. 附錄：各層規模 | 層／子系統 | 檔案數 | 行數 | | --- | ---: | ---: | -| `gui/` | 90 | 26,699 | -| `utils/mcp_server/` | 21 | 17,354 | -| `utils/remote_desktop/` | 56 | 11,990 | -| `utils/executor/` | 6 | 9,081 | -| `utils/usb/` | 17 | 4,281 | -| `je_auto_control/`（頂層 3 檔） | 3 | 2,367 | -| `utils/accessibility/` | 13 | 2,835 | -| `wrapper/` | 19 | 3,514 | -| `windows/` | 23 | 1,906 | -| `utils/rest_api/` | 8 | 1,751 | -| `utils/agent/` | 8 | 1,250 | +| `gui/` | 91 | 26,795 | +| `utils/mcp_server/` | 31 | 17,656 | +| `utils/remote_desktop/` | 56 | 12,559 | +| `utils/executor/` | 7 | 9,402 | +| `utils/usb/` | 17 | 4,472 | +| `je_auto_control/`（頂層 3 檔） | 3 | 2,395 | +| `utils/accessibility/` | 14 | 2,890 | +| `wrapper/` | 19 | 3,615 | +| `windows/` | 23 | 1,957 | +| `utils/rest_api/` | 8 | 1,793 | +| `utils/agent/` | 8 | 1,438 | | `linux_with_x11/` | 19 | 1,236 | | `linux_wayland/` | 17 | 2,870 | -| `utils/triggers/` | 4 | 1,152 | -| `utils/ocr/` | 9 | 1,113 | -| `utils/usbip/` | 5 | 920 | -| `utils/assertion/` | 3 | 863 | -| `osx/` | 17 | 915 | +| `utils/triggers/` | 4 | 1,241 | +| `utils/ocr/` | 9 | 1,126 | +| `utils/usbip/` | 5 | 945 | +| `utils/assertion/` | 3 | 881 | +| `osx/` | 17 | 919 | | `autocontrol-lsp/` | 8 | 744 | -| `utils/hotkey/` | 7 | 727 | -| 其餘模組（約 286 個 `utils/` 子套件 + `android/`／`ios/`／周邊小工具） | 673 | 47,683 | -| **總計** | **1,026** | **141,251** | +| `utils/hotkey/` | 7 | 837 | +| 其餘模組（約 286 個 `utils/` 子套件 + `android/`／`ios/`／周邊小工具） | 677 | 50,987 | +| **總計** | **1,043** | **146,758** | diff --git a/dev_requirements.txt b/dev_requirements.txt index d64208db..ab10d821 100644 --- a/dev_requirements.txt +++ b/dev_requirements.txt @@ -9,6 +9,12 @@ qt-material==2.17 mss==10.2.0 defusedxml==0.7.1 +# WebRTC ([webrtc] extra) — without these, eleven `utils/remote_desktop` +# modules raise ImportError at import and their tests skip, so the coverage +# figure a developer measures is ~4 points under the one CI enforces. +aiortc>=1.14.0 +av>=14.0.0 + # Office I/O ([office] extra) — exercised by the headless Office tests. openpyxl==3.1.5 python-docx==1.2.0 @@ -25,4 +31,7 @@ pytest-rerunfailures==15.1 # `pytest --cov` under-reports by ~24 points. Measure with # `coverage run -m pytest`; see the comment in .github/workflows/quality.yml. coverage==7.15.4 +# The complexity limit in CLAUDE.md is measured by this, both in the +# pre-commit list and by test/unit_test/headless/test_complexity_budget.py. +radon==6.0.1 mypy>=1.15 diff --git a/docs/source/Eng/doc/cli/cli_doc.rst b/docs/source/Eng/doc/cli/cli_doc.rst index 18d3fe87..49397d34 100644 --- a/docs/source/Eng/doc/cli/cli_doc.rst +++ b/docs/source/Eng/doc/cli/cli_doc.rst @@ -69,6 +69,9 @@ Execute all files in a directory python -m je_auto_control --execute_dir "path/to/action_files/" python -m je_auto_control -d "path/to/action_files/" +Every ``.json`` file under the directory runs, subdirectories included, in +sorted path order; links that lead outside the directory are not followed. + Execute a JSON string directly ------------------------------ diff --git a/docs/source/Eng/doc/mcp_server/mcp_server_doc.rst b/docs/source/Eng/doc/mcp_server/mcp_server_doc.rst index 44c2e332..62d2fdb4 100644 --- a/docs/source/Eng/doc/mcp_server/mcp_server_doc.rst +++ b/docs/source/Eng/doc/mcp_server/mcp_server_doc.rst @@ -88,13 +88,26 @@ Every tool carries the MCP 2025-06-18 ``annotations`` block read-only queries and require user confirmation before destructive ones. +A tool is destructive when it sends input, runs an action list, script or +code (now, or later from a scheduler, trigger, hotkey, watch or voice +command), deletes data, sends data off the machine, or loosens a security +control (egress, USB ACL, approvals, secret leases, hosting a remote +session). A tool that writes a file at a path the caller chooses is never +read-only, and a read-only tool given a ``db`` that does not exist answers +with an empty result instead of creating the file. ``ac_assert_http`` only +sends ``GET`` or ``HEAD``. + +A ``tools/call`` argument that the tool's input schema does not declare is +refused with ``-32602`` (invalid params) before the tool runs. + Resources, prompts, sampling ============================ Resources - ``autocontrol://files/`` — every JSON action file in the workspace root (re-targets when the client publishes - ``roots/list``). + ``roots/list``). Only a plain ``*.json`` name is readable; other + files in the root, subdirectories and ``:`` stream names are not. - ``autocontrol://history`` — recent run-history snapshot. - ``autocontrol://commands`` — full ``AC_*`` executor catalogue. - ``autocontrol://screen/live`` — base64 PNG screenshots, with @@ -253,6 +266,19 @@ box), start the same dispatcher behind HTTP: Bearer token can also come from ``JE_AUTOCONTROL_MCP_TOKEN``. +Browser requests are refused unless they come from this machine: a request +whose ``Origin`` header is not a loopback origin gets 403, and when the +server is bound to loopback so does one whose ``Host`` header does not name +loopback (DNS rebinding). Clients that are not browsers send no ``Origin`` +and are unaffected. To let a browser-based client on another origin in, list +its exact origins in ``JE_AUTOCONTROL_MCP_ALLOWED_ORIGINS`` +(comma-separated, e.g. ``https://tool.example:8443``). + +With ``JE_AUTOCONTROL_MCP_CONFIRM_DESTRUCTIVE=1``, a client that advertised +``elicitation`` must have its session's event stream open for a destructive +call to be confirmed; without one the call is refused rather than run. Only +the session a prompt was sent to can answer it. + Sessions ======== @@ -346,8 +372,10 @@ Audit log Set ``JE_AUTOCONTROL_MCP_AUDIT=/path/to/audit.jsonl`` to append one JSONL record per ``tools/call``: timestamp, tool name, sanitised -arguments (``password`` / ``token`` / ``secret`` / ``api_key`` / -``authorization`` are redacted), status (``ok`` / ``error`` / +arguments (``password`` / ``passphrase`` / ``token`` / ``secret`` / +``api_key`` / ``key`` / ``authorization`` and similar names are redacted at +any depth, and action lists are masked like the executor log), status +(``ok`` / ``error`` / ``cancelled``), duration, optional error text, and optional auto-screenshot artifact path (see below). @@ -440,4 +468,5 @@ Security notes normalised via ``os.path.realpath``; the resource provider blocks path traversal at the boundary. - Subprocess calls (``ac_launch_process`` / ``ac_shell``) accept - argv lists or ``shlex.split`` parses — never an OS shell. + argv lists or a command line (POSIX-split, or passed to + ``CreateProcess`` as written on Windows) — never an OS shell. diff --git a/docs/source/Eng/doc/new_features/new_features_doc.rst b/docs/source/Eng/doc/new_features/new_features_doc.rst index 9e9857e1..655ac978 100644 --- a/docs/source/Eng/doc/new_features/new_features_doc.rst +++ b/docs/source/Eng/doc/new_features/new_features_doc.rst @@ -132,6 +132,12 @@ start with ``AC_``. Each one becomes a new executor command:: # Now usable from JSON: # [["AC_greet", {"name": "world"}]] +A file that fails to import is logged and skipped; the rest of the directory +still loads. ``register_plugin_commands`` skips (and logs) a value that is not a +function and a name that already belongs to a built-in command such as +``AC_click_mouse``; pass ``allow_override=True`` to replace one on purpose. +A plugin may always re-register its own commands (a reload). + GUI: **Plugins** tab (browse directory, one-click register). .. warning:: @@ -386,7 +392,10 @@ false. ``AC_break`` / ``AC_continue`` work as in any loop:: runs (on success, on a caught error, or while a ``reraise`` / loop break/continue propagates). The error text is exposed to ``error_var`` for the ``catch`` branch to inspect, and ``reraise=true`` re-raises after -cleanup:: +cleanup. A failure anywhere inside ``body`` counts, including inside a loop, +an ``AC_if_*`` branch or a macro it runs: nested bodies inherit the strictness +of the list running them, which also makes ``AC_retry`` retry them and +``execute_action(..., raise_on_error=True)`` raise from them:: executor.execute_action([ ["AC_try", { @@ -544,9 +553,9 @@ Connection approval + view-only mode ------------------------------------ Optional callback gates every incoming session AnyDesk-style. -Returning ``"view_only"`` admits the viewer but drops their ``INPUT`` -messages; returning a falsy value (or raising) sends ``AUTH_FAIL`` -"rejected by host":: +Returning ``"view_only"`` admits the viewer but drops their ``INPUT``, +``CLIPBOARD`` and file-transfer messages; returning a falsy value (or +raising) sends ``AUTH_FAIL`` "rejected by host":: from je_auto_control import RemoteDesktopHost, PendingViewer @@ -561,7 +570,8 @@ IP allowlist (CIDR + exact IPs) ------------------------------- Reject peers outside the configured ranges *before* TLS / auth runs, -so attackers can't probe further:: +so attackers can't probe further. Invalid entries are dropped with a +warning; a list whose every entry is invalid admits nobody:: host = RemoteDesktopHost( token="tok", diff --git a/docs/source/Eng/doc/new_features/v103_features_doc.rst b/docs/source/Eng/doc/new_features/v103_features_doc.rst index 86b6eaf7..da18c566 100644 --- a/docs/source/Eng/doc/new_features/v103_features_doc.rst +++ b/docs/source/Eng/doc/new_features/v103_features_doc.rst @@ -29,7 +29,8 @@ Headless API ``in_progress`` (a duplicate before completion), or ``completed`` (replay the stored response); reusing a key with a different ``request`` fingerprint raises ``IdempotencyConflict`` (Stripe's HTTP-400 behaviour). ``complete`` records the -response, ``get`` reads a live record, and ``save`` / ``load`` persist the store +response, ``release`` drops an ``in_progress`` key whose work failed so a retry +runs it, ``get`` reads a live record, and ``save`` / ``load`` persist the store as JSON. ``request_fingerprint`` is a stable, order-independent SHA-256 of a payload. diff --git a/docs/source/Eng/doc/new_features/v10_features_doc.rst b/docs/source/Eng/doc/new_features/v10_features_doc.rst index 9ad7cdbe..59c3e6de 100644 --- a/docs/source/Eng/doc/new_features/v10_features_doc.rst +++ b/docs/source/Eng/doc/new_features/v10_features_doc.rst @@ -45,7 +45,9 @@ Dispatcher / performer item = q.get_next() ``get_next`` atomically claims the oldest ``new`` item (marking it -``in_progress``) so multiple performers don't double-process. +``in_progress``) so multiple performers don't double-process. ``complete`` and +``fail`` only settle an ``in_progress`` item; an unknown id or an item in any +other state raises, so finished work is never requeued. Failure semantics @@ -67,9 +69,14 @@ Executor commands ================= * ``AC_queue_add`` — enqueue ``data`` (dedup by ``reference``). -* ``AC_queue_next`` — claim the next item (or null when drained). -* ``AC_queue_complete`` — mark an item successful. -* ``AC_queue_fail`` — fail with ``kind`` (``application`` / ``business``). +* ``AC_queue_next`` — claim the next item (or null when drained). The item + carries a ``claim`` number; with ``stale_after_s`` an abandoned item is + reclaimed, which counts as a retry, and one abandoned ``max_retries`` times is + marked ``failed``. +* ``AC_queue_complete`` — mark an item successful. Pass the item's ``claim`` so + a performer whose item was reclaimed meanwhile is refused. +* ``AC_queue_fail`` — fail with ``kind`` (``application`` / ``business``); takes + ``claim`` the same way. * ``AC_queue_stats`` — per-status counts. The same ``db`` file + ``name`` identify a queue, so a dispatcher script diff --git a/docs/source/Eng/doc/new_features/v135_features_doc.rst b/docs/source/Eng/doc/new_features/v135_features_doc.rst index 5965407b..a30aafa6 100644 --- a/docs/source/Eng/doc/new_features/v135_features_doc.rst +++ b/docs/source/Eng/doc/new_features/v135_features_doc.rst @@ -34,8 +34,13 @@ The building blocks are ``to_grayscale``, ``upscale`` (``scale`` / ``interp``), ``denoise``, ``enhance_contrast`` (CLAHE), ``deskew`` and ``detect_skew_angle`` (returns the measured text-skew in degrees, clamped to ``±max_angle``). ``preprocess_image`` chains any of the named ``steps`` — ``grayscale``, -``upscale``, ``binarize``, ``denoise``, ``deskew``, ``contrast`` — in order; -unknown step names raise ``ValueError``. +``upscale``, ``binarize`` (Otsu), ``adaptive_mean`` / ``adaptive_gaussian`` +(tuned by ``block_size`` / ``c``), ``denoise``, ``deskew``, ``contrast`` — in +order; unknown step names raise ``ValueError``. Colour images are handled in +OpenCV's BGR order: files are read that way and PIL images and screen grabs are +converted, so grayscale weights red and blue correctly; paths may contain +non-ASCII characters. ``deskew`` works on light-on-dark text too, and +``upscale`` refuses a scale that is not a positive number. Executor command ---------------- diff --git a/docs/source/Eng/doc/new_features/v148_features_doc.rst b/docs/source/Eng/doc/new_features/v148_features_doc.rst index db96db9d..896fad2d 100644 --- a/docs/source/Eng/doc/new_features/v148_features_doc.rst +++ b/docs/source/Eng/doc/new_features/v148_features_doc.rst @@ -25,7 +25,9 @@ Headless API ``check(condition, message)`` records a pass/fail and never raises (it returns the bool, so you can branch on it); ``check_equal(actual, expected, message)`` is the equality shortcut. ``failures`` lists the failed messages, ``passed`` counts the -passes, and ``assert_all()`` raises ``AutoControlActionException`` aggregating them. +passes, and ``assert_all()`` raises ``SoftAssertionsFailed`` aggregating them -- an +``AutoControlAssertionException`` (a suite scores it *failed*, and a lenient run does +not swallow it) that is also the ``AutoControlActionException`` raised before. The context manager calls ``assert_all`` on a clean exit (and never masks an exception already propagating). Pass ``raise_on_exit=False`` to collect without auto-raising. diff --git a/docs/source/Eng/doc/new_features/v179_features_doc.rst b/docs/source/Eng/doc/new_features/v179_features_doc.rst index 8aeb2de1..3ac1b384 100644 --- a/docs/source/Eng/doc/new_features/v179_features_doc.rst +++ b/docs/source/Eng/doc/new_features/v179_features_doc.rst @@ -5,17 +5,19 @@ Every matcher in ``visual_match`` converts to grayscale first, so a red versus g indicator of identical shape is *indistinguishable* to ``match_template`` — the discriminating signal is thrown away. ``color_region`` finds blobs of a *known* colour but cannot template-match a multi-colour glyph by appearance. ``color_match`` matches on the HSV -hue / saturation channels using a colour-*distance* metric (``TM_SQDIFF_NORMED``, not a +hue / saturation channels using a colour-*distance* metric (squared distance, not a correlation — correlation normalises away the absolute hue, so a red→green edge and a black→blue edge would score the same), locating colour-discriminated targets that grayscale -matching collapses. +matching collapses. The score is ``1 -`` the root-mean-square distance, each channel scaled to +its full range; hue is compared the short way round the colour wheel, so hue 1 and hue 179 +are both red. It reuses ``color_region``'s RGB loaders and ``visual_match``'s resize / NMS / ``Match``. The ``haystack`` is injectable; the search is unit-testable on synthetic arrays. Imports no ``PySide6``. -Note: like any window metric, a *flat* single-colour patch has no per-channel variance — for -solid colour blobs use ``find_color_region``; ``color_match`` is for targets with colour +Note: a *flat* single-colour patch matches every area of that colour — for solid colour +blobs ``find_color_region`` is the better tool; ``color_match`` is for targets with colour *structure*. Headless API diff --git a/docs/source/Eng/doc/new_features/v185_features_doc.rst b/docs/source/Eng/doc/new_features/v185_features_doc.rst index 134cf3fd..5814f8fb 100644 --- a/docs/source/Eng/doc/new_features/v185_features_doc.rst +++ b/docs/source/Eng/doc/new_features/v185_features_doc.rst @@ -34,8 +34,10 @@ Headless API set_clipboard_rtf("Paste me as styled text") # Windows set_clipboard_csv([["Name", "Qty"], ["Pen", "3"]], delimiter="\t") # TSV -``build_rtf`` escapes braces / backslashes, turns newlines into ``\par`` and -non-ASCII characters into ``\uNNNN?`` escapes (the output is pure ASCII). +``build_rtf`` escapes braces / backslashes, turns line breaks (``\n``, ``\r\n`` +or a lone ``\r``) into ``\par`` and non-ASCII characters into ``\uN?`` escapes of +their signed 16-bit UTF-16 units (the output is pure ASCII); ``rtf_to_text`` +honours ``\ucN`` and joins surrogate pairs. ``set_clipboard_rtf`` / ``set_clipboard_csv`` also seed plain text by default so plain editors still paste something; ``get_clipboard_rtf`` returns the raw RTF string (feed it to ``rtf_to_text``) and ``get_clipboard_csv`` returns rows. diff --git a/docs/source/Eng/doc/new_features/v20_features_doc.rst b/docs/source/Eng/doc/new_features/v20_features_doc.rst index 61c56290..3f4fd20b 100644 --- a/docs/source/Eng/doc/new_features/v20_features_doc.rst +++ b/docs/source/Eng/doc/new_features/v20_features_doc.rst @@ -19,11 +19,14 @@ text and pre-stress layout *before* any real translation exists:: from je_auto_control import pseudo_localize, pseudo_localize_catalog - pseudo_localize("Hello {name}") # "⟦Hèllo {name}········⟧" + pseudo_localize("Hello {name}") # "⟦Hèllò {name}··⟧" pseudo_localize_catalog({"save": "Save", "cancel": "Cancel"}) -Placeholders (``{name}`` / ``{{x}}`` / ``%s`` / ``%d``) are preserved -verbatim; ``expansion`` controls the padding fraction; the ``⟦…⟧`` brackets +Placeholders (``{name}`` / ``{{x}}`` / ``{0}``, printf conversions such as +``%s`` / ``%(user)s`` / ``%1$s``), HTML tags and the structure of ICU +``plural`` / ``select`` arguments are preserved verbatim, while the text of each +ICU case is localized; ``expansion`` is the padding as a fraction of the visible +text; the ``⟦…⟧`` brackets make truncation visible. Exposed as ``AC_pseudo_localize`` / ``ac_pseudo_localize``. Untranslated (un-accented) strings in a screen are a sign of unexternalized, hardcoded text. diff --git a/docs/source/Eng/doc/new_features/v21_features_doc.rst b/docs/source/Eng/doc/new_features/v21_features_doc.rst index eaa16bef..cd48da95 100644 --- a/docs/source/Eng/doc/new_features/v21_features_doc.rst +++ b/docs/source/Eng/doc/new_features/v21_features_doc.rst @@ -27,7 +27,8 @@ variables:: result = run_resumable(actions, run_id="nightly-invoices", store=store) result["resumed_from"] # 0 on a fresh run, N when resuming after a crash -On normal completion the checkpoint is cleared. The store is injectable, so +On normal completion the checkpoint is cleared. A failing step raises and +leaves the checkpoint on that step, so the next call runs it again. The store is injectable, so resume is unit-tested deterministically without a real crash: ``CheckpointStore.save`` / ``load`` / ``clear``. diff --git a/docs/source/Eng/doc/new_features/v2_features_doc.rst b/docs/source/Eng/doc/new_features/v2_features_doc.rst index e756c820..b7d9bafb 100644 --- a/docs/source/Eng/doc/new_features/v2_features_doc.rst +++ b/docs/source/Eng/doc/new_features/v2_features_doc.rst @@ -202,7 +202,9 @@ Computer-use high-level API --------------------------- Wraps :class:`ComputerUseAgentBackend` + :class:`AgentLoop` so a -single call drives Anthropic's official ``computer_20250124`` tool:: +single call drives Anthropic's computer-use tool (``computer_20251124`` on +``claude-opus-5`` by default, sent under its ``computer-use-2025-11-24`` beta; +``tool_type=`` picks another version and ``beta=`` names its beta):: from je_auto_control import run_computer_use result = run_computer_use( @@ -236,7 +238,11 @@ Chat-ops bot Transport-agnostic ``CommandRouter`` plus a polling Slack adapter so ``/run