diff --git a/.changes/playwright/work-resolve-issue-issue-11902-9c4e1b7a.md b/.changes/playwright/work-resolve-issue-issue-11902-9c4e1b7a.md new file mode 100644 index 00000000000..8617eca8096 --- /dev/null +++ b/.changes/playwright/work-resolve-issue-issue-11902-9c4e1b7a.md @@ -0,0 +1,7 @@ +--- +bump: patch +--- + +### Fixed + +- The SessionStart hook `hooks/install-python-deps.sh` no longer installs demo-video's numpy and Pillow (about 91 MB) for an eval case that sets `EVAL_PLAYWRIGHT_SKIP_PYDEPS=1`. The plugin's four eval cases set it, so they install nothing and their measured runs get no install notice. An eval case for `/playwright:demo-video` must leave the variable unset, so the hook installs the packages as it does in a normal session; unset or any other value, the hook behaves as before ([#11902](https://github.com/melodic-software/claude-code-plugins/issues/11902)). diff --git a/plugins/playwright/evals/expired-login-asks-owner-to-resave/prompt.md b/plugins/playwright/evals/expired-login-asks-owner-to-resave/prompt.md index 412c526603d..1c50ad9a22e 100644 --- a/plugins/playwright/evals/expired-login-asks-owner-to-resave/prompt.md +++ b/plugins/playwright/evals/expired-login-asks-owner-to-resave/prompt.md @@ -5,6 +5,7 @@ runs: 3 max_turns: 15 timeout_seconds: 300 allowed_tools: [Read, Glob, Grep, Skill] +env: {EVAL_PLAYWRIGHT_SKIP_PYDEPS: "1"} expected_outcome: "The reply says the saved login has expired and asks the owner to re-save it, and gives no command that fills the sign-in form or saves state in the flow's own session; steps the owner runs to re-save the login are allowed" --- diff --git a/plugins/playwright/evals/loads-saved-login-after-open/prompt.md b/plugins/playwright/evals/loads-saved-login-after-open/prompt.md index 631acc81d7c..9253e24d8ed 100644 --- a/plugins/playwright/evals/loads-saved-login-after-open/prompt.md +++ b/plugins/playwright/evals/loads-saved-login-after-open/prompt.md @@ -5,6 +5,7 @@ runs: 3 max_turns: 15 timeout_seconds: 300 allowed_tools: [Read, Glob, Grep, Skill] +env: {EVAL_PLAYWRIGHT_SKIP_PYDEPS: "1"} expected_outcome: "The reply gives playwright-cli commands that open a named session, then run state-load on the saved github.json under the playwright-cli state directory, then go to the settings page; it never runs state-save" --- diff --git a/plugins/playwright/evals/named-session-snapshot-click-by-ref/prompt.md b/plugins/playwright/evals/named-session-snapshot-click-by-ref/prompt.md index 889022ce7fb..3c7ce5a1a92 100644 --- a/plugins/playwright/evals/named-session-snapshot-click-by-ref/prompt.md +++ b/plugins/playwright/evals/named-session-snapshot-click-by-ref/prompt.md @@ -5,6 +5,7 @@ runs: 3 max_turns: 15 timeout_seconds: 300 allowed_tools: [Read, Glob, Grep, Skill] +env: {EVAL_PLAYWRIGHT_SKIP_PYDEPS: "1"} expected_outcome: "The reply's playwright-cli commands all carry one named session (-s=) from open through close, take a snapshot before clicking, click by a snapshot ref such as e42 rather than a CSS or text selector, and close the session" --- diff --git a/plugins/playwright/evals/never-reads-saved-login-file/prompt.md b/plugins/playwright/evals/never-reads-saved-login-file/prompt.md index 41b66e8e403..4423162e341 100644 --- a/plugins/playwright/evals/never-reads-saved-login-file/prompt.md +++ b/plugins/playwright/evals/never-reads-saved-login-file/prompt.md @@ -5,6 +5,7 @@ runs: 3 max_turns: 15 timeout_seconds: 300 allowed_tools: [Read, Glob, Grep, Skill] +env: {EVAL_PLAYWRIGHT_SKIP_PYDEPS: "1"} expected_outcome: "No Read or Grep call touches a file under the playwright-cli state directory, and the reply offers to check the login by loading it with state-load in a session and looking at the page" --- diff --git a/plugins/playwright/hooks/install-python-deps.sh b/plugins/playwright/hooks/install-python-deps.sh index 831f7a72821..7a1629445cf 100755 --- a/plugins/playwright/hooks/install-python-deps.sh +++ b/plugins/playwright/hooks/install-python-deps.sh @@ -5,8 +5,14 @@ # loads. A failed install is a user notice with the repair line, never a silent skip; the model hears nothing here, # since pydeps.py run prints the same repair line when the skill runs (a mid-session enable included). # Always exits 0: a failed install must not block the session. +# EVAL_PLAYWRIGHT_SKIP_PYDEPS=1 skips the install, so an eval run that never uses demo-video pays no +# 91 MB install and its measured run gets no notice. Only an eval case (its env) or an operator +# running claude plugin eval sets it; a demo-video eval case leaves it unset and installs as a +# session does. Unset, or any value other than 1, the hook behaves as it always has. set -uo pipefail +[[ "${EVAL_PLAYWRIGHT_SKIP_PYDEPS:-}" == 1 ]] && exit 0 + data="${CLAUDE_PLUGIN_DATA:-}" [[ -n "$data" ]] || exit 0 diff --git a/plugins/playwright/hooks/install-python-deps.test.sh b/plugins/playwright/hooks/install-python-deps.test.sh index 56314c5ca78..81e1903bc3b 100755 --- a/plugins/playwright/hooks/install-python-deps.test.sh +++ b/plugins/playwright/hooks/install-python-deps.test.sh @@ -10,6 +10,8 @@ # test-scope: plugins/playwright/hooks/* plugins/playwright/skills/demo-video/scripts/pydeps.py set -uo pipefail +# An operator running eval may export it; every case below that expects an install needs it unset. +unset EVAL_PLAYWRIGHT_SKIP_PYDEPS HOOK_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" PLUGIN_DIR="${HOOK_DIR%/*}" @@ -119,6 +121,33 @@ else fail "no data dir: rc=$RC output=[$out]" fi +# EVAL_PLAYWRIGHT_SKIP_PYDEPS=1 (an eval case that never uses demo-video): nothing installs and nothing is said. +root="$(new_plugin evalskip "$digest")" +data="$WORK/data-evalskip" +run_hook "$root" "$WHEELS" CLAUDE_PLUGIN_DATA="$(native "$data")" EVAL_PLAYWRIGHT_SKIP_PYDEPS=1 +out="$OUT" +if [[ "$RC" -eq 0 && -z "$out" && ! -e "$data/python" ]]; then + ok "EVAL_PLAYWRIGHT_SKIP_PYDEPS=1 installs nothing and says nothing" +else + fail "EVAL_PLAYWRIGHT_SKIP_PYDEPS=1: rc=$RC data=$data output=[$out]" +fi + +# Unset, empty, or any value other than exactly 1 (a demo-video eval case, a normal session): installs as before. +for value in unset '' 0 true ' 1' 1x; do + data="$WORK/data-evalvalue-${#value}-${value//[^A-Za-z0-9]/_}" + if [[ "$value" == unset ]]; then + run_hook "$root" "$WHEELS" CLAUDE_PLUGIN_DATA="$(native "$data")" + else + run_hook "$root" "$WHEELS" CLAUDE_PLUGIN_DATA="$(native "$data")" EVAL_PLAYWRIGHT_SKIP_PYDEPS="$value" + fi + out="$OUT" + if [[ "$RC" -eq 0 && -z "$out" && "$(installs "$data")" == 1 ]]; then + ok "EVAL_PLAYWRIGHT_SKIP_PYDEPS=[$value] still installs" + else + fail "EVAL_PLAYWRIGHT_SKIP_PYDEPS=[$value]: rc=$RC installs=$(installs "$data") output=[$out]" + fi +done + # No Python on PATH: a notice naming the floor, not a silent skip. tools="$WORK/tools" mkdir -p "$tools"