From 261b2ea1784b5ebab249b45584cfecb58aee7a21 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 3 Nov 2026 19:52:27 +0000 Subject: [PATCH 01/64] fix(client-reports): keep Control's URL out of branding, fix period lengths, explicit From Branding: the compiler fell back to config('app.url') for the stored support_url whenever both the reports and the shared Companion support URLs were blank. Control runs on a private host, so every report carried a localhost address that would leak the moment any view rendered it. The support URL now falls back to the agency website and then null. The dead non-branding-manager branch (and its unused Settings dependency) is gone. The branding manager itself is unchanged: a blank Companion support URL still hides Companion's support button. Settings: the reports support URL and the shared support / agency website URLs must now be public http(s) addresses. Localhost, loopback, private and reserved IPs, single-label and internal-only names, and the app's own URL are rejected; blank is still accepted. The reports tab help text states the real fallback order. Metadata: Carbon 3's diffInDays returns a float, and the period end is the last microsecond of the day, so meta.period_days was stored as 31.99... for a 31-day month and uptime total minutes came out one short. Both are now whole numbers (31 / 44,640 for a 31-day month, 30 / 43,200 for 30 days). SCHEMA_VERSION is bumped to 4. Email: the envelope now sets From to the configured sending address under the report brand's company name, so the inbox no longer depends on MAIL_FROM_NAME (by default the internal product name). The operator's "Email version" page shows that From line. The plain-text part now prints the footer text when one is set, matching the HTML part. Wording: template section descriptions promised data the report never contains (referrers, page views, TTFB, translation upgrades, restore readiness, form deliverability). They now describe what is compiled. The contact-form testing docs no longer claim inbox delivery verification: scheduled tests run in lab mode with the email suppressed, and live runs only record whether the site reported the email as sent. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Q7rExNMgxzTNb3pagVUZrg --- .../CompanionSettingsController.php | 62 +++++++++- .../views/emails/report-text.blade.php | 4 + .../resources/views/reports/email.blade.php | 1 + .../Controllers/ClientReportsController.php | 7 +- .../src/Mail/ClientReportMail.php | 16 +++ .../src/Models/ClientReportTemplate.php | 16 +-- .../src/Services/ClientReportCompiler.php | 49 ++++---- resources/docs/features/client-reports.md | 8 +- .../docs/features/contact-form-testing.md | 21 ++-- resources/views/settings/companion.blade.php | 1 + .../CompanionSettingsControllerTest.php | 71 ++++++++++- .../Modules/ClientReportCompilerTest.php | 114 ++++++++++++++++++ .../Feature/Modules/ClientReportMailTest.php | 38 ++++++ .../Modules/ClientReportTemplatesTest.php | 19 +++ .../Modules/ClientReportsControllerTest.php | 18 +++ 15 files changed, 396 insertions(+), 49 deletions(-) create mode 100644 tests/Feature/Modules/ClientReportCompilerTest.php diff --git a/app/Http/Controllers/CompanionSettingsController.php b/app/Http/Controllers/CompanionSettingsController.php index 6a892467..acb86219 100644 --- a/app/Http/Controllers/CompanionSettingsController.php +++ b/app/Http/Controllers/CompanionSettingsController.php @@ -7,6 +7,7 @@ use App\Models\Site; use App\Services\Companion\CompanionBrandingManager; use App\Services\Process\BackgroundArtisan; +use Closure; use Illuminate\Http\JsonResponse; use Illuminate\Http\RedirectResponse; use Illuminate\Http\Request; @@ -139,7 +140,7 @@ public function update(Request $request, CompanionBrandingManager $brandingManag 'enabled' => 'nullable|boolean', 'company_name' => 'nullable|string|max:120', 'support_email' => 'nullable|email|max:120', - 'support_url' => 'nullable|string|max:255', + 'support_url' => self::publicUrlRules(), 'primary_color' => CompanionBrandingManager::hexColorRules(), 'accent_color' => CompanionBrandingManager::hexColorRules(), 'footer_text' => 'nullable|string|max:500', @@ -209,9 +210,9 @@ public function update(Request $request, CompanionBrandingManager $brandingManag $validated = $request->validate([ 'enabled' => 'nullable|boolean', 'company_name' => 'nullable|string|max:120', - 'company_url' => 'nullable|string|max:255', + 'company_url' => self::publicUrlRules(), 'support_email' => 'nullable|email|max:120', - 'support_url' => 'nullable|string|max:255', + 'support_url' => self::publicUrlRules(), 'plugin_name' => 'nullable|string|max:120', 'plugin_description' => 'nullable|string|max:500', 'menu_title' => 'nullable|string|max:60', @@ -419,4 +420,59 @@ public function sendTestEmail(Request $request, CompanionBrandingManager $brandi ], 500); } } + + /** + * Rules for an address clients are sent to (agency website, support + * portal). Blank is allowed. A value must be a public http(s) URL: not + * localhost, a private or reserved IP, an internal-only name, or + * Control's own URL (APP_URL), which runs on a private host clients + * can't reach. + * + * @return list + */ + private static function publicUrlRules(): array + { + return [ + 'nullable', + 'string', + 'max:255', + 'url:http,https', + function (string $attribute, mixed $value, Closure $fail): void { + $host = is_string($value) ? parse_url($value, PHP_URL_HOST) : null; + + // Not a URL at all: the url rule already says so. + if (! is_string($host) || $host === '') { + return; + } + + if (! self::isPublicHost($host)) { + $fail('The :attribute must be a public web address clients can reach, not localhost, a private network address or this app\'s own URL.'); + } + }, + ]; + } + + private static function isPublicHost(string $host): bool + { + $host = rtrim(strtolower(trim($host, '[]')), '.'); + + if (filter_var($host, FILTER_VALIDATE_IP) !== false) { + return filter_var($host, FILTER_VALIDATE_IP, FILTER_FLAG_GLOBAL_RANGE) !== false; + } + + // Single-label names (localhost, intranet hosts) don't resolve publicly. + if (! str_contains($host, '.')) { + return false; + } + + foreach (['.localhost', '.local', '.internal', '.lan', '.home.arpa', '.test', '.invalid'] as $suffix) { + if (str_ends_with($host, $suffix)) { + return false; + } + } + + $appHost = parse_url((string) config('app.url'), PHP_URL_HOST); + + return ! is_string($appHost) || $host !== rtrim(strtolower($appHost), '.'); + } } diff --git a/modules/ClientReports/resources/views/emails/report-text.blade.php b/modules/ClientReports/resources/views/emails/report-text.blade.php index 12073d3d..0cd2eecb 100644 --- a/modules/ClientReports/resources/views/emails/report-text.blade.php +++ b/modules/ClientReports/resources/views/emails/report-text.blade.php @@ -114,6 +114,10 @@ - {!! \Illuminate\Support\Carbon::parse($entry['worked_on'])->format('M j') !!}: {!! $entry['description'] !!} ({!! $entry['hours'] !!}h) @endforeach @endif +@if ($branding['footer_text'] !== '') + +{!! $branding['footer_text'] !!} +@endif Questions about anything in this report? Just reply to this email{!! $branding['support_email'] !== '' ? ' or write to '.$branding['support_email'] : '' !!}. diff --git a/modules/ClientReports/resources/views/reports/email.blade.php b/modules/ClientReports/resources/views/reports/email.blade.php index e4cc0e39..acf8d5e9 100644 --- a/modules/ClientReports/resources/views/reports/email.blade.php +++ b/modules/ClientReports/resources/views/reports/email.blade.php @@ -25,6 +25,7 @@
+
From{{ $from ?? '' }}
To{{ $toLabel }}
@if (! empty($report->sent_to))
Sent to{{ implode(', ', $report->sent_to) }}
diff --git a/modules/ClientReports/src/Http/Controllers/ClientReportsController.php b/modules/ClientReports/src/Http/Controllers/ClientReportsController.php index 57015720..07474388 100644 --- a/modules/ClientReports/src/Http/Controllers/ClientReportsController.php +++ b/modules/ClientReports/src/Http/Controllers/ClientReportsController.php @@ -6,6 +6,7 @@ use App\Models\Site; use Illuminate\Http\RedirectResponse; use Illuminate\Http\Request; +use Illuminate\Mail\Mailables\Address; use Illuminate\Support\Carbon; use Illuminate\View\View; use Modules\ClientReports\Mail\ClientReportMail; @@ -107,10 +108,14 @@ public function email(ClientReport $report): View $report->load('site.client', 'schedule'); $mail = new ClientReportMail($report); $content = $mail->content(); + $envelope = $mail->envelope(); + // With no From on the envelope, Laravel sends from the global address. + $from = $envelope->from ?? new Address((string) config('mail.from.address'), (string) config('mail.from.name')); return view('client-reports::reports.email', [ 'report' => $report, - 'subject' => $mail->envelope()->subject, + 'from' => $from->name !== null && $from->name !== '' ? "{$from->name} <{$from->address}>" : $from->address, + 'subject' => $envelope->subject, 'html' => $mail->render(), // Plain-text clients get this part instead. 'text' => view($content->text, $content->with)->render(), diff --git a/modules/ClientReports/src/Mail/ClientReportMail.php b/modules/ClientReports/src/Mail/ClientReportMail.php index 423fa37a..03e12e56 100644 --- a/modules/ClientReports/src/Mail/ClientReportMail.php +++ b/modules/ClientReports/src/Mail/ClientReportMail.php @@ -36,11 +36,27 @@ public function envelope(): Envelope : []; return new Envelope( + from: $this->fromAddress(), subject: sprintf('Your %s website report — %s', $this->report->period_start->format('F Y'), $this->domain()), replyTo: $replyTo, ); } + /** + * The configured sending address under the report brand's company name, + * so the inbox shows the agency rather than whatever MAIL_FROM_NAME says + * (by default the internal product name). Null leaves Laravel's global + * From in place when no sending address is configured. + */ + public function fromAddress(): ?Address + { + $address = (string) config('mail.from.address'); + + return filter_var($address, FILTER_VALIDATE_EMAIL) + ? new Address($address, $this->branding()['company_name']) + : null; + } + public function content(): Content { return new Content( diff --git a/modules/ClientReports/src/Models/ClientReportTemplate.php b/modules/ClientReports/src/Models/ClientReportTemplate.php index d9d67aa3..cfbc9cf2 100644 --- a/modules/ClientReports/src/Models/ClientReportTemplate.php +++ b/modules/ClientReports/src/Models/ClientReportTemplate.php @@ -41,42 +41,42 @@ protected function casts(): array public const SECTIONS = [ 'updates' => [ 'label' => 'Updates & Upgrades', - 'description' => 'Plugins, themes, core updates and translation upgrades applied.', + 'description' => 'Plugin, theme and WordPress core updates applied during the period, and any still waiting at its end.', 'icon' => 'fa-circle-arrow-up', ], 'uptime' => [ 'label' => 'Uptime & Availability', - 'description' => 'Availability percentage, outage counts, and total downtime.', + 'description' => 'Uptime percentage, outage count, total downtime and the most recent outages, for sites the uptime monitor checks.', 'icon' => 'fa-heart-pulse', ], 'security' => [ 'label' => 'Security & Firewall', - 'description' => 'Checksum malware scans, file integrity, and blocked malicious requests.', + 'description' => 'Latest malware scan, WordPress core file check, blocklist check and known plugin vulnerabilities, plus attackers blocked when the site reports its blocks.', 'icon' => 'fa-shield-halved', ], 'performance' => [ 'label' => 'Performance Benchmarks', - 'description' => 'PageSpeed scores, time to first byte (TTFB), and page load speed.', + 'description' => 'Latest mobile and desktop performance score and page load time (LCP) from tests run during the period.', 'icon' => 'fa-gauge-high', ], 'forms' => [ 'label' => 'Contact Form Testing', - 'description' => 'Automated synthetic form submission success and deliverability rates.', + 'description' => 'Automated test submissions and whether each form accepted them. Scheduled tests suppress the email, so delivery is only reported for live tests.', 'icon' => 'fa-envelope-circle-check', ], 'traffic' => [ 'label' => 'Traffic Analytics', - 'description' => 'Monthly unique visits, top referrers, and page view statistics.', + 'description' => 'Visits, page and file requests served, and data transferred (when the host reports it).', 'icon' => 'fa-chart-line', ], 'backups' => [ 'label' => 'Backups & Snapshots', - 'description' => 'Completed backup runs, cloud storage locations, and restore readiness.', + 'description' => 'Backups recorded from the site\'s own backup source: where they are stored and the last 10 days of the period.', 'icon' => 'fa-box-archive', ], 'work_log' => [ 'label' => 'Work log', - 'description' => 'Manual hours logged by the team during the report period.', + 'description' => 'Hours logged in the site\'s work log during the period; left out when there are none.', 'icon' => 'fa-clock', ], ]; diff --git a/modules/ClientReports/src/Services/ClientReportCompiler.php b/modules/ClientReports/src/Services/ClientReportCompiler.php index d8dd7f80..1d1d7518 100644 --- a/modules/ClientReports/src/Services/ClientReportCompiler.php +++ b/modules/ClientReports/src/Services/ClientReportCompiler.php @@ -18,12 +18,11 @@ use App\Services\Backups\SiteBackupRecorder; use App\Services\Companion\CompanionBrandingManager; use App\Services\Security\PluginVulnerabilityMatcher; -use App\Support\Settings; use Illuminate\Support\Carbon; class ClientReportCompiler { - public const SCHEMA_VERSION = 3; + public const SCHEMA_VERSION = 4; /** How many of the period's final days the backups list shows. */ public const BACKUP_LIST_DAYS = 10; @@ -42,7 +41,6 @@ class ClientReportCompiler protected ?bool $vulnerabilityFeedFresh = null; public function __construct( - protected Settings $settings, protected PluginVulnerabilityMatcher $vulnerabilities, protected SiteBackupRecorder $backups, ) {} @@ -74,7 +72,9 @@ public function compile( 'compiled_at' => now()->toIso8601String(), 'period_start' => $start->toDateString(), 'period_end' => $end->toDateString(), - 'period_days' => $start->diffInDays($end) + 1, + // Carbon 3's diffInDays is a float, and $end is the last + // microsecond of the day, so round to whole days (31 for October). + 'period_days' => (int) round($start->diffInDays($end)), 'custom_notes' => $customNotes, // Bumped when a section's meaning changes, so the views can // tell reports compiled under the old (overstating) rules apart. @@ -119,30 +119,28 @@ public function compile( return $data; } + /** + * The report's branding, captured at compile time. The support URL falls + * back to the agency website and never to Control's own URL: Control runs + * on a private host (often localhost) that clients can't reach. A blank + * Companion support URL is meaningful there (it hides the support + * button), so the fallback lives here rather than in the branding manager. + */ protected function compileBranding(): array { - if (class_exists(CompanionBrandingManager::class)) { - $reportsBranding = app(CompanionBrandingManager::class)->getReportsBranding(); - - return [ - 'company_name' => $reportsBranding['company_name'] ?: config('app.name', 'Clockwork Control'), - 'support_url' => $reportsBranding['support_url'] ?: config('app.url'), - 'support_email' => $reportsBranding['support_email'] ?: config('mail.from.address'), - 'logo_url' => $reportsBranding['logo_url'], - 'primary_color' => $reportsBranding['primary_color'], - 'accent_color' => $reportsBranding['accent_color'], - 'footer_text' => $reportsBranding['footer_text'], - ]; - } + $reportsBranding = app(CompanionBrandingManager::class)->getReportsBranding(); + + $supportUrl = trim($reportsBranding['support_url']); + $companyUrl = trim($reportsBranding['company_url']); return [ - 'company_name' => $this->settings->get('companion.branding.company_name') ?: ($this->settings->get('companion.company_name') ?: config('app.name', 'Clockwork Control')), - 'support_url' => $this->settings->get('companion.branding.support_url') ?: ($this->settings->get('companion.support_url') ?: config('app.url')), - 'support_email' => $this->settings->get('companion.branding.support_email') ?: ($this->settings->get('companion.support_email') ?: config('mail.from.address')), - 'logo_url' => $this->settings->get('companion.branding.logo_url') ?: $this->settings->get('companion.logo_url'), - 'primary_color' => '#2D2062', - 'accent_color' => '#7EFF83', - 'footer_text' => '', + 'company_name' => $reportsBranding['company_name'] ?: config('app.name', 'Clockwork Control'), + 'support_url' => $supportUrl !== '' ? $supportUrl : ($companyUrl !== '' ? $companyUrl : null), + 'support_email' => $reportsBranding['support_email'] ?: config('mail.from.address'), + 'logo_url' => $reportsBranding['logo_url'], + 'primary_color' => $reportsBranding['primary_color'], + 'accent_color' => $reportsBranding['accent_color'], + 'footer_text' => $reportsBranding['footer_text'], ]; } @@ -295,7 +293,8 @@ public static function isNoOpUpdate(string $summary): bool */ protected function compileUptime(Site $site, Carbon $start, Carbon $end): array { - $totalMinutes = max(1, (int) $start->diffInMinutes($end)); + // $end is the last microsecond of the day: round, or a 31-day month is 44,639 minutes. + $totalMinutes = max(1, (int) round($start->diffInMinutes($end))); $periodEnd = $end->copy()->min(now()); $priorEvent = SiteUptimeEvent::query() diff --git a/resources/docs/features/client-reports.md b/resources/docs/features/client-reports.md index 947c5cb6..8e96987d 100644 --- a/resources/docs/features/client-reports.md +++ b/resources/docs/features/client-reports.md @@ -2,7 +2,7 @@ title: Client reports section: Features order: 101 -updated: 2026-10-22 +updated: 2026-11-03 author: Aaron Reimann tags: [client-reports, reporting, email, care-plans, scheduling, templates] tracks: [modules/ClientReports/**] @@ -59,7 +59,7 @@ The rule for every section: **if we didn't check it during the period, the repor | `backups` | `site_backups`: what the site's **real** backup source reported, recorded daily by `clockwork:record-site-backups` (05:30). Pressable → Pressable's backup API (files daily, plus one database backup per day); SpinupWP → the DigitalOcean Spaces bucket SpinupWP writes to; unhosted sites → our own S3 archives (Companion uploads). The email lists the **last 10 days of the period**, one row per day, plus how many days that month had a backup. That count is left out when our records for the site start after the period's first day (`covers_period` false): Pressable keeps only ~8 days, so the month we started recording would otherwise read as "backed up on 9 days". A day's size counts one backup of each kind (the newest scheduled one), so extra on-demand or hourly backups don't inflate it. | In period | Section hidden from the client. The preview shows **No backups found** (checked, none), **No backups in the last 10 days** (backups stopped mid-period) or **Could not check** (read failed, or no clean check in 48h), with the error. | | `work_log` | `site_work_logs` (hours entered on the site page) | In period | Section hidden | -Every compiled report carries `meta.schema_version` (currently **3**: 2 = honesty fixes, 3 = recorded backups list). Reports compiled before the honesty fixes have no version. The views treat their vulnerability line as "Not checked" and hide their forms numbers, because those were computed wrongly. +Every compiled report carries `meta.schema_version` (currently **4**: 2 = honesty fixes, 3 = recorded backups list, 4 = whole-number period lengths and the support URL fix below). `meta.period_days` is the period's length in whole days (31 for October). Reports compiled before the honesty fixes have no version. The views treat their vulnerability line as "Not checked" and hide their forms numbers, because those were computed wrongly. ### Backup recording @@ -107,13 +107,13 @@ Careful: `--force` and **Send now** send for real on an `auto` schedule. On a de ## The email -`ClientReportMail` puts the whole report in the email body: an HTML view plus a plain-text one (`emails/report*.blade.php`). The logo is embedded inline via `BrandLogo` so it still shows when remote images are blocked. Update rows are cleaned up into client-friendly lines ("TranslatePress 3.3.6 → 3.3.7"). Reply-To is the report brand's support email. +`ClientReportMail` puts the whole report in the email body: an HTML view plus a plain-text one (`emails/report*.blade.php`). The logo is embedded inline via `BrandLogo` so it still shows when remote images are blocked. Update rows are cleaned up into client-friendly lines ("TranslatePress 3.3.6 → 3.3.7"). The From line is the configured sending address (`MAIL_FROM_ADDRESS`) under the report brand's company name, so the inbox shows the agency whatever `MAIL_FROM_NAME` says; the **Email version** page shows it. Reply-To is the report brand's support email. The footer text, when set, appears in both the HTML and plain-text parts. There's deliberately **no "view online" link**: Control lives on a private network clients can't reach. That's also why the public `/reports/view/{token}` route exists but isn't linked anywhere useful. ## Branding -Colours, logo, company name and support details come from **Settings → Companion & White Label → Client Reports** (`/settings/companion?tab=reports`, stored as `reports.branding.*`). Blank fields fall back to the shared Companion identity. See [Companion & White Label](/docs/features/companion-branding). +Colours, logo, company name and support details come from **Settings → Companion & White Label → Client Reports** (`/settings/companion?tab=reports`, stored as `reports.branding.*`). Blank fields fall back to the shared Companion identity. The support URL falls back from the reports value to the shared Companion support URL, then to the Agency Website; it never falls back to Control's own URL (`APP_URL`), which is private. The settings form only accepts public http(s) addresses there, not localhost or private-network hosts. The email doesn't link to it today; it's stored with each report. See [Companion & White Label](/docs/features/companion-branding). The **primary colour** fills the header banner of both the email and the operator preview (and the section underlines); the logo sits on it, so use a logo that reads on that colour. The preview keeps the colours when printed or saved as a PDF. Because branding is captured in the snapshot, regenerate a report to see a colour change. diff --git a/resources/docs/features/contact-form-testing.md b/resources/docs/features/contact-form-testing.md index 20f01925..14498343 100644 --- a/resources/docs/features/contact-form-testing.md +++ b/resources/docs/features/contact-form-testing.md @@ -2,13 +2,13 @@ title: Contact form testing section: Features order: 70 -updated: 2026-10-04 +updated: 2026-11-03 author: Aaron Reimann tags: [contact-forms, companion, testing, care-plan, slack, modules] tracks: [modules/ContactForms/src/Commands/{TestContactForms,DetectContactForms,SyncCompanionFormSubscriptions}.php, modules/ContactForms/src/ContactFormsServiceProvider.php, modules/ContactForms/src/ContactFormTester.php, modules/ContactForms/src/FormsController.php, app/Http/Controllers/FormsController.php, app/Models/ContactFormTest.php, resources/views/dashboard/forms/index.blade.php] --- -A care-plan benefit that confirms a site's contact forms are still firing. Companion injects a marker and triggers the form; we verify the receiving email lands. Failures and recoveries fire a chat alert. Care-plan sites can configure up to **3 forms per site**, each on its own schedule. Packaged as the `clockwork/contact-forms` module (`modules/ContactForms`) in the **Maintenance & QA** category. +A care-plan benefit that confirms a site's contact forms still accept submissions. Companion submits the form with a marker and reports whether the form plugin accepted it. Scheduled tests suppress the outgoing email, so they don't test delivery (see [Lab vs live](#lab-vs-live)). Failures and recoveries fire a chat alert. Care-plan sites can configure up to **3 forms per site**, each on its own schedule. Packaged as the `clockwork/contact-forms` module (`modules/ContactForms`) in the **Maintenance & QA** category. ## Module architecture & enablement @@ -24,10 +24,17 @@ Contact Form Testing is decoupled into a standalone module (`modules/ContactForm 2. Companion is installed on the site with the `'contact-form-test'` capability advertised. 3. Operator opens the per-site **Forms** tab and adds up to 3 form-tests (form ID, frequency: weekly default, optional URL). 4. Daily at 06:00 UTC, `clockwork:test-contact-forms` walks every `contact_form_tests` row whose frequency makes it due (weekly ≥ 7 days since last run; daily ≥ 1 day) and POSTs to Companion's `/test-contact-form`. -5. Companion submits the form with a marker string in the message body. We watch the destination inbox for the marker. If the marker arrives → pass. If not within the timeout → fail. +5. Companion submits the form with a marker string in the message body, in **lab** mode (the scheduled default): the outgoing email is suppressed. The run passes when the form plugin accepts the submission and fails when it rejects it or Companion can't be reached. Each run writes a row to `contact_form_test_runs` keyed by `contact_form_test_id` so per-form history is preserved. Per-form state (pass/fail/streak/last_test_at) lives on the `contact_form_tests` row. +## Lab vs live + +- **Lab** (`--mode=lab`, the default for the daily schedule and the Forms tab's **Test now**): Companion submits the form with the outgoing email suppressed (`mail_outcome = 'suppressed'`). This proves the form accepts a submission, not that its email is delivered. +- **Live** (`clockwork:test-contact-forms --mode=live`, run by hand): the email goes out and the run records whether the site reported it as sent (`mail_outcome = 'sent'` or `'failed'`). Nothing watches the destination inbox, so even a live pass doesn't prove the email arrived. + +Client reports follow the same split: lab runs are shown as "accepted N of M" and never as delivered; only live runs produce a delivery rate. See [Features → Client reports](/docs/features/client-reports). + ## Where to look - **`/forms`** — fleet-wide table of every configured form-test across every care-plan site (top-nav). Its search box filters instantly as you type, no need to hit Enter — same client-side pattern as [Features → Sites (fleet view)](/docs/features/sites-fleet-view#what-you-see); see that page for how it works and its "current page only" limitation. @@ -43,7 +50,7 @@ Per-site Forms tab: 2. Click **Install Companion** on the Settings tab if not already. 3. **Detect contact forms** runs daily — Companion scans for active form plugins and caches what it finds. 4. On the Forms tab, **+ Add form**: pick a form ID from the dropdown (or paste one), URL optional, frequency = Weekly. -5. **Test now** to verify end-to-end before relying on the schedule. +5. **Test now** to confirm the form accepts a submission before relying on the schedule (a lab run, so no email is sent). Repeat for up to 3 forms. If a client needs more than 3 or a daily cadence, they contact support — daily is exposed via the `?admin=1` query string on the Forms tab. @@ -70,11 +77,11 @@ The HMAC secret can be rotated from the Settings tab too — useful when offboar ## Why this is care-plan-only -Form testing is non-trivial to run reliably (Companion install per site, HMAC rotation, marker round-trip via real SMTP). Bundling it with the care plan keeps the operations surface area aligned with the customers who already opted into managed maintenance, rather than spinning up a separate billing line. +Form testing is non-trivial to run reliably (Companion install per site, HMAC rotation, a real submission through each form plugin). Bundling it with the care plan keeps the operations surface area aligned with the customers who already opted into managed maintenance, rather than spinning up a separate billing line. ## What off-plan clients see (Companion 1.21.5+) -Off-plan clients still see the **Forms** admin page in their `Tools → Clockwork` menu — it's a deliberate sales surface. Detection runs for everyone, so they can see Clockwork already inventoried their contact forms. The Monitor toggles are visually present but locked with a "Care plan" pill; the upsell banner at the top names the specific features they'd get (weekly tests, email-delivery verification, the Mattermost ping when a form silently breaks). Re-detect still works so they can demo it. If the site previously had a care plan and has subscriptions, those render as "Forms you were monitoring" with a "Care plan paused" pill — Stop-monitoring still works for cleanup, Test-now is gated. +Off-plan clients still see the **Forms** admin page in their `Tools → Clockwork` menu — it's a deliberate sales surface. Detection runs for everyone, so they can see Clockwork already inventoried their contact forms. The Monitor toggles are visually present but locked with a "Care plan" pill; the upsell banner at the top names the specific features they'd get (weekly test submissions, the Mattermost ping when a form stops accepting them). The banner text lives in Companion; it must not promise email-delivery verification, which the scheduled tests don't do. Re-detect still works so they can demo it. If the site previously had a care plan and has subscriptions, those render as "Forms you were monitoring" with a "Care plan paused" pill — Stop-monitoring still works for cleanup, Test-now is gated. This is the same upsell pattern Performance / Security / Backups use — see [decision #27 in the architecture notes](/docs/architecture/clockwork-decisions) for the underlying principle ("show what they're missing, don't hide the feature"). Defense-in-depth: the `wp_ajax_clockwork_companion_subscribe_form` and `wp_ajax_clockwork_companion_test_form_now` endpoints refuse with `403 + error: care_plan_required` when off-plan, so a clever admin can't bypass the disabled UI. @@ -83,5 +90,5 @@ This is the same upsell pattern Performance / Security / Backups use — see [de - **Care-plan flag is the master gate.** Removing the care plan on a site stops its form tests immediately (next scheduler pass skips the rows). The rows themselves stay — re-enabling the care plan resumes testing without re-configuration. - **Companion install is intentionally manual.** Each site gets the plugin via the per-site button. - **Detect + test commands are no-ops** on sites without Companion. Safe to leave running fleet-wide. -- **The marker must round-trip to a real inbox.** If the form's destination email is bounced or the box is unreachable, we'll see the form as failing even though the form plugin itself is fine. +- **A pass doesn't mean the email arrived.** Scheduled (lab) runs suppress the email, and live runs only record whether the site reported it as sent. A form whose notification email bounces or goes to spam can still pass. - **Schema migration safety net:** the seven `sites.contact_form_test_*` columns are still in the table for one release cycle as a backup; they're no longer read. A follow-up migration drops them. diff --git a/resources/views/settings/companion.blade.php b/resources/views/settings/companion.blade.php index 08bd14dc..2418af22 100644 --- a/resources/views/settings/companion.blade.php +++ b/resources/views/settings/companion.blade.php @@ -1067,6 +1067,7 @@ class="w-full px-3 py-2 text-sm border border-[var(--color-border)] rounded-md f +

Leave empty to use the Companion Support Portal URL, then your Agency Website (now: ). Must be a public http(s) address clients can reach, not localhost or a private address.

diff --git a/tests/Feature/Controllers/CompanionSettingsControllerTest.php b/tests/Feature/Controllers/CompanionSettingsControllerTest.php index dec0f1d2..e7b65926 100644 --- a/tests/Feature/Controllers/CompanionSettingsControllerTest.php +++ b/tests/Feature/Controllers/CompanionSettingsControllerTest.php @@ -7,6 +7,7 @@ use App\Services\Companion\CompanionBrandingManager; use App\Services\Process\BackgroundArtisan; use App\Services\Process\BackgroundArtisanResult; +use App\Support\Settings; use Illuminate\Http\UploadedFile; use Illuminate\Support\Facades\Mail; use Illuminate\Support\Facades\Queue; @@ -323,7 +324,8 @@ ->assertSee('Primary Brand Color') ->assertSee('Accent Strip') ->assertSee('Website Care Report') - ->assertSee('Email Header Background', false); + ->assertSee('Email Header Background', false) + ->assertSee('Leave empty to use the Companion Support Portal URL, then your Agency Website', false); }); it('saves custom reports branding and redirects to reports tab', function () { @@ -382,6 +384,73 @@ $response->assertSessionHasErrors(['support_email']); }); + it('rejects a reports support URL clients could not reach', function (string $url) { + $user = User::factory()->create(); + + $this->actingAs($user)->patch(route('settings.companion.update'), [ + 'tab' => 'reports', + 'support_url' => $url, + ])->assertSessionHasErrors(['support_url']); + + expect(app(CompanionBrandingManager::class)->getReportsBranding()['support_url'])->not->toBe($url); + })->with([ + 'localhost' => 'http://localhost:8000', + 'loopback IPv4' => 'http://127.0.0.1:8000/support', + 'loopback IPv6' => 'http://[::1]:8000/', + 'private network' => 'http://10.0.0.5/help', + 'link-local' => 'http://169.254.10.10/', + 'single-label host' => 'http://intranet/help', + 'internal-only name' => 'https://control.internal/help', + 'mDNS name' => 'https://office.local/help', + 'not http(s)' => 'ftp://example.com/support', + 'not a URL' => 'support page', + ]); + + it('rejects this app\'s own URL as the reports support URL', function () { + config(['app.url' => 'https://control.example.com']); + $user = User::factory()->create(); + + $this->actingAs($user)->patch(route('settings.companion.update'), [ + 'tab' => 'reports', + 'support_url' => 'https://control.example.com/help', + ])->assertSessionHasErrors(['support_url']); + }); + + it('accepts a blank reports support URL, and a public one', function () { + $user = User::factory()->create(); + $manager = app(CompanionBrandingManager::class); + + $this->actingAs($user)->patch(route('settings.companion.update'), [ + 'tab' => 'reports', + 'support_url' => '', + ])->assertSessionHasNoErrors()->assertRedirect(route('settings.companion.index', ['tab' => 'reports'])); + expect(app(Settings::class)->get('reports.branding.support_url'))->toBe(''); + + $this->actingAs($user)->patch(route('settings.companion.update'), [ + 'tab' => 'reports', + 'support_url' => 'https://example.com/support', + ])->assertSessionHasNoErrors(); + expect($manager->getReportsBranding()['support_url'])->toBe('https://example.com/support'); + }); + + it('rejects localhost for the shared support and agency website URLs but still allows a blank support URL', function () { + $user = User::factory()->create(); + + $this->actingAs($user)->patch(route('settings.companion.update'), [ + 'company_url' => 'http://localhost:8000', + 'support_url' => 'http://127.0.0.1/help', + ])->assertSessionHasErrors(['company_url', 'support_url']); + + $this->actingAs($user)->patch(route('settings.companion.update'), [ + 'company_url' => 'https://example.com', + 'support_url' => '', + ])->assertSessionHasNoErrors(); + + $branding = app(CompanionBrandingManager::class)->get(); + expect($branding['company_url'])->toBe('https://example.com') + ->and($branding['support_url'])->toBe(''); + }); + it('resets reports styling to defaults', function () { $user = User::factory()->create(); $manager = app(CompanionBrandingManager::class); diff --git a/tests/Feature/Modules/ClientReportCompilerTest.php b/tests/Feature/Modules/ClientReportCompilerTest.php new file mode 100644 index 00000000..c7a3592a --- /dev/null +++ b/tests/Feature/Modules/ClientReportCompilerTest.php @@ -0,0 +1,114 @@ +travelTo(Carbon::parse('2026-11-02 06:00')); +}); + +function compileMonth(Site $site, string $start, string $end, ?array $sections = null): array +{ + return app(ClientReportCompiler::class)->compile($site, Carbon::parse($start), Carbon::parse($end), null, $sections); +} + +describe('support URL', function () { + it('falls back to the agency website, never to the app URL, when both support URLs are blank', function () { + config(['app.url' => 'http://localhost:8000']); + app(Settings::class)->putMany([ + 'reports.branding.support_url' => '', + 'companion.branding.support_url' => '', + 'companion.branding.company_url' => 'https://agency.example.com', + ]); + + $branding = compileMonth(Site::factory()->create(), '2026-10-01', '2026-10-31', [])['branding']; + + expect($branding['support_url'])->toBe('https://agency.example.com') + ->and($branding['support_url'])->not->toContain('localhost'); + }); + + it('is null rather than the app URL when there is no agency website either', function () { + config(['app.url' => 'http://localhost:8000']); + $this->mock(CompanionBrandingManager::class, fn ($mock) => $mock->shouldReceive('getReportsBranding')->andReturn([ + 'enabled' => false, + 'company_name' => 'Example Agency', + 'company_url' => '', + 'support_email' => 'support@example.com', + 'support_url' => '', + 'logo_url' => '', + 'primary_color' => '#2D2062', + 'accent_color' => '#7EFF83', + 'footer_text' => '', + 'is_custom' => false, + ])); + + $branding = compileMonth(Site::factory()->create(), '2026-10-01', '2026-10-31', [])['branding']; + + expect($branding['support_url'])->toBeNull(); + }); + + it('uses the shared Companion support URL when the reports one is blank', function () { + app(Settings::class)->putMany([ + 'reports.branding.support_url' => '', + 'companion.branding.support_url' => 'https://help.example.com', + 'companion.branding.company_url' => 'https://agency.example.com', + ]); + + expect(compileMonth(Site::factory()->create(), '2026-10-01', '2026-10-31', [])['branding']['support_url']) + ->toBe('https://help.example.com'); + }); + + it('uses the reports support URL when one is set', function () { + app(Settings::class)->putMany([ + 'reports.branding.support_url' => 'https://example.com/support', + 'companion.branding.support_url' => 'https://help.example.com', + ]); + + expect(compileMonth(Site::factory()->create(), '2026-10-01', '2026-10-31', [])['branding']['support_url']) + ->toBe('https://example.com/support'); + }); +}); + +describe('period length', function () { + it('counts a 31-day month as 31 whole days and 44,640 minutes', function () { + $site = Site::factory()->create(['uptime_monitoring_enabled' => true, 'uptime_last_checked_at' => now()]); + + $data = compileMonth($site, '2026-10-01', '2026-10-31', ['uptime']); + + expect($data['meta']['period_days'])->toBe(31) + ->and($data['uptime']['total_period_minutes'])->toBe(31 * 24 * 60); + }); + + it('counts a 30-day month as 30 whole days and 43,200 minutes', function () { + $site = Site::factory()->create(['uptime_monitoring_enabled' => true, 'uptime_last_checked_at' => now()]); + + $data = compileMonth($site, '2026-09-01', '2026-09-30', ['uptime']); + + expect($data['meta']['period_days'])->toBe(30) + ->and($data['uptime']['total_period_minutes'])->toBe(30 * 24 * 60); + }); + + it('counts a single-day period as one day', function () { + expect(compileMonth(Site::factory()->create(), '2026-10-15', '2026-10-15', [])['meta']['period_days'])->toBe(1); + }); +}); + +it('stamps the current schema version', function () { + expect(ClientReportCompiler::SCHEMA_VERSION)->toBe(4) + ->and(compileMonth(Site::factory()->create(), '2026-10-01', '2026-10-31', [])['meta']['schema_version'])->toBe(4); +}); diff --git a/tests/Feature/Modules/ClientReportMailTest.php b/tests/Feature/Modules/ClientReportMailTest.php index 550766e9..3d5be4b9 100644 --- a/tests/Feature/Modules/ClientReportMailTest.php +++ b/tests/Feature/Modules/ClientReportMailTest.php @@ -3,6 +3,7 @@ use App\Models\Site; use Illuminate\Support\Carbon; use Illuminate\Support\Facades\Http; +use Illuminate\Support\Facades\Mail; use Modules\ClientManagement\Models\Client; use Modules\ClientReports\Mail\ClientReportMail; use Modules\ClientReports\Models\ClientReport; @@ -119,6 +120,43 @@ function clientReportMailFixture(array $overrides = []): ClientReport $mail->assertHasReplyTo('support@clockwork.test'); }); +it('sends from the configured address under the report brand name, not MAIL_FROM_NAME', function () { + config(['mail.from.address' => 'reports@example.com', 'mail.from.name' => 'Internal Control App']); + $mail = new ClientReportMail(clientReportMailFixture(['branding' => ['company_name' => 'Example Agency']])); + + $mail->assertFrom('reports@example.com', 'Example Agency'); + expect($mail->envelope()->from?->name)->toBe('Example Agency'); + + // The header that actually goes out, through the array transport. + Mail::to('client@example.com')->send($mail); + $from = app('mail.manager')->mailer()->getSymfonyTransport()->messages()->last()->getOriginalMessage()->getFrom(); + + expect($from)->toHaveCount(1) + ->and($from[0]->getAddress())->toBe('reports@example.com') + ->and($from[0]->getName())->toBe('Example Agency'); +}); + +it('leaves the global From in place when no sending address is configured', function () { + config(['mail.from.address' => null]); + + expect((new ClientReportMail(clientReportMailFixture()))->envelope()->from)->toBeNull(); +}); + +it('prints the footer text in the plain-text part when one is set', function () { + $mail = new ClientReportMail(clientReportMailFixture(['branding' => ['footer_text' => 'Example Agency, 1 Example Street']])); + $content = $mail->content(); + $text = view($content->text, $content->with)->render(); + + expect($text)->toContain("Example Agency, 1 Example Street\n\nQuestions about anything in this report?") + ->and($mail->render())->toContain('Example Agency, 1 Example Street'); + + // No footer set: no stray blank block before the sign-off. + $plain = new ClientReportMail(clientReportMailFixture()); + $content = $plain->content(); + + expect(view($content->text, $content->with)->render())->not->toContain("\n\n\nQuestions about anything"); +}); + it('cleans raw action-log summaries', function (string $summary, ?string $target, string $expected) { expect(ClientReportMail::cleanUpdateSummary($summary, $target))->toBe($expected); })->with([ diff --git a/tests/Feature/Modules/ClientReportTemplatesTest.php b/tests/Feature/Modules/ClientReportTemplatesTest.php index d90dfff8..dda2997c 100644 --- a/tests/Feature/Modules/ClientReportTemplatesTest.php +++ b/tests/Feature/Modules/ClientReportTemplatesTest.php @@ -52,6 +52,25 @@ ->assertSee('Security & Firewall', false); }); + it('describes each section by what the report actually contains', function () { + $descriptions = collect(ClientReportTemplate::SECTIONS)->pluck('description')->implode(' '); + + // Promises the compiler never delivers: no referrer, page-view, TTFB, + // translation or restore data is compiled, and scheduled form tests + // suppress the email, so they can't speak to delivery. + foreach (['referrer', 'page view', 'unique visits', 'TTFB', 'time to first byte', 'translation', 'deliverability', 'restore readiness'] as $promise) { + expect(stripos($descriptions, $promise))->toBeFalse("Section descriptions still promise \"{$promise}\"."); + } + + expect(ClientReportTemplate::SECTIONS['forms']['description'])->toContain('delivery is only reported for live tests'); + + $this->mockIssueCounterZero(); + $this->actingAs(User::factory()->create()) + ->get(route('client-reports.templates.create')) + ->assertOk() + ->assertSee(ClientReportTemplate::SECTIONS['forms']['description']); + }); + it('stores a new custom template with selected sections', function () { $user = User::factory()->create(); diff --git a/tests/Feature/Modules/ClientReportsControllerTest.php b/tests/Feature/Modules/ClientReportsControllerTest.php index 621c12cf..36d22f4f 100644 --- a/tests/Feature/Modules/ClientReportsControllerTest.php +++ b/tests/Feature/Modules/ClientReportsControllerTest.php @@ -183,6 +183,24 @@ ->assertSee(route('client-reports.email', $report), false); }); + it('shows the From line the client will see: the sending address under the report brand name', function () { + config(['mail.from.address' => 'reports@example.com', 'mail.from.name' => 'Internal Control App']); + app(CompanionBrandingManager::class)->saveReportsBranding(['company_name' => 'Example Agency']); + $user = User::factory()->create(); + $site = Site::factory()->create(['domain' => 'from-line.example.com']); + $report = ClientReport::create([ + 'site_id' => $site->id, 'title' => 'From line', 'period_start' => now()->subMonth()->startOfMonth(), + 'period_end' => now()->subMonth()->endOfMonth(), + 'sections_data' => app(ClientReportCompiler::class)->compile($site, now()->subMonth()->startOfMonth(), now()->subMonth()->endOfMonth()), + 'client_email' => 'a@example.com', 'status' => 'generated', + ]); + + $this->actingAs($user)->get(route('client-reports.email', $report)) + ->assertOk() + ->assertSee('Example Agency ') + ->assertDontSee('Internal Control App'); + }); + it('shows the plain-text part too, and says so when everyone already has the report', function () { $user = User::factory()->create(); $site = Site::factory()->create(['domain' => 'text-part.test']); From b331d208cedf4f71d39cdf8451d6f9cf1e1e75a4 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 3 Nov 2026 20:30:27 +0000 Subject: [PATCH 02/64] fix(contact-forms): add grace window to daily/weekly form-test due check The scheduled form-test run starts at a fixed time each day, but each test's last_test_at is stamped when the test finishes, a few minutes later and at a slightly different minute every day. The due filter used a strict 24-hour / 7-day cutoff, so whenever today's run reached the filter earlier in the day than the previous test had finished, the row was treated as not yet due and skipped. Daily tests ended up running roughly every other day and weekly tests every eighth day, which also under-counted the checks shown in monthly client reports. Daily rows are now due once last_test_at is more than 20 hours old and weekly rows once it is more than 7 days minus 4 hours old. The margins absorb the run-time drift without letting a row run twice in the same daily window or turning weekly into a six-day cadence. Adds a regression test frozen just after the scheduled start time with the previous runs stamped a few minutes later in the day, and updates the cadence step in the contact-form testing docs. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Q7rExNMgxzTNb3pagVUZrg --- .../src/Commands/TestContactForms.php | 15 +++++-- .../docs/features/contact-form-testing.md | 2 +- .../Feature/Console/TestContactFormsTest.php | 45 +++++++++++++++++++ 3 files changed, 58 insertions(+), 4 deletions(-) diff --git a/modules/ContactForms/src/Commands/TestContactForms.php b/modules/ContactForms/src/Commands/TestContactForms.php index c9e2e32d..801b7193 100644 --- a/modules/ContactForms/src/Commands/TestContactForms.php +++ b/modules/ContactForms/src/Commands/TestContactForms.php @@ -106,14 +106,23 @@ private function targetTests() /** * "Is this row due?" — never-tested rows always qualify; otherwise the - * row must be older than its frequency window. + * row must be older than its frequency window, less a grace margin. + * + * The scheduled run starts at a fixed time each day, but last_test_at + * is stamped when each test finishes — a few minutes later, and at a + * slightly different minute every day. A strict 24h / 7d cutoff skips + * a row whenever today's run reaches this filter earlier in the day + * than the previous test finished, so daily rows ran roughly every + * other day and weekly rows every eighth day. The margins absorb that + * drift while still keeping a row from running twice in one window: + * daily is due after 20 hours, weekly after 7 days minus 4 hours. * * @param Builder $q */ private function applyDueFilter($q): void { - $dailyCutoff = now()->subDay(); - $weeklyCutoff = now()->subDays(7); + $dailyCutoff = now()->subHours(20); + $weeklyCutoff = now()->subDays(7)->addHours(4); $q->where(function ($q) use ($dailyCutoff, $weeklyCutoff) { $q->whereNull('last_test_at') diff --git a/resources/docs/features/contact-form-testing.md b/resources/docs/features/contact-form-testing.md index 20f01925..764c5768 100644 --- a/resources/docs/features/contact-form-testing.md +++ b/resources/docs/features/contact-form-testing.md @@ -23,7 +23,7 @@ Contact Form Testing is decoupled into a standalone module (`modules/ContactForm 1. The site is on a **care plan** (gate enforced at the service, command, and controller). 2. Companion is installed on the site with the `'contact-form-test'` capability advertised. 3. Operator opens the per-site **Forms** tab and adds up to 3 form-tests (form ID, frequency: weekly default, optional URL). -4. Daily at 06:00 UTC, `clockwork:test-contact-forms` walks every `contact_form_tests` row whose frequency makes it due (weekly ≥ 7 days since last run; daily ≥ 1 day) and POSTs to Companion's `/test-contact-form`. +4. Daily at 06:00 UTC, `clockwork:test-contact-forms` walks every `contact_form_tests` row whose frequency makes it due and POSTs to Companion's `/test-contact-form`. Never-tested rows are always due; daily rows are due once the last run is more than 20 hours old, weekly rows once it is more than 7 days minus 4 hours old. The grace window absorbs the few minutes each run takes, so a test that finished at 06:12 yesterday is still due when today's run starts. 5. Companion submits the form with a marker string in the message body. We watch the destination inbox for the marker. If the marker arrives → pass. If not within the timeout → fail. Each run writes a row to `contact_form_test_runs` keyed by `contact_form_test_id` so per-form history is preserved. Per-form state (pass/fail/streak/last_test_at) lives on the `contact_form_tests` row. diff --git a/tests/Feature/Console/TestContactFormsTest.php b/tests/Feature/Console/TestContactFormsTest.php index 1060b2f7..fb524187 100644 --- a/tests/Feature/Console/TestContactFormsTest.php +++ b/tests/Feature/Console/TestContactFormsTest.php @@ -3,6 +3,7 @@ use App\Models\ContactFormTest; use App\Models\Site; use App\Services\Forms\ContactFormTester; +use Illuminate\Support\Carbon; /* |-------------------------------------------------------------------------- @@ -168,6 +169,50 @@ $this->artisan('clockwork:test-contact-forms')->assertSuccessful(); }); + it('treats rows as due when the previous run finished later in the day than this run starts', function () { + // Regression: the scheduled run starts at a fixed time but + // last_test_at is stamped a few minutes later when each test + // finishes. A strict 24h / 7d cutoff skipped these rows. + $this->travelTo(Carbon::parse('2026-03-10 06:05:00', 'UTC')); + + $site = Site::factory()->create(['care_plan_enabled' => true]); + + $daily = ContactFormTest::factory()->create([ + 'site_id' => $site->id, + 'slot' => 1, + 'frequency' => ContactFormTest::FREQUENCY_DAILY, + 'last_test_at' => Carbon::parse('2026-03-09 06:12:00', 'UTC'), + ]); + $weekly = ContactFormTest::factory()->create([ + 'site_id' => $site->id, + 'slot' => 2, + 'frequency' => ContactFormTest::FREQUENCY_WEEKLY, + 'last_test_at' => Carbon::parse('2026-03-03 06:12:00', 'UTC'), + ]); + // The grace window must not turn weekly into every six days. + ContactFormTest::factory()->create([ + 'site_id' => $site->id, + 'slot' => 3, + 'frequency' => ContactFormTest::FREQUENCY_WEEKLY, + 'last_test_at' => Carbon::parse('2026-03-04 06:12:00', 'UTC'), + ]); + + $this->mock(ContactFormTester::class, function ($mock) use ($daily, $weekly) { + $mock->shouldReceive('test') + ->once() + ->withArgs(fn (ContactFormTest $t) => $t->is($daily)) + ->andReturn(['result' => ContactFormTester::RESULT_SUCCESS, 'message' => 'ok']); + $mock->shouldReceive('test') + ->once() + ->withArgs(fn (ContactFormTest $t) => $t->is($weekly)) + ->andReturn(['result' => ContactFormTester::RESULT_SUCCESS, 'message' => 'ok']); + }); + + $this->artisan('clockwork:test-contact-forms') + ->expectsOutputToContain('passed=2, failed=0, skipped=0, transitions=0') + ->assertSuccessful(); + }); + it('--force bypasses the due filter for an otherwise-fresh row', function () { $site = Site::factory()->create(['care_plan_enabled' => true]); $fresh = ContactFormTest::factory()->create([ From 26b9ba3e2939e1285dfe5f769901d3450560b0ec Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 3 Nov 2026 20:45:28 +0000 Subject: [PATCH 03/64] fix(pressable): record Pressable traffic before client reports, without zero-filling history The Pressable traffic job ran at 06:37 UTC, after the 06:00 client-report run, so a month's report compiled from a last-day row holding only the first hours of that day. It now runs at 04:45 UTC: Pressable's buckets are UTC days, so the previous day is complete by then, and the ~15-20 minute run ends well before 06:00. A test pins that ordering. The command also upserted the zero-filled 30-day Companion payload into site_traffic_daily, so any day the API had no bucket for (for example before a site moved to Pressable) overwrote earlier history, including the previous host's access-log rows, with zeros. It now writes only the days, and per day only the metric family (requests or uniques), that Pressable actually returned. Empty buckets count as no data. Only the last 28 days are written, because the API's "Past 1 month" window can begin part-way through a day as recent as 28 days ago. The Companion payload stays zero-filled over 30 days, as its chart expects. The history write now happens before, and independently of, the Companion capability check and push, and covers every Pressable site with a Pressable site id rather than only Companion-equipped ones. A Companion outage or a site without Companion no longer leaves its client report without traffic. Top paths are fetched only for sites that get the push. Docs updated for the new time and behaviour. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Q7rExNMgxzTNb3pagVUZrg --- .../Commands/PressableTrafficReport.php | 240 +++++++++++---- .../src/PressableServiceProvider.php | 10 +- .../docs/features/traffic-and-capacity.md | 6 +- resources/docs/integrations/pressable.md | 4 +- .../docs/reference/api-endpoints-we-call.md | 4 +- resources/docs/reference/artisan-commands.md | 4 +- resources/docs/reference/scheduled-jobs.md | 4 +- routes/console.php | 6 +- .../Console/PressableTrafficReportTest.php | 283 +++++++++++++++++- 9 files changed, 486 insertions(+), 75 deletions(-) diff --git a/app/Console/Commands/PressableTrafficReport.php b/app/Console/Commands/PressableTrafficReport.php index 3531b364..6c683a7b 100644 --- a/app/Console/Commands/PressableTrafficReport.php +++ b/app/Console/Commands/PressableTrafficReport.php @@ -39,13 +39,39 @@ * "Past 1 month" returns clean daily (86400s) buckets, but Edge Logs * metrics (requests, status, etc.) over the same range return 8-hour * (28800s) buckets instead — aggregated into calendar days here. + * + * Two jobs, deliberately independent of each other: + * 1. History. Every Pressable site with a pressable_site_id gets the days + * Pressable actually returned upserted into site_traffic_daily, which + * is what the monthly client report and /capacity read. This runs + * whether or not Companion is installed, reachable, or new enough, so a + * Companion problem never leaves a report without traffic. + * 2. Companion push. Sites that can take it also get the zero-filled + * 30-day payload for their wp-admin Traffic page. */ class PressableTrafficReport extends Command { protected $signature = 'clockwork:pressable-traffic-report {--site= : Limit to a single site (id or domain)}'; - protected $description = 'Push a real daily traffic rollup (sourced from Pressable\'s own metrics API) to each Companion-equipped Pressable site.'; + protected $description = 'Record daily traffic for every Pressable site from Pressable\'s own metrics API, and push the 30-day rollup to each Companion-equipped one.'; + + /** Days in the zero-filled Companion payload, today included. */ + private const PAYLOAD_DAYS = 30; + + /** + * Days written to site_traffic_daily, today included. Shorter than the + * payload on purpose: the API's "Past 1 month" window starts part-way + * through a day that can be as recent as 28 days ago (after February), + * and that first, partial day must not overwrite a complete row an + * earlier run wrote. Every day is still rewritten 28 times while it + * stays in the window, so history coverage is unchanged. + */ + private const HISTORY_DAYS = 28; + + private const REQUEST_COLUMNS = ['requests', 'status_2xx', 'status_3xx', 'status_4xx', 'status_5xx']; + + private const VISIT_COLUMNS = ['unique_ips', 'visits']; /** * Path-categorisation rules ported from RollupTraffic::applyBucketFilter() @@ -72,31 +98,51 @@ public function handle(PressableClient $pressable): int $sites = $this->resolveSites(); if ($sites->isEmpty()) { - $this->warn('No Companion-equipped Pressable sites matched.'); + $this->warn('No Pressable sites matched.'); return self::SUCCESS; } - $stats = ['ok' => 0, 'failed' => 0, 'skipped' => 0]; + // recorded = history written; pushed = also pushed to Companion; + // push_skipped = recorded, but the site can't take the push. + $stats = ['recorded' => 0, 'pushed' => 0, 'push_skipped' => 0, 'failed' => 0]; foreach ($sites as $site) { - $caps = $site->companion_capabilities ?? []; - if (! in_array('traffic-report', $caps, true)) { - $stats['skipped']++; - $this->line(" [skip] {$site->domain} — capability 'traffic-report' not advertised (Companion 1.16.0+ required)"); + try { + $requestsByDay = $this->requestsByDayAndStatus($pressable, $site->pressable_site_id); + $visitsByDay = $this->uniquesByDay($pressable, $site->pressable_site_id); + } catch (Throwable $e) { + $this->fetchFailed($site, $e, $stats); continue; } try { - $report = $this->buildReport($pressable, $site); + $days = $this->recordDailyTraffic($site, $requestsByDay, $visitsByDay); } catch (Throwable $e) { $stats['failed']++; - Log::warning('companion.pressable_traffic_report.fetch_failed', [ + Log::warning('companion.pressable_traffic_report.record_failed', [ 'site' => $site->domain, 'error' => $e->getMessage(), ]); - $this->warn(" [fail] {$site->domain}: Pressable fetch failed — {$e->getMessage()}"); + $this->warn(" [fail] {$site->domain}: recording daily traffic failed — {$e->getMessage()}"); + + continue; + } + $stats['recorded']++; + + $skipReason = $this->pushSkipReason($site); + if ($skipReason !== null) { + $stats['push_skipped']++; + $this->line(" [rec] {$site->domain} — days={$days}; push skipped: {$skipReason}"); + + continue; + } + + try { + $report = $this->buildReport($pressable, $site, $requestsByDay, $visitsByDay); + } catch (Throwable $e) { + $this->fetchFailed($site, $e, $stats); continue; } @@ -114,42 +160,18 @@ public function handle(PressableClient $pressable): int continue; } - if (! empty($report['daily'])) { - $now = now(); - $upsertRows = array_map(fn (array $r) => [ - 'site_id' => $site->id, - 'date' => $r['date'], - 'requests' => (int) ($r['requests'] ?? 0), - 'unique_ips' => (int) ($r['visits'] ?? 0), - 'visits' => (int) ($r['visits'] ?? 0), - 'bytes_sent' => 0, - 'status_2xx' => (int) ($r['status_2xx'] ?? 0), - 'status_3xx' => (int) ($r['status_3xx'] ?? 0), - 'status_4xx' => (int) ($r['status_4xx'] ?? 0), - 'status_5xx' => (int) ($r['status_5xx'] ?? 0), - 'created_at' => $now, - 'updated_at' => $now, - ], $report['daily']); - - SiteTrafficDaily::upsert( - $upsertRows, - ['site_id', 'date'], - ['requests', 'unique_ips', 'visits', 'status_2xx', 'status_3xx', 'status_4xx', 'status_5xx', 'updated_at'] - ); - } - - $stats['ok']++; + $stats['pushed']++; $today = (int) ($report['totals']['today'] ?? 0); $month = (int) ($report['totals']['month_30d'] ?? 0); - $rows = count($report['daily']); - $this->line(" [ok] {$site->domain} — today={$today} 30d={$month} rows={$rows}"); + $this->line(" [ok] {$site->domain} — today={$today} 30d={$month} days={$days}"); } $msg = sprintf( - 'companion.pressable_traffic_report.push complete: ok=%d failed=%d skipped=%d', - $stats['ok'], + 'companion.pressable_traffic_report complete: recorded=%d pushed=%d push_skipped=%d failed=%d', + $stats['recorded'], + $stats['pushed'], + $stats['push_skipped'], $stats['failed'], - $stats['skipped'], ); Log::info($msg); $this->info($msg); @@ -158,13 +180,108 @@ public function handle(PressableClient $pressable): int } /** + * @param array $stats + */ + private function fetchFailed(Site $site, Throwable $e, array &$stats): void + { + $stats['failed']++; + Log::warning('companion.pressable_traffic_report.fetch_failed', [ + 'site' => $site->domain, + 'error' => $e->getMessage(), + ]); + $this->warn(" [fail] {$site->domain}: Pressable fetch failed — {$e->getMessage()}"); + } + + /** + * Why this site can't take the Companion push, or null when it can. + */ + private function pushSkipReason(Site $site): ?string + { + // Raw value: a null check doesn't need the encrypted secret decrypted. + if (! $site->companion_installed || $site->getRawOriginal('companion_secret') === null) { + return 'Companion not installed'; + } + + if (! in_array('traffic-report', $site->companion_capabilities ?? [], true)) { + return "capability 'traffic-report' not advertised (Companion 1.16.0+ required)"; + } + + return null; + } + + /** + * Upsert into site_traffic_daily only what Pressable actually returned. + * + * A day the API has no bucket for (before the site moved to Pressable, + * or a gap on Pressable's side) is left alone rather than written as + * zeros, so it can't erase an earlier row — including one the previous + * host's access-log rollup wrote. The same goes per metric family: a + * day with request buckets but no uniques bucket updates only the + * request columns. bytes_sent and top_paths are never updated; Pressable + * has no per-day figure for either. + * + * @param array $requestsByDay + * @param array $visitsByDay + * @return int days written + */ + private function recordDailyTraffic(Site $site, array $requestsByDay, array $visitsByDay): int + { + $from = Carbon::today()->subDays(self::HISTORY_DAYS - 1)->toDateString(); + $to = Carbon::today()->toDateString(); + + $dates = array_filter( + array_unique([...array_keys($requestsByDay), ...array_keys($visitsByDay)]), + fn ($date) => $date >= $from && $date <= $to, + ); + + $now = now(); + $batches = []; + foreach ($dates as $date) { + $statuses = $requestsByDay[$date] ?? null; + $visits = $visitsByDay[$date] ?? null; + + $update = [ + ...($statuses !== null ? self::REQUEST_COLUMNS : []), + ...($visits !== null ? self::VISIT_COLUMNS : []), + 'updated_at', + ]; + + $key = implode(',', $update); + $batches[$key]['update'] = $update; + // Columns outside $update only take effect when the row is new. + $batches[$key]['rows'][] = [ + 'site_id' => $site->id, + 'date' => $date, + 'requests' => $statuses !== null ? array_sum($statuses) : 0, + 'unique_ips' => $visits ?? 0, + 'visits' => $visits ?? 0, + 'bytes_sent' => 0, + 'status_2xx' => $statuses['2xx'] ?? 0, + 'status_3xx' => $statuses['3xx'] ?? 0, + 'status_4xx' => $statuses['4xx'] ?? 0, + 'status_5xx' => $statuses['5xx'] ?? 0, + 'created_at' => $now, + 'updated_at' => $now, + ]; + } + + foreach ($batches as $batch) { + SiteTrafficDaily::upsert($batch['rows'], ['site_id', 'date'], $batch['update']); + } + + return count($dates); + } + + /** + * @param array $requestsByDay + * @param array $visitsByDay * @return array */ - private function buildReport(PressableClient $pressable, Site $site): array + private function buildReport(PressableClient $pressable, Site $site, array $requestsByDay, array $visitsByDay): array { $pressableSiteId = $site->pressable_site_id; - $daily = $this->buildDailyRows($pressable, $pressableSiteId); + $daily = $this->buildDailyRows($requestsByDay, $visitsByDay); $today = Carbon::today()->toDateString(); $byDate = collect($daily)->keyBy('date'); @@ -196,15 +313,19 @@ private function buildReport(PressableClient $pressable, Site $site): array } /** + * The Companion payload's `daily` array: every day of the window, with + * zeros for days Pressable returned nothing for, because the wp-admin + * chart expects a continuous series. Never written to the DB as-is — + * see recordDailyTraffic(). + * + * @param array $requestsByDay + * @param array $visitsByDay * @return array> */ - private function buildDailyRows(PressableClient $pressable, string $pressableSiteId): array + private function buildDailyRows(array $requestsByDay, array $visitsByDay): array { - $requestsByDay = $this->requestsByDayAndStatus($pressable, $pressableSiteId); - $visitsByDay = $this->uniquesByDay($pressable, $pressableSiteId); - $today = Carbon::today(); - $start = $today->copy()->subDays(29); + $start = $today->copy()->subDays(self::PAYLOAD_DAYS - 1); $daily = []; for ($cursor = $start->copy(); $cursor->lte($today); $cursor->addDay()) { @@ -227,7 +348,8 @@ private function buildDailyRows(PressableClient $pressable, string $pressableSit /** * Edge Logs family: requests × http_status, auto-resolved to 8-hour * buckets over a 1-month window (not daily) — aggregated into calendar - * days here. + * days here. A bucket with no counts in it is not data, so a day made + * only of empty buckets is left out, the same as a day with none. * * @return array */ @@ -237,10 +359,15 @@ private function requestsByDayAndStatus(PressableClient $pressable, string $pres $out = []; foreach ($periods as $period) { + $dimension = $period['dimension'] ?? []; + if (! is_array($dimension) || $dimension === []) { + continue; + } + $date = gmdate('Y-m-d', (int) ($period['timestamp'] ?? 0)); $out[$date] ??= ['2xx' => 0, '3xx' => 0, '4xx' => 0, '5xx' => 0]; - foreach ($period['dimension'] ?? [] as $status => $count) { + foreach ($dimension as $status => $count) { $bucket = match (true) { (int) $status >= 200 && (int) $status < 300 => '2xx', (int) $status >= 300 && (int) $status < 400 => '3xx', @@ -260,7 +387,7 @@ private function requestsByDayAndStatus(PressableClient $pressable, string $pres /** * Uniques & Views family: uniques × hostname, real daily (86400s) * buckets. Summed across every hostname variant (bare + www) for one - * unified per-day visitor count. + * unified per-day visitor count. Empty buckets are skipped, as above. * * @return array */ @@ -270,9 +397,14 @@ private function uniquesByDay(PressableClient $pressable, string $pressableSiteI $out = []; foreach ($periods as $period) { + $counts = $period['hostname']['uniques'] ?? $period['dimension'] ?? []; + if (! is_array($counts) || $counts === []) { + continue; + } + $date = gmdate('Y-m-d', (int) ($period['timestamp'] ?? 0)); $sum = 0; - foreach ($period['hostname']['uniques'] ?? $period['dimension'] ?? [] as $count) { + foreach ($counts as $count) { $sum += (int) $count; } $out[$date] = ($out[$date] ?? 0) + $sum; @@ -342,13 +474,15 @@ private function categorisePath(string $path): string } /** + * Every Pressable site the API can be asked about, Companion or not — + * the history write needs all of them; pushSkipReason() decides which + * also get the Companion push. + * * @return Collection */ private function resolveSites(): Collection { $q = Site::query() - ->where('companion_installed', true) - ->whereNotNull('companion_secret') ->where('hosting_provider', Site::HOSTING_PROVIDER_PRESSABLE) ->whereNotNull('pressable_site_id'); diff --git a/modules/Pressable/src/PressableServiceProvider.php b/modules/Pressable/src/PressableServiceProvider.php index bf169093..ac7cb8bb 100644 --- a/modules/Pressable/src/PressableServiceProvider.php +++ b/modules/Pressable/src/PressableServiceProvider.php @@ -68,8 +68,16 @@ public function scheduledTasks(Schedule $schedule): void ->withoutOverlapping(60) ->onOneServer(); + // Also writes site_traffic_daily, which the 06:00 client-report run + // reads, so it must finish well before then: a run after the + // reports leaves the last day of the month with only the hours + // recorded the morning before. Times here are UTC; Pressable's + // buckets are UTC days, so by 04:45 yesterday is complete with + // margin for Pressable's own lag, and the ~15-20 min run ends long + // before 06:00. The capability check reads the previous day's 06:25 + // refresh, which only matters the day a site first upgrades. $schedule->command('clockwork:pressable-traffic-report') - ->dailyAt('06:37') + ->dailyAt('04:45') ->withoutOverlapping(60) ->onOneServer(); diff --git a/resources/docs/features/traffic-and-capacity.md b/resources/docs/features/traffic-and-capacity.md index bf60985d..9f5cf506 100644 --- a/resources/docs/features/traffic-and-capacity.md +++ b/resources/docs/features/traffic-and-capacity.md @@ -2,7 +2,7 @@ title: Traffic + capacity section: Features order: 120 -updated: 2026-09-28 +updated: 2026-11-03 author: Aaron Reimann tags: [traffic, capacity, visits, analytics, pressable, eol, php] tracks: [app/Console/Commands/RollupTraffic.php, app/Http/Controllers/CapacityController.php, app/Services/Runtime/**, app/Console/Commands/RefreshRuntimeEol.php, app/Console/Commands/PressableTrafficReport.php, modules/Pressable/src/PressableClient.php] @@ -90,11 +90,11 @@ So if you need to backfill rollups for a date older than 30 days, the data isn't ## Pressable traffic reporting — pushed to Companion, not this page -`clockwork:pressable-traffic-report` (daily 06:37) builds a 30-day **daily** rollup for each Companion-equipped Pressable site — sourced from Pressable's time-series metrics API (`POST /sites/{id}/metrics`, via `PressableClient::siteMetrics()`). Pressable's `/metrics` endpoint provides per-day requests (by HTTP status), daily unique visitors, and per-path request counts, enabling the command to build the standard `{totals: {today, month_30d}, daily: [...]}` payload shape expected by `PushCompanionTraffic`. Companion's `TrafficPage` renders the normal chart display seamlessly. +`clockwork:pressable-traffic-report` (daily 04:45) builds a 30-day **daily** rollup for each Companion-equipped Pressable site — sourced from Pressable's time-series metrics API (`POST /sites/{id}/metrics`, via `PressableClient::siteMetrics()`). Pressable's `/metrics` endpoint provides per-day requests (by HTTP status), daily unique visitors, and per-path request counts, enabling the command to build the standard `{totals: {today, month_30d}, daily: [...]}` payload shape expected by `PushCompanionTraffic`. Companion's `TrafficPage` renders the normal chart display seamlessly. Two real Pressable API quirks the command has to work around: metrics/dimensions must come from the same "family" to combine (mixing `views` with `http_status` silently returns empty rather than erroring), and the auto-selected time resolution differs by family for the same date range (daily buckets for Uniques & Views, 8-hour buckets for Edge Logs — aggregated into calendar days here). -This command also upserts the same daily rows into Clockwork's `site_traffic_daily` table, which is what `/capacity` reads for Pressable MTD / rolling / trending numbers. The dashboard itself makes **one** Pressable `GET /account` per hour (5-minute negative cache on failure) and otherwise stays on local SQL. +Before any push, the command also upserts daily rows into Clockwork's `site_traffic_daily` table for **every** Pressable site with a Pressable site id, Companion or not. This is what `/capacity` reads for Pressable MTD / rolling / trending numbers, and what monthly client reports read, which is why it runs before the 06:00 report run. Only the days and metrics Pressable actually returned are written, over the last 28 days; the zero-filled days of the Companion chart are never stored, so a day Pressable has nothing for (for example before a site moved to Pressable) keeps its earlier row. `bytes_sent` and `top_paths` are never overwritten. The dashboard itself makes **one** Pressable `GET /account` per hour (5-minute negative cache on failure) and otherwise stays on local SQL. The per-site `/sites/{id}/traffic` tab still stays hidden for Pressable sites (no nginx access log to source it from). Clients see the chart in Companion wp-admin (`/traffic-report`) via this push; operators see Pressable quota on `/capacity` from the same `site_traffic_daily` rows. diff --git a/resources/docs/integrations/pressable.md b/resources/docs/integrations/pressable.md index 97a214d7..cb19c3b9 100644 --- a/resources/docs/integrations/pressable.md +++ b/resources/docs/integrations/pressable.md @@ -2,7 +2,7 @@ title: Pressable section: Integrations order: 21 -updated: 2026-10-04 +updated: 2026-11-03 author: Aaron Reimann tags: [integrations, pressable, hosting, wordpress] tracks: [modules/Pressable/src/**, app/Console/Commands/{ImportPressable,PressableTest,InstallCompanionPressable,PressableBackupsReport,PressableTrafficReport,PressableSecuritySummaryReport}.php] @@ -110,7 +110,7 @@ Handles Pressable's 30-second no-output kill. Live-tested both the success path | Cadence | Command | |---|---| | daily 06:32 | `clockwork:pressable-backups-report` — pushes real backup run history to Companion's Backups page. | -| daily 06:37 | `clockwork:pressable-traffic-report` — pushes period-total traffic stats to Companion's Traffic page. | +| daily 04:45 | `clockwork:pressable-traffic-report` — records each Pressable site's daily traffic into `site_traffic_daily` (read by client reports, so it runs before the 06:00 report run), then pushes the 30-day chart to Companion's Traffic page where supported. | | daily 06:39 | `clockwork:pressable-security-summary-report` — pushes plugin/theme vulnerability alerts + Defensive Mode status to Companion's Security page. Pressable-only capability, no SpinupWP parity intent. | `clockwork:import-pressable` and `clockwork:install-companion-pressable` are **not** scheduled — the site inventory doesn't churn the way it needs a daily re-import yet, and Companion rollout to new Pressable sites is still a deliberate per-site/per-batch step. Run them manually as needed. See [Reference → Scheduled jobs](/docs/reference/scheduled-jobs) and [Reference → Artisan commands](/docs/reference/artisan-commands). diff --git a/resources/docs/reference/api-endpoints-we-call.md b/resources/docs/reference/api-endpoints-we-call.md index 41526a8f..e024f042 100644 --- a/resources/docs/reference/api-endpoints-we-call.md +++ b/resources/docs/reference/api-endpoints-we-call.md @@ -2,7 +2,7 @@ title: APIs we call section: Reference order: 10 -updated: 2026-09-28 +updated: 2026-11-03 author: Aaron Reimann tags: [reference, api, integrations] tracks: [app/Services/*/*Client.php, modules/*/src/*Client.php, app/Services/Companion/ClockworkCompanionClient.php] @@ -108,7 +108,7 @@ Daily import at 03:30 (`clockwork:import-spinupwp`). | POST | `/sites/{id}/wordpress/commands`, `/sites/{id}/wordpress/wpcli` | Fire-and-forget shell / wp-cli commands — see `PressableCommandRunner` for how this becomes synchronous (result read back from the activity log, not the response body). | | GET | `/sites/{id}/backups`, `/backups/fs`, `/backups/db` | Backup run history — `clockwork:pressable-backups-report` (06:32 daily). | | GET | `/sites/{id}/reports/performance/latest` | Pressable's own Lighthouse report — see [Integrations → Pressable](/docs/integrations/pressable). | -| GET | `/sites/{id}/statistics` | Page-view stats — `clockwork:pressable-traffic-report` (06:37 daily). | +| GET | `/sites/{id}/statistics` | Page-view stats — `clockwork:pressable-traffic-report` (04:45 daily). | | GET | `/sites/{id}/security-alerts/{plugins,themes}` | Known-vulnerability + Defensive Mode status — `clockwork:pressable-security-summary-report` (06:39 daily), Pressable-only, no SpinupWP equivalent. | | POST | `/sites/{id}/metrics`, `/sites/{id}/logs/activity` | CPU/MySQL resource metrics and the activity-log poll `PressableCommandRunner` uses for command results. | diff --git a/resources/docs/reference/artisan-commands.md b/resources/docs/reference/artisan-commands.md index 84b95937..2ee0529d 100644 --- a/resources/docs/reference/artisan-commands.md +++ b/resources/docs/reference/artisan-commands.md @@ -2,7 +2,7 @@ title: Artisan commands section: Reference order: 50 -updated: 2026-10-29 +updated: 2026-11-03 author: Aaron Reimann tags: [reference, artisan, cli, modules] tracks: [app/Console/Commands/**, modules/*/src/Commands/**, modules/*/src/Console/*.php, modules/*/src/Console/Commands/**] @@ -111,7 +111,7 @@ Every `clockwork:*` command, alphabetical, with a one-line summary and an exampl | `clockwork:push-companion-branding` | Push white-label branding (logo, company name, colours, support email) to Companion/Renegade sites. `--site=` limits to one; `--verify` reads live branding first and re-pushes only drifted sites (scheduled weekly, Mon 06:50). | `php artisan clockwork:push-companion-branding --verify` | | `clockwork:push-companion-backups` | Push SpinupWP config + DO Spaces history per site. | `php artisan clockwork:push-companion-backups` | | `clockwork:pressable-backups-report` | Pressable counterpart — pushes real backup run history from Pressable's own `/backups` endpoint. No 30/90-day retention filter applied (Pressable's API has no pagination to request more than it hands back). | `php artisan clockwork:pressable-backups-report` | -| `clockwork:pressable-traffic-report` | Pushes Pressable's period-total traffic stats (today/yesterday/current+last month/last 12 months) to Companion's Traffic page. No daily breakdown — structurally different from the nginx-log rollup the SpinupWP command sources from. | `php artisan clockwork:pressable-traffic-report` | +| `clockwork:pressable-traffic-report` | Reads daily requests (by status) and unique visitors from Pressable's metrics API. Writes the days Pressable returned into `site_traffic_daily` for every Pressable site, Companion or not, then pushes the 30-day chart to Companion's Traffic page on sites that support it. `--site=` limits to one site (id or domain). Runs daily at 04:45. | `php artisan clockwork:pressable-traffic-report --site=42` | | `clockwork:pressable-security-summary-report` | Pushes Pressable's own plugin/theme CVE feed + Defensive Mode status to Companion's Security page. Pressable-only capability, no SpinupWP equivalent. | `php artisan clockwork:pressable-security-summary-report` | | `clockwork:record-site-backups` | Record each active site's backups from its real source into `site_backups`: Pressable's backup API, SpinupWP's DigitalOcean Spaces bucket, or our S3 archives for unhosted relay sites. Also writes a `site_backup_checks` row for each site that has a backup source, saying whether the read worked. Sites with no source are skipped. Rows are kept after the source prunes them. Exits as failed if any site couldn't be read. `--site=` limits to one site (id or domain). Runs daily at 05:30. | `php artisan clockwork:record-site-backups --site=42` | | `clockwork:push-backup-relay-targets` | Writes the backup-relay-enabled site list to S3. Direct-to-S3 (Companion) sites are left out because the droplet can't reach them; Control backs those up with `clockwork:backup-relay-run --direct-only`. The file goes to (`{S3_BUCKET}/{prefix}/targets.json`) for the standalone backup-relay droplet to read — no direct connection to that droplet. See [Features → Backup relay](/docs/features/backup-relay). | `php artisan clockwork:push-backup-relay-targets` | diff --git a/resources/docs/reference/scheduled-jobs.md b/resources/docs/reference/scheduled-jobs.md index f2c250cd..d03eaf80 100644 --- a/resources/docs/reference/scheduled-jobs.md +++ b/resources/docs/reference/scheduled-jobs.md @@ -2,7 +2,7 @@ title: Scheduled jobs section: Reference order: 30 -updated: 2026-10-15 +updated: 2026-11-03 author: Aaron Reimann tags: [reference, scheduler, cron] tracks: [routes/console.php, modules/*/src/*ServiceProvider.php] @@ -114,6 +114,7 @@ The scheduler itself is watched by `clockwork:scheduler-heartbeat` (every minute | 04:45 | `clockwork:run-performance-scans --strategy=mobile --weekly-rotation` | Lighthouse run for tonight's 1/7th of the care-plan fleet, leaving out Pressable sites (read nightly at 05:05/05:15 instead). GTmetrix primary, PSI fallback. Each site gets one scan per week. Scheduled at 04:45 to land just after GTmetrix's daily credit refill (credits don't bank). `--strategy` is display-only on GTmetrix's tier; kept for row continuity. | | 04:45 | `clockwork:capture-site-screenshots` | Refresh each site's homepage screenshot via Automattic's mShots service, feeding the visual fleet grid view. `withoutOverlapping(30)`, `runInBackground()`. | | 04:45 | `clockwork:detect-wp-plugins` | SSH wp-cli probe for active LLAR/Wordfence per WP site. | +| 04:45 | `clockwork:pressable-traffic-report` | Pressable's daily traffic from its metrics API, via `PressableServiceProvider::scheduledTasks()`. Writes the days Pressable returned into `site_traffic_daily` for every Pressable site, then pushes the 30-day chart to sites whose Companion supports it. Runs before the 06:00 client reports, which read those rows, so the last day of the month is complete. | | 04:50 | `clockwork:detect-contact-forms` | Companion-aware contact-form detection. | | 04:55 | `clockwork:sync-companion-form-subscriptions` | Reconcile client-picked form-test subscriptions (Companion wp-admin Forms tab) into `contact_form_tests`. Slots between detect (04:50) and test (06:00). | | 04:58 | `clockwork:backup-relay-run` (or `clockwork:push-backup-relay-targets`) | Off-host archival to S3 Glacier IR. Per-site cadence (`sites.backup_relay_frequency`) is honoured in in-repo mode; custom/unhosted sites default to daily. `--force` / Backup Now skips the interval. External-agent mode writes `targets.json` instead, leaving out direct-to-S3 (Companion) sites, which the 04:30 `--direct-only` pass covers. See [Features → Backup relay](/docs/features/backup-relay). | @@ -134,7 +135,6 @@ The scheduler itself is watched by `clockwork:scheduler-heartbeat` (every minute | 06:30 | `clockwork:push-companion-backups` | Fetch SpinupWP backup config + DO Spaces history → POST to each Companion site. | | 06:32 | `clockwork:pressable-backups-report` | Pressable counterpart — pulls real backup run history straight from Pressable's `/backups` API (no separate inventory-import dependency, unlike the SpinupWP version). | | 06:35 | `clockwork:push-companion-traffic` | Push yesterday's traffic rollup to each Companion-equipped site. | -| 06:37 | `clockwork:pressable-traffic-report` | Pressable counterpart — pulls page-view period totals straight from Pressable's stats API (no nginx access-log rollup dependency). | | 06:39 | `clockwork:pressable-security-summary-report` | Pressable-only: known plugin/theme vulnerabilities + Defensive Mode status. No SpinupWP equivalent — pure additional capability, not a parity fix. | | 06:40 | `clockwork:push-gatekeeper-settings` | Daily catch-up sync for Gatekeeper login lockout settings. Pushes policy defaults and site overrides to Companion and Renegade across the fleet. | | 06:40 | `clockwork:pull-backup-relay-report` | External-agent mode only: reads the backup-relay droplet's last run summary back from S3 and records it into `backup_relay_runs`. Also alerts if no run has completed within 6 days. See [Features → Backup relay](/docs/features/backup-relay). | diff --git a/routes/console.php b/routes/console.php index 673f0340..e7a39e1a 100644 --- a/routes/console.php +++ b/routes/console.php @@ -381,8 +381,10 @@ ->withoutOverlapping() ->onOneServer(); -// Pressable counterpart (06:37) moved to Modules\Pressable\PressableServiceProvider::scheduledTasks() (Phase 7) -// — pulls page-view stats straight from Pressable's API (no nginx +// Pressable counterpart (clockwork:pressable-traffic-report) moved to +// Modules\Pressable\PressableServiceProvider::scheduledTasks() (Phase 7) and +// now runs at 04:45, ahead of the 06:00 client reports that read its rows +// — pulls daily traffic straight from Pressable's API (no nginx // access-log rollup dependency, unlike the SpinupWP version). // clockwork:pressable-security-summary-report (06:39, Pressable-only: known plugin/theme diff --git a/tests/Feature/Console/PressableTrafficReportTest.php b/tests/Feature/Console/PressableTrafficReportTest.php index 6b23d28f..0eacac35 100644 --- a/tests/Feature/Console/PressableTrafficReportTest.php +++ b/tests/Feature/Console/PressableTrafficReportTest.php @@ -4,7 +4,10 @@ use App\Models\Site; use App\Models\SiteTrafficDaily; +use Illuminate\Console\Scheduling\Event; +use Illuminate\Console\Scheduling\Schedule; use Illuminate\Support\Carbon; +use Illuminate\Support\Facades\DB; use Illuminate\Support\Facades\Http; use Illuminate\Support\Facades\Log; use Modules\Pressable\PressableClient; @@ -18,6 +21,9 @@ | ->siteMetrics() calls (requests+http_status, uniques+hostname — both | "Past 1 month" — and requests+path for "Past 1 day" top-paths) and POSTs | it to each Companion-equipped Pressable site's /traffic-report endpoint. +| Independently of that push, every Pressable site gets the days Pressable +| actually returned upserted into site_traffic_daily (the client-report +| history) — never the zero-filled days of the Companion payload. | PressableClient is mocked directly as a collaborator (same reasoning as | PressableBackupsReportTest's docblock) since the three calls differ only | by argument, not by method name — each is matched here via a distinct @@ -37,11 +43,13 @@ function ptrSite(array $overrides = []): Site /** * Wires up the three distinct siteMetrics() call shapes the command makes, * each matched by its own ->with() so mismatched args fail loudly rather - * than silently returning the wrong fixture. + * than silently returning the wrong fixture. The top-paths call only feeds + * the Companion payload, so it happens once when the site gets the push and + * never when it doesn't. */ -function ptrMockPressable(Site $site, ?callable $configure = null): void +function ptrMockPressable(Site $site, ?callable $configure = null, bool $pushes = true): void { - test()->mock(PressableClient::class, function ($mock) use ($site, $configure) { + test()->mock(PressableClient::class, function ($mock) use ($site, $configure, $pushes) { $mock->shouldReceive('isConfigured')->andReturn(true); $today = Carbon::today(); @@ -63,6 +71,7 @@ function ptrMockPressable(Site $site, ?callable $configure = null): void $mock->shouldReceive('siteMetrics') ->with($site->pressable_site_id, ['requests'], ['path'], 'Past 1 day') + ->times($pushes ? 1 : 0) ->andReturn([ ['dimension' => ['/' => 100, '/wp-admin/' => 20, '/wp-content/uploads/img.jpg' => 5]], ]); @@ -84,6 +93,55 @@ function ptrFakeTraffic(Site $site, bool $ok = true, int $status = 200): void ]); } +/** + * Mocks the two "Past 1 month" series (requests by status, uniques by + * hostname) with the given periods; the top-paths call returns nothing. + * + * @param list> $requestPeriods + * @param list> $uniquePeriods + */ +function ptrMockSeries(Site $site, array $requestPeriods, array $uniquePeriods): void +{ + test()->mock(PressableClient::class, function ($mock) use ($site, $requestPeriods, $uniquePeriods) { + $mock->shouldReceive('isConfigured')->andReturn(true); + $mock->shouldReceive('siteMetrics')->with($site->pressable_site_id, ['requests'], ['http_status'], 'Past 1 month')->andReturn($requestPeriods); + $mock->shouldReceive('siteMetrics')->with($site->pressable_site_id, ['uniques'], ['hostname'], 'Past 1 month')->andReturn($uniquePeriods); + $mock->shouldReceive('siteMetrics')->with($site->pressable_site_id, ['requests'], ['path'], 'Past 1 day')->andReturn([]); + }); +} + +/** + * Seeds a site_traffic_daily row the way MySQL's DATE column stores it (a + * bare Y-m-d), so the command's (site_id, date) upsert key matches it on + * SQLite too. + * + * @param array $values + */ +function ptrSeedDay(Site $site, Carbon $day, array $values): void +{ + DB::table('site_traffic_daily')->insert(array_merge([ + 'site_id' => $site->id, + 'date' => $day->toDateString(), + 'requests' => 0, + 'unique_ips' => 0, + 'visits' => 0, + 'bytes_sent' => 0, + 'status_2xx' => 0, + 'status_3xx' => 0, + 'status_4xx' => 0, + 'status_5xx' => 0, + 'created_at' => now()->subMonth(), + 'updated_at' => now()->subMonth(), + ], $values)); +} + +function ptrRow(Site $site, Carbon $day): ?SiteTrafficDaily +{ + return SiteTrafficDaily::where('site_id', $site->id) + ->whereDate('date', $day->toDateString()) + ->first(); +} + describe('clockwork:pressable-traffic-report — happy path', function () { it('aggregates Edge Logs + Uniques&Views metrics into a 30-day rollup with correct totals and categorised top paths', function () { $site = ptrSite(); @@ -111,7 +169,21 @@ function ptrFakeTraffic(Site $site, bool $ok = true, int $status = 200): void && $body['top_paths_date'] === Carbon::today()->toDateString(); }); - expect(SiteTrafficDaily::where('site_id', $site->id)->count())->toBe(30); + // Only the two days Pressable returned are stored — not the 28 + // zero-filled ones the Companion chart needs. + expect(SiteTrafficDaily::where('site_id', $site->id)->count())->toBe(2); + + $today = ptrRow($site, Carbon::today()); + expect($today->requests)->toBe(35) + ->and($today->status_2xx)->toBe(30) + ->and($today->status_4xx)->toBe(5) + ->and($today->visits)->toBe(40) + ->and($today->unique_ips)->toBe(40) + ->and($today->bytes_sent)->toBe(0); + + $earlier = ptrRow($site, Carbon::today()->subDays(3)); + expect($earlier->requests)->toBe(10) + ->and($earlier->visits)->toBe(10); }); it('reports has_data=false and zeroed totals when Pressable returns no metrics at all', function () { @@ -132,21 +204,197 @@ function ptrFakeTraffic(Site $site, bool $ok = true, int $status = 200): void $body = json_decode($request->body(), true); return $body['has_data'] === false + && count($body['daily']) === 30 && $body['totals'] === ['today' => 0, 'week_7d' => 0, 'month_30d' => 0] && $body['top_paths'] === ['pages' => [], 'api' => [], 'uploads' => []]; }); + + expect(SiteTrafficDaily::where('site_id', $site->id)->count())->toBe(0); }); - it('skips a site that does not advertise the traffic-report capability', function () { + it('records history but does not push for a site that does not advertise the traffic-report capability', function () { $site = ptrSite(['companion_capabilities' => ['snapshot']]); + ptrMockPressable($site, pushes: false); + + Http::fake(); - $this->mock(PressableClient::class, fn ($mock) => $mock->shouldReceive('isConfigured')->andReturn(true)); + $this->artisan('clockwork:pressable-traffic-report') + ->expectsOutputToContain("capability 'traffic-report' not advertised") + ->assertSuccessful(); + + Http::assertNothingSent(); + expect(SiteTrafficDaily::where('site_id', $site->id)->count())->toBe(2); + }); + + it('records history for a Pressable site without Companion, without trying to push', function () { + $site = Site::factory()->pressable()->create(); + ptrMockPressable($site, pushes: false); Http::fake(); - $this->artisan('clockwork:pressable-traffic-report')->assertSuccessful(); + $this->artisan('clockwork:pressable-traffic-report') + ->expectsOutputToContain('push skipped: Companion not installed') + ->assertSuccessful(); Http::assertNothingSent(); + expect(ptrRow($site, Carbon::today())->requests)->toBe(35) + ->and(ptrRow($site, Carbon::today()->subDays(3))->visits)->toBe(10); + }); + + it('ignores sites that are not on Pressable or have no Pressable site id', function () { + Site::factory()->spinupwp()->withCompanionInstalled()->create(); + Site::factory()->pressable()->withCompanionInstalled()->create(['pressable_site_id' => null]); + + $this->mock(PressableClient::class, function ($mock) { + $mock->shouldReceive('isConfigured')->andReturn(true); + $mock->shouldReceive('siteMetrics')->never(); + }); + + Http::fake(); + + $this->artisan('clockwork:pressable-traffic-report') + ->expectsOutputToContain('No Pressable sites matched.') + ->assertSuccessful(); + + Http::assertNothingSent(); + expect(SiteTrafficDaily::count())->toBe(0); + }); +}); + +describe('clockwork:pressable-traffic-report — stored history', function () { + it('leaves days Pressable returned nothing for untouched, so earlier history is never overwritten with zeros', function () { + $site = ptrSite(); + $missing = Carbon::today()->subDays(5); + $returned = Carbon::today()->subDays(3); + + // Rows an earlier host's access-log rollup wrote. + ptrSeedDay($site, $missing, [ + 'requests' => 500, 'unique_ips' => 60, 'visits' => 50, 'bytes_sent' => 1234567, + 'status_2xx' => 480, 'status_4xx' => 20, 'top_paths' => json_encode([['path' => '/', 'hits' => 9]]), + ]); + ptrSeedDay($site, $returned, [ + 'requests' => 999, 'visits' => 99, 'bytes_sent' => 7654321, + 'top_paths' => json_encode([['path' => '/about/', 'hits' => 4]]), + ]); + + ptrMockPressable($site); + ptrFakeTraffic($site); + + $this->artisan('clockwork:pressable-traffic-report')->assertSuccessful(); + + $untouched = ptrRow($site, $missing); + expect($untouched->requests)->toBe(500) + ->and($untouched->visits)->toBe(50) + ->and($untouched->unique_ips)->toBe(60) + ->and($untouched->bytes_sent)->toBe(1234567) + ->and($untouched->status_2xx)->toBe(480) + ->and($untouched->status_4xx)->toBe(20) + ->and($untouched->top_paths)->toBe([['path' => '/', 'hits' => 9]]); + + // A returned day takes Pressable's counts but keeps what Pressable + // has no per-day figure for. + $updated = ptrRow($site, $returned); + expect($updated->requests)->toBe(10) + ->and($updated->visits)->toBe(10) + ->and($updated->bytes_sent)->toBe(7654321) + ->and($updated->top_paths)->toBe([['path' => '/about/', 'hits' => 4]]); + + // The Companion chart still gets the full zero-filled series. + Http::assertSent(fn ($request) => count(json_decode($request->body(), true)['daily']) === 30); + }); + + it('updates only the metric family Pressable returned for a day', function () { + $site = ptrSite(); + $requestsOnly = Carbon::today()->subDays(2); + $uniquesOnly = Carbon::today()->subDays(4); + + ptrSeedDay($site, $requestsOnly, ['requests' => 300, 'status_2xx' => 300, 'visits' => 77, 'unique_ips' => 88]); + ptrSeedDay($site, $uniquesOnly, ['requests' => 400, 'status_2xx' => 390, 'status_5xx' => 10, 'visits' => 66, 'unique_ips' => 70]); + + ptrMockSeries( + $site, + requestPeriods: [['timestamp' => $requestsOnly->copy()->setTime(8, 0)->timestamp, 'dimension' => ['200' => 12, '301' => 3]]], + uniquePeriods: [['timestamp' => $uniquesOnly->timestamp, 'dimension' => ['example.com' => 21]]], + ); + ptrFakeTraffic($site); + + $this->artisan('clockwork:pressable-traffic-report')->assertSuccessful(); + + $a = ptrRow($site, $requestsOnly); + expect($a->requests)->toBe(15) + ->and($a->status_2xx)->toBe(12) + ->and($a->status_3xx)->toBe(3) + ->and($a->visits)->toBe(77) + ->and($a->unique_ips)->toBe(88); + + $b = ptrRow($site, $uniquesOnly); + expect($b->requests)->toBe(400) + ->and($b->status_2xx)->toBe(390) + ->and($b->status_5xx)->toBe(10) + ->and($b->visits)->toBe(21) + ->and($b->unique_ips)->toBe(21); + }); + + it('treats empty buckets as no data', function () { + $site = ptrSite(); + $seeded = Carbon::today()->subDays(6); + $unseeded = Carbon::today()->subDays(7); + + ptrSeedDay($site, $seeded, ['requests' => 250, 'visits' => 25, 'unique_ips' => 30, 'status_2xx' => 250]); + + ptrMockSeries( + $site, + requestPeriods: [ + ['timestamp' => $seeded->copy()->setTime(8, 0)->timestamp, 'dimension' => []], + ['timestamp' => $unseeded->copy()->setTime(8, 0)->timestamp, 'dimension' => []], + ], + uniquePeriods: [ + ['timestamp' => $seeded->timestamp, 'dimension' => []], + ['timestamp' => $unseeded->timestamp], + ], + ); + ptrFakeTraffic($site); + + $this->artisan('clockwork:pressable-traffic-report')->assertSuccessful(); + + $row = ptrRow($site, $seeded); + expect($row->requests)->toBe(250) + ->and($row->visits)->toBe(25) + ->and($row->unique_ips)->toBe(30) + ->and(ptrRow($site, $unseeded))->toBeNull(); + }); + + it('stores only the last 28 days, while the Companion payload still carries all 30', function () { + $site = ptrSite(); + $oldest = Carbon::today()->subDays(29); + $edge = Carbon::today()->subDays(27); + + ptrMockSeries( + $site, + requestPeriods: [ + ['timestamp' => $oldest->copy()->setTime(16, 0)->timestamp, 'dimension' => ['200' => 7]], + ['timestamp' => $edge->copy()->setTime(16, 0)->timestamp, 'dimension' => ['200' => 9]], + ], + uniquePeriods: [ + ['timestamp' => $oldest->timestamp, 'dimension' => ['example.com' => 3]], + ['timestamp' => $edge->timestamp, 'dimension' => ['example.com' => 4]], + ], + ); + ptrFakeTraffic($site); + + $this->artisan('clockwork:pressable-traffic-report')->assertSuccessful(); + + expect(ptrRow($site, $oldest))->toBeNull() + ->and(ptrRow($site, $edge)->requests)->toBe(9) + ->and(SiteTrafficDaily::where('site_id', $site->id)->count())->toBe(1); + + Http::assertSent(function ($request) use ($oldest) { + $daily = collect(json_decode($request->body(), true)['daily'])->keyBy('date'); + + return $daily->count() === 30 + && $daily->get($oldest->toDateString())['requests'] === 7 + && $daily->get($oldest->toDateString())['visits'] === 3; + }); }); }); @@ -180,9 +428,10 @@ function ptrFakeTraffic(Site $site, bool $ok = true, int $status = 200): void Http::assertNothingSent(); Log::shouldHaveReceived('warning')->with('companion.pressable_traffic_report.fetch_failed', \Mockery::type('array')); + expect(SiteTrafficDaily::where('site_id', $site->id)->count())->toBe(0); }); - it('logs and continues (still SUCCESS overall) when the push to Companion itself fails', function () { + it('logs and continues (still SUCCESS overall) when the push to Companion itself fails, keeping the recorded history', function () { Log::spy(); $site = ptrSite(); ptrMockPressable($site); @@ -192,6 +441,7 @@ function ptrFakeTraffic(Site $site, bool $ok = true, int $status = 200): void $this->artisan('clockwork:pressable-traffic-report')->assertSuccessful(); Log::shouldHaveReceived('warning')->with('companion.pressable_traffic_report.push_failed', \Mockery::type('array')); + expect(SiteTrafficDaily::where('site_id', $site->id)->count())->toBe(2); }); }); @@ -215,3 +465,20 @@ function ptrFakeTraffic(Site $site, bool $ok = true, int $status = 200): void Http::assertNotSent(fn ($request) => str_contains($request->url(), $other->domain)); }); }); + +describe('clockwork:pressable-traffic-report — schedule', function () { + it('runs early enough to finish before the client-report run that reads its rows', function () { + $find = fn (string $command): Event => collect(app(Schedule::class)->events()) + ->first(fn (Event $event) => str_ends_with((string) $event->command, $command)); + + $from = Carbon::parse('2030-03-01 00:00:00', 'UTC'); + $traffic = Carbon::instance($find('clockwork:pressable-traffic-report')->nextRunDate($from)); + $reports = Carbon::instance($find('clockwork:send-client-reports')->nextRunDate($from)); + + // Same morning, at least an hour ahead of the reports (the run takes + // ~15-20 min), and late enough that the previous UTC day is complete. + expect($traffic->toDateString())->toBe($reports->toDateString()) + ->and($traffic->diffInMinutes($reports))->toBeGreaterThanOrEqual(60) + ->and($from->diffInMinutes($traffic))->toBeGreaterThanOrEqual(120); + }); +}); From e6d9ad0551b104d2ba5d5050c2075f1e955e985d Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 3 Nov 2026 19:59:13 +0000 Subject: [PATCH 04/64] fix(uptime): treat HTTP 429 as a blocked probe, not downtime A site behind a host rate limiter or a security plugin's anti-bot challenge can answer the uptime monitor with 429 Too Many Requests while real visitors load it fine. The prober treated 429 as a plain failure, so two in a row moved the site to down and a long block showed up as a multi-day outage in uptime figures and client reports. - UptimeProber returns UptimeProbeResult::blocked() for 429 (not a success, not a plain failure; Retry-After kept, capped). - UptimeStateUpdater routes blocked probes to a new probe_blocked state: the failure streak is not incremented, last-up is not set, and the site never goes down on a 429. Entering the state records a probe_blocked site_uptime_events row and an action log entry, so stats and the event feeds never count it as downtime. A down site that starts answering 429 has its outage closed (recovery is not claimed). A real outage while blocked still goes down after the usual threshold; a success, maintenance or a down transition ends the blocked stretch. - After N hours blocked (Monitoring -> Settings, default 6) one uptime_probe_blocked ops chat notice is sent per stretch through ChatNotifier. It honours the per-site ignore flag, is skipped for inactive sites by the dispatcher, can be switched off per channel like other events, and never reaches client Slack or SMS. - New sites columns uptime_probe_blocked_since and uptime_probe_blocked_alerted_at (migration). - Monitoring board, site widget, sites list and recheck endpoint show the blocked state instead of down/unknown; docs updated. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Q7rExNMgxzTNb3pagVUZrg --- app/Console/Commands/CheckSiteUptime.php | 11 +- app/Http/Controllers/MonitoringController.php | 19 +- app/Http/Controllers/SitesController.php | 10 +- app/Models/Site.php | 21 +- app/Models/SiteUptimeEvent.php | 11 +- app/Services/Chat/ChatNotifier.php | 7 + app/Services/Chat/ChatNotifierDispatcher.php | 9 + app/Services/Uptime/UptimeProbeResult.php | 28 + app/Services/Uptime/UptimeProber.php | 17 +- app/Services/Uptime/UptimeStateUpdater.php | 156 ++++- ...0000_add_uptime_probe_blocked_to_sites.php | 53 ++ .../ClientSlack/src/ClientSlackNotifier.php | 5 + .../Core/src/Support/WebhookChatNotifier.php | 38 ++ resources/docs/features/uptime-monitoring.md | 22 +- .../site/widgets/_widget-uptime.blade.php | 13 +- resources/views/dashboard/sites.blade.php | 1 + resources/views/monitoring/index.blade.php | 31 +- resources/views/monitoring/settings.blade.php | 20 + tests/Feature/Chat/ChatNotifierGatingTest.php | 10 +- .../Controllers/MonitoringControllerTest.php | 74 +++ .../Controllers/SitesControllerTest.php | 20 + .../Feature/Uptime/UptimeProbeBlockedTest.php | 531 ++++++++++++++++++ 22 files changed, 1078 insertions(+), 29 deletions(-) create mode 100644 database/migrations/2026_11_03_120000_add_uptime_probe_blocked_to_sites.php create mode 100644 tests/Feature/Uptime/UptimeProbeBlockedTest.php diff --git a/app/Console/Commands/CheckSiteUptime.php b/app/Console/Commands/CheckSiteUptime.php index 451f9905..8cfed5cc 100644 --- a/app/Console/Commands/CheckSiteUptime.php +++ b/app/Console/Commands/CheckSiteUptime.php @@ -47,6 +47,7 @@ public function handle(UptimeProber $prober, UptimeStateUpdater $updater, Indexa $up = 0; $down = 0; + $blocked = 0; $seoFailures = 0; $seoLastError = null; $start = microtime(true); @@ -66,7 +67,13 @@ public function handle(UptimeProber $prober, UptimeStateUpdater $updater, Indexa $seoFailures++; $seoLastError = $seoError->getMessage(); } - $probe->succeeded ? $up++ : $down++; + if ($probe->isProbeBlocked) { + $blocked++; + } elseif ($probe->succeeded) { + $up++; + } else { + $down++; + } } catch (Throwable $e) { // Synthesize a transport-failed probe and run it through the // state machine so consecutive_failures still ticks up — a @@ -88,7 +95,7 @@ public function handle(UptimeProber $prober, UptimeStateUpdater $updater, Indexa } $elapsed = (int) round(microtime(true) - $start); - $this->info("Done. up={$up} down={$down} elapsed={$elapsed}s"); + $this->info("Done. up={$up} down={$down} blocked={$blocked} elapsed={$elapsed}s"); if ($seoFailures > 0) { $this->warn("SEO indexability piggyback failed on {$seoFailures} of {$sites->count()} sites. Last error: {$seoLastError}"); diff --git a/app/Http/Controllers/MonitoringController.php b/app/Http/Controllers/MonitoringController.php index 8c03a7f7..ce370c0b 100644 --- a/app/Http/Controllers/MonitoringController.php +++ b/app/Http/Controllers/MonitoringController.php @@ -43,6 +43,10 @@ class MonitoringController extends Controller public const DEFAULT_FAILURE_THRESHOLD = UptimeStateUpdater::FAILURE_THRESHOLD_FOR_DOWN; + public const SETTING_PROBE_BLOCKED_ALERT_HOURS = UptimeStateUpdater::SETTING_PROBE_BLOCKED_ALERT_HOURS; + + public const DEFAULT_PROBE_BLOCKED_ALERT_HOURS = UptimeStateUpdater::PROBE_BLOCKED_ALERT_HOURS; + public function index(UptimeStatsCalculator $calc): View { // Sites whose servers aren't ignored AND have monitoring enabled — @@ -64,7 +68,8 @@ public function index(UptimeStatsCalculator $calc): View $excused = $this->siteOutageIsExcused($site, $latestDownEvents); $tier = match (true) { $site->uptime_state === 'down' && $site->uptime_ignored_at === null && ! $excused => 1, - $site->uptime_state === 'unknown' => 2, + // Rate-limiting the probe: like unknown, we can't see it. + $site->uptime_state === 'unknown', $site->uptime_state === Site::UPTIME_STATE_PROBE_BLOCKED => 2, $site->uptime_state === 'down' && $excused => 3, $site->uptime_state === 'down' => 4, $site->uptime_state === 'maintenance' => 5, @@ -94,6 +99,7 @@ public function index(UptimeStatsCalculator $calc): View $currentlyUp = $sites->where('uptime_state', 'up')->count(); $currentlyMaintenance = $sites->where('uptime_state', 'maintenance')->count(); $unknown = $sites->where('uptime_state', 'unknown')->count(); + $currentlyProbeBlocked = $sites->where('uptime_state', Site::UPTIME_STATE_PROBE_BLOCKED)->count(); $currentlyIgnored = $sites->whereNotNull('uptime_ignored_at')->count(); // Fleet aggregate uptime: arithmetic mean of per-site uptimes, @@ -123,6 +129,7 @@ public function index(UptimeStatsCalculator $calc): View 'currentlyUp', 'currentlyMaintenance', 'unknown', + 'currentlyProbeBlocked', 'currentlyIgnored', 'avg7d', 'avg30d', @@ -136,6 +143,7 @@ public function settings(Settings $settings, DomainIgnoreList $ignoreList): View { $intervalMin = (int) $settings->get(self::SETTING_INTERVAL_MIN, self::DEFAULT_INTERVAL_MIN); $failureThreshold = (int) $settings->get(self::SETTING_FAILURE_THRESHOLD, self::DEFAULT_FAILURE_THRESHOLD); + $probeBlockedAlertHours = (int) $settings->get(self::SETTING_PROBE_BLOCKED_ALERT_HOURS, self::DEFAULT_PROBE_BLOCKED_ALERT_HOURS); // Sites toggled off (per-site override). Surface so the operator can quickly // re-enable from the same screen as the global settings. @@ -164,6 +172,7 @@ public function settings(Settings $settings, DomainIgnoreList $ignoreList): View return view('monitoring.settings', compact( 'intervalMin', 'failureThreshold', + 'probeBlockedAlertHours', 'disabledSites', 'schedulerHeartbeat', 'ignoredPatterns', @@ -203,6 +212,8 @@ public function updateSettings(Request $request, Settings $settings): RedirectRe $validated = $request->validate([ 'interval_minutes' => 'required|integer|in:1,5,10,15', 'failure_threshold' => 'required|integer|min:1|max:6', + // Optional so a form posted without it keeps the stored value. + 'probe_blocked_alert_hours' => 'sometimes|integer|min:1|max:168', 'ignored_domains' => 'nullable|string|max:10000', ]); @@ -217,11 +228,13 @@ public function updateSettings(Request $request, Settings $settings): RedirectRe ]); } - $settings->putMany([ + $settings->putMany(array_merge([ self::SETTING_INTERVAL_MIN => $validated['interval_minutes'], self::SETTING_FAILURE_THRESHOLD => $validated['failure_threshold'], DomainIgnoreList::SETTING_KEY => $parsed['patterns'], - ]); + ], isset($validated['probe_blocked_alert_hours']) ? [ + self::SETTING_PROBE_BLOCKED_ALERT_HOURS => (int) $validated['probe_blocked_alert_hours'], + ] : [])); return redirect() ->route('monitoring.settings') diff --git a/app/Http/Controllers/SitesController.php b/app/Http/Controllers/SitesController.php index 5a161b75..b0c17ba9 100644 --- a/app/Http/Controllers/SitesController.php +++ b/app/Http/Controllers/SitesController.php @@ -111,7 +111,7 @@ public function index(Request $request): View break; case 'status': - $statusOrder = "CASE sites.uptime_state WHEN 'down' THEN 1 WHEN 'maintenance' THEN 2 WHEN 'unknown' THEN 3 WHEN 'up' THEN 4 ELSE 5 END"; + $statusOrder = "CASE sites.uptime_state WHEN 'down' THEN 1 WHEN 'maintenance' THEN 2 WHEN 'unknown' THEN 3 WHEN 'probe_blocked' THEN 3 WHEN 'up' THEN 4 ELSE 5 END"; $sitesQuery->select('sites.*') ->orderByRaw("{$statusOrder} ".($dir === 'desc' ? 'DESC' : 'ASC')) ->orderBy('sites.domain', 'asc'); @@ -677,9 +677,11 @@ public function recheckUptime(Site $site, UptimeProber $prober, UptimeStateUpdat return response()->json([ 'ok' => true, - 'message' => $probe->succeeded - ? "Up (HTTP {$probe->statusCode})" - : 'Down ('.($probe->statusCode ? "HTTP {$probe->statusCode}" : ($probe->error ?? 'no response')).')', + 'message' => match (true) { + $probe->isProbeBlocked => "Probe blocked (HTTP {$probe->statusCode}) — not counted as down", + $probe->succeeded => "Up (HTTP {$probe->statusCode})", + default => 'Down ('.($probe->statusCode ? "HTTP {$probe->statusCode}" : ($probe->error ?? 'no response')).')', + }, 'state' => $site->uptime_state, 'status_code' => $probe->statusCode, ]); diff --git a/app/Models/Site.php b/app/Models/Site.php index f4b09dfb..f45777b3 100644 --- a/app/Models/Site.php +++ b/app/Models/Site.php @@ -102,13 +102,15 @@ * @property ?string $auto_updates_paused_reason short note shown as a tooltip in the fleet listing * @property ?Carbon $auto_updates_last_run_at last time the nightly loop touched this site * @property bool $uptime_monitoring_enabled - * @property string $uptime_state up|down|maintenance|unknown + * @property string $uptime_state up|down|maintenance|probe_blocked|unknown * @property ?Carbon $uptime_last_checked_at * @property ?Carbon $uptime_last_up_at * @property ?int $uptime_last_status_code * @property int $uptime_consecutive_failures * @property ?Carbon $uptime_down_since * @property ?Carbon $uptime_maintenance_since + * @property ?Carbon $uptime_probe_blocked_since start of the current unbroken run of HTTP 429 probe answers + * @property ?Carbon $uptime_probe_blocked_alerted_at when ops was told about that run (one notice per run) * @property ?Carbon $uptime_ignored_at * @property ?string $uptime_ignore_reason * @property bool $uptime_sla_exempt @@ -255,6 +257,12 @@ class Site extends Model public const UPTIME_STATE_UNKNOWN = 'unknown'; + /** + * The site rate-limits the probe (HTTP 429), so we can't tell whether it + * is up. Not an outage: it never counts as downtime. + */ + public const UPTIME_STATE_PROBE_BLOCKED = 'probe_blocked'; + protected $fillable = [ 'spinupwp_id', 'pressable_site_id', @@ -344,6 +352,8 @@ class Site extends Model 'uptime_consecutive_failures', 'uptime_down_since', 'uptime_maintenance_since', + 'uptime_probe_blocked_since', + 'uptime_probe_blocked_alerted_at', 'uptime_ignored_at', 'uptime_ignore_reason', 'uptime_sla_exempt', @@ -443,6 +453,8 @@ protected function casts(): array 'uptime_consecutive_failures' => 'integer', 'uptime_down_since' => 'datetime', 'uptime_maintenance_since' => 'datetime', + 'uptime_probe_blocked_since' => 'datetime', + 'uptime_probe_blocked_alerted_at' => 'datetime', 'uptime_ignored_at' => 'datetime', 'uptime_sla_exempt' => 'boolean', 'uptime_skip_body_check' => 'boolean', @@ -1404,6 +1416,11 @@ public function isMaintenance(): bool return $this->uptime_state === self::UPTIME_STATE_MAINTENANCE; } + public function isUptimeProbeBlocked(): bool + { + return $this->uptime_state === self::UPTIME_STATE_PROBE_BLOCKED; + } + /** * Return a consolidated status color for visual cards ('green', 'yellow', 'red'). */ @@ -1417,7 +1434,7 @@ public function healthColor(): string return 'red'; } - if ($this->uptime_state === 'unknown' || $this->uptime_state === self::UPTIME_STATE_MAINTENANCE || $this->sslState() === self::SSL_STATE_YELLOW) { + if ($this->uptime_state === 'unknown' || $this->uptime_state === self::UPTIME_STATE_MAINTENANCE || $this->uptime_state === self::UPTIME_STATE_PROBE_BLOCKED || $this->sslState() === self::SSL_STATE_YELLOW) { return 'yellow'; } diff --git a/app/Models/SiteUptimeEvent.php b/app/Models/SiteUptimeEvent.php index 332a160b..d12ec3de 100644 --- a/app/Models/SiteUptimeEvent.php +++ b/app/Models/SiteUptimeEvent.php @@ -8,11 +8,12 @@ use Illuminate\Support\Carbon; /** - * One row per uptime state transition (up→down or down→up). NOT one per probe. + * One row per uptime state transition (up→down, down→up, into maintenance, + * into probe_blocked). NOT one per probe. * * @property int $id * @property int $site_id - * @property string $event_type 'down' or 'up' + * @property string $event_type 'down', 'up', 'maintenance' or 'probe_blocked' * @property ?int $status_code * @property ?string $error * @property ?int $response_time_ms @@ -36,6 +37,12 @@ class SiteUptimeEvent extends Model public const TYPE_MAINTENANCE = 'maintenance'; + /** + * The site started rate-limiting the probe (HTTP 429). Opens a stretch of + * "can't tell" — never downtime. Closed by the next up/down/maintenance. + */ + public const TYPE_PROBE_BLOCKED = 'probe_blocked'; + public const REASON_CLIENT_DNS = 'client_dns'; public const REASON_DOMAIN_EXPIRED = 'domain_expired'; diff --git a/app/Services/Chat/ChatNotifier.php b/app/Services/Chat/ChatNotifier.php index c44aa654..27e8594c 100644 --- a/app/Services/Chat/ChatNotifier.php +++ b/app/Services/Chat/ChatNotifier.php @@ -41,6 +41,11 @@ interface ChatNotifier 'description' => 'Uptime probe detected the site finished scheduled maintenance and is back up.', 'default' => true, ], + 'uptime_probe_blocked' => [ + 'label' => 'Uptime probe blocked', + 'description' => 'A site has rate-limited the uptime probe (HTTP 429) for longer than the hours set on Monitoring → Settings, so its real status cannot be confirmed. Sent once per blocked stretch.', + 'default' => true, + ], 'ssl_state_changed' => [ 'label' => 'SSL state changed', 'description' => 'Certificate state moved between green / yellow / red.', @@ -186,6 +191,8 @@ public function siteEnteredMaintenance(Site $site, ?int $statusCode, ?string $re public function siteExitedMaintenance(Site $site, ?int $maintenanceSec): bool; + public function uptimeProbeBlocked(Site $site, int $blockedSec, ?int $statusCode, ?string $retryAfter = null): bool; + public function pluginUpdateFailed(Site $site, PluginUpdateJob $job): bool; public function pluginUpdateAutoIgnored(Site $site, PluginUpdateJob $job, int $failures): bool; diff --git a/app/Services/Chat/ChatNotifierDispatcher.php b/app/Services/Chat/ChatNotifierDispatcher.php index fa56ce48..022ae6bb 100644 --- a/app/Services/Chat/ChatNotifierDispatcher.php +++ b/app/Services/Chat/ChatNotifierDispatcher.php @@ -125,6 +125,15 @@ public function siteExitedMaintenance(Site $site, ?int $maintenanceSec): bool return $this->dispatch(fn (ChatNotifier $n) => $n->siteExitedMaintenance($site, $maintenanceSec)); } + /** + * A blocked probe is a monitoring gap, not an outage, so it is routed like + * the other routine per-site notices and stays quiet for inactive sites. + */ + public function uptimeProbeBlocked(Site $site, int $blockedSec, ?int $statusCode, ?string $retryAfter = null): bool + { + return $this->dispatchForSite($site, fn (ChatNotifier $n) => $n->uptimeProbeBlocked($site, $blockedSec, $statusCode, $retryAfter)); + } + public function pluginUpdateFailed(Site $site, PluginUpdateJob $job): bool { return $this->dispatchForSite($site, fn (ChatNotifier $n) => $n->pluginUpdateFailed($site, $job)); diff --git a/app/Services/Uptime/UptimeProbeResult.php b/app/Services/Uptime/UptimeProbeResult.php index 3e50950d..80e68f81 100644 --- a/app/Services/Uptime/UptimeProbeResult.php +++ b/app/Services/Uptime/UptimeProbeResult.php @@ -12,6 +12,11 @@ * - succeeded=false → status_code is set if we got a response (4xx/5xx), * or null if the request never completed (timeout / DNS / refused / TLS fail). * In either failure case, `error` carries a short human description. + * - isProbeBlocked=true (always with succeeded=false) → the site answered + * HTTP 429: a rate limiter or anti-bot challenge refused the probe. That + * says nothing either way about whether real visitors can reach the site, + * so UptimeStateUpdater neither counts it as a failure nor as proof of + * life (see blocked()). */ final class UptimeProbeResult { @@ -24,6 +29,7 @@ public function __construct( public readonly ?string $xRobotsTagHeader = null, public readonly bool $isMaintenance = false, public readonly ?string $retryAfter = null, + public readonly bool $isProbeBlocked = false, ) {} public static function success(int $statusCode, int $responseTimeMs, ?string $body = null, ?string $xRobotsTagHeader = null): self @@ -61,6 +67,28 @@ public static function authProtected(int $statusCode, int $responseTimeMs, strin return new self(true, $statusCode, $responseTimeMs, $error, $body, $xRobotsTagHeader); } + /** + * The site rate-limited the probe (HTTP 429). Deliberately NOT a success: + * a host can throttle real visitors too, so this must not set last-up or + * clear a failure streak. Deliberately NOT a plain failure either: a rate + * limiter or anti-bot challenge that singles out automated clients would + * otherwise read as a multi-day outage. The state updater treats it as + * "no signal" and tracks it separately. + */ + public static function blocked(int $statusCode, int $responseTimeMs, string $reason, ?string $retryAfter = null, ?string $body = null, ?string $xRobotsTagHeader = null): self + { + return new self( + succeeded: false, + statusCode: $statusCode, + responseTimeMs: $responseTimeMs, + error: $reason, + body: $body, + xRobotsTagHeader: $xRobotsTagHeader, + retryAfter: $retryAfter, + isProbeBlocked: true, + ); + } + public static function badStatus(int $statusCode, int $responseTimeMs, string $error, ?string $body = null, ?string $xRobotsTagHeader = null): self { return new self(false, $statusCode, $responseTimeMs, $error, $body, $xRobotsTagHeader); diff --git a/app/Services/Uptime/UptimeProber.php b/app/Services/Uptime/UptimeProber.php index 64b7b417..593612dd 100644 --- a/app/Services/Uptime/UptimeProber.php +++ b/app/Services/Uptime/UptimeProber.php @@ -29,7 +29,8 @@ * job; uptime is reachability only. * * Treats 2xx, 3xx, and auth-protected 4xx (401 with WWW-Authenticate, 403) - * as success; everything else as failure. + * as success; 429 as "probe blocked" (neither success nor failure — see + * UptimeProbeResult::blocked()); everything else as failure. */ class UptimeProber { @@ -100,6 +101,20 @@ public function probe(string $url, int $timeoutSec = 10, ?string $requireKeyword return UptimeProbeResult::authProtected($status, $elapsed, 'HTTP 403 (forbidden — likely WAF or allowlist)', $body, $xRobotsTag); } + // Rate-limited (RFC 6585 §4). A host rate limiter or a security + // plugin's anti-bot challenge is refusing the probe itself, which + // tells us nothing about whether real visitors get through. Report it + // as blocked so the state machine neither opens an outage on it nor + // treats it as proof the site is up. + if ($status === 429) { + // Capped: the header is remote-controlled and the reason lands in + // a 500-char column. + $retryAfter = $response->header('Retry-After') !== '' ? mb_substr($response->header('Retry-After'), 0, 64) : null; + $reason = 'HTTP 429 (rate-limited — probe blocked'.($retryAfter !== null ? " · Retry-After: {$retryAfter}" : '').')'; + + return UptimeProbeResult::blocked($status, $elapsed, $reason, $retryAfter, $body, $xRobotsTag); + } + // Scheduled maintenance mode (RFC 7231 §6.6.4): WordPress core and // reputable maintenance plugins (SeedProd, Kadence, WP Maintenance Mode) // emit 503 + Retry-After (or recognizable maintenance copy) for SEO diff --git a/app/Services/Uptime/UptimeStateUpdater.php b/app/Services/Uptime/UptimeStateUpdater.php index af5f3ce6..5b62dfe9 100644 --- a/app/Services/Uptime/UptimeStateUpdater.php +++ b/app/Services/Uptime/UptimeStateUpdater.php @@ -27,6 +27,15 @@ * - unknown → up: instant, no event/alert (initial state, not a transition) * - unknown → down: 2 failures, but emits a "first detected down" event * (we still want to know about a site we've never seen up) + * - any → probe_blocked: the site answered HTTP 429 (a rate limiter or + * anti-bot challenge refused the probe). That is neither a failure nor + * proof of life: the failure streak is not incremented, last-up is not + * set, and the site never goes down on it. Entering the state records a + * probe_blocked event (which closes an open outage: the server is + * answering again, so the probe can no longer vouch it is down). Staying + * blocked for the configured number of hours sends one ops chat notice. + * A sub-threshold failure while blocked leaves the blocked stretch open; + * a success, maintenance, or a real down transition ends it. */ class UptimeStateUpdater { @@ -38,6 +47,15 @@ class UptimeStateUpdater */ public const FAILURE_THRESHOLD_FOR_DOWN = 2; + /** + * Hours a site may keep rate-limiting the probe before ops gets a chat + * notice. Runtime value comes from the Settings key below (Monitoring → + * Settings); this is the default. + */ + public const PROBE_BLOCKED_ALERT_HOURS = 6; + + public const SETTING_PROBE_BLOCKED_ALERT_HOURS = 'monitoring.uptime_probe_blocked_alert_hours'; + public function __construct( protected ChatNotifier $mattermost, protected ActionLogger $actionLogger, @@ -50,7 +68,9 @@ public function update(Site $site, UptimeProbeResult $probe): void $now = Carbon::now(); $previousState = $site->uptime_state ?? 'unknown'; - if ($probe->isMaintenance) { + if ($probe->isProbeBlocked) { + $this->handleBlocked($site, $probe, $previousState, $now); + } elseif ($probe->isMaintenance) { $this->handleMaintenance($site, $probe, $previousState, $now); } elseif ($probe->succeeded) { $this->handleSuccess($site, $probe, $previousState, $now); @@ -58,10 +78,122 @@ public function update(Site $site, UptimeProbeResult $probe): void $this->handleFailure($site, $probe, $previousState, $now); } - $site->forceFill([ + $fields = [ 'uptime_last_checked_at' => $now, 'uptime_last_status_code' => $probe->statusCode, - ])->save(); + ]; + + // The blocked stretch ends once the site has left the probe_blocked + // state on a real answer (up, maintenance, or a confirmed outage). + if (! $probe->isProbeBlocked && $site->uptime_state !== Site::UPTIME_STATE_PROBE_BLOCKED) { + $fields['uptime_probe_blocked_since'] = null; + $fields['uptime_probe_blocked_alerted_at'] = null; + } + + $site->forceFill($fields)->save(); + } + + /** + * HTTP 429: the site is refusing the probe, so we can't tell whether it is + * up. Never counts toward the failure streak, never sets last-up, never + * opens an outage. + */ + private function handleBlocked(Site $site, UptimeProbeResult $probe, string $previousState, Carbon $now): void + { + $isTransition = $previousState !== Site::UPTIME_STATE_PROBE_BLOCKED; + $wasDown = $previousState === Site::UPTIME_STATE_DOWN; + $downSince = $site->uptime_down_since; + $blockedSince = $site->uptime_probe_blocked_since ?? $now; + + $fields = [ + 'uptime_state' => Site::UPTIME_STATE_PROBE_BLOCKED, + 'uptime_probe_blocked_since' => $blockedSince, + 'uptime_down_since' => null, + 'uptime_maintenance_since' => null, + ]; + + // Closing an outage resets the streak the way a success would; + // otherwise a single blip after this would count as the second + // failure of an outage that has already ended. + if ($wasDown) { + $fields['uptime_consecutive_failures'] = 0; + } + + $site->forceFill($fields)->save(); + + if ($isTransition) { + $this->recordEvent($site, SiteUptimeEvent::TYPE_PROBE_BLOCKED, $probe, $now); + $this->logProbeBlocked($site, $probe, $wasDown && $downSince !== null ? (int) abs($downSince->diffInSeconds($now)) : null); + } + + $this->maybeNotifyProbeBlocked($site, $probe, $blockedSince, $now); + } + + private function logProbeBlocked(Site $site, UptimeProbeResult $probe, ?int $closedOutageSec): void + { + $summary = $closedOutageSec !== null + ? sprintf('%s answers again but is blocking the uptime probe (HTTP %d) — outage closed after %d min; recovery not confirmed', $site->domain, (int) $probe->statusCode, (int) round($closedOutageSec / 60)) + : sprintf('%s is blocking the uptime probe (HTTP %d) — not counted as downtime', $site->domain, (int) $probe->statusCode); + + $this->actionLogger->record( + actionType: ActionLog::TYPE_UPTIME_TRANSITION, + summary: $summary, + site: $site, + target: Site::UPTIME_STATE_PROBE_BLOCKED, + details: array_filter([ + 'status_code' => $probe->statusCode, + 'retry_after' => $probe->retryAfter, + 'closed_outage_sec' => $closedOutageSec, + ], fn ($v) => $v !== null), + ok: false, + error: $probe->error, + elapsedMs: $probe->responseTimeMs, + actor: 'scheduled', + ); + } + + /** + * One chat notice per blocked stretch, once it has lasted the configured + * number of hours. Ignored sites stay silent (same rule as down alerts); + * the dispatcher also silences inactive sites. + */ + private function maybeNotifyProbeBlocked(Site $site, UptimeProbeResult $probe, Carbon $blockedSince, Carbon $now): void + { + if ($site->uptime_probe_blocked_alerted_at !== null || $site->isUptimeIgnored()) { + return; + } + + $blockedSec = (int) abs($blockedSince->diffInSeconds($now)); + if ($blockedSec < $this->probeBlockedAlertHours() * 3600) { + return; + } + + $site->forceFill(['uptime_probe_blocked_alerted_at' => $now])->save(); + + try { + $this->mattermost->uptimeProbeBlocked($site, $blockedSec, $probe->statusCode, $probe->retryAfter); + } catch (\Throwable $e) { + Log::warning('uptime.mattermost_probe_blocked_failed', ['site_id' => $site->id, 'error' => $e->getMessage()]); + } + + $this->actionLogger->record( + actionType: ActionLog::TYPE_UPTIME_TRANSITION, + summary: sprintf('%s has blocked the uptime probe for %d h (HTTP %d) — ops notified', $site->domain, intdiv($blockedSec, 3600), (int) $probe->statusCode), + site: $site, + target: Site::UPTIME_STATE_PROBE_BLOCKED, + details: ['status_code' => $probe->statusCode, 'blocked_sec' => $blockedSec, 'notified' => true], + ok: false, + error: $probe->error, + elapsedMs: $probe->responseTimeMs, + actor: 'scheduled', + ); + } + + private function probeBlockedAlertHours(): int + { + $hours = (int) app(Settings::class)->get(self::SETTING_PROBE_BLOCKED_ALERT_HOURS, self::PROBE_BLOCKED_ALERT_HOURS); + + return max(1, $hours); } private function handleMaintenance(Site $site, UptimeProbeResult $probe, string $previousState, Carbon $now): void @@ -88,8 +220,10 @@ private function handleSuccess(Site $site, UptimeProbeResult $probe, string $pre { $isRecoveryFromDown = $previousState === 'down'; $isRecoveryFromMaint = $previousState === 'maintenance'; + $isRecoveryFromBlocked = $previousState === Site::UPTIME_STATE_PROBE_BLOCKED; $downSince = $site->uptime_down_since; $maintSince = $site->uptime_maintenance_since; + $blockedSince = $site->uptime_probe_blocked_since; $site->forceFill([ 'uptime_state' => 'up', @@ -131,6 +265,22 @@ private function handleSuccess(Site $site, UptimeProbeResult $probe, string $pre if (! $site->isUptimeIgnored()) { $this->fireMaintenanceRecoveryNotification($site, $maintSec, $probe); } + } elseif ($isRecoveryFromBlocked) { + // Closes the blocked stretch in the event log. No chat: nothing + // was down, and the stretch was never reported as an outage. + $blockedSec = $blockedSince !== null ? (int) abs($blockedSince->diffInSeconds($now)) : null; + $this->recordEvent($site, SiteUptimeEvent::TYPE_UP, $probe, $now); + + $this->actionLogger->record( + actionType: ActionLog::TYPE_UPTIME_TRANSITION, + summary: "{$site->domain} uptime probe gets through again (was blocked for ".($blockedSec !== null ? (int) round($blockedSec / 60) : 0).' min)', + site: $site, + target: 'up', + details: ['status_code' => $probe->statusCode, 'probe_blocked_sec' => $blockedSec], + ok: true, + elapsedMs: $probe->responseTimeMs, + actor: 'scheduled', + ); } } diff --git a/database/migrations/2026_11_03_120000_add_uptime_probe_blocked_to_sites.php b/database/migrations/2026_11_03_120000_add_uptime_probe_blocked_to_sites.php new file mode 100644 index 00000000..19f89c27 --- /dev/null +++ b/database/migrations/2026_11_03_120000_add_uptime_probe_blocked_to_sites.php @@ -0,0 +1,53 @@ + ! Schema::hasColumn('sites', $c))); + if ($missing === []) { + return; + } + + if (DB::getDriverName() === 'mysql') { + // INSTANT add at end of table (sites is hit continuously by the + // uptime probe and the dashboard). + DB::statement('SET SESSION lock_wait_timeout = 15'); + $adds = implode(', ', array_map(fn (string $c) => "ADD COLUMN {$c} TIMESTAMP NULL", $missing)); + DB::statement("ALTER TABLE sites {$adds}, ALGORITHM=INSTANT"); + + return; + } + + Schema::table('sites', function (Blueprint $table) use ($missing) { + foreach ($missing as $column) { + $table->timestamp($column)->nullable(); + } + }); + } + + public function down(): void + { + Schema::table('sites', function (Blueprint $table) { + $table->dropColumn(['uptime_probe_blocked_since', 'uptime_probe_blocked_alerted_at']); + }); + } +}; diff --git a/modules/ClientSlack/src/ClientSlackNotifier.php b/modules/ClientSlack/src/ClientSlackNotifier.php index 68140920..1db42b04 100644 --- a/modules/ClientSlack/src/ClientSlackNotifier.php +++ b/modules/ClientSlack/src/ClientSlackNotifier.php @@ -122,6 +122,11 @@ public function siteExitedMaintenance(Site $site, ?int $maintenanceSec): bool return false; } + public function uptimeProbeBlocked(Site $site, int $blockedSec, ?int $statusCode, ?string $retryAfter = null): bool + { + return false; + } + public function contactFormTestFailed(Site $site, string $reason, int $streak, ?string $formId = null): bool { $webhook = $this->webhookUrl($site); diff --git a/modules/Core/src/Support/WebhookChatNotifier.php b/modules/Core/src/Support/WebhookChatNotifier.php index 25a0def0..91c69cc0 100644 --- a/modules/Core/src/Support/WebhookChatNotifier.php +++ b/modules/Core/src/Support/WebhookChatNotifier.php @@ -8,6 +8,7 @@ use App\Models\Site; use App\Models\SiteSecurityScan; use App\Services\Chat\ChatNotifier; +use App\Services\Uptime\UptimeProber; use App\Support\Settings; use Illuminate\Support\Carbon; use Illuminate\Support\Facades\Http; @@ -588,6 +589,43 @@ public function siteExitedMaintenance(Site $site, ?int $maintenanceSec): bool return $this->send($title, [$attachment]); } + /** + * The uptime probe has been rate-limited (HTTP 429) for a long stretch. + * Not an outage alert: the state machine does not count this as + * downtime, but it can't confirm the site is up either, so a person + * should look. Fires once per blocked stretch. + */ + public function uptimeProbeBlocked(Site $site, int $blockedSec, ?int $statusCode, ?string $retryAfter = null): bool + { + if (! $this->isEventEnabled('uptime_probe_blocked')) { + return false; + } + + $duration = $this->formatDowntime($blockedSec); + $detail = $statusCode !== null ? "HTTP {$statusCode}" : 'rate-limited'; + $title = sprintf(':no_entry: %s is blocking the uptime probe (%s for %s)', $site->domain, $detail, $duration); + + $body = 'The site has kept answering the uptime probe with "Too Many Requests" for this long, so Clockwork cannot tell whether it is up. ' + .'This is not counted as downtime. A host rate limiter or a security plugin\'s anti-bot challenge is the usual cause: ' + .'allowlist the uptime monitor\'s User-Agent ('.UptimeProber::userAgent().') there. ' + .'If real visitors are being throttled too, treat it as an outage.'; + + $attachment = [ + 'fallback' => $title, + 'color' => '#e0a800', + 'title' => $title, + 'text' => $body, + 'fields' => array_values(array_filter([ + ['title' => 'Site', 'value' => $site->domain, 'short' => true], + ['title' => 'Server', 'value' => $site->server !== null ? $site->server->name : 'n/a', 'short' => true], + ['title' => 'Blocked for', 'value' => $duration, 'short' => true], + $retryAfter !== null ? ['title' => 'Retry-After', 'value' => mb_strimwidth($retryAfter, 0, 64, '…'), 'short' => true] : null, + ])), + ]; + + return $this->send($title, [$attachment]); + } + /** * Per-failure ping for the nightly auto-update loop. Fires only when * the queued job is part of a `nightly-*` batch — manual bulk-update diff --git a/resources/docs/features/uptime-monitoring.md b/resources/docs/features/uptime-monitoring.md index 612536f4..0d96c784 100644 --- a/resources/docs/features/uptime-monitoring.md +++ b/resources/docs/features/uptime-monitoring.md @@ -2,7 +2,7 @@ title: Uptime monitoring section: Features order: 30 -updated: 2026-10-02 +updated: 2026-11-03 author: Aaron Reimann tags: [monitoring, uptime, alerts, hosting, pressable, slack, twilio, subscribers] tracks: [app/Services/Uptime/**, app/Support/Monitoring/**, app/Http/Controllers/MonitoringController.php, app/Http/Controllers/SitesController.php, app/Console/Commands/CheckSiteUptime.php, app/Services/Twilio/OnCallResolver.php, app/Mail/SiteDownClientMail.php, app/Mail/SiteUpClientMail.php] @@ -26,14 +26,14 @@ Monitoring queries use `Site::hostMonitored()` across the fleet, ensuring both s A site is **up** if it responds with HTTP 2xx or 3xx within 10 seconds — or if it returns 401/403 indicating it's alive but auth-protected (see below). Anything else is **down**: - HTTP 5xx -- HTTP 4xx other than 401 (with `WWW-Authenticate`) and 403 +- HTTP 4xx other than 401 (with `WWW-Authenticate`), 403, and 429 (see the rate-limit carve-out below) - Connection refused, DNS failure, TLS handshake failure - Request timeout - HTTP 200/3xx whose body is a white screen (under 50 visible characters after tag-stripping) or contains WordPress/PHP fatal signatures (`There has been a critical error`, `Fatal error`, etc.). Optional per-site `uptime_require_keyword` on the site Settings tab also fails a 200 that does not contain that string. Sites that are *legitimately* near-empty on a logged-out GET (parked static HTML, a JS-rendered SPA shell, a gated/private homepage) can skip only the length check via `uptime_skip_body_check` on the same Settings tab — signatures and the required keyword still apply. The probe does not issue a second HTTP request — it inspects the body already fetched. A site has to fail **two probes in a row** (about 10 minutes at the default cadence) before it transitions to `down` and fires the Mattermost alert. Recovery is instant — the moment the next probe succeeds, we transition back to `up` and fire the recovery alert. -The 2-failure threshold is the de-jitter logic. A single bad probe (CF blip, momentary network flicker, a brief WAF rate-limit) won't fire an alert. +The 2-failure threshold is the de-jitter logic. A single bad probe (CF blip, momentary network flicker) won't fire an alert. ## Outage fault attribution & SLA exemption ("Not Our Fault" / Client DNS) @@ -67,6 +67,19 @@ The per-site Overview status pill shows **"Up · auth required"** with a lock ic If you want to exempt our probe from a CF WAF rule, the User-Agent is `Clockwork-Uptime/1.0` plus `(+CLOCKWORK_OPERATOR_CONTACT_EMAIL)` if that's set (see `reference/env-vars`) — `UptimeProber::userAgent()` builds it dynamically as of the modularization roadmap's Phase 8, replacing what used to be a hardcoded email address. Whitelist whatever your instance's actual User-Agent string is in the WAF and the probe will receive a 200 instead. +## The 429 rate-limit carve-out ("probe blocked") + +Some hosts and security plugins rate-limit or challenge automated clients and answer the probe with **HTTP 429 Too Many Requests** while real visitors load the site fine. Counting that as down turned a host's anti-bot rule into a multi-day "outage" in uptime numbers and client reports, so a 429 is now its own result: + +- `UptimeProber` returns `UptimeProbeResult::blocked()` (`isProbeBlocked = true`, `succeeded = false`, `Retry-After` kept when sent). +- `UptimeStateUpdater` moves the site to the **`probe_blocked`** state and records a `probe_blocked` row in `site_uptime_events`. It does **not** add to `uptime_consecutive_failures`, does **not** set `uptime_last_up_at`, and never transitions the site to `down`. Blocked time never counts as downtime. +- A site that was `down` and starts answering 429 leaves `down`: the server is answering again, so the outage is closed (and the failure streak reset), but recovery is not confirmed, so no "back up" alert goes out. +- `sites.uptime_probe_blocked_since` marks the start of the blocked stretch. A single failed probe while blocked does not end it; a success, maintenance, or a real down transition does. A real outage while blocked still goes `down` after the usual failure threshold. +- If a site keeps blocking for **6 hours** (Monitoring → Settings, "Notify when a site blocks the probe for"), one `uptime_probe_blocked` chat notice goes to the ops channels (Mattermost / Slack, toggleable per channel like every other event). It respects the per-site rules: ignored sites stay silent and inactive sites are skipped. One notice per blocked stretch (`sites.uptime_probe_blocked_alerted_at`); it never pages on-call SMS or client contacts. +- On `/monitoring` the site shows a yellow **BLOCKED** pill and a "probe blocked" filter; the per-site widget shows **Probe blocked**. + +The fix is usually on the host: allowlist the `Clockwork-Uptime/1.0` User-Agent (and the Control server's IP) in the host's rate limiter or the security plugin's anti-bot settings. If real visitors are throttled too, treat it as an outage. + ## The 503 maintenance mode carve-out WordPress core automatic updates (`.maintenance`), popular maintenance plugins (SeedProd, Kadence, WP Maintenance Mode), and hosting panel maintenance modes legitimately return **HTTP 503 Service Unavailable** for SEO protection (RFC 7231 §6.6.4). @@ -111,11 +124,12 @@ How: per-site **Settings** tab → "Ignore uptime alerts" section. There's a rea ## Tuning the global settings -`/monitoring/settings` exposes three knobs. Change them carefully — they affect every site. +`/monitoring/settings` exposes four knobs. Change them carefully — they affect every site. - **Probe interval** — how often we hit each site. Default 5 minutes. The minimum is 1 minute (gives ManageWP-tier sensitivity), max is 15 minutes (gentlest, lowest log noise). Changes only apply on the next `schedule:work` restart. - **Failure threshold** — how many failures in a row trigger the down transition. Default 2. Lower values = more sensitive (1 = alert on first miss, you'll get noise). Higher values = fewer alerts (6 = ~30 minutes at default cadence before you hear about it). Threshold changes apply on the next probe — no restart needed. - **Ignored domains** — wildcard patterns that remove matching sites from uptime monitoring entirely (see the section above). Applies on the next probe — no restart needed. +- **Probe-blocked notice** — hours a site may keep answering HTTP 429 before the one-off ops chat notice (1–168, default 6). Applies on the next probe. ## Alert channels & dispatch diff --git a/resources/views/dashboard/site/widgets/_widget-uptime.blade.php b/resources/views/dashboard/site/widgets/_widget-uptime.blade.php index 37a5d0fd..18cb62f4 100644 --- a/resources/views/dashboard/site/widgets/_widget-uptime.blade.php +++ b/resources/views/dashboard/site/widgets/_widget-uptime.blade.php @@ -2,6 +2,7 @@ $isUp = $site->uptime_state === 'up'; $isDown = $site->uptime_state === 'down'; $isMaintenance = $site->uptime_state === 'maintenance'; + $isProbeBlocked = $site->uptime_state === 'probe_blocked'; $latestDown = $isDown ? $site->latestDownEvent() : null; $isNotOurFault = $isDown && ($site->uptime_sla_exempt || (bool) $latestDown?->is_sla_exempt); $exemptionReason = $site->uptime_exemption_reason ?: $latestDown?->exemption_reason; @@ -34,6 +35,10 @@ Maintenance + @elseif ($isProbeBlocked) + + Probe blocked + @else {{ $authProtected ? 'Up (Auth)' : 'Operational' }} @@ -47,7 +52,7 @@ - DOWN @elseif ($isMaintenance) + @elseif ($isProbeBlocked) + 429 @else UP @endif @@ -87,6 +94,8 @@ @if ($site->uptime_last_status_code) (HTTP {{ $site->uptime_last_status_code }}) @endif @elseif ($isMaintenance && $site->uptime_maintenance_since) In maintenance for {{ $site->uptime_maintenance_since->diffForHumans(['parts' => 2, 'short' => true]) }} + @elseif ($isProbeBlocked) + Probe rate-limited (HTTP 429){{ $site->uptime_probe_blocked_since ? ' since ' . $site->uptime_probe_blocked_since->diffForHumans(['parts' => 2, 'short' => true]) : '' }} · not counted as downtime @elseif ($site->uptime_last_up_at) Up for {{ $site->uptime_last_up_at->diffForHumans(['parts' => 2, 'short' => true]) }} @else @@ -110,6 +119,8 @@ ↑ Up @elseif ($event->event_type === 'maintenance') 🔧 Maint + @elseif ($event->event_type === 'probe_blocked') + ⊘ Blocked @elseif ($event->is_sla_exempt) ↓ Down (Excused) @else diff --git a/resources/views/dashboard/sites.blade.php b/resources/views/dashboard/sites.blade.php index c67a2b00..3bf828e6 100644 --- a/resources/views/dashboard/sites.blade.php +++ b/resources/views/dashboard/sites.blade.php @@ -223,6 +223,7 @@ class="inline-flex items-center gap-1.5 hover:text-[var(--color-ink-strong)] tra 'up' => ['class' => 'status-green', 'icon' => 'fa-circle-check', 'tooltip' => 'Uptime Online'], 'down' => ['class' => 'status-red', 'icon' => 'fa-circle-exclamation', 'tooltip' => 'Site Down'], 'maintenance' => ['class' => 'status-yellow', 'icon' => 'fa-wrench', 'tooltip' => 'In Maintenance'], + 'probe_blocked' => ['class' => 'status-yellow', 'icon' => 'fa-hand', 'tooltip' => 'Uptime probe blocked (HTTP 429)'], default => ['class' => 'status-unknown', 'icon' => 'fa-circle-question', 'tooltip' => 'Uptime Unknown'], }; @endphp diff --git a/resources/views/monitoring/index.blade.php b/resources/views/monitoring/index.blade.php index 34368212..ffb903db 100644 --- a/resources/views/monitoring/index.blade.php +++ b/resources/views/monitoring/index.blade.php @@ -228,7 +228,7 @@ class="btn-primary text-xs py-1 px-3 cursor-pointer">
{{ $unknown }} unknown @php - $exemptTotal = $currentlyNotOurFault + $currentlyMaintenance + $currentlyIgnored; + $exemptTotal = $currentlyNotOurFault + $currentlyMaintenance + $currentlyIgnored + ($currentlyProbeBlocked ?? 0); @endphp @if ($exemptTotal > 0) · {{ $exemptTotal }} excused / other @@ -276,7 +276,7 @@ class="absolute right-3 top-1/2 -translate-y-1/2 text-[var(--color-ink-soft)] ho
{{-- Quick filter chips for states --}} - @if ($currentlyDown > 0 || $currentlyNotOurFault > 0 || $currentlyMaintenance > 0 || $currentlyIgnored > 0) + @if ($currentlyDown > 0 || $currentlyNotOurFault > 0 || $currentlyMaintenance > 0 || $currentlyIgnored > 0 || ($currentlyProbeBlocked ?? 0) > 0)
Filter: + @endif @if ($currentlyIgnored > 0)
+ {{-- Probe-blocked notice --}} +
+ +
+ + hours +
+ @error('probe_blocked_alert_hours') +

{{ $message }}

+ @enderror +

+ A site that answers the probe with HTTP 429 (Too Many Requests) is rate-limiting the monitor, not down. + It shows as "blocked", never counts as downtime, and never fires a down alert. If it keeps blocking + for this many hours in a row, one chat notice is sent so someone can allowlist the monitor or check + whether real visitors are being throttled too. Default: 6. +

+
+ {{-- Domain ignore list --}}
diff --git a/tests/Feature/Chat/ChatNotifierGatingTest.php b/tests/Feature/Chat/ChatNotifierGatingTest.php index 64c59359..9c7c079a 100644 --- a/tests/Feature/Chat/ChatNotifierGatingTest.php +++ b/tests/Feature/Chat/ChatNotifierGatingTest.php @@ -30,7 +30,7 @@ | an inactive site: | sslStateChanged, contactFormTestFailed, | contactFormTestRecovered, companionUnreachable, companionReachable, -| pluginUpdateFailed +| pluginUpdateFailed, uptimeProbeBlocked (a monitoring gap, not an outage) | | NEVER gated — routed straight through dispatch() even for an inactive | site, because the class's own comment block says these are active-incident @@ -163,6 +163,13 @@ public function siteExitedMaintenance(Site $site, ?int $maintenanceSec): bool return $this->returns; } + public function uptimeProbeBlocked(Site $site, int $blockedSec, ?int $statusCode, ?string $retryAfter = null): bool + { + $this->calls[] = __FUNCTION__; + + return $this->returns; + } + public function pluginUpdateFailed(Site $site, PluginUpdateJob $job): bool { $this->calls[] = __FUNCTION__; @@ -285,6 +292,7 @@ public function emailAuthDegraded(string $domain, array $findings, ?string $prev PluginUpdateJob::factory()->create(['site_id' => $s->id]), 5 )], + 'uptimeProbeBlocked' => [fn (ChatNotifierDispatcher $d, Site $s) => $d->uptimeProbeBlocked($s, 6 * 3600, 429, '120')], ]); /** diff --git a/tests/Feature/Controllers/MonitoringControllerTest.php b/tests/Feature/Controllers/MonitoringControllerTest.php index 535bceec..1705e2cf 100644 --- a/tests/Feature/Controllers/MonitoringControllerTest.php +++ b/tests/Feature/Controllers/MonitoringControllerTest.php @@ -68,6 +68,36 @@ ->assertViewHas('currentlyMaintenance', 1); }); + it('renders a site that rate-limits the probe as blocked, not down, with its event in the feed', function () { + $server = Server::factory()->create(); + $site = Site::factory()->spinupwp()->create([ + 'server_id' => $server->id, + 'domain' => 'ratelimited.example.com', + 'uptime_monitoring_enabled' => true, + 'uptime_state' => Site::UPTIME_STATE_PROBE_BLOCKED, + 'uptime_probe_blocked_since' => now()->subHours(3), + ]); + SiteUptimeEvent::create([ + 'site_id' => $site->id, + 'event_type' => SiteUptimeEvent::TYPE_PROBE_BLOCKED, + 'status_code' => 429, + 'error' => 'HTTP 429 (rate-limited — probe blocked)', + 'event_at' => now()->subHours(3), + ]); + + $response = $this->actingAs(User::factory()->create()) + ->get(route('monitoring.index')); + + $response->assertOk() + ->assertSee('ratelimited.example.com') + ->assertSee('BLOCKED') + ->assertSee('Probe blocked (HTTP 429) for 3h') + ->assertSee('1 probe blocked') + ->assertSee('not counted as downtime') + ->assertViewHas('currentlyProbeBlocked', 1) + ->assertViewHas('currentlyDown', 0); + }); + it('renders separate 7d and 30d fleet uptime headline figures', function () { $server = Server::factory()->create(); $site = Site::factory()->spinupwp()->create([ @@ -173,6 +203,50 @@ $response->assertSessionHasErrors('interval_minutes'); }); + it('shows the probe-blocked notice hours on the settings page, defaulting to 6', function () { + $this->actingAs(User::factory()->create()) + ->get(route('monitoring.settings')) + ->assertOk() + ->assertSee('Notify when a site blocks the probe for') + ->assertSee('name="probe_blocked_alert_hours"', false) + ->assertViewHas('probeBlockedAlertHours', 6); + }); + + it('saves the probe-blocked notice hours', function () { + $this->actingAs(User::factory()->create()) + ->patch(route('monitoring.settings.update'), [ + 'interval_minutes' => 5, + 'failure_threshold' => 2, + 'probe_blocked_alert_hours' => 12, + ]) + ->assertRedirect(route('monitoring.settings')); + + expect((int) app(Settings::class)->get('monitoring.uptime_probe_blocked_alert_hours'))->toBe(12); + }); + + it('keeps the stored probe-blocked hours when the field is not posted', function () { + app(Settings::class)->put('monitoring.uptime_probe_blocked_alert_hours', 9); + + $this->actingAs(User::factory()->create()) + ->patch(route('monitoring.settings.update'), [ + 'interval_minutes' => 5, + 'failure_threshold' => 2, + ]) + ->assertRedirect(route('monitoring.settings')); + + expect((int) app(Settings::class)->get('monitoring.uptime_probe_blocked_alert_hours'))->toBe(9); + }); + + it('rejects out-of-range probe-blocked hours', function () { + $this->actingAs(User::factory()->create()) + ->patch(route('monitoring.settings.update'), [ + 'interval_minutes' => 5, + 'failure_threshold' => 2, + 'probe_blocked_alert_hours' => 0, + ]) + ->assertSessionHasErrors('probe_blocked_alert_hours'); + }); + it('classifies active outage as Not Our Fault without setting standing site SLA policy', function () { $server = Server::factory()->create(); $site = Site::factory()->spinupwp()->create([ diff --git a/tests/Feature/Controllers/SitesControllerTest.php b/tests/Feature/Controllers/SitesControllerTest.php index 9e036f91..b6bc7159 100644 --- a/tests/Feature/Controllers/SitesControllerTest.php +++ b/tests/Feature/Controllers/SitesControllerTest.php @@ -9,6 +9,7 @@ use App\Models\SiteIngestExclusion; use App\Models\SiteUptimeEvent; use App\Models\User; +use App\Services\Chat\ChatNotifier; use App\Services\Companion\CompanionInstaller; use App\Services\DigitalOcean\SpacesClient; use App\Services\Fail2ban\Fail2banClient; @@ -465,6 +466,25 @@ function companionSite(array $overrides = []): Site $response->assertOk()->assertJsonPath('ok', true)->assertJsonPath('status_code', 200); }); + + it('reports a rate-limited probe as blocked rather than down', function () { + [, $site] = spinupSite(['uptime_monitoring_enabled' => true, 'uptime_state' => 'up']); + + $this->mock(UptimeProber::class) + ->shouldReceive('probe') + ->once() + ->andReturn(UptimeProbeResult::blocked(429, 90, 'HTTP 429 (rate-limited — probe blocked)')); + + $this->mock(ChatNotifier::class)->shouldNotReceive('siteWentDown'); + + $response = $this->actingAs(User::factory()->create())->post(route('sites.uptime.recheck', $site)); + + $response->assertOk() + ->assertJsonPath('ok', true) + ->assertJsonPath('status_code', 429) + ->assertJsonPath('state', Site::UPTIME_STATE_PROBE_BLOCKED) + ->assertJsonPath('message', 'Probe blocked (HTTP 429) — not counted as down'); + }); }); describe('refreshWpPlugins', function () { diff --git a/tests/Feature/Uptime/UptimeProbeBlockedTest.php b/tests/Feature/Uptime/UptimeProbeBlockedTest.php new file mode 100644 index 00000000..430888c5 --- /dev/null +++ b/tests/Feature/Uptime/UptimeProbeBlockedTest.php @@ -0,0 +1,531 @@ +mock(ChatNotifier::class); + $chat->shouldNotReceive('siteWentDown'); + $chat->shouldNotReceive('siteWentUp'); + + $sms = test()->mock(SmsNotifier::class); + $sms->shouldNotReceive('siteWentDown'); + $sms->shouldNotReceive('siteWentUp'); + + return $chat; +} + +describe('UptimeProber on HTTP 429', function () { + it('returns a blocked result that is neither a success nor a plain failure, keeping Retry-After', function () { + Http::fake([ + 'https://blocked.example.com/' => Http::response('Too Many Requests', 429, ['Retry-After' => '120']), + ]); + + $result = (new UptimeProber)->probe('https://blocked.example.com/'); + + expect($result->isProbeBlocked)->toBeTrue() + ->and($result->succeeded)->toBeFalse() + ->and($result->isMaintenance)->toBeFalse() + ->and($result->statusCode)->toBe(429) + ->and($result->retryAfter)->toBe('120') + ->and($result->error)->toContain('HTTP 429') + ->and($result->error)->toContain('Retry-After: 120'); + }); + + it('returns a blocked result without Retry-After when the header is absent', function () { + Http::fake([ + 'https://blocked.example.com/' => Http::response('Too Many Requests', 429), + ]); + + $result = (new UptimeProber)->probe('https://blocked.example.com/'); + + expect($result->isProbeBlocked)->toBeTrue() + ->and($result->retryAfter)->toBeNull() + ->and($result->error)->toBe('HTTP 429 (rate-limited — probe blocked)'); + }); + + it('caps an oversized Retry-After header', function () { + Http::fake([ + 'https://blocked.example.com/' => Http::response('Too Many Requests', 429, ['Retry-After' => str_repeat('9', 400)]), + ]); + + $result = (new UptimeProber)->probe('https://blocked.example.com/'); + + expect(mb_strlen((string) $result->retryAfter))->toBe(64) + ->and(mb_strlen((string) $result->error))->toBeLessThan(500); + }); + + it('still treats other 4xx responses as failures, not blocked probes', function () { + Http::fake([ + 'https://blocked.example.com/' => Http::response('Not Found', 404), + ]); + + $result = (new UptimeProber)->probe('https://blocked.example.com/'); + + expect($result->isProbeBlocked)->toBeFalse() + ->and($result->succeeded)->toBeFalse() + ->and($result->statusCode)->toBe(404); + }); +}); + +describe('UptimeStateUpdater with blocked probes', function () { + it('moves an up site to probe_blocked without touching the failure streak or last-up, and records the event', function () { + $lastUp = now()->subMinutes(10)->startOfSecond(); + $site = Site::factory()->create([ + 'uptime_state' => 'up', + 'uptime_consecutive_failures' => 1, + 'uptime_last_up_at' => $lastUp, + 'uptime_down_since' => null, + ]); + + $chat = quietUptimeChannels(); + $chat->shouldNotReceive('uptimeProbeBlocked'); + + app(UptimeStateUpdater::class)->update($site, blockedProbe('120')); + $site->refresh(); + + expect($site->uptime_state)->toBe(Site::UPTIME_STATE_PROBE_BLOCKED) + ->and($site->uptime_consecutive_failures)->toBe(1) + ->and($site->uptime_last_up_at?->timestamp)->toBe($lastUp->timestamp) + ->and($site->uptime_down_since)->toBeNull() + ->and($site->uptime_probe_blocked_since)->not->toBeNull() + ->and($site->uptime_probe_blocked_alerted_at)->toBeNull() + ->and($site->uptime_last_status_code)->toBe(429) + ->and($site->uptime_last_checked_at)->not->toBeNull() + ->and($site->healthColor())->toBe('yellow'); + + $events = SiteUptimeEvent::where('site_id', $site->id)->get(); + expect($events)->toHaveCount(1) + ->and($events->first()->event_type)->toBe(SiteUptimeEvent::TYPE_PROBE_BLOCKED) + ->and($events->first()->status_code)->toBe(429) + ->and($events->first()->is_sla_exempt)->toBeFalse(); + + expect(ActionLog::where('site_id', $site->id)->where('target', Site::UPTIME_STATE_PROBE_BLOCKED)->count())->toBe(1); + }); + + it('never takes a site down however many 429s arrive in a row, and records the stretch once', function () { + $site = Site::factory()->create(['uptime_state' => 'up', 'uptime_consecutive_failures' => 0]); + + $chat = quietUptimeChannels(); + $chat->shouldNotReceive('uptimeProbeBlocked'); + + $updater = app(UptimeStateUpdater::class); + $updater->update($site, blockedProbe()); + $site->refresh(); + $blockedSince = $site->uptime_probe_blocked_since; + + for ($i = 0; $i < 10; $i++) { + $this->travel(10)->minutes(); + $updater->update($site, blockedProbe()); + } + $site->refresh(); + + expect($site->uptime_state)->toBe(Site::UPTIME_STATE_PROBE_BLOCKED) + ->and($site->uptime_consecutive_failures)->toBe(0) + ->and($site->uptime_down_since)->toBeNull() + ->and($site->uptime_probe_blocked_since?->timestamp)->toBe($blockedSince->timestamp) + ->and(SiteUptimeEvent::where('site_id', $site->id)->count())->toBe(1); + }); + + it('does not set last-up for a site first seen blocked', function () { + $site = Site::factory()->create([ + 'uptime_state' => 'unknown', + 'uptime_last_up_at' => null, + ]); + + quietUptimeChannels(); + + app(UptimeStateUpdater::class)->update($site, blockedProbe()); + $site->refresh(); + + expect($site->uptime_state)->toBe(Site::UPTIME_STATE_PROBE_BLOCKED) + ->and($site->uptime_last_up_at)->toBeNull(); + }); + + it('closes an open outage when a down site starts answering 429, without claiming recovery', function () { + $site = Site::factory()->create([ + 'uptime_state' => 'down', + 'uptime_consecutive_failures' => 3, + 'uptime_down_since' => now()->subMinutes(30), + 'uptime_last_up_at' => now()->subMinutes(40), + ]); + $lastUp = $site->uptime_last_up_at; + + quietUptimeChannels(); + + app(UptimeStateUpdater::class)->update($site, blockedProbe()); + $site->refresh(); + + expect($site->uptime_state)->toBe(Site::UPTIME_STATE_PROBE_BLOCKED) + ->and($site->uptime_down_since)->toBeNull() + ->and($site->uptime_consecutive_failures)->toBe(0) + ->and($site->uptime_last_up_at?->timestamp)->toBe($lastUp->timestamp); + + expect(SiteUptimeEvent::where('site_id', $site->id)->value('event_type'))->toBe(SiteUptimeEvent::TYPE_PROBE_BLOCKED); + + $log = ActionLog::where('site_id', $site->id)->where('target', Site::UPTIME_STATE_PROBE_BLOCKED)->first(); + expect($log)->not->toBeNull() + ->and($log->summary)->toContain('outage closed after 30 min') + ->and($log->summary)->toContain('recovery not confirmed'); + }); + + it('moves a site in maintenance to probe_blocked and clears the maintenance timestamp', function () { + $site = Site::factory()->create([ + 'uptime_state' => 'maintenance', + 'uptime_maintenance_since' => now()->subMinutes(20), + ]); + + $chat = quietUptimeChannels(); + $chat->shouldNotReceive('siteExitedMaintenance'); + + app(UptimeStateUpdater::class)->update($site, blockedProbe()); + $site->refresh(); + + expect($site->uptime_state)->toBe(Site::UPTIME_STATE_PROBE_BLOCKED) + ->and($site->uptime_maintenance_since)->toBeNull(); + }); + + it('returns a blocked site to up on the next success, closing the stretch quietly', function () { + $site = Site::factory()->create([ + 'uptime_state' => Site::UPTIME_STATE_PROBE_BLOCKED, + 'uptime_probe_blocked_since' => now()->subHours(7), + 'uptime_probe_blocked_alerted_at' => now()->subHour(), + 'uptime_last_up_at' => null, + ]); + + $chat = quietUptimeChannels(); + $chat->shouldNotReceive('uptimeProbeBlocked'); + + app(UptimeStateUpdater::class)->update($site, UptimeProbeResult::success(200, 80)); + $site->refresh(); + + expect($site->uptime_state)->toBe('up') + ->and($site->uptime_last_up_at)->not->toBeNull() + ->and($site->uptime_probe_blocked_since)->toBeNull() + ->and($site->uptime_probe_blocked_alerted_at)->toBeNull(); + + expect(SiteUptimeEvent::where('site_id', $site->id)->value('event_type'))->toBe(SiteUptimeEvent::TYPE_UP); + }); + + it('keeps the blocked stretch open through a single failure below the threshold', function () { + $since = now()->subHours(2)->startOfSecond(); + $site = Site::factory()->create([ + 'uptime_state' => Site::UPTIME_STATE_PROBE_BLOCKED, + 'uptime_consecutive_failures' => 0, + 'uptime_probe_blocked_since' => $since, + ]); + + quietUptimeChannels(); + + app(UptimeStateUpdater::class)->update($site, UptimeProbeResult::transportFailed('Connection timed out')); + $site->refresh(); + + expect($site->uptime_state)->toBe(Site::UPTIME_STATE_PROBE_BLOCKED) + ->and($site->uptime_consecutive_failures)->toBe(1) + ->and($site->uptime_probe_blocked_since?->timestamp)->toBe($since->timestamp) + ->and(SiteUptimeEvent::where('site_id', $site->id)->count())->toBe(0); + }); + + it('still takes a blocked site down on a real outage, ending the blocked stretch', function () { + $site = Site::factory()->create([ + 'uptime_state' => Site::UPTIME_STATE_PROBE_BLOCKED, + 'uptime_consecutive_failures' => 0, + 'uptime_probe_blocked_since' => now()->subHours(2), + ]); + + $chat = $this->mock(ChatNotifier::class); + $chat->shouldReceive('siteWentDown') + ->once() + ->withArgs(fn (Site $s, ?int $code) => $s->is($site) && $code === 502) + ->andReturn(true); + $chat->shouldNotReceive('uptimeProbeBlocked'); + + $sms = $this->mock(SmsNotifier::class); + $sms->shouldReceive('siteWentDown')->once()->andReturn(false); + + $updater = app(UptimeStateUpdater::class); + $updater->update($site, UptimeProbeResult::badStatus(502, 90, 'HTTP 502')); + $updater->update($site, UptimeProbeResult::badStatus(502, 90, 'HTTP 502')); + $site->refresh(); + + expect($site->uptime_state)->toBe('down') + ->and($site->uptime_down_since)->not->toBeNull() + ->and($site->uptime_probe_blocked_since)->toBeNull() + ->and(SiteUptimeEvent::where('site_id', $site->id)->value('event_type'))->toBe(SiteUptimeEvent::TYPE_DOWN); + }); +}); + +describe('probe-blocked ops notice', function () { + it('sends one notice once the site has blocked the probe for the default 6 hours', function () { + $site = Site::factory()->create(['uptime_state' => 'up']); + + $chat = quietUptimeChannels(); + $chat->shouldReceive('uptimeProbeBlocked') + ->once() + ->withArgs(fn (Site $s, int $sec, ?int $code, ?string $retryAfter) => $s->is($site) + && $sec >= 6 * 3600 + && $code === 429 + && $retryAfter === '120') + ->andReturn(true); + + $updater = app(UptimeStateUpdater::class); + + // Ten-minute cadence: nothing until the stretch reaches six hours... + $updater->update($site, blockedProbe('120')); + for ($i = 0; $i < 35; $i++) { + $this->travel(10)->minutes(); + $updater->update($site, blockedProbe('120')); + } + expect($site->refresh()->uptime_probe_blocked_alerted_at)->toBeNull(); + + // ...the probe at the six-hour mark sends it... + $this->travel(10)->minutes(); + $updater->update($site, blockedProbe('120')); + expect($site->refresh()->uptime_probe_blocked_alerted_at)->not->toBeNull(); + + // ...and that stays the only notice, however long it keeps blocking. + for ($i = 0; $i < 12; $i++) { + $this->travel(10)->minutes(); + $updater->update($site, blockedProbe('120')); + } + + expect(ActionLog::where('site_id', $site->id)->where('summary', 'like', '%ops notified%')->count())->toBe(1); + }); + + it('uses the hours configured on Monitoring settings', function () { + app(Settings::class)->put(UptimeStateUpdater::SETTING_PROBE_BLOCKED_ALERT_HOURS, 2); + + $site = Site::factory()->create([ + 'uptime_state' => Site::UPTIME_STATE_PROBE_BLOCKED, + 'uptime_probe_blocked_since' => now()->subHours(2)->subMinute(), + ]); + + $chat = quietUptimeChannels(); + $chat->shouldReceive('uptimeProbeBlocked')->once()->andReturn(true); + + app(UptimeStateUpdater::class)->update($site, blockedProbe()); + }); + + it('stays quiet below the configured hours', function () { + $site = Site::factory()->create([ + 'uptime_state' => Site::UPTIME_STATE_PROBE_BLOCKED, + 'uptime_probe_blocked_since' => now()->subHours(5)->subMinutes(50), + ]); + + $chat = quietUptimeChannels(); + $chat->shouldNotReceive('uptimeProbeBlocked'); + + app(UptimeStateUpdater::class)->update($site, blockedProbe()); + + expect($site->refresh()->uptime_probe_blocked_alerted_at)->toBeNull(); + }); + + it('respects the per-site ignore flag', function () { + $site = Site::factory()->create([ + 'uptime_state' => Site::UPTIME_STATE_PROBE_BLOCKED, + 'uptime_probe_blocked_since' => now()->subHours(8), + 'uptime_ignored_at' => now()->subDay(), + ]); + + $chat = quietUptimeChannels(); + $chat->shouldNotReceive('uptimeProbeBlocked'); + + app(UptimeStateUpdater::class)->update($site, blockedProbe()); + + expect($site->refresh()->uptime_probe_blocked_alerted_at)->toBeNull(); + }); + + it('can notify again for a new blocked stretch after the site got through', function () { + $site = Site::factory()->create([ + 'uptime_state' => Site::UPTIME_STATE_PROBE_BLOCKED, + 'uptime_probe_blocked_since' => now()->subHours(8), + 'uptime_probe_blocked_alerted_at' => now()->subHours(2), + ]); + + $chat = quietUptimeChannels(); + $chat->shouldReceive('uptimeProbeBlocked')->once()->andReturn(true); + + $updater = app(UptimeStateUpdater::class); + + // Already notified for this stretch: silent. + $updater->update($site, blockedProbe()); + + // A success ends it; a fresh six-hour stretch notifies once more. + $updater->update($site, UptimeProbeResult::success(200, 70)); + $updater->update($site, blockedProbe()); + $this->travel(6)->hours(); + $updater->update($site, blockedProbe()); + }); + + it('is a gated chat event that names the site and the duration in generic terms', function () { + config([ + 'clockwork.mattermost.enabled' => true, + 'clockwork.mattermost.webhook_url' => 'https://chat.example.com/hooks/test', + ]); + Http::fake(['chat.example.com/*' => Http::response('ok', 200)]); + + $site = Site::factory()->create(['domain' => 'blocked.example.com']); + + expect(ChatNotifier::EVENTS)->toHaveKey('uptime_probe_blocked'); + + $sent = app(MattermostNotifier::class)->uptimeProbeBlocked($site, 6 * 3600 + 600, 429, '120'); + + expect($sent)->toBeTrue(); + Http::assertSent(function ($request) { + $title = (string) $request['text']; + $body = (string) ($request['attachments'][0]['text'] ?? ''); + + return str_contains($title, 'blocked.example.com') + && str_contains($title, 'HTTP 429') + && str_contains($title, '6h 10m') + && str_contains($body, 'not counted as downtime'); + }); + }); + + it('is not sent when the event is switched off for the channel', function () { + config([ + 'clockwork.mattermost.enabled' => true, + 'clockwork.mattermost.webhook_url' => 'https://chat.example.com/hooks/test', + ]); + Http::fake(); + app(Settings::class)->put(MattermostNotifier::SETTINGS_KEY, ['uptime_probe_blocked' => false]); + + $site = Site::factory()->create(); + + expect(app(MattermostNotifier::class)->uptimeProbeBlocked($site, 7 * 3600, 429))->toBeFalse(); + Http::assertNothingSent(); + }); + + it('never reaches the client-facing Slack channel', function () { + $site = Site::factory()->create(); + + expect((new ClientSlackNotifier)->uptimeProbeBlocked($site, 7 * 3600, 429))->toBeFalse(); + }); +}); + +describe('uptime stats with blocked stretches', function () { + it('never counts a blocked stretch as downtime', function () { + $site = Site::factory()->create(['uptime_state' => Site::UPTIME_STATE_PROBE_BLOCKED]); + + SiteUptimeEvent::create([ + 'site_id' => $site->id, + 'event_type' => SiteUptimeEvent::TYPE_PROBE_BLOCKED, + 'status_code' => 429, + 'error' => 'HTTP 429 (rate-limited — probe blocked)', + 'event_at' => now()->subHours(20), + ]); + + $pct = app(UptimeStatsCalculator::class)->siteUptime($site, now()->subDay(), now()); + + expect($pct)->toBe(100.0); + }); + + it('stops counting an outage once the site answers 429', function () { + $site = Site::factory()->create([ + 'uptime_state' => 'down', + 'uptime_consecutive_failures' => 2, + 'uptime_down_since' => now()->subHours(12), + ]); + SiteUptimeEvent::create([ + 'site_id' => $site->id, + 'event_type' => SiteUptimeEvent::TYPE_DOWN, + 'status_code' => 502, + 'event_at' => now()->subHours(12), + ]); + + quietUptimeChannels(); + + // Down for 12h, then blocked for the following 12h. + app(UptimeStateUpdater::class)->update($site, blockedProbe()); + $this->travel(12)->hours(); + + $pct = app(UptimeStatsCalculator::class)->siteUptime($site, now()->subDay(), now()); + + expect($pct)->toBe(50.0); + }); +}); + +describe('clockwork:check-site-uptime with a rate-limited site', function () { + it('reports the site as blocked rather than down', function () { + Http::fake(['*' => Http::response('Too Many Requests', 429)]); + + $site = Site::factory()->create([ + 'domain' => 'blocked.example.com', + 'uptime_monitoring_enabled' => true, + 'uptime_state' => 'up', + ]); + + quietUptimeChannels(); + + $this->artisan('clockwork:check-site-uptime', ['--site' => $site->id]) + ->expectsOutputToContain('up=0 down=0 blocked=1') + ->assertExitCode(0); + + expect($site->refresh()->uptime_state)->toBe(Site::UPTIME_STATE_PROBE_BLOCKED); + }); +}); + +describe('site dashboard with a blocked probe', function () { + it('labels the uptime widget and its event list as blocked, not down', function () { + $this->mockIssueCounterZero(); + + $site = Site::factory()->create([ + 'domain' => 'blocked.example.com', + 'uptime_monitoring_enabled' => true, + 'uptime_state' => Site::UPTIME_STATE_PROBE_BLOCKED, + 'uptime_probe_blocked_since' => now()->subHours(2), + 'uptime_last_status_code' => 429, + ]); + SiteUptimeEvent::create([ + 'site_id' => $site->id, + 'event_type' => SiteUptimeEvent::TYPE_PROBE_BLOCKED, + 'status_code' => 429, + 'error' => 'HTTP 429 (rate-limited — probe blocked)', + 'event_at' => now()->subHours(2), + ]); + + $this->actingAs(User::factory()->create()) + ->get(route('sites.show', $site)) + ->assertOk() + ->assertSee('Probe blocked') + ->assertSee('⊘ Blocked') + ->assertSee('not counted as downtime') + ->assertDontSee('↓ Down'); + }); +}); From 31d737d4ceedb847185839e23e07113daebdb262 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 3 Nov 2026 20:09:38 +0000 Subject: [PATCH 05/64] fix(updates): retry auto-paused plugins, prioritise CVEs, run nightly once per local date Vulnerable-first ordering never matched. The nightly runner looked up each Companion slug ("dir/file.php") in a map keyed by the vulnerability matcher's directory-only slug, so is_vulnerable was false for every plugin in a folder. It now compares the directory part, and it keeps the highest patched_in per directory for the pause logic below. Auto-failure pauses never expired. Once a plugin failed N nightly runs it was dropped before any vulnerability check, so later releases, including security fixes, were never attempted. filterIgnored still honours manual ignores, but lets an auto_failure pause through when: - the offered version differs from the streak's last_target_version (one attempt per new release, because a failed attempt records it as the new last target), or - the installed version is vulnerable and the offered version is at or above the highest patched_in of the matching advisories. A successful update that reaches its target version now deletes the slug's auto_failure ignore, clears the streak's pause markers, and re-pushes the exceptions list so the wp-admin "paused" notice clears. The chained 01:30 America/New_York nightly fired twice on the DST fall-back night, so failures were counted twice in one day. The command now skips (with a log line) when a fleet-wide nightly batch was already queued during the current local date, judged by created_at so 23/25-hour days are handled. --force overrides. --site runs are not guarded and get a "nightly--site-" batch id, so they never block the fleet run. Batch ids now carry the local date (the same as before for the scheduled run). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Q7rExNMgxzTNb3pagVUZrg --- .../Commands/RunNightlyPluginUpdates.php | 265 +++++++++++-- app/Models/PluginUpdateIgnore.php | 8 +- .../Updates/UpdateFailureStreakRecorder.php | 66 +++- resources/docs/features/updates.md | 10 +- resources/docs/reference/artisan-commands.md | 2 +- resources/docs/reference/scheduled-jobs.md | 2 +- routes/console.php | 10 +- .../Console/RunNightlyPluginUpdatesTest.php | 373 +++++++++++++++++- .../Updates/UpdateFailureStreakTest.php | 182 ++++++++- 9 files changed, 870 insertions(+), 48 deletions(-) diff --git a/app/Console/Commands/RunNightlyPluginUpdates.php b/app/Console/Commands/RunNightlyPluginUpdates.php index 781872b9..729708f1 100644 --- a/app/Console/Commands/RunNightlyPluginUpdates.php +++ b/app/Console/Commands/RunNightlyPluginUpdates.php @@ -4,6 +4,7 @@ use App\Jobs\RunPluginUpdate; use App\Jobs\RunThemeUpdate; +use App\Models\PluginUpdateFailureStreak; use App\Models\PluginUpdateIgnore; use App\Models\PluginUpdateJob; use App\Models\Site; @@ -11,21 +12,70 @@ use Illuminate\Console\Attributes\Description; use Illuminate\Console\Attributes\Signature; use Illuminate\Console\Command; +use Illuminate\Contracts\Database\Query\Builder as QueryBuilder; use Illuminate\Database\Eloquent\Collection; use Illuminate\Support\Carbon; use Illuminate\Support\Facades\DB; use Illuminate\Support\Facades\Log; use Illuminate\Support\Str; -#[Signature('clockwork:run-nightly-plugin-updates {--dry-run : print candidates only, no DB writes / dispatch} {--site= : limit to one site (id or domain)}')] +#[Signature('clockwork:run-nightly-plugin-updates {--dry-run : print candidates only, no DB writes / dispatch} {--site= : limit to one site (id or domain)} {--force : queue even if a fleet-wide nightly batch already ran for today\'s local date}')] #[Description('Queue plugin AND theme updates for every care-plan site that has either pending. Fired nightly at 02:00 ET; workers chew through the queue during the 4-hour window. Name predates theme support (added 2026-08-31) — kept as-is to avoid renaming a live scheduled job.')] class RunNightlyPluginUpdates extends Command { + /** + * Timezone the nightly is scheduled in (routes/console.php). "Today" + * for the once-per-night guard and the date in the batch id both use + * this zone, so a night maps to exactly one local calendar date. + */ + public const LOCAL_TIMEZONE = 'America/New_York'; + + /** + * Every nightly batch id starts with this. UpdateFailureStreakRecorder, + * AbstractRunUpdate's chat ping, the stale-job reaper and the summary + * mail all key off the prefix, so it must not change. + */ + public const BATCH_PREFIX = 'nightly-'; + + /** + * Marks a batch queued by an operator's --site run, so it never counts + * as (or blocks) the fleet-wide nightly for that date. + */ + public const SITE_BATCH_MARKER = 'site-'; + + public const BYPASS_NEW_VERSION = 'new_version'; + + public const BYPASS_SECURITY_FIX = 'security_fix'; + public function handle(PluginVulnerabilityMatcher $vulnMatcher): int { $isDryRun = (bool) $this->option('dry-run'); $siteFilter = $this->option('site'); + // Once per local night. The chained schedule fires at 01:30 + // America/New_York, and on the DST fall-back day 01:30 happens + // twice, so the whole refresh + update chain ran twice and every + // failure was counted twice toward the auto-pause threshold. + // Operator --site runs are explicit and are not guarded; --force + // overrides the guard for a deliberate fleet-wide re-run. + if (! $siteFilter && ! $this->option('force')) { + $existingBatch = $this->fleetBatchForLocalDate(); + if ($existingBatch !== null) { + $localDate = Carbon::now(self::LOCAL_TIMEZONE)->format('Y-m-d'); + if ($isDryRun) { + $this->warn("Nightly batch {$existingBatch} already ran for {$localDate}; a real run would be skipped (use --force to override)."); + } else { + $this->info("Nightly batch {$existingBatch} already ran for {$localDate}. Skipping (use --force to override)."); + Log::info('plugin_update.nightly_already_ran', [ + 'local_date' => $localDate, + 'batch_id' => $existingBatch, + ]); + + return self::SUCCESS; + } + } + } + $candidates = $this->loadCandidateSites($siteFilter); if ($candidates->isEmpty()) { @@ -67,6 +117,10 @@ public function handle(PluginVulnerabilityMatcher $vulnMatcher): int if ($slug === '') { continue; } + // The snapshot slug is "directory/file.php"; the matcher keys + // on the directory only, so compare on that. + $dirSlug = self::directorySlug($slug); + $isVulnerable = array_key_exists($dirSlug, $vulnSlugs); $tuples[] = [ 'site' => $site, 'kind' => PluginUpdateJob::KIND_PLUGIN, @@ -74,7 +128,9 @@ public function handle(PluginVulnerabilityMatcher $vulnMatcher): int 'name' => $entry['name'] ?? null, 'before_version' => $entry['version'] ?? null, 'target_version' => $entry['new_version'] ?? null, - 'is_vulnerable' => isset($vulnSlugs[$slug]), + 'is_vulnerable' => $isVulnerable, + 'patched_in' => $isVulnerable ? $vulnSlugs[$dirSlug] : null, + 'pause_bypass' => null, ]; } @@ -91,6 +147,8 @@ public function handle(PluginVulnerabilityMatcher $vulnMatcher): int 'before_version' => $entry['version'] ?? null, 'target_version' => $entry['new_version'] ?? null, 'is_vulnerable' => false, + 'patched_in' => null, + 'pause_bypass' => null, ]; } } @@ -101,7 +159,8 @@ public function handle(PluginVulnerabilityMatcher $vulnMatcher): int return self::SUCCESS; } - // Drop tuples in PluginUpdateIgnore. + // Drop tuples in PluginUpdateIgnore (auto-failure pauses can be let + // through for a new release or a security fix; see filterIgnored). $tuples = $this->filterIgnored($tuples); // Drop tuples that already have a live PluginUpdateJob for this kind/slug. @@ -128,6 +187,7 @@ public function handle(PluginVulnerabilityMatcher $vulnMatcher): int $vulnerableCount = collect($tuples)->where('is_vulnerable', true)->count(); $sitesTouched = collect($tuples)->pluck('site.id')->unique()->count(); + $retried = array_values(array_filter($tuples, fn ($t) => $t['pause_bypass'] !== null)); $this->info(sprintf( 'Queue: %d tuples (%d vulnerable-first) across %d site%s.', @@ -137,18 +197,29 @@ public function handle(PluginVulnerabilityMatcher $vulnMatcher): int $sitesTouched === 1 ? '' : 's', )); + if ($retried !== []) { + $this->info(sprintf( + 'Retrying %d auto-paused update%s (%d new release, %d security fix).', + count($retried), + count($retried) === 1 ? '' : 's', + count(array_filter($retried, fn ($t) => $t['pause_bypass'] === self::BYPASS_NEW_VERSION)), + count(array_filter($retried, fn ($t) => $t['pause_bypass'] === self::BYPASS_SECURITY_FIX)), + )); + } + if ($isDryRun) { // Print the first ~25 tuples so the operator can sanity-check // ordering without flooding stdout on a 200-row night. foreach (array_slice($tuples, 0, 25) as $t) { $this->line(sprintf( - ' %s %s [%s] %s (%s → %s)', + ' %s %s [%s] %s (%s → %s)%s', $t['is_vulnerable'] ? '!CVE' : ' ', str_pad((string) $t['site']->domain, 36), $t['kind'], $t['slug'], $t['before_version'] ?: '?', $t['target_version'] ?: '?', + $t['pause_bypass'] !== null ? " [auto-paused, retry: {$t['pause_bypass']}]" : '', )); } if (count($tuples) > 25) { @@ -160,11 +231,26 @@ public function handle(PluginVulnerabilityMatcher $vulnMatcher): int // Real run. One batch_id for the whole night so the summary command // can find them with a LIKE 'nightly-%' query and the dashboard can - // group them as a single batch. - $batchId = 'nightly-'.Carbon::now()->format('Y-m-d').'-'.Str::random(8); + // group them as a single batch. The date is the local night's date + // (same zone as the once-per-night guard above). + $batchId = self::BATCH_PREFIX + .Carbon::now(self::LOCAL_TIMEZONE)->format('Y-m-d') + .'-'.($siteFilter ? self::SITE_BATCH_MARKER : '') + .Str::random(8); $accepted = 0; $failed = 0; + foreach ($retried as $t) { + Log::info('plugin_update.nightly_retrying_auto_paused', [ + 'site_id' => $t['site']->id, + 'kind' => $t['kind'], + 'slug' => $t['slug'], + 'target_version' => $t['target_version'], + 'reason' => $t['pause_bypass'], + 'batch_id' => $batchId, + ]); + } + foreach ($tuples as $t) { try { DB::transaction(function () use ($t, $batchId) { @@ -302,7 +388,12 @@ private function pendingThemes(Site $site): array } /** - * @return array slug => true map (for fast lookup) + * Plugins on this site whose installed version has a known + * vulnerability, keyed by directory slug (the matcher's plugin_slug). + * The value is the highest patched_in among those advisories that + * have a fix newer than the installed version, or null when none do. + * + * @return array directory slug => highest patched_in */ private function vulnerableSlugsForSite(Site $site, PluginVulnerabilityMatcher $matcher): array { @@ -310,9 +401,19 @@ private function vulnerableSlugsForSite(Site $site, PluginVulnerabilityMatcher $ try { foreach ($matcher->forSite($site) as $row) { $slug = (string) ($row['plugin_slug'] ?? ''); - if ($slug !== '') { - $hits[$slug] = true; + if ($slug === '') { + continue; } + + $highest = $hits[$slug] ?? null; + $patchedIn = ($row['patch_available'] ?? false) + ? trim((string) ($row['vulnerability']->patched_in ?? '')) + : ''; + if ($patchedIn !== '' && ($highest === null || version_compare($patchedIn, $highest, '>'))) { + $highest = $patchedIn; + } + + $hits[$slug] = $highest; } } catch (\Throwable $e) { // Vuln matcher failure is not fatal for the auto-update loop — @@ -351,7 +452,16 @@ private function stampEligibleSites(?string $siteFilter): void } /** - * Drop tuples whose (site, plugin) is in PluginUpdateIgnore. + * Drop tuples whose (site, kind, slug) is in PluginUpdateIgnore. + * + * Manual ignores always win. An auto-failure pause (the slug failed the + * nightly loop N times in a row) is let through, and the tuple tagged + * with why, when either: + * - the offered version differs from the version that kept failing + * (a new release since the pause; one attempt per new version, + * because a failed attempt records it as the new last target), or + * - the installed version is vulnerable and the offered version is + * at or above the highest patched_in, so the update is the fix. * * @param list> $tuples * @return list> @@ -366,22 +476,127 @@ private function filterIgnored(array $tuples): array // in principle share a slug string, and an ignore entry for one // kind must never suppress the other. $triples = collect($tuples)->map(fn ($t) => [$t['site']->id, $t['kind'], $t['slug']])->all(); - $ignored = PluginUpdateIgnore::query() - ->where(function ($q) use ($triples) { - foreach ($triples as [$siteId, $kind, $slug]) { - $q->orWhere(function ($qq) use ($siteId, $kind, $slug) { - $qq->where('site_id', $siteId)->where('target_kind', $kind)->where('target_slug', $slug); - }); - } - }) - ->get(['site_id', 'target_kind', 'target_slug']) - ->mapWithKeys(fn ($row) => ["{$row->target_kind}:{$row->site_id}:{$row->target_slug}" => true]) - ->all(); + $ignores = PluginUpdateIgnore::query() + ->where(fn ($q) => $this->whereAnyTriple($q, $triples)) + ->get(['site_id', 'target_kind', 'target_slug', 'source']) + ->keyBy(fn ($row) => "{$row->target_kind}:{$row->site_id}:{$row->target_slug}"); - return array_values(array_filter( - $tuples, - fn ($t) => ! isset($ignored["{$t['kind']}:{$t['site']->id}:{$t['slug']}"]), - )); + if ($ignores->isEmpty()) { + return $tuples; + } + + $autoPaused = $ignores->filter(fn (PluginUpdateIgnore $row) => $row->isAutoFailure()); + $streaks = $autoPaused->isEmpty() + ? collect() + : PluginUpdateFailureStreak::query() + ->where(fn ($q) => $this->whereAnyTriple( + $q, + $autoPaused->map(fn ($row) => [$row->site_id, $row->target_kind, $row->target_slug])->values()->all(), + )) + ->get(['site_id', 'target_kind', 'target_slug', 'last_target_version']) + ->keyBy(fn ($row) => "{$row->target_kind}:{$row->site_id}:{$row->target_slug}"); + + $kept = []; + foreach ($tuples as $t) { + $key = "{$t['kind']}:{$t['site']->id}:{$t['slug']}"; + $ignore = $ignores->get($key); + if ($ignore === null) { + $kept[] = $t; + + continue; + } + + if (! $ignore->isAutoFailure()) { + continue; + } + + $bypass = $this->autoPauseBypassReason($t, $streaks->get($key)); + if ($bypass !== null) { + $t['pause_bypass'] = $bypass; + $kept[] = $t; + } + } + + return $kept; + } + + /** + * Why an auto-failure pause should not hold this tuple back tonight, + * or null to keep it paused. + * + * @param array $tuple + */ + private function autoPauseBypassReason(array $tuple, ?PluginUpdateFailureStreak $streak): ?string + { + $offered = trim((string) ($tuple['target_version'] ?? '')); + if ($offered === '') { + return null; + } + + $patchedIn = (string) ($tuple['patched_in'] ?? ''); + if ($tuple['is_vulnerable'] && $patchedIn !== '' && version_compare($offered, $patchedIn, '>=')) { + return self::BYPASS_SECURITY_FIX; + } + + // Without a recorded failing version there is nothing to compare + // against, so the pause stands. + $failedVersion = trim((string) ($streak?->last_target_version ?? '')); + if ($failedVersion !== '' && $offered !== $failedVersion) { + return self::BYPASS_NEW_VERSION; + } + + return null; + } + + /** + * The first fleet-wide nightly batch already queued during the current + * local (LOCAL_TIMEZONE) calendar day, or null. Looks at created_at + * rather than the date in the batch id, so it is right on 23- and + * 25-hour DST days and for batches named before the id used the local + * date. --site batches are excluded so an operator's single-site run + * never stops the fleet's nightly. + */ + private function fleetBatchForLocalDate(): ?string + { + $appTimezone = (string) config('app.timezone'); + $localStart = Carbon::now(self::LOCAL_TIMEZONE)->startOfDay(); + $localEnd = $localStart->copy()->addDay(); + + $batchId = PluginUpdateJob::query() + ->where('batch_id', 'like', self::BATCH_PREFIX.'%') + ->where('batch_id', 'not like', self::BATCH_PREFIX.'%-'.self::SITE_BATCH_MARKER.'%') + ->where('created_at', '>=', $localStart->copy()->setTimezone($appTimezone)) + ->where('created_at', '<', $localEnd->copy()->setTimezone($appTimezone)) + ->orderBy('id') + ->value('batch_id'); + + return is_string($batchId) ? $batchId : null; + } + + /** + * Directory part of a Companion plugin slug ("dir/file.php" → "dir"); + * a single-file plugin ("hello.php") is returned as-is. Same rule as + * PluginVulnerabilityMatcher. + */ + private static function directorySlug(string $slug): string + { + $dir = strtok($slug, '/'); + + return $dir === false ? $slug : $dir; + } + + /** + * OR together one (site_id, target_kind, target_slug) match per triple. + * + * @param list $triples + */ + private function whereAnyTriple(QueryBuilder $query, array $triples): void + { + foreach ($triples as [$siteId, $kind, $slug]) { + $query->orWhere(function ($qq) use ($siteId, $kind, $slug) { + $qq->where('site_id', $siteId)->where('target_kind', $kind)->where('target_slug', $slug); + }); + } } /** diff --git a/app/Models/PluginUpdateIgnore.php b/app/Models/PluginUpdateIgnore.php index 631aa6ef..226c7024 100644 --- a/app/Models/PluginUpdateIgnore.php +++ b/app/Models/PluginUpdateIgnore.php @@ -10,9 +10,11 @@ * "This (site, target) pair should never appear as a pending update." * Exact analogue of ManageWP Orion's "Ignored" tab. * - * No expiry semantics — a future migration can add `expires_at` if "ignore - * this version only" becomes a use case. Today: ignore is forever until - * removed. + * No expiry column. A manual ignore holds until an operator removes it. An + * auto_failure pause is softer: the nightly loop still tries a newly + * released version or a version that fixes a known vulnerability + * (RunNightlyPluginUpdates::filterIgnored), and a successful update to the + * target version deletes the row (UpdateFailureStreakRecorder::record). */ class PluginUpdateIgnore extends Model { diff --git a/app/Services/Updates/UpdateFailureStreakRecorder.php b/app/Services/Updates/UpdateFailureStreakRecorder.php index c66c2512..0b82afc9 100644 --- a/app/Services/Updates/UpdateFailureStreakRecorder.php +++ b/app/Services/Updates/UpdateFailureStreakRecorder.php @@ -24,7 +24,11 @@ * client_visible=true). * * Other plugins on the site continue their regular care-plan updates. - * Any successful update (nightly or manual) resets the streak. + * Any successful update (nightly or manual) resets the streak. A successful + * update that reaches its target version also lifts an auto-failure pause + * for that slug and re-pushes the exceptions list, so the wp-admin notice + * clears. (The nightly loop retries auto-paused slugs when a new release or + * a security fix is offered; see RunNightlyPluginUpdates::filterIgnored.) */ class UpdateFailureStreakRecorder { @@ -49,6 +53,10 @@ public function record(PluginUpdateJob $job, ?array $result = null, ?Throwable $ if ($job->status === PluginUpdateJob::STATUS_COMPLETE) { $this->resetStreak($job->site_id, $job->target_kind, $job->target_slug); + if ($this->reachedTargetVersion($job)) { + $this->liftAutoPause($job); + } + return; } @@ -102,6 +110,62 @@ public function resetStreak(int $siteId, string $targetKind, string $targetSlug) ->update(['consecutive_failures' => 0]); } + /** + * Did the job land on (or past) the version it was queued for? Needs + * both versions: a completed job without them proves nothing about + * whether the version that kept failing is now installed. + */ + private function reachedTargetVersion(PluginUpdateJob $job): bool + { + $target = trim((string) ($job->target_version ?? '')); + $after = trim((string) ($job->after_version ?? '')); + + return $target !== '' && $after !== '' && version_compare($after, $target, '>='); + } + + /** + * Remove this slug's auto-failure pause after a successful update: + * delete the ignore row (manual ignores are left alone), clear the + * streak's pause markers, and push the shorter exceptions list so the + * client's wp-admin stops showing the plugin as paused. + */ + private function liftAutoPause(PluginUpdateJob $job): void + { + $deleted = PluginUpdateIgnore::query() + ->where('site_id', $job->site_id) + ->where('target_kind', $job->target_kind) + ->where('target_slug', $job->target_slug) + ->where('source', PluginUpdateIgnore::SOURCE_AUTO_FAILURE) + ->delete(); + + if ($deleted === 0) { + return; + } + + PluginUpdateFailureStreak::query() + ->where('site_id', $job->site_id) + ->where('target_kind', $job->target_kind) + ->where('target_slug', $job->target_slug) + ->update([ + 'consecutive_failures' => 0, + 'ignored_at' => null, + 'ignore_id' => null, + ]); + + Log::info('updates.auto_pause_lifted', [ + 'site_id' => $job->site_id, + 'kind' => $job->target_kind, + 'slug' => $job->target_slug, + 'version' => $job->after_version, + 'job_id' => $job->id, + ]); + + $site = $job->site; + if ($site instanceof Site) { + $this->maybePushExceptions($site); + } + } + private function incrementStreak(PluginUpdateJob $job): void { $streak = PluginUpdateFailureStreak::firstOrNew([ diff --git a/resources/docs/features/updates.md b/resources/docs/features/updates.md index e60b170c..10c85dc8 100644 --- a/resources/docs/features/updates.md +++ b/resources/docs/features/updates.md @@ -2,7 +2,7 @@ title: Updates section: Features order: 35 -updated: 2026-09-28 +updated: 2026-11-03 author: Aaron Reimann tags: [updates, plugins, themes, wp-core, translations, care-plan, pressable] tracks: [app/Services/Updates/UpdateGrouping.php, app/Services/Updates/UpdateFailureStreakRecorder.php, app/Http/Controllers/UpdatesController.php, app/Http/Controllers/MaintenanceHistoryController.php, app/Http/Controllers/SitesController.php, app/Jobs/**, app/Models/PluginUpdateJob.php, app/Models/PluginUpdateIgnore.php, app/Models/PluginUpdateFailureStreak.php, app/Console/Commands/RunNightlyPluginUpdates.php, app/Console/Commands/PushUpdateExceptions.php, app/Console/Commands/NightlyUpdateSummary.php, app/Console/Commands/RefreshCompanionSnapshot.php, app/Console/Commands/DetectStuckCompanionState.php, app/Console/Commands/ReapStaleUpdateJobs.php, app/Services/Companion/ClockworkCompanionClient.php, app/Services/Companion/CompanionInstaller.php, modules/Pressable/src/PressableCompanionInstaller.php, database/migrations/**add_state_to_plugin_update_jobs*, database/migrations/**plugin_update_failure_streaks*, database/migrations/**add_failure_tracking_to_plugin_update_ignores_table*] @@ -110,8 +110,8 @@ When a stubborn plugin or theme fails the nightly update queue night after night - **Kill switch**: `updates.auto_ignore_enabled` (default `true`). When set to `false`, consecutive failure streaks are still tracked for operator visibility, but auto-ignore rows are never created. #### Reset and Resume -- Any **successful update** (nightly or manual) resets `consecutive_failures` back to 0. -- When an auto-ignored plugin ships a new version, it remains paused until resumed (badged as "New version available — still paused") to prevent endless retries on broken plugins. +- Any **successful update** (nightly or manual) resets `consecutive_failures` back to 0. When it also lands on its target version, an auto-ignore for that slug is removed, the streak's pause markers are cleared, and the shorter exceptions list is pushed so the wp-admin notice clears. Manual ignores are never removed this way. +- An auto-ignore does not block forever. The nightly loop still tries an auto-ignored plugin or theme when the snapshot offers a **different version** from the one that kept failing (`plugin_update_failure_streaks.last_target_version`), once per new version, since a failed retry records that version as the new last target. It also tries an auto-ignored plugin whose installed version is **vulnerable** when the offered version is at or above the highest `patched_in` of the matching advisories. Otherwise the slug stays paused until resumed or until one of those conditions is met. - Clicking **Resume** (or unignore) in Control deletes the ignore record, resets the consecutive failure streak to 0, and immediately pushes an updated exceptions payload to the WordPress site. - Converting an auto-ignore into a **manual ignore** keeps nightly retries paused but sets `client_visible=false` and pushes an updated exceptions list so the slug drops out of wp-admin. - Manual ignores are never rewritten as auto-failure, even if the slug later fails nightly. @@ -165,8 +165,8 @@ Plugins-only, opt-in per site, runs every night between 02:00 and 06:00 ET. The ### How it works -1. `clockwork:run-nightly-plugin-updates` (02:00 ET) discovers every care-plan site with `auto_updates_paused = false` and `companion_installed = true`, queries its Companion snapshot for pending plugin updates, dedupes against existing `plugin_update_jobs` rows and `plugin_update_ignores`, and queues one `PluginUpdateJob` per plugin with `batch_id = "nightly-YYYY-MM-DD-..."`. The run reuses the same `plugin-updates` Supervisor queue as manual batches. At the end of every run — whether or not any jobs were queued — the loop stamps `auto_updates_last_run_at` on **all eligible** care-plan sites. This gives a daily heartbeat: "3 hours ago" means the loop ran and found nothing to do; a stale or null timestamp means the loop stopped firing, not that the site was up to date. -2. Vulnerable plugins (matched by `PluginVulnerabilityMatcher`) are sorted first so security patches land before cosmetic updates. +1. `clockwork:run-nightly-plugin-updates` (02:00 ET) discovers every care-plan site with `auto_updates_paused = false` and `companion_installed = true`, queries its Companion snapshot for pending plugin updates, dedupes against existing `plugin_update_jobs` rows and `plugin_update_ignores`, and queues one `PluginUpdateJob` per plugin with `batch_id = "nightly-YYYY-MM-DD-..."` (the America/New_York date; a `--site` run uses `nightly-YYYY-MM-DD-site-...`). The fleet-wide run happens at most once per local date: if a fleet-wide nightly batch was already queued today (as on the DST fall-back night, when 01:30 happens twice), it logs and skips. `--force` overrides that, and `--site` runs are neither blocked nor counted. The run reuses the same `plugin-updates` Supervisor queue as manual batches. At the end of every run — whether or not any jobs were queued — the loop stamps `auto_updates_last_run_at` on **all eligible** care-plan sites. This gives a daily heartbeat: "3 hours ago" means the loop ran and found nothing to do; a stale or null timestamp means the loop stopped firing, not that the site was up to date. +2. Vulnerable plugins (matched by `PluginVulnerabilityMatcher` on the plugin's directory, the part of the snapshot's `dir/file.php` slug before the slash) are sorted first so security patches land before cosmetic updates. 3. `clockwork:nightly-update-summary` (06:15 ET) reads every job whose `batch_id` starts with `nightly-` since the last summary run and emails the operator a success/failure breakdown. No jobs → no email. 4. If any job ends in `failed` and its `batch_id` is a nightly batch, `AbstractRunUpdate` fires a Mattermost `pluginUpdateFailed()` ping immediately — manual bulk failures stay quiet since the operator is watching the page. diff --git a/resources/docs/reference/artisan-commands.md b/resources/docs/reference/artisan-commands.md index 2ee0529d..22cf4dc4 100644 --- a/resources/docs/reference/artisan-commands.md +++ b/resources/docs/reference/artisan-commands.md @@ -121,7 +121,7 @@ Every `clockwork:*` command, alphabetical, with a one-line summary and an exampl | `clockwork:rotate-companion-secret` | Rotate per-site HMAC secret. | `php artisan clockwork:rotate-companion-secret --all` | | `clockwork:detect-contact-forms` | Companion-aware contact-form detection. | `php artisan clockwork:detect-contact-forms` | | `clockwork:test-contact-forms` | Run the due contact-form tests across the fleet (care-plan only). `--site=X` bypasses the care-plan filter; `--form=ID` targets a single contact_form_tests row; `--force` bypasses the frequency-due check. | `php artisan clockwork:test-contact-forms --force --site=example.com` | -| `clockwork:run-nightly-plugin-updates` | Nightly care-plan auto-update path — runs `wp plugin update` per opted-in site. Has no fixed time: it's chained via `->then()` and starts as soon as the 01:30 ET `clockwork:refresh-companion-snapshot` run exits. Per-site opt-out toggle on the site Updates tab. | `php artisan clockwork:run-nightly-plugin-updates` | +| `clockwork:run-nightly-plugin-updates` | Nightly care-plan auto-update path — runs `wp plugin update` per opted-in site. Has no fixed time: it's chained via `->then()` and starts as soon as the 01:30 ET `clockwork:refresh-companion-snapshot` run exits. Per-site opt-out toggle on the site Updates tab. The fleet-wide run happens at most once per America/New_York date; `--force` re-runs anyway, `--site=` limits to one site (not guarded), `--dry-run` only lists candidates. | `php artisan clockwork:run-nightly-plugin-updates` | | `clockwork:nightly-update-summary` | Email summary of the previous night's auto-update run. Runs 06:15 ET. | `php artisan clockwork:nightly-update-summary` | | `clockwork:push-companion-traffic` | Push the previous day's traffic rollup to each Companion-equipped site for display in the WP admin. | `php artisan clockwork:push-companion-traffic` | | `clockwork:ensure-queue-worker` | Watchdog for the `com.clockwork.queue` launchd service — checks `launchctl list` for a live PID and kickstarts the worker if it has crashed or been throttled into a backoff loop. Scheduled every 5 min so recovery gap ≤ 5 min. A successful revival is log-only; if the kickstart itself fails, fires a `queue_worker_restart_failed` Mattermost/Slack alert (every queued job in the app is stuck at that point). Crontab itself is watched by `clockwork:scheduler-heartbeat` (Health + metrics). | `php artisan clockwork:ensure-queue-worker` | diff --git a/resources/docs/reference/scheduled-jobs.md b/resources/docs/reference/scheduled-jobs.md index d03eaf80..9dd6682f 100644 --- a/resources/docs/reference/scheduled-jobs.md +++ b/resources/docs/reference/scheduled-jobs.md @@ -80,7 +80,7 @@ The scheduler itself is watched by `clockwork:scheduler-heartbeat` (every minute | 01:30 | `clockwork:refresh-companion-snapshot` | Pull `/snapshot` from each Companion-equipped site → cache to `sites.companion_snapshot`. (Was every-15-min, then 03:00 UTC — daily is enough; manual update batches trigger their own refresh.) Chained via `->then()`: `clockwork:run-nightly-plugin-updates` fires immediately after this command's process actually exits, not at a fixed clock offset — see the next row. | | 01:45 | `clockwork:ensure-companion-trust-proxy` | Idempotently inject `CLOCKWORK_COMPANION_TRUST_PROXY` into wp-config.php on care-plan auto-update sites so Companion's rate limiter sees real IPs behind CF. Independently scheduled at a fixed time, not chained — unlike the update loop below, a stale snapshot here just means the constant gets injected a day late, not a silently dropped update. | | 01:50 | `clockwork:detect-stuck-companion-state` | Fleet-wide sweep for a failed install never retried (>24h) or an installed Companion gone silent (>3 days) — alerts on the transition into and out of "stuck" so unmonitored or failed installs are caught automatically. | -| chained | `clockwork:run-nightly-plugin-updates` | Care-plan auto-update path — runs `wp plugin update` for each opted-in site. Per-site opt-out toggle on the site Updates tab. **Not its own `Schedule::command()` entry** — chained via `->then()` off `clockwork:refresh-companion-snapshot` above (`routes/console.php`), ensuring the full snapshot refresh completes across the entire fleet before update queries evaluate candidates. | +| chained | `clockwork:run-nightly-plugin-updates` | Care-plan auto-update path — runs `wp plugin update` for each opted-in site. Per-site opt-out toggle on the site Updates tab. **Not its own `Schedule::command()` entry** — chained via `->then()` off `clockwork:refresh-companion-snapshot` above (`routes/console.php`), ensuring the full snapshot refresh completes across the entire fleet before update queries evaluate candidates. On the DST fall-back night 01:30 ET happens twice and the chain fires twice; the command skips the second run because a fleet-wide nightly batch already exists for that local date (`--force` overrides). | ## Daily — security scan window (02:00 UTC) diff --git a/routes/console.php b/routes/console.php index e7a39e1a..a607176b 100644 --- a/routes/console.php +++ b/routes/console.php @@ -258,9 +258,13 @@ // stands at read time). ->then() guarantees the update loop only starts once // refresh has actually exited, however long that takes, instead of racing a // fixed offset. The chained call intentionally skips withoutOverlapping()/ -// onOneServer() of its own — it inherits both from this parent event, since -// the whole refresh+then-update sequence only ever fires once per this -// event's single daily occurrence. +// onOneServer() of its own — it inherits both from this parent event. +// +// 01:30 local happens twice on the DST fall-back day, so this event fires +// twice that night. The update command guards itself: it skips when a +// fleet-wide nightly batch already exists for the local date (see +// RunNightlyPluginUpdates::fleetBatchForLocalDate()), so failures are not +// counted twice toward the auto-pause threshold. Schedule::command('clockwork:refresh-companion-snapshot') ->dailyAt('01:30') ->timezone('America/New_York') diff --git a/tests/Feature/Console/RunNightlyPluginUpdatesTest.php b/tests/Feature/Console/RunNightlyPluginUpdatesTest.php index d13f0695..3a941634 100644 --- a/tests/Feature/Console/RunNightlyPluginUpdatesTest.php +++ b/tests/Feature/Console/RunNightlyPluginUpdatesTest.php @@ -4,10 +4,12 @@ use App\Jobs\RunPluginUpdate; use App\Jobs\RunThemeUpdate; +use App\Models\PluginUpdateFailureStreak; use App\Models\PluginUpdateIgnore; use App\Models\PluginUpdateJob; use App\Models\PluginVulnerability; use App\Models\Site; +use Illuminate\Support\Carbon; use Illuminate\Support\Facades\Queue; /* @@ -189,31 +191,386 @@ function themeSnapshot(array $themes): array }); describe('CVE-priority ordering', function () { - it('assigns the same nightly batch_id to every tuple queued in one run, with the vulnerable plugin present', function () { + it('dispatches the vulnerable plugin first, matching the snapshot\'s dir/file.php slug to the advisory\'s directory slug', function () { Queue::fake(); + // Advisories key on the plugin directory; Companion snapshots use + // "directory/file.php". The vulnerable plugin sits on the + // alphabetically later site and after a safe plugin, so only the + // CVE priority can put it first. PluginVulnerability::factory()->create([ 'slug' => 'vuln-plugin', 'from_version' => '0', - 'to_version' => '2.0.0', + 'to_version' => '1.0', + 'to_inclusive' => true, + 'patched_in' => '1.1', ]); + $first = nightlySite([ + 'domain' => 'aaa.example.com', + 'wp_plugin_updates' => true, + 'companion_snapshot' => pluginSnapshot([ + ['slug' => 'safe-plugin/safe-plugin.php', 'name' => 'Safe Plugin', 'version' => '1.0', 'new_version' => '1.1', 'update_available' => true], + ]), + ]); + $second = nightlySite([ + 'domain' => 'zzz.example.com', + 'wp_plugin_updates' => true, + 'companion_snapshot' => pluginSnapshot([ + ['slug' => 'other-plugin/other-plugin.php', 'name' => 'Other Plugin', 'version' => '1.0', 'new_version' => '1.1', 'update_available' => true], + ['slug' => 'vuln-plugin/vuln-plugin.php', 'name' => 'Vuln Plugin', 'version' => '1.0', 'new_version' => '1.1', 'update_available' => true], + ]), + ]); + + $this->artisan('clockwork:run-nightly-plugin-updates') + ->expectsOutputToContain('(1 vulnerable-first)') + ->assertSuccessful(); + + $dispatched = Queue::pushed(RunPluginUpdate::class) + ->map(fn ($job) => PluginUpdateJob::find($job->jobRowId)) + ->values(); + + expect($dispatched->map(fn ($row) => [$row->site_id, $row->target_slug])->all())->toBe([ + [$second->id, 'vuln-plugin/vuln-plugin.php'], + [$first->id, 'safe-plugin/safe-plugin.php'], + [$second->id, 'other-plugin/other-plugin.php'], + ]); + + // One batch for the whole night. + expect($dispatched->pluck('batch_id')->unique())->toHaveCount(1); + }); + + it('marks the vulnerable plugin in the dry-run listing', function () { + Queue::fake(); + + PluginVulnerability::factory()->create([ + 'slug' => 'vuln-plugin', + 'from_version' => '0', + 'to_version' => '1.0', + 'to_inclusive' => true, + 'patched_in' => '1.1', + ]); + + nightlySite([ + 'wp_plugin_updates' => true, + 'companion_snapshot' => pluginSnapshot([ + ['slug' => 'vuln-plugin/vuln-plugin.php', 'name' => 'Vuln Plugin', 'version' => '1.0', 'new_version' => '1.1', 'update_available' => true], + ]), + ]); + + $this->artisan('clockwork:run-nightly-plugin-updates', ['--dry-run' => true]) + ->expectsOutputToContain('!CVE') + ->assertSuccessful(); + + Queue::assertNothingPushed(); + }); +}); + +/** + * Seed an auto-failure pause the way UpdateFailureStreakRecorder leaves it: + * an auto_failure ignore row plus a streak recording the version that kept + * failing. Pass null to leave the streak out. + */ +function pauseAfterFailures(Site $site, string $slug, ?string $failedVersion, string $kind = PluginUpdateJob::KIND_PLUGIN): PluginUpdateIgnore +{ + $ignore = PluginUpdateIgnore::create([ + 'site_id' => $site->id, + 'target_kind' => $kind, + 'target_slug' => $slug, + 'source' => PluginUpdateIgnore::SOURCE_AUTO_FAILURE, + 'failure_count' => 5, + 'client_visible' => true, + 'note' => 'Automatically paused after 5 consecutive nightly update failures.', + ]); + + if ($failedVersion !== null) { + PluginUpdateFailureStreak::create([ + 'site_id' => $site->id, + 'target_kind' => $kind, + 'target_slug' => $slug, + 'consecutive_failures' => 5, + 'last_from_version' => '1.0.0', + 'last_target_version' => $failedVersion, + 'last_failed_at' => now()->subDay(), + 'ignored_at' => now()->subDay(), + 'ignore_id' => $ignore->id, + ]); + } + + return $ignore; +} + +function vulnerableAdvisory(string $dirSlug, string $affectedUpTo, string $patchedIn): PluginVulnerability +{ + return PluginVulnerability::factory()->create([ + 'slug' => $dirSlug, + 'from_version' => '0', + 'from_inclusive' => true, + 'to_version' => $affectedUpTo, + 'to_inclusive' => true, + 'patched_in' => $patchedIn, + ]); +} + +describe('auto-failure pauses', function () { + it('keeps an auto-paused plugin paused while the version that kept failing is still the one offered', function () { + Queue::fake(); + + $site = nightlySite([ + 'wp_plugin_updates' => true, + 'companion_snapshot' => pluginSnapshot([ + ['slug' => 'stuck-plugin/stuck-plugin.php', 'name' => 'Stuck Plugin', 'version' => '1.0.0', 'new_version' => '1.1.0', 'update_available' => true], + ]), + ]); + pauseAfterFailures($site, 'stuck-plugin/stuck-plugin.php', '1.1.0'); + + $this->artisan('clockwork:run-nightly-plugin-updates')->assertSuccessful(); + + expect(PluginUpdateJob::query()->where('site_id', $site->id)->count())->toBe(0); + Queue::assertNothingPushed(); + }); + + it('retries an auto-paused plugin once a newer release than the failing version is offered, leaving the pause in place', function () { + Queue::fake(); + + $site = nightlySite([ + 'wp_plugin_updates' => true, + 'companion_snapshot' => pluginSnapshot([ + ['slug' => 'stuck-plugin/stuck-plugin.php', 'name' => 'Stuck Plugin', 'version' => '1.0.0', 'new_version' => '1.2.0', 'update_available' => true], + ]), + ]); + $ignore = pauseAfterFailures($site, 'stuck-plugin/stuck-plugin.php', '1.1.0'); + + $this->artisan('clockwork:run-nightly-plugin-updates') + ->expectsOutputToContain('Retrying 1 auto-paused update (1 new release, 0 security fix)') + ->assertSuccessful(); + + $row = PluginUpdateJob::query()->where('site_id', $site->id)->sole(); + expect($row->target_slug)->toBe('stuck-plugin/stuck-plugin.php') + ->and($row->target_version)->toBe('1.2.0') + ->and($row->batch_id)->toStartWith('nightly-'); + Queue::assertPushed(RunPluginUpdate::class, fn ($job) => $job->jobRowId === $row->id); + + // Only a successful update lifts the pause. + expect(PluginUpdateIgnore::query()->whereKey($ignore->id)->exists())->toBeTrue(); + }); + + it('retries an auto-paused theme once a newer release is offered', function () { + Queue::fake(); + + $site = nightlySite([ + 'wp_theme_updates' => true, + 'companion_snapshot' => themeSnapshot([ + ['slug' => 'stuck-theme', 'name' => 'Stuck Theme', 'version' => '2.0', 'new_version' => '2.2', 'update_available' => true], + ]), + ]); + pauseAfterFailures($site, 'stuck-theme', '2.1', PluginUpdateJob::KIND_THEME); + + $this->artisan('clockwork:run-nightly-plugin-updates')->assertSuccessful(); + + $row = PluginUpdateJob::query()->where('site_id', $site->id)->sole(); + expect($row->target_kind)->toBe(PluginUpdateJob::KIND_THEME) + ->and($row->target_version)->toBe('2.2'); + Queue::assertPushed(RunThemeUpdate::class, fn ($job) => $job->jobRowId === $row->id); + }); + + it('retries an auto-paused vulnerable plugin when the offered version contains the fix, even if that version failed before', function () { + Queue::fake(); + + vulnerableAdvisory('vuln-plugin', '1.0.0', '1.1.0'); + + $site = nightlySite([ + 'wp_plugin_updates' => true, + 'companion_snapshot' => pluginSnapshot([ + ['slug' => 'vuln-plugin/vuln-plugin.php', 'name' => 'Vuln Plugin', 'version' => '1.0.0', 'new_version' => '1.1.0', 'update_available' => true], + ]), + ]); + pauseAfterFailures($site, 'vuln-plugin/vuln-plugin.php', '1.1.0'); + + $this->artisan('clockwork:run-nightly-plugin-updates') + ->expectsOutputToContain('Retrying 1 auto-paused update (0 new release, 1 security fix)') + ->assertSuccessful(); + + $row = PluginUpdateJob::query()->where('site_id', $site->id)->sole(); + expect($row->target_version)->toBe('1.1.0'); + Queue::assertPushed(RunPluginUpdate::class, 1); + }); + + it('keeps a vulnerable plugin paused when the offered version is below the highest patched_in', function () { + Queue::fake(); + + // Two advisories hit 1.0.0; the offered 1.5.0 fixes only the first. + vulnerableAdvisory('vuln-plugin', '1.4.0', '1.5.0'); + vulnerableAdvisory('vuln-plugin', '1.9.0', '2.0.0'); + + $site = nightlySite([ + 'wp_plugin_updates' => true, + 'companion_snapshot' => pluginSnapshot([ + ['slug' => 'vuln-plugin/vuln-plugin.php', 'name' => 'Vuln Plugin', 'version' => '1.0.0', 'new_version' => '1.5.0', 'update_available' => true], + ]), + ]); + pauseAfterFailures($site, 'vuln-plugin/vuln-plugin.php', '1.5.0'); + + $this->artisan('clockwork:run-nightly-plugin-updates')->assertSuccessful(); + + expect(PluginUpdateJob::query()->where('site_id', $site->id)->count())->toBe(0); + Queue::assertNothingPushed(); + }); + + it('keeps an auto-pause that has no recorded failing version', function () { + Queue::fake(); + $site = nightlySite([ 'wp_plugin_updates' => true, 'companion_snapshot' => pluginSnapshot([ - ['slug' => 'safe-plugin', 'name' => 'Safe Plugin', 'version' => '1.0', 'new_version' => '1.1', 'update_available' => true], - ['slug' => 'vuln-plugin', 'name' => 'Vuln Plugin', 'version' => '1.0', 'new_version' => '1.1', 'update_available' => true], + ['slug' => 'stuck-plugin/stuck-plugin.php', 'name' => 'Stuck Plugin', 'version' => '1.0.0', 'new_version' => '1.2.0', 'update_available' => true], ]), ]); + pauseAfterFailures($site, 'stuck-plugin/stuck-plugin.php', null); $this->artisan('clockwork:run-nightly-plugin-updates')->assertSuccessful(); - $rows = PluginUpdateJob::query()->where('site_id', $site->id)->get(); - expect($rows)->toHaveCount(2); - $batchIds = $rows->pluck('batch_id')->unique(); - expect($batchIds)->toHaveCount(1); + Queue::assertNothingPushed(); + }); + + it('never retries a manually ignored plugin, even for a new release that fixes a vulnerability', function () { + Queue::fake(); + + vulnerableAdvisory('held-plugin', '1.0.0', '1.1.0'); + + $site = nightlySite([ + 'wp_plugin_updates' => true, + 'companion_snapshot' => pluginSnapshot([ + ['slug' => 'held-plugin/held-plugin.php', 'name' => 'Held Plugin', 'version' => '1.0.0', 'new_version' => '1.2.0', 'update_available' => true], + ]), + ]); + PluginUpdateIgnore::create([ + 'site_id' => $site->id, + 'target_kind' => PluginUpdateJob::KIND_PLUGIN, + 'target_slug' => 'held-plugin/held-plugin.php', + 'source' => PluginUpdateIgnore::SOURCE_MANUAL, + 'client_visible' => false, + 'note' => 'Hold for now', + ]); + PluginUpdateFailureStreak::create([ + 'site_id' => $site->id, + 'target_kind' => PluginUpdateJob::KIND_PLUGIN, + 'target_slug' => 'held-plugin/held-plugin.php', + 'consecutive_failures' => 5, + 'last_target_version' => '1.1.0', + ]); + + $this->artisan('clockwork:run-nightly-plugin-updates')->assertSuccessful(); + + expect(PluginUpdateJob::query()->where('site_id', $site->id)->count())->toBe(0); + Queue::assertNothingPushed(); + }); +}); + +/** + * Mark every queued job failed so nothing is live: a second run would then + * re-queue everything if it were allowed to. + */ +function failAllNightlyJobs(): void +{ + PluginUpdateJob::query()->update(['status' => PluginUpdateJob::STATUS_FAILED]); +} + +function pendingUpdateSite(): Site +{ + return nightlySite([ + 'wp_plugin_updates' => true, + 'companion_snapshot' => pluginSnapshot([ + ['slug' => 'some-plugin/some-plugin.php', 'name' => 'Some Plugin', 'version' => '1.0', 'new_version' => '1.1', 'update_available' => true], + ]), + ]); +} + +describe('once per local night', function () { + it('runs only once on the DST fall-back night, when 01:30 local happens twice', function () { + Queue::fake(); + $site = pendingUpdateSite(); + + // 01:45 EDT, before the clocks go back. + $this->travelTo(Carbon::parse('2026-11-01 05:45:00', 'UTC')); + $this->artisan('clockwork:run-nightly-plugin-updates')->assertSuccessful(); + $firstBatch = PluginUpdateJob::query()->where('site_id', $site->id)->sole()->batch_id; + expect($firstBatch)->toStartWith('nightly-2026-11-01-'); + failAllNightlyJobs(); + + // 01:45 EST, an hour later: the scheduler fires the chain again. + $this->travelTo(Carbon::parse('2026-11-01 06:45:00', 'UTC')); + $this->artisan('clockwork:run-nightly-plugin-updates') + ->expectsOutputToContain("Nightly batch {$firstBatch} already ran for 2026-11-01") + ->assertSuccessful(); + expect(PluginUpdateJob::query()->where('site_id', $site->id)->count())->toBe(1); + + // The next night runs normally. + $this->travelTo(Carbon::parse('2026-11-02 06:45:00', 'UTC')); + $this->artisan('clockwork:run-nightly-plugin-updates')->assertSuccessful(); + expect(PluginUpdateJob::query()->where('site_id', $site->id)->count())->toBe(2) + ->and(PluginUpdateJob::query()->latest('id')->first()->batch_id)->toStartWith('nightly-2026-11-02-'); Queue::assertPushed(RunPluginUpdate::class, 2); }); + + it('decides "today" by the local date, not the UTC date', function () { + Queue::fake(); + $site = pendingUpdateSite(); + + // 21:30 EST on Nov 2 is already Nov 3 in UTC. + $this->travelTo(Carbon::parse('2026-11-03 02:30:00', 'UTC')); + $this->artisan('clockwork:run-nightly-plugin-updates')->assertSuccessful(); + expect(PluginUpdateJob::query()->where('site_id', $site->id)->sole()->batch_id)->toStartWith('nightly-2026-11-02-'); + failAllNightlyJobs(); + + // 01:40 EST on Nov 3 is a new local night, so it runs. + $this->travelTo(Carbon::parse('2026-11-03 06:40:00', 'UTC')); + $this->artisan('clockwork:run-nightly-plugin-updates')->assertSuccessful(); + expect(PluginUpdateJob::query()->where('site_id', $site->id)->count())->toBe(2) + ->and(PluginUpdateJob::query()->latest('id')->first()->batch_id)->toStartWith('nightly-2026-11-03-'); + }); + + it('re-runs with --force, and only reports the skip under --dry-run', function () { + Queue::fake(); + $site = pendingUpdateSite(); + + $this->travelTo(Carbon::parse('2026-11-04 06:40:00', 'UTC')); + $this->artisan('clockwork:run-nightly-plugin-updates')->assertSuccessful(); + failAllNightlyJobs(); + + $this->travelTo(Carbon::parse('2026-11-04 15:00:00', 'UTC')); + $this->artisan('clockwork:run-nightly-plugin-updates', ['--dry-run' => true]) + ->expectsOutputToContain('a real run would be skipped') + ->assertSuccessful(); + expect(PluginUpdateJob::query()->where('site_id', $site->id)->count())->toBe(1); + + $this->artisan('clockwork:run-nightly-plugin-updates', ['--force' => true])->assertSuccessful(); + expect(PluginUpdateJob::query()->where('site_id', $site->id)->count())->toBe(2); + }); + + it('does not guard a --site run, and a --site run does not block the fleet-wide nightly', function () { + Queue::fake(); + $site = pendingUpdateSite(); + + // Operator runs one site just after local midnight. + $this->travelTo(Carbon::parse('2026-11-04 05:10:00', 'UTC')); + $this->artisan('clockwork:run-nightly-plugin-updates', ['--site' => (string) $site->id])->assertSuccessful(); + $siteBatch = PluginUpdateJob::query()->where('site_id', $site->id)->sole()->batch_id; + expect($siteBatch)->toStartWith('nightly-2026-11-04-site-'); + failAllNightlyJobs(); + + // The scheduled fleet run that night still goes ahead. + $this->travelTo(Carbon::parse('2026-11-04 06:40:00', 'UTC')); + $this->artisan('clockwork:run-nightly-plugin-updates')->assertSuccessful(); + $fleetBatch = PluginUpdateJob::query()->latest('id')->first()->batch_id; + expect($fleetBatch)->toStartWith('nightly-2026-11-04-') + ->and($fleetBatch)->not->toContain('-site-'); + failAllNightlyJobs(); + + // A later --site run is still allowed. + $this->artisan('clockwork:run-nightly-plugin-updates', ['--site' => (string) $site->id])->assertSuccessful(); + expect(PluginUpdateJob::query()->where('site_id', $site->id)->count())->toBe(3); + }); }); describe('--dry-run', function () { diff --git a/tests/Feature/Updates/UpdateFailureStreakTest.php b/tests/Feature/Updates/UpdateFailureStreakTest.php index 9a7620f3..56cea9b5 100644 --- a/tests/Feature/Updates/UpdateFailureStreakTest.php +++ b/tests/Feature/Updates/UpdateFailureStreakTest.php @@ -153,7 +153,7 @@ function fakeNightlyJob(Site $site, string $slug = 'stubborn-plugin', array $ove ], ]); - PluginUpdateIgnore::create([ + $ignore = PluginUpdateIgnore::create([ 'site_id' => $site->id, 'target_kind' => PluginUpdateJob::KIND_PLUGIN, 'target_slug' => 'stubborn-plugin', @@ -163,12 +163,192 @@ function fakeNightlyJob(Site $site, string $slug = 'stubborn-plugin', array $ove 'note' => 'Auto-ignored', ]); + // The snapshot still offers the version that kept failing. + PluginUpdateFailureStreak::create([ + 'site_id' => $site->id, + 'target_kind' => PluginUpdateJob::KIND_PLUGIN, + 'target_slug' => 'stubborn-plugin', + 'consecutive_failures' => 5, + 'last_from_version' => '1.0.0', + 'last_target_version' => '1.1.0', + 'ignored_at' => now(), + 'ignore_id' => $ignore->id, + ]); + $this->artisan('clockwork:run-nightly-plugin-updates')->assertSuccessful(); Queue::assertNothingPushed(); expect(PluginUpdateJob::where('site_id', $site->id)->where('target_slug', 'stubborn-plugin')->count())->toBe(0); }); + it('lifts the auto-pause, clears the streak and re-pushes exceptions when an update reaches its target version', function () { + $site = makeExampleSite(['companion_installed' => true]); + + $ignore = PluginUpdateIgnore::create([ + 'site_id' => $site->id, + 'target_kind' => PluginUpdateJob::KIND_PLUGIN, + 'target_slug' => 'stubborn-plugin', + 'source' => PluginUpdateIgnore::SOURCE_AUTO_FAILURE, + 'failure_count' => 5, + 'client_visible' => true, + ]); + // A second paused plugin on the same site stays in the pushed list. + PluginUpdateIgnore::create([ + 'site_id' => $site->id, + 'target_kind' => PluginUpdateJob::KIND_PLUGIN, + 'target_slug' => 'other-plugin', + 'source' => PluginUpdateIgnore::SOURCE_AUTO_FAILURE, + 'failure_count' => 5, + 'client_visible' => true, + ]); + $streak = PluginUpdateFailureStreak::create([ + 'site_id' => $site->id, + 'target_kind' => PluginUpdateJob::KIND_PLUGIN, + 'target_slug' => 'stubborn-plugin', + 'consecutive_failures' => 5, + 'last_target_version' => '1.1.0', + 'ignored_at' => now(), + 'ignore_id' => $ignore->id, + ]); + + Http::fake([ + "https://{$site->domain}/wp-json/clockwork/v1/update-exceptions*" => Http::response(['ok' => true], 200), + ]); + + // A nightly retry of a newer release that went through. + $job = fakeNightlyJob($site, 'stubborn-plugin', [ + 'target_version' => '1.2.0', + 'status' => PluginUpdateJob::STATUS_COMPLETE, + 'after_version' => '1.2.0', + ]); + app(UpdateFailureStreakRecorder::class)->record($job, [ + 'ok' => true, + 'slug' => 'stubborn-plugin', + 'before_version' => '1.0.0', + 'after_version' => '1.2.0', + ]); + + expect(PluginUpdateIgnore::whereKey($ignore->id)->exists())->toBeFalse() + ->and(PluginUpdateIgnore::where('site_id', $site->id)->where('target_slug', 'other-plugin')->exists())->toBeTrue(); + + $streak->refresh(); + expect($streak->consecutive_failures)->toBe(0) + ->and($streak->ignored_at)->toBeNull() + ->and($streak->ignore_id)->toBeNull(); + + Http::assertSent(function ($request) use ($site) { + return strtok($request->url(), '?') === "https://{$site->domain}/wp-json/clockwork/v1/update-exceptions" + && collect($request['items'] ?? [])->pluck('slug')->all() === ['other-plugin']; + }); + }); + + it('lifts the auto-pause when a nightly retry of a new release succeeds through the update job', function () { + $site = makeExampleSite(); + + $ignore = PluginUpdateIgnore::create([ + 'site_id' => $site->id, + 'target_kind' => PluginUpdateJob::KIND_PLUGIN, + 'target_slug' => 'stubborn-plugin', + 'source' => PluginUpdateIgnore::SOURCE_AUTO_FAILURE, + 'failure_count' => 5, + 'client_visible' => true, + ]); + PluginUpdateFailureStreak::create([ + 'site_id' => $site->id, + 'target_kind' => PluginUpdateJob::KIND_PLUGIN, + 'target_slug' => 'stubborn-plugin', + 'consecutive_failures' => 5, + 'last_target_version' => '1.1.0', + 'ignored_at' => now(), + 'ignore_id' => $ignore->id, + ]); + + Http::fake([ + "https://{$site->domain}/wp-json/clockwork/v1/plugins/update*" => Http::response([ + 'ok' => true, + 'slug' => 'stubborn-plugin', + 'before_version' => '1.0.0', + 'after_version' => '1.2.0', + 'was_active' => true, + 'reactivated' => true, + 'elapsed_ms' => 400, + ], 200), + ]); + + $job = fakeNightlyJob($site, 'stubborn-plugin', ['target_version' => '1.2.0']); + (new RunPluginUpdate($job->id))->handle(app(ActionLogger::class)); + + expect($job->refresh()->status)->toBe(PluginUpdateJob::STATUS_COMPLETE) + ->and(PluginUpdateIgnore::whereKey($ignore->id)->exists())->toBeFalse(); + + $streak = PluginUpdateFailureStreak::where('site_id', $site->id)->where('target_slug', 'stubborn-plugin')->first(); + expect($streak->consecutive_failures)->toBe(0) + ->and($streak->ignored_at)->toBeNull(); + }); + + it('keeps the auto-pause when a completed job cannot show it reached the target version', function () { + $site = makeExampleSite(['companion_installed' => true]); + + $ignore = PluginUpdateIgnore::create([ + 'site_id' => $site->id, + 'target_kind' => PluginUpdateJob::KIND_PLUGIN, + 'target_slug' => 'stubborn-plugin', + 'source' => PluginUpdateIgnore::SOURCE_AUTO_FAILURE, + 'failure_count' => 5, + 'client_visible' => true, + ]); + $streak = PluginUpdateFailureStreak::create([ + 'site_id' => $site->id, + 'target_kind' => PluginUpdateJob::KIND_PLUGIN, + 'target_slug' => 'stubborn-plugin', + 'consecutive_failures' => 5, + 'last_target_version' => '1.1.0', + 'ignored_at' => now(), + 'ignore_id' => $ignore->id, + ]); + + Http::fake(); + + $job = fakeNightlyJob($site, 'stubborn-plugin', [ + 'status' => PluginUpdateJob::STATUS_COMPLETE, + 'after_version' => null, + ]); + app(UpdateFailureStreakRecorder::class)->record($job, ['ok' => true, 'slug' => 'stubborn-plugin']); + + expect(PluginUpdateIgnore::whereKey($ignore->id)->exists())->toBeTrue(); + $streak->refresh(); + expect($streak->consecutive_failures)->toBe(0) + ->and($streak->ignored_at)->not->toBeNull() + ->and($streak->ignore_id)->toBe($ignore->id); + Http::assertNothingSent(); + }); + + it('never lifts a manual ignore after a successful update', function () { + $site = makeExampleSite(['companion_installed' => true]); + + $ignore = PluginUpdateIgnore::create([ + 'site_id' => $site->id, + 'target_kind' => PluginUpdateJob::KIND_PLUGIN, + 'target_slug' => 'held-plugin', + 'source' => PluginUpdateIgnore::SOURCE_MANUAL, + 'client_visible' => false, + 'note' => 'Hold this one', + ]); + + Http::fake(); + + $job = fakeNightlyJob($site, 'held-plugin', [ + 'requested_by_user_id' => User::factory()->create()->id, + 'batch_id' => (string) Str::uuid(), + 'status' => PluginUpdateJob::STATUS_COMPLETE, + 'after_version' => '1.1.0', + ]); + app(UpdateFailureStreakRecorder::class)->record($job, ['ok' => true, 'slug' => 'held-plugin', 'after_version' => '1.1.0']); + + expect(PluginUpdateIgnore::whereKey($ignore->id)->exists())->toBeTrue(); + Http::assertNothingSent(); + }); + it('resets streak to 0 on any successful update after prior failures', function () { $site = makeExampleSite(); From 2c17e464e90955a4b3f9764e21221217d57c040a Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 3 Nov 2026 20:22:08 +0000 Subject: [PATCH 06/64] fix(updates): surface real upgrader errors, redact download URLs, pause site on fs errors Companion and Renegade builds in the field return ok=true and upgrade_completed=true even when the WordPress upgrader logged an "Error: ..." line (could not copy file, could not create directory, download unauthorized, bad zip). AbstractRunUpdate treated every such stalled no-op as "WordPress's own update-checker no longer offered this update (premium/licensed)", hiding the real cause from the job error, the failure streak, the auto-ignore and the ops chat alert, and still ran post-update verify/repair. Inside the stalled no-op branch the job now looks for the last "Error:" line in the upgrader output. When there is one, the error records that line plus the line before it (never a download line), upgrade_completed is set to false so verify/repair is skipped, and a reason_code is stored (fs_permission, download_unauthorized, bad_package, no_package, upgrader_error). Without an Error: line the old explanation is kept with reason_code not_offered. When Companion reports ok=false itself, its error text is kept and the reason code is still derived from the output. The action-log summary now follows the job's verdict, so a failed upgrade is no longer logged as "Plugin updated: 1.0 -> 1.0". reason_code is a new nullable column on plugin_update_jobs and plugin_update_failure_streaks. The upgrader's "Downloading update from " lines carry licence keys, vendor tokens and signed download URLs. A new UpgraderOutput helper redacts every update result before it is stored or sent (job messages and error, action_log details/summary/error and their Companion activity mirror, streaks, ignores, chat): download URLs are cut to scheme://host/[redacted], other URLs lose credentials, query string and fragment, and JWTs, bearer tokens and secret-named key=value pairs are masked. The single-plugin update endpoint applies the same redaction. A file-permission failure is a site problem, not a plugin one. When an fs_permission streak reaches the threshold and no plugin or theme update on the site has succeeded since the streak began, the recorder now sets the site's existing auto_updates_paused flag once (with a "Filesystem not writable" reason), writes one auto_updates_toggled action log and sends one ops alert, instead of auto-ignoring each plugin and pushing client-visible pause notices. An already-paused site is left alone. Other reason codes keep the per-plugin auto-ignore. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Q7rExNMgxzTNb3pagVUZrg --- app/Http/Controllers/SitesController.php | 5 +- app/Jobs/AbstractRunUpdate.php | 72 ++++- app/Models/PluginUpdateFailureStreak.php | 1 + app/Models/PluginUpdateJob.php | 1 + .../Updates/UpdateFailureStreakRecorder.php | 162 +++++++++- app/Support/UpgraderOutput.php | 299 ++++++++++++++++++ ...plugin_update_jobs_and_failure_streaks.php | 43 +++ resources/docs/features/updates.md | 26 +- .../Controllers/SitesControllerTest.php | 33 ++ tests/Feature/Jobs/UpdateJobPipelineTest.php | 253 +++++++++++++++ .../Updates/UpdateFailureStreakTest.php | 234 ++++++++++++++ tests/Unit/Support/UpgraderOutputTest.php | 235 ++++++++++++++ 12 files changed, 1349 insertions(+), 15 deletions(-) create mode 100644 app/Support/UpgraderOutput.php create mode 100644 database/migrations/2026_11_03_120000_add_reason_code_to_plugin_update_jobs_and_failure_streaks.php create mode 100644 tests/Unit/Support/UpgraderOutputTest.php diff --git a/app/Http/Controllers/SitesController.php b/app/Http/Controllers/SitesController.php index b0c17ba9..49c11013 100644 --- a/app/Http/Controllers/SitesController.php +++ b/app/Http/Controllers/SitesController.php @@ -36,6 +36,7 @@ use App\Services\Uptime\UptimeStateUpdater; use App\Support\CompanionExclusion; use App\Support\FleetSelfIps; +use App\Support\UpgraderOutput; use Carbon\CarbonImmutable; use Illuminate\Http\JsonResponse; use Illuminate\Http\RedirectResponse; @@ -1567,7 +1568,9 @@ public function updatePlugin(Site $site, Request $request, ActionLogger $logger) } try { - $result = (new ClockworkCompanionClient($site))->updatePlugin($slug); + // Redacted before it is logged or returned: the upgrader's + // download lines carry licence keys and signed URLs. + $result = UpgraderOutput::redact((new ClockworkCompanionClient($site))->updatePlugin($slug)); } catch (Throwable $e) { $logger->record( actionType: ActionLog::TYPE_PLUGIN_UPDATE, diff --git a/app/Jobs/AbstractRunUpdate.php b/app/Jobs/AbstractRunUpdate.php index c2837135..c98ec998 100644 --- a/app/Jobs/AbstractRunUpdate.php +++ b/app/Jobs/AbstractRunUpdate.php @@ -8,6 +8,7 @@ use App\Services\Chat\ChatNotifier; use App\Services\Companion\ClockworkCompanionClient; use App\Services\Updates\UpdateFailureStreakRecorder; +use App\Support\UpgraderOutput; use Illuminate\Bus\Queueable; use Illuminate\Contracts\Queue\ShouldQueue; use Illuminate\Foundation\Bus\Dispatchable; @@ -125,7 +126,14 @@ public function handle(ActionLogger $logger): void } $client = new ClockworkCompanionClient($site); - $result = $this->run($site, $row, $client); + + // Redact before anything is derived from or persisted out of the + // result: the upgrader's "Downloading update from " lines + // carry licence keys and signed download URLs, and everything + // below (messages + error columns, action_log details/summary, + // the Companion activity mirror, the failure streak, chat) reads + // from $result. + $result = UpgraderOutput::redact($this->run($site, $row, $client)); $ok = (bool) ($result['ok'] ?? false); @@ -148,6 +156,16 @@ public function handle(ActionLogger $logger): void // while target_version was 9.0.1. Only applies when we had a // real target_version to compare against (core/plugin/theme; // translations has none, so this never fires for that kind). + // + // Not every stalled no-op is a stale update offer, though. The + // Companion/Renegade versions in the field also report ok=true + // (and upgrade_completed=true) when the WordPress upgrader + // itself failed and logged "Error: …" — "Could not copy file", + // "Could not create directory", "Download failed. Unauthorized", + // a bad zip. Those must surface as what they are (file + // permissions, licence, package), not as "no longer offered", + // or ops chase a premium licence while the real problem is + // file ownership. reason_code records the cause for grouping. $afterVersion = $result['after_version'] ?? null; $beforeVersion = $result['before_version'] ?? $row->before_version; if ( @@ -158,11 +176,38 @@ public function handle(ActionLogger $logger): void && $afterVersion !== $row->target_version ) { $ok = false; - $result['error'] = sprintf( - "WordPress's own update-checker no longer offered this update when the job ran (still on %s, expected %s). Common for premium/licensed plugins whose update source went stale or unreachable between our last check and now. Nothing was changed on the site.", - $afterVersion, - $row->target_version, - ); + $errorLines = UpgraderOutput::errorLines($result['messages'] ?? null); + + if ($errorLines !== []) { + $result['reason_code'] = UpgraderOutput::classify($errorLines); + $result['error'] = sprintf( + '%s (WordPress upgrader error; still on %s, expected %s.)', + implode(' — ', $errorLines), + $afterVersion, + $row->target_version, + ); + // The upgrader stopped with an error and the version did + // not move, so the file swap did not happen whatever + // Companion claimed — there is nothing to verify/repair. + $result['upgrade_completed'] = false; + } else { + $result['reason_code'] = UpgraderOutput::REASON_NOT_OFFERED; + $result['error'] = sprintf( + "WordPress's own update-checker no longer offered this update when the job ran (still on %s, expected %s). Common for premium/licensed plugins whose update source went stale or unreachable between our last check and now. Nothing was changed on the site.", + $afterVersion, + $row->target_version, + ); + } + } elseif (! $ok && ! isset($result['reason_code'])) { + // Companion already reported the failure itself (ok=false, + // its own error text kept as-is). Still record the cause + // when the upgrader output says what it was, so a failure + // reported correctly is grouped the same way as one caught + // by the stalled check above. + $errorLines = UpgraderOutput::errorLines($result['messages'] ?? null); + if ($errorLines !== []) { + $result['reason_code'] = UpgraderOutput::classify($errorLines); + } } // 'upgrade_completed' (Companion >= 1.30.2) tells us the file @@ -228,10 +273,14 @@ public function handle(ActionLogger $logger): void 'messages' => $result['messages'] ?? null, 'repairs' => $result['repairs'] ?? null, 'error' => $ok ? null : (string) ($result['error'] ?? 'unknown error'), + 'reason_code' => $ok ? null : ($result['reason_code'] ?? null), 'completed_at' => now(), ])->save(); - $summary = $this->actionLogSummary($row, $result); + // The summary follows the job's own verdict, not Companion's raw + // `ok`: a stalled no-op or an upgrader error must not be logged + // (and mirrored to wp-admin) as "Plugin updated: 1.0.0 → 1.0.0". + $summary = $this->actionLogSummary($row, ['ok' => $ok] + $result); $repairs = $result['repairs'] ?? []; if (is_array($repairs) && $repairs !== []) { $repairCount = count($repairs); @@ -258,19 +307,22 @@ public function handle(ActionLogger $logger): void ); } catch (Throwable $e) { $caughtException = $e; + // A transport exception can quote the response body, which may + // hold upgrader output — redact it like any other result text. + $exceptionMessage = UpgraderOutput::redactText($e->getMessage()); $row->forceFill([ 'status' => PluginUpdateJob::STATUS_FAILED, - 'error' => $e->getMessage(), + 'error' => $exceptionMessage, 'completed_at' => now(), ])->save(); $logger->record( actionType: $this->actionLogType(), - summary: "Update threw: {$e->getMessage()}", + summary: "Update threw: {$exceptionMessage}", site: $site, target: $this->actionLogTarget($row), ok: false, - error: $e->getMessage(), + error: $exceptionMessage, actor: $row->requested_by_user_id !== null ? "user:{$row->requested_by_user_id}" : 'auto', ); diff --git a/app/Models/PluginUpdateFailureStreak.php b/app/Models/PluginUpdateFailureStreak.php index 1fe4446b..d4df476e 100644 --- a/app/Models/PluginUpdateFailureStreak.php +++ b/app/Models/PluginUpdateFailureStreak.php @@ -23,6 +23,7 @@ class PluginUpdateFailureStreak extends Model 'last_target_version', 'last_from_version', 'last_error', + 'reason_code', 'last_failed_at', 'last_job_id', 'ignored_at', diff --git a/app/Models/PluginUpdateJob.php b/app/Models/PluginUpdateJob.php index bace91ad..afed4998 100644 --- a/app/Models/PluginUpdateJob.php +++ b/app/Models/PluginUpdateJob.php @@ -65,6 +65,7 @@ class PluginUpdateJob extends Model 'reactivated', 'elapsed_ms', 'error', + 'reason_code', 'messages', 'state_before', 'repairs', diff --git a/app/Services/Updates/UpdateFailureStreakRecorder.php b/app/Services/Updates/UpdateFailureStreakRecorder.php index 0b82afc9..7259b9c0 100644 --- a/app/Services/Updates/UpdateFailureStreakRecorder.php +++ b/app/Services/Updates/UpdateFailureStreakRecorder.php @@ -11,6 +11,7 @@ use App\Services\Chat\ChatNotifier; use App\Services\Companion\ClockworkCompanionClient; use App\Support\Settings; +use App\Support\UpgraderOutput; use Illuminate\Support\Carbon; use Illuminate\Support\Facades\Log; use Throwable; @@ -82,7 +83,7 @@ public function record(PluginUpdateJob $job, ?array $result = null, ?Throwable $ return; } - $this->incrementStreak($job); + $this->incrementStreak($job, $result); } /** @@ -166,7 +167,15 @@ private function liftAutoPause(PluginUpdateJob $job): void } } - private function incrementStreak(PluginUpdateJob $job): void + /** + * Count one more plugin-level nightly failure and, at the threshold, + * pause it. A file-permission failure (reason_code fs_permission) that + * affects the whole site pauses the site once instead of auto-ignoring + * the plugin — see pauseSiteForFilesystemFailure(). + * + * @param array|null $result + */ + private function incrementStreak(PluginUpdateJob $job, ?array $result = null): void { $streak = PluginUpdateFailureStreak::firstOrNew([ 'site_id' => $job->site_id, @@ -177,7 +186,11 @@ private function incrementStreak(PluginUpdateJob $job): void $streak->consecutive_failures = (int) $streak->consecutive_failures + 1; $streak->last_target_version = $job->target_version; $streak->last_from_version = $job->before_version; - $streak->last_error = mb_strimwidth((string) ($job->error ?? 'unknown error'), 0, 500, '…'); + // last_error is copied into the ignore row, the chat alerts and the + // site pause log, so it gets the same redaction as the job's own + // columns even if the job error was written by another path. + $streak->last_error = mb_strimwidth(UpgraderOutput::redactText((string) ($job->error ?? 'unknown error')), 0, 500, '…'); + $streak->reason_code = $job->reason_code ?? ($result['reason_code'] ?? null); $streak->last_failed_at = Carbon::now(); $streak->last_job_id = $job->id; $streak->save(); @@ -187,10 +200,153 @@ private function incrementStreak(PluginUpdateJob $job): void $enabled = (bool) $settings->get('updates.auto_ignore_enabled', true); if ($enabled && $streak->consecutive_failures >= $threshold) { + if ( + $streak->reason_code === UpgraderOutput::REASON_FS_PERMISSION + && $this->isSiteWideFilesystemFailure($job, $streak) + ) { + $this->pauseSiteForFilesystemFailure($job, $streak); + + return; + } + $this->applyAutoIgnore($job, $streak); } } + /** + * A file-permission failure is a site problem when no plugin or theme + * update on the site has succeeded since this streak began. If one has, + * the site's files are writable in general and only this plugin's own + * files are off, so the plugin keeps the normal per-plugin auto-ignore + * rather than stopping updates for every other plugin on the site. + */ + private function isSiteWideFilesystemFailure(PluginUpdateJob $job, PluginUpdateFailureStreak $streak): bool + { + // The oldest nightly failure still inside the current streak. + $streakStartJobId = PluginUpdateJob::query() + ->where('site_id', $job->site_id) + ->where('target_kind', $job->target_kind) + ->where('target_slug', $job->target_slug) + ->where('status', PluginUpdateJob::STATUS_FAILED) + ->where('batch_id', 'like', 'nightly-%') + ->orderByDesc('id') + ->skip(max(0, (int) $streak->consecutive_failures - 1)) + ->value('id') ?? $job->id; + + return ! PluginUpdateJob::query() + ->where('site_id', $job->site_id) + ->whereIn('target_kind', [PluginUpdateJob::KIND_PLUGIN, PluginUpdateJob::KIND_THEME]) + ->where('status', PluginUpdateJob::STATUS_COMPLETE) + ->where('id', '>', $streakStartJobId) + ->exists(); + } + + /** + * WordPress cannot write the site's files ("Could not copy file", + * "Could not create directory …/wp-content/upgrade/…"), so every plugin + * on the site fails the same way, night after night. Pausing each plugin + * as it crosses the threshold turns one site problem into a pile of + * per-plugin pauses (each with its own chat alert and a client-visible + * "paused" notice in wp-admin) and hides the real cause. Instead, pause + * nightly updates for the whole site once, through the same + * auto_updates_paused flag operators use, with one action log and one + * ops alert. No per-plugin ignore is written and nothing is pushed to + * wp-admin. Ops fix the file ownership/permissions and resume the site + * with the existing per-site toggle. + */ + private function pauseSiteForFilesystemFailure(PluginUpdateJob $job, PluginUpdateFailureStreak $streak): void + { + $site = $job->site; + if (! $site instanceof Site) { + return; + } + + $reason = mb_strimwidth(sprintf( + 'Filesystem not writable: %d nightly updates in a row failed with a file-permission error. Fix file ownership/permissions, then resume.', + $streak->consecutive_failures, + ), 0, 255, '…'); + + // Conditional update: the recorder runs after the per-site update + // lock is released, so two of the site's jobs can reach this point + // together; only the one that flips the flag logs and alerts. A site + // that is already paused (by an operator, or an earlier failure) + // keeps its existing reason and gets no second alert. + $flipped = Site::query() + ->whereKey($site->id) + ->where('auto_updates_paused', false) + ->update([ + 'auto_updates_paused' => true, + 'auto_updates_paused_reason' => $reason, + ]); + + if ($flipped === 0) { + return; + } + + $site->refresh(); + + $name = $job->target_name ?: $job->target_slug; + $kindLabel = $job->target_kind === PluginUpdateJob::KIND_THEME ? 'theme' : 'plugin'; + + try { + app(ActionLogger::class)->record( + actionType: ActionLog::TYPE_AUTO_UPDATES_TOGGLED, + summary: "Paused nightly auto-updates on {$site->domain}: filesystem not writable ({$streak->consecutive_failures} failed nightly updates in a row)", + site: $site, + target: 'paused', + details: [ + 'previous' => false, + 'now' => true, + 'reason' => $reason, + 'reason_code' => UpgraderOutput::REASON_FS_PERMISSION, + 'target_kind' => $job->target_kind, + 'target_slug' => $job->target_slug, + 'streak' => $streak->consecutive_failures, + 'from_version' => $streak->last_from_version, + 'target_version' => $streak->last_target_version, + 'last_error' => $streak->last_error, + 'job_id' => $job->id, + ], + ok: true, + actor: 'auto' + ); + } catch (Throwable $e) { + Log::warning('updates.site_pause_action_log_failed', [ + 'site_id' => $site->id, + 'error' => $e->getMessage(), + ]); + } + + // Same gate as the per-plugin alerts (ChatNotifierDispatcher:: + // dispatchForSite): inactive sites stay quiet. + if ($site->is_inactive) { + return; + } + + try { + $title = sprintf(':pause_button: Paused nightly auto-updates on %s: filesystem not writable', $site->domain); + app(ChatNotifier::class)->send($title, [[ + 'fallback' => $title, + 'color' => '#f59e0b', + 'title' => $title, + 'text' => sprintf( + '%d nightly updates in a row failed because WordPress could not write the files, so nightly updates are paused for the whole site instead of plugin by plugin. Make wp-content/plugins and wp-content/upgrade writable by the PHP user, then resume auto-updates for the site.', + $streak->consecutive_failures, + ), + 'fields' => [ + ['title' => 'Site', 'value' => $site->domain, 'short' => true], + ['title' => 'Last failure', 'value' => sprintf('%s %s (%s → %s)', $kindLabel, $name, $streak->last_from_version ?: '?', $streak->last_target_version ?: '?'), 'short' => true], + ['title' => 'Last error', 'value' => mb_strimwidth((string) $streak->last_error, 0, 280, '…'), 'short' => false], + ], + ]]); + } catch (Throwable $e) { + Log::warning('chat_notifier.site_auto_updates_paused_failed', [ + 'site_id' => $site->id, + 'error' => $e->getMessage(), + ]); + } + } + private function applyAutoIgnore(PluginUpdateJob $job, PluginUpdateFailureStreak $streak): void { $site = $job->site; diff --git a/app/Support/UpgraderOutput.php b/app/Support/UpgraderOutput.php new file mode 100644 index 00000000..26ad3318 --- /dev/null +++ b/app/Support/UpgraderOutput.php @@ -0,0 +1,299 @@ +…". For premium plugins that URL carries the licence key, a + * vendor authorisation token or JWT, or a signed, expiring download link — + * in the query string, and sometimes in the path. Everything Control keeps + * or sends from an update result (plugin_update_jobs.messages and .error, + * action_logs details/summary/error, failure streaks, ignores, chat alerts + * and the Companion activity mirror) goes through redact()/redactText() + * first: + * - a download URL is cut down to scheme://host/[redacted]; + * - any other URL loses its credentials, query string and fragment, and + * token-like path segments are masked; + * - JWTs, bearer tokens and secret-named key=value pairs are masked + * wherever they appear. + * + * Classification. Companion/Renegade versions in the field report ok=true + * even when the upgrader logged "Error: …" and nothing changed on the site. + * errorLines() pulls out the real error — only the last "Error:" line plus + * the line before it, which carries the detail (e.g. the file that could + * not be copied) — never a download line. classify() maps those lines to a + * reason code so failures can be grouped by cause. + */ +final class UpgraderOutput +{ + /** The site's files or wp-content/upgrade are not writable by PHP. A site problem, not a plugin one. */ + public const REASON_FS_PERMISSION = 'fs_permission'; + + /** The vendor refused the download (licence inactive or expired, 401/403). */ + public const REASON_DOWNLOAD_UNAUTHORIZED = 'download_unauthorized'; + + /** The download was not a usable zip (vendor returned an error page, truncated file). */ + public const REASON_BAD_PACKAGE = 'bad_package'; + + /** WordPress had no download URL for the update (typical of an unlicensed premium plugin). */ + public const REASON_NO_PACKAGE = 'no_package'; + + /** No upgrader error at all: WordPress simply no longer offered the update when the job ran. */ + public const REASON_NOT_OFFERED = 'not_offered'; + + /** An upgrader error that matches none of the known causes. */ + public const REASON_UPGRADER_ERROR = 'upgrader_error'; + + public const REDACTED = '[redacted]'; + + private const MAX_LINE_LENGTH = 300; + + /** + * Parameter names whose values are secrets wherever a name=value pair + * shows up (query strings that escaped URL parsing, HTML-escaped + * `&key=` runs, error text). + */ + private const SECRET_PARAM_NAMES = 'license|licence|license_key|licence_key|license_hash|license_id|key|api_key|apikey|site_key|token|access_token|auth|authorization|password|passwd|pwd|secret|signature|sig|purchase_code|purchase_key|code|x-amz-signature|x-amz-credential|x-amz-security-token|awsaccesskeyid|key-pair-id'; + + /** + * A URL, optionally preceded by the "Downloading … from " phrase the + * upgrader logs before fetching a package. No /u modifier on purpose: + * an invalid UTF-8 byte must not make preg_* bail out and hand back + * the unredacted text. + */ + private const URL_PATTERN = '~(Downloading\b[^\r\n]{0,80}?\bfrom\s+)?((?:https?|ftp)://[^\s<>"\'`]+)~i'; + + /** + * Recursively redact every string in a value (an update result, a + * messages list, action_log details). Keys and non-string scalars are + * left alone. + */ + public static function redact(mixed $value): mixed + { + if (is_string($value)) { + return self::redactText($value); + } + + if (is_array($value)) { + foreach ($value as $key => $item) { + $value[$key] = self::redact($item); + } + } + + return $value; + } + + public static function redactText(string $text): string + { + if ($text === '') { + return $text; + } + + $redacted = preg_replace_callback(self::URL_PATTERN, function (array $m): string { + [$url, $suffix] = self::splitTrailing($m[2]); + $isDownload = ($m[1] ?? '') !== ''; + + return $m[1].($isDownload ? self::redactDownloadUrl($url) : self::redactUrl($url)).$suffix; + }, $text); + + if ($redacted === null) { + // Regex engine gave up (backtrack limit, bad input). Fail closed. + $redacted = preg_replace('~(?:https?|ftp)://\S+~i', '[url '.self::REDACTED.']', $text); + if ($redacted === null) { + return self::REDACTED; + } + } + + // A JWT (or any base64url-encoded JSON object) starts with "eyJ". + $redacted = preg_replace('/eyJ[A-Za-z0-9_-]{8,}(?:\.[A-Za-z0-9_-]*){0,2}/', self::REDACTED, $redacted) ?? self::REDACTED; + $redacted = preg_replace('/\b(Bearer|Basic)\s+[A-Za-z0-9._~+\/=-]{8,}/i', '$1 '.self::REDACTED, $redacted) ?? self::REDACTED; + + return preg_replace( + '/(?]+/i', + '$1='.self::REDACTED, + $redacted, + ) ?? self::REDACTED; + } + + /** + * Strip credentials, query string and fragment from a URL and mask + * token-like path segments. Keeps scheme, host, port and the rest of + * the path so the URL stays useful for diagnosis. + */ + public static function redactUrl(string $url): string + { + $parts = parse_url($url); + if ($parts === false || ! isset($parts['scheme'], $parts['host'])) { + $url = preg_replace('~^([a-z][a-z0-9+.-]*://)[^/?#]*@~i', '$1', $url) ?? ''; + + return preg_replace('~[?#].*$~s', '', $url) ?? self::REDACTED; + } + + $segments = explode('/', (string) ($parts['path'] ?? '')); + foreach ($segments as $i => $segment) { + if ($segment !== '' && self::looksLikeSecret($segment)) { + $segments[$i] = self::REDACTED; + } + } + + return self::origin($parts).implode('/', $segments); + } + + public static function isErrorLine(string $line): bool + { + return preg_match('/^\s*Error:/i', $line) === 1; + } + + public static function isDownloadLine(string $line): bool + { + return preg_match('/^\s*Downloading\b/i', $line) === 1; + } + + /** + * The upgrader's real error: the last line starting with "Error:" plus + * the line just before it (unless that one is a download line), cleaned + * of HTML and redacted. Empty when the output holds no "Error:" line. + * + * @return list + */ + public static function errorLines(mixed $messages): array + { + if (is_string($messages)) { + $messages = preg_split('/\R/', $messages) ?: []; + } + if (! is_array($messages)) { + return []; + } + + $lines = []; + foreach ($messages as $message) { + if (! is_scalar($message)) { + continue; + } + $line = self::cleanLine((string) $message); + if ($line !== '') { + $lines[] = $line; + } + } + + for ($i = count($lines) - 1; $i >= 0; $i--) { + if (! self::isErrorLine($lines[$i])) { + continue; + } + + $out = []; + if ($i > 0 && ! self::isDownloadLine($lines[$i - 1])) { + $out[] = $lines[$i - 1]; + } + $out[] = $lines[$i]; + + return $out; + } + + return []; + } + + /** + * Map upgrader error lines (see errorLines()) to a reason code. + * + * @param list $lines + */ + public static function classify(array $lines): string + { + $text = strtolower(implode("\n", $lines)); + + return match (true) { + (bool) preg_match('/could not (copy|create|remove|delete|move|write)|could not access filesystem|filesystem error|unable to locate (wordpress|needed folder)|not writable|permission denied|could not be copied|inconsistent file permissions/', $text) => self::REASON_FS_PERMISSION, + (bool) preg_match('/unauthori[sz]ed|forbidden|not authori[sz]ed|\b40[13]\b|licen[cs]e|purchase code/', $text) => self::REASON_DOWNLOAD_UNAUTHORIZED, + (bool) preg_match('/pclzip|incompatible archive|central dir|not a valid zip|package could not be installed|contains no files|could not extract|unable to unzip/', $text) => self::REASON_BAD_PACKAGE, + (bool) preg_match('/valid url was not provided|package not available|no update package/', $text) => self::REASON_NO_PACKAGE, + default => self::REASON_UPGRADER_ERROR, + }; + } + + private static function cleanLine(string $line): string + { + $line = html_entity_decode(strip_tags($line), ENT_QUOTES | ENT_HTML5, 'UTF-8'); + $line = self::redactText($line); + $line = trim(preg_replace('/\s+/', ' ', $line) ?? ''); + + return mb_strimwidth($line, 0, self::MAX_LINE_LENGTH, '…'); + } + + /** + * Download URLs are where licence keys, vendor tokens and signatures + * live — in the query string and, for some vendors, in the path — so + * only the origin survives. + */ + private static function redactDownloadUrl(string $url): string + { + $parts = parse_url($url); + if ($parts === false || ! isset($parts['scheme'], $parts['host'])) { + return '[url '.self::REDACTED.']'; + } + + return self::origin($parts).'/'.self::REDACTED; + } + + /** + * @param array $parts + */ + private static function origin(array $parts): string + { + return strtolower((string) $parts['scheme']).'://'.$parts['host'] + .(isset($parts['port']) ? ':'.$parts['port'] : ''); + } + + /** + * Peel trailing punctuation and the upgrader's ellipsis off a matched + * URL, so "…/pkg.zip…" keeps its "…" after redaction. + * + * @return array{0: string, 1: string} + */ + private static function splitTrailing(string $url): array + { + $suffix = ''; + while (preg_match('/(…|…|\xE2\x80\xA6|\.\.\.|[.,;:!?)\]}])$/', $url, $m) === 1) { + $suffix = $m[1].$suffix; + $url = substr($url, 0, -strlen($m[1])); + } + + return [$url, $suffix]; + } + + /** + * A path segment that is (or contains) a JWT, or a long run of mixed + * letters and digits — the shape of licence keys, hashes and download + * tokens. Ordinary slugs and versioned zip names stay readable. + */ + private static function looksLikeSecret(string $segment): bool + { + $segment = rawurldecode($segment); + + if (preg_match('/eyJ[A-Za-z0-9_-]{8,}/', $segment) === 1) { + return true; + } + + foreach (preg_split('/[^A-Za-z0-9_-]+/', $segment) ?: [] as $run) { + $length = strlen($run); + if ($length < 12) { + continue; + } + $hasDigit = preg_match('/\d/', $run) === 1; + if (! $hasDigit) { + continue; + } + if ($length >= 16 && preg_match('/[A-Za-z]/', $run) === 1) { + return true; + } + if (preg_match('/[a-z]/', $run) === 1 && preg_match('/[A-Z]/', $run) === 1) { + return true; + } + } + + return false; + } +} diff --git a/database/migrations/2026_11_03_120000_add_reason_code_to_plugin_update_jobs_and_failure_streaks.php b/database/migrations/2026_11_03_120000_add_reason_code_to_plugin_update_jobs_and_failure_streaks.php new file mode 100644 index 00000000..61263f32 --- /dev/null +++ b/database/migrations/2026_11_03_120000_add_reason_code_to_plugin_update_jobs_and_failure_streaks.php @@ -0,0 +1,43 @@ +string('reason_code', 32)->nullable()->after('error'); + }); + + Schema::table('plugin_update_failure_streaks', function (Blueprint $table) { + $table->string('reason_code', 32)->nullable()->after('last_error'); + }); + } + + public function down(): void + { + Schema::table('plugin_update_failure_streaks', function (Blueprint $table) { + $table->dropColumn('reason_code'); + }); + + Schema::table('plugin_update_jobs', function (Blueprint $table) { + $table->dropColumn('reason_code'); + }); + } +}; diff --git a/resources/docs/features/updates.md b/resources/docs/features/updates.md index 10c85dc8..ddc17c9e 100644 --- a/resources/docs/features/updates.md +++ b/resources/docs/features/updates.md @@ -107,7 +107,15 @@ When a stubborn plugin or theme fails the nightly update queue night after night #### Threshold and Kill Switch - **Threshold**: Default 5 consecutive nightly failures (min 3, max 20), configurable via the `updates.auto_ignore_after_failures` setting. -- **Kill switch**: `updates.auto_ignore_enabled` (default `true`). When set to `false`, consecutive failure streaks are still tracked for operator visibility, but auto-ignore rows are never created. +- **Kill switch**: `updates.auto_ignore_enabled` (default `true`). When set to `false`, consecutive failure streaks are still tracked for operator visibility, but auto-ignore rows are never created (and the site-level pause below never fires). + +#### File-permission failures pause the site, not each plugin +Each streak stores the `reason_code` of its latest failure (see *Stalled no-op detection*). When a streak reaches the threshold with `reason_code = fs_permission` ("Could not copy file", "Could not create directory …/wp-content/upgrade/…") and no plugin or theme update on that site has succeeded since the streak began, the problem is the site's file ownership, not the plugin. Instead of auto-ignoring each plugin in turn, Control: +- sets `sites.auto_updates_paused = true` with a "Filesystem not writable…" reason (the same flag as the manual per-site pause, so the nightly loop skips the whole site); +- writes one `auto_updates_toggled` action log (`actor = auto`, `details.reason_code = fs_permission`); +- sends one ops chat alert. + +No per-plugin ignore is created and nothing is pushed to wp-admin. Fix the file ownership/permissions for the PHP user, then resume the site with the per-site auto-update toggle. If the site is already paused (by an operator or an earlier failure), nothing changes and no second alert is sent. If another update on the site did succeed during the streak, the plugin gets the normal per-plugin auto-ignore. #### Reset and Resume - Any **successful update** (nightly or manual) resets `consecutive_failures` back to 0. When it also lands on its target version, an auto-ignore for that slug is removed, the streak's pause markers are cleared, and the shorter exceptions list is pushed so the wp-admin notice clears. Manual ignores are never removed this way. @@ -231,6 +239,7 @@ WordPress's `validate_current_theme()` can fire during the filesystem replacemen |---|---|---| | `state_before` | JSON? | Active plugins + stylesheet/template captured from local snapshot before the update runs. | | `repairs` | JSON? | Array of `{type, slug, detail}` objects from the post-update verify call. `null` if no repairs were needed or the site doesn't support post-update-verify yet. | +| `reason_code` | string? | Cause of a failed job: `fs_permission`, `download_unauthorized`, `bad_package`, `no_package`, `not_offered` or `upgrader_error`. `null` for successes and for failures with no upgrader evidence (transport, reactivation). Also stored on `plugin_update_failure_streaks` for the latest counted failure. | ### Stalled no-op detection @@ -242,6 +251,21 @@ Root cause: for licensed plugins where update feeds may require an active licens This is a Laravel-side reinterpretation of Companion's response — no Companion plugin change or fleet redeploy involved. The underlying flakiness (a premium plugin's own update source going stale) isn't something this app can fix directly; it usually means the plugin's license/purchase code needs attention in that site's wp-admin. +**Upgrader errors are not "no longer offered".** Companion/Renegade builds in the field also return `ok: true` (and `upgrade_completed: true`) when the WordPress upgrader itself failed and logged an `Error: …` line. Inside the stalled-no-op branch, Control looks for that line in `messages`. When there is one, the job's `error` is the last `Error:` line plus the line before it (the detail, e.g. which file could not be copied), `upgrade_completed` is set to `false` so post-update verify/repair is skipped, and `reason_code` records the cause: + +| `reason_code` | Upgrader output | Usually means | +|---|---|---| +| `fs_permission` | "Could not copy file", "Could not create directory", "Could not remove the old …" | PHP cannot write `wp-content/plugins` or `wp-content/upgrade` (file ownership). A site problem: see the site-level pause above. | +| `download_unauthorized` | "Download failed. Unauthorized / Forbidden", licence wording | Premium licence inactive or expired on that site. | +| `bad_package` | PCLZIP errors, "The package could not be installed" | The vendor returned something that is not a zip (often a licence error page). | +| `no_package` | "A valid URL was not provided", "Update package not available" | WordPress had no download URL (unlicensed premium plugin). | +| `upgrader_error` | Any other `Error:` line | Read the error text. | +| `not_offered` | No `Error:` line at all | The original stalled no-op described above. | + +When Companion reports the failure itself (`ok: false`), its own error text is kept and `reason_code` is still derived from the upgrader output. The action-log summary follows the job's verdict, so a stalled no-op is logged as "Plugin update failed", not "Plugin updated: 1.0 → 1.0". + +**Download URLs are redacted.** The upgrader's "Downloading update from …" lines carry licence keys, vendor tokens/JWTs and signed download URLs. `App\Support\UpgraderOutput::redact()` runs on every update result before anything is stored or sent (`plugin_update_jobs.messages`/`error`, action-log details/summary/error and their Companion activity mirror, failure streaks, ignores, chat alerts): download URLs are cut to `scheme://host/[redacted]`, other URLs lose credentials, query string and fragment (token-like path segments are masked), and JWTs, bearer tokens and secret-named `key=value` pairs are masked anywhere. Download lines are never copied into error text. Rows written before this change still hold the unredacted lines. + ## What's NOT here - **Safe Update / backup-before-update / rollback**. Not building this. If an update breaks a site, recover from the host's normal backup (BlogVault, SpinupWP). diff --git a/tests/Feature/Controllers/SitesControllerTest.php b/tests/Feature/Controllers/SitesControllerTest.php index b6bc7159..78979421 100644 --- a/tests/Feature/Controllers/SitesControllerTest.php +++ b/tests/Feature/Controllers/SitesControllerTest.php @@ -939,6 +939,39 @@ function companionSite(array $overrides = []): Site $response->assertOk()->assertJsonPath('ok', true)->assertJsonPath('after_version', '5.4'); expect(ActionLog::query()->where('site_id', $site->id)->where('action_type', ActionLog::TYPE_PLUGIN_UPDATE)->exists())->toBeTrue(); }); + + it('keeps the licence key in the upgrader download line out of the action log and the response', function () { + $site = companionSite(['companion_capabilities' => ['updates']]); + $fakeLicense = 'FAKE-LICENSE-1234-5678-9012'; + + Http::fake([ + "https://{$site->domain}/wp-json/clockwork/v1/plugins/update*" => Http::response([ + 'ok' => true, + 'slug' => 'sample-pro/sample-pro.php', + 'before_version' => '2.0.0', + 'after_version' => '2.1.0', + 'was_active' => true, + 'reactivated' => true, + 'messages' => [ + "Downloading update from https://updates.example.com/?action=package_download&license={$fakeLicense}&domain=client.example.org…", + 'Unpacking the update…', + 'Plugin updated successfully.', + ], + 'elapsed_ms' => 1500, + ], 200, ['Content-Type' => 'application/json']), + ]); + + $response = $this->actingAs(User::factory()->create()) + ->post(route('sites.companion.plugin-update', $site), ['slug' => 'sample-pro/sample-pro.php']); + + $response->assertOk() + ->assertJsonPath('messages.0', 'Downloading update from https://updates.example.com/[redacted]…'); + expect($response->getContent())->not->toContain($fakeLicense); + + $log = ActionLog::query()->where('site_id', $site->id)->where('action_type', ActionLog::TYPE_PLUGIN_UPDATE)->first(); + expect($log->details['messages'][0])->toBe('Downloading update from https://updates.example.com/[redacted]…') + ->and(json_encode($log->details))->not->toContain($fakeLicense); + }); }); describe('flushPressableObjectCache', function () { diff --git a/tests/Feature/Jobs/UpdateJobPipelineTest.php b/tests/Feature/Jobs/UpdateJobPipelineTest.php index 2ecca2ca..a8c5b414 100644 --- a/tests/Feature/Jobs/UpdateJobPipelineTest.php +++ b/tests/Feature/Jobs/UpdateJobPipelineTest.php @@ -8,6 +8,7 @@ use App\Models\Site; use App\Services\ActionLog\ActionLogger; use App\Services\Chat\ChatNotifier; +use Illuminate\Http\Client\ConnectionException; use Illuminate\Support\Facades\Cache; use Illuminate\Support\Facades\Http; use Illuminate\Support\Str; @@ -192,6 +193,258 @@ function fakePluginUpdateEndpoint(Site $site, array $response, int $status = 200 }); }); +/* +| Realistic upgrader output as Companion returns it in `messages`. The +| licence key and signature are invented; hosts are reserved example +| domains. Field Companion/Renegade versions answer ok=true and +| upgrade_completed=true for these even though the upgrader failed. +*/ +const PIPELINE_FAKE_LICENSE = 'FAKE-LICENSE-9999-8888-7777'; + +function licensedDownloadLine(): string +{ + return 'Downloading update from https://updates.example.com/?action=package_download&license=' + .PIPELINE_FAKE_LICENSE.'&domain=client.example.org…'; +} + +/** + * @param list $messages + * @return array + */ +function stalledUpgraderResponse(array $messages, string $version = '1.0.0'): array +{ + return [ + 'ok' => true, + 'upgrade_completed' => true, + 'slug' => 'sample-plugin/sample-plugin.php', + 'before_version' => $version, + 'after_version' => $version, + 'was_active' => true, + 'reactivated' => true, + 'messages' => $messages, + 'elapsed_ms' => 900, + ]; +} + +function makeVerifyCapableSite(): Site +{ + // post-update-verify capability + a snapshot means verifyAndRepair() + // WOULD run whenever the job believes the upgrade completed. + return makeCompanionSite([ + 'companion_capabilities' => ['updates', 'post-update-verify'], + 'companion_snapshot' => [ + 'plugins' => ['plugins' => [ + ['slug' => 'sample-plugin/sample-plugin.php', 'name' => 'Sample Plugin', 'version' => '1.0.0', 'active' => true], + ]], + 'themes' => ['items' => []], + ], + 'companion_snapshot_at' => now(), + ]); +} + +function fakeUpdateAndVerify(Site $site, array $updateResponse): void +{ + Http::fake([ + "https://{$site->domain}/wp-json/clockwork/v1/plugins/update*" => Http::response($updateResponse, 200), + "https://{$site->domain}/wp-json/clockwork/v1/post-update-verify*" => Http::response(['ok' => true, 'repairs' => []], 200), + ]); +} + +describe('AbstractRunUpdate stalled no-op with a real upgrader error', function () { + it('records the upgrader error and fs_permission instead of "no longer offered", skips verify, and keeps the licence key out of everything it stores', function () { + $site = makeVerifyCapableSite(); + $job = PluginUpdateJob::factory()->for($site)->create([ + 'target_slug' => 'sample-plugin/sample-plugin.php', + 'target_name' => 'Sample Plugin', + 'before_version' => '1.0.0', + 'target_version' => '1.1.0', + 'batch_id' => 'nightly-'.now()->format('Y-m-d'), + ]); + + fakeUpdateAndVerify($site, stalledUpgraderResponse([ + licensedDownloadLine(), + 'Unpacking the update…', + 'Could not copy file. sample-plugin/includes/class-admin.php', + 'Error: Could not copy file.', + ])); + + $this->mock(ChatNotifier::class, function ($mock) { + $mock->shouldReceive('pluginUpdateFailed') + ->once() + ->withArgs(fn (Site $s, PluginUpdateJob $j) => str_contains((string) $j->error, 'Could not copy file. sample-plugin/includes/class-admin.php') + && ! str_contains((string) $j->error, PIPELINE_FAKE_LICENSE)) + ->andReturn(true); + }); + + (new RunPluginUpdate($job->id))->handle(app(ActionLogger::class)); + + $job->refresh(); + expect($job->status)->toBe(PluginUpdateJob::STATUS_FAILED) + ->and($job->reason_code)->toBe('fs_permission') + ->and($job->error)->toBe('Could not copy file. sample-plugin/includes/class-admin.php — Error: Could not copy file. (WordPress upgrader error; still on 1.0.0, expected 1.1.0.)') + ->and($job->error)->not->toContain('no longer offered') + ->and($job->messages[0])->toBe('Downloading update from https://updates.example.com/[redacted]…') + ->and(json_encode($job->messages))->not->toContain(PIPELINE_FAKE_LICENSE); + + // The upgrader failed, so there was no file swap to verify. + Http::assertNotSent(fn ($request) => str_contains($request->url(), '/post-update-verify')); + + $log = ActionLog::where('action_type', ActionLog::TYPE_PLUGIN_UPDATE)->where('site_id', $site->id)->first(); + expect($log->ok)->toBeFalse() + ->and($log->details['reason_code'])->toBe('fs_permission') + ->and($log->details['upgrade_completed'])->toBeFalse() + ->and($log->summary)->toContain('Could not copy file.') + ->and(json_encode($log->details))->not->toContain(PIPELINE_FAKE_LICENSE) + ->and($log->summary.$log->error)->not->toContain(PIPELINE_FAKE_LICENSE); + }); + + it('classifies a refused licensed download as download_unauthorized', function () { + $site = makeCompanionSite(); + $job = PluginUpdateJob::factory()->for($site)->create([ + 'target_slug' => 'sample-pro/sample-pro.php', + 'before_version' => '3.0.0', + 'target_version' => '4.0.0', + 'batch_id' => (string) Str::uuid(), + ]); + + fakeUpdateAndVerify($site, stalledUpgraderResponse([ + licensedDownloadLine(), + 'Download failed. Unauthorized', + 'Error: Download failed.', + ], '3.0.0')); + + (new RunPluginUpdate($job->id))->handle(app(ActionLogger::class)); + + $job->refresh(); + expect($job->status)->toBe(PluginUpdateJob::STATUS_FAILED) + ->and($job->reason_code)->toBe('download_unauthorized') + ->and($job->error)->toStartWith('Download failed. Unauthorized — Error: Download failed.') + ->and($job->error)->not->toContain(PIPELINE_FAKE_LICENSE); + }); + + it('never copies the download line into the error, even when it directly precedes the Error: line', function () { + $site = makeCompanionSite(); + $job = PluginUpdateJob::factory()->for($site)->create([ + 'target_slug' => 'sample-pro/sample-pro.php', + 'before_version' => '3.0.0', + 'target_version' => '4.0.0', + 'batch_id' => (string) Str::uuid(), + ]); + + fakeUpdateAndVerify($site, stalledUpgraderResponse([ + licensedDownloadLine(), + 'Error: Download failed.', + ], '3.0.0')); + + (new RunPluginUpdate($job->id))->handle(app(ActionLogger::class)); + + $job->refresh(); + expect($job->error)->toStartWith('Error: Download failed. (WordPress upgrader error;') + ->and($job->error)->not->toContain('Downloading') + ->and($job->error)->not->toContain('updates.example.com') + ->and($job->reason_code)->toBe('upgrader_error'); + }); + + it('keeps the "no longer offered" explanation, as reason not_offered, when the upgrader logged no error', function () { + $site = makeCompanionSite(); + $job = PluginUpdateJob::factory()->for($site)->create([ + 'target_slug' => 'js_composer/js_composer.php', + 'before_version' => '8.7.2', + 'target_version' => '9.0.1', + 'batch_id' => (string) Str::uuid(), + ]); + + fakeUpdateAndVerify($site, stalledUpgraderResponse(['Plugin is already up to date.'], '8.7.2')); + + (new RunPluginUpdate($job->id))->handle(app(ActionLogger::class)); + + $job->refresh(); + expect($job->status)->toBe(PluginUpdateJob::STATUS_FAILED) + ->and($job->reason_code)->toBe('not_offered') + ->and($job->error)->toContain("WordPress's own update-checker no longer offered this update") + ->and($job->error)->toContain('still on 8.7.2, expected 9.0.1'); + + // Logged as the failure it is, not as "Plugin updated: 8.7.2 → 8.7.2". + $log = ActionLog::where('action_type', ActionLog::TYPE_PLUGIN_UPDATE)->where('site_id', $site->id)->first(); + expect($log->ok)->toBeFalse() + ->and($log->summary)->toStartWith('Plugin update failed:'); + }); + + it('does not fail an update that reached its target just because an Error: line is present', function () { + $site = makeCompanionSite(); + $job = PluginUpdateJob::factory()->for($site)->create([ + 'target_slug' => 'sample-plugin/sample-plugin.php', + 'before_version' => '1.0.0', + 'target_version' => '1.1.0', + 'batch_id' => (string) Str::uuid(), + ]); + + fakeUpdateAndVerify($site, array_merge(stalledUpgraderResponse([ + licensedDownloadLine(), + 'Error: Could not remove the old translation file.', + 'Plugin updated successfully.', + ]), ['after_version' => '1.1.0'])); + + (new RunPluginUpdate($job->id))->handle(app(ActionLogger::class)); + + $job->refresh(); + expect($job->status)->toBe(PluginUpdateJob::STATUS_COMPLETE) + ->and($job->error)->toBeNull() + ->and($job->reason_code)->toBeNull() + ->and(json_encode($job->messages))->not->toContain(PIPELINE_FAKE_LICENSE); + }); + + it('keeps Companion\'s own error when it reports ok=false, but still records the upgrader reason code', function () { + $site = makeCompanionSite(); + $job = PluginUpdateJob::factory()->for($site)->create([ + 'target_slug' => 'sample-plugin/sample-plugin.php', + 'before_version' => '1.0.0', + 'target_version' => '1.1.0', + 'batch_id' => (string) Str::uuid(), + ]); + + fakeUpdateAndVerify($site, array_merge(stalledUpgraderResponse([ + licensedDownloadLine(), + 'Unpacking the update…', + 'Could not create directory. /var/www/html/wp-content/upgrade/sample-plugin_1.1.0-AbC12d-x9Yz', + 'Error: Could not create directory.', + ]), ['ok' => false, 'upgrade_completed' => false, 'error' => 'Could not create directory.'])); + + (new RunPluginUpdate($job->id))->handle(app(ActionLogger::class)); + + $job->refresh(); + expect($job->status)->toBe(PluginUpdateJob::STATUS_FAILED) + ->and($job->error)->toBe('Could not create directory.') + ->and($job->reason_code)->toBe('fs_permission'); + }); + + it('redacts URLs quoted in a transport exception message', function () { + $site = makeCompanionSite(); + $job = PluginUpdateJob::factory()->for($site)->create([ + 'target_slug' => 'sample-plugin/sample-plugin.php', + 'batch_id' => (string) Str::uuid(), + ]); + + Http::fake([ + "https://{$site->domain}/wp-json/clockwork/v1/plugins/update*" => function () { + throw new ConnectionException( + 'cURL error 28 while fetching https://updates.example.com/pkg.zip?license='.PIPELINE_FAKE_LICENSE + ); + }, + ]); + + (new RunPluginUpdate($job->id))->handle(app(ActionLogger::class)); + + $job->refresh(); + expect($job->status)->toBe(PluginUpdateJob::STATUS_FAILED) + ->and($job->error)->toContain('cURL error 28') + ->and($job->error)->not->toContain(PIPELINE_FAKE_LICENSE); + + $log = ActionLog::where('site_id', $site->id)->first(); + expect($log->summary.$log->error)->not->toContain(PIPELINE_FAKE_LICENSE); + }); +}); + describe('AbstractRunUpdate per-site locking', function () { it('re-queues with a 15s delay instead of running when another update already holds the site lock', function () { $site = makeCompanionSite(); diff --git a/tests/Feature/Updates/UpdateFailureStreakTest.php b/tests/Feature/Updates/UpdateFailureStreakTest.php index 56cea9b5..ca6cad48 100644 --- a/tests/Feature/Updates/UpdateFailureStreakTest.php +++ b/tests/Feature/Updates/UpdateFailureStreakTest.php @@ -11,10 +11,12 @@ use App\Models\Site; use App\Models\User; use App\Services\ActionLog\ActionLogger; +use App\Services\Chat\ChatNotifier; use App\Services\Updates\UpdateFailureStreakRecorder; use App\Support\IssueCounter; use App\Support\Settings; use Illuminate\Http\Client\ConnectionException; +use Illuminate\Http\Client\Request; use Illuminate\Support\Facades\Http; use Illuminate\Support\Facades\Queue; use Illuminate\Support\Str; @@ -49,6 +51,43 @@ function fakeNightlyJob(Site $site, string $slug = 'stubborn-plugin', array $ove ], $overrides)); } +/* +| Upgrader output as Companion returns it for a failed upgrade it still +| reports as ok=true (before == after, below the target). The licence key +| is invented; hosts are reserved example domains. +*/ +const STREAK_FAKE_LICENSE = 'FAKE-LICENSE-5555-4444-3333'; + +/** + * @param list $messages + * @return array + */ +function stalledWithUpgraderError(array $messages): array +{ + return [ + 'ok' => true, + 'upgrade_completed' => true, + 'before_version' => '1.0.0', + 'after_version' => '1.0.0', + 'was_active' => true, + 'reactivated' => true, + 'messages' => $messages, + ]; +} + +/** + * @return list + */ +function copyFailedMessages(string $slug): array +{ + return [ + 'Downloading update from https://downloads.example.org/packages/'.$slug.'.1.1.0.zip?license='.STREAK_FAKE_LICENSE.'…', + 'Unpacking the update…', + "Could not copy file. {$slug}/includes/class-main.php", + 'Error: Could not copy file.', + ]; +} + describe('Update failure streaks and auto-ignore', function () { it('increments streak on nightly failure with companion error response, does not auto-ignore before threshold', function () { $site = makeExampleSite(); @@ -768,6 +807,35 @@ function fakeNightlyJob(Site $site, string $slug = 'stubborn-plugin', array $ove expect($payload['items'])->toBeEmpty(); }); + it('stores the reason code and a redacted last_error on the streak and the auto-ignore', function () { + $site = makeExampleSite(); + + Http::fake([ + "https://{$site->domain}/wp-json/clockwork/v1/plugins/update*" => Http::response(stalledWithUpgraderError([ + 'Downloading update from https://vendor.example.net/download?license_key='.STREAK_FAKE_LICENSE.'…', + 'Download failed. Unauthorized', + 'Error: Download failed.', + ]), 200), + ]); + + for ($i = 1; $i <= 5; $i++) { + $job = fakeNightlyJob($site, 'licensed-plugin'); + (new RunPluginUpdate($job->id))->handle(app(ActionLogger::class)); + } + + $streak = PluginUpdateFailureStreak::where('site_id', $site->id)->where('target_slug', 'licensed-plugin')->first(); + expect($streak->reason_code)->toBe('download_unauthorized') + ->and($streak->last_error)->toStartWith('Download failed. Unauthorized — Error: Download failed.') + ->and($streak->last_error)->not->toContain(STREAK_FAKE_LICENSE); + + // A licence problem is specific to the plugin: per-plugin pause as before. + $ignore = PluginUpdateIgnore::where('site_id', $site->id)->where('target_slug', 'licensed-plugin')->first(); + expect($ignore)->not->toBeNull() + ->and($ignore->source)->toBe(PluginUpdateIgnore::SOURCE_AUTO_FAILURE) + ->and($ignore->last_error)->not->toContain(STREAK_FAKE_LICENSE) + ->and($site->refresh()->auto_updates_paused)->toBeFalse(); + }); + it('pushes an updated exceptions list when an auto-ignore is converted to a manual ignore', function () { $site = makeExampleSite(['companion_installed' => true]); $user = User::factory()->create(); @@ -804,3 +872,169 @@ function fakeNightlyJob(Site $site, string $slug = 'stubborn-plugin', array $ove }); }); }); + +describe('File-permission failures pause the site, not each plugin', function () { + it('pauses the whole site once, with one action log and one ops alert, instead of auto-ignoring each plugin', function () { + $site = makeExampleSite(); + + Http::fake([ + "https://{$site->domain}/wp-json/clockwork/v1/plugins/update*" => fn (Request $request) => Http::response( + stalledWithUpgraderError(copyFailedMessages((string) $request['slug'])), + 200, + ), + ]); + + $this->mock(ChatNotifier::class, function ($mock) use ($site) { + $mock->shouldReceive('pluginUpdateFailed')->andReturn(true); + $mock->shouldReceive('pluginUpdateAutoIgnored')->never(); + $mock->shouldReceive('send') + ->once() + ->withArgs(fn (string $text, array $attachments) => str_contains($text, $site->domain) + && str_contains($text, 'filesystem not writable') + && ! str_contains((string) json_encode($attachments), STREAK_FAKE_LICENSE)) + ->andReturn(true); + }); + + // Two plugins on the same site fail the same way every night; both + // cross the threshold on night 5. + for ($night = 1; $night <= 5; $night++) { + foreach (['alpha-plugin', 'beta-plugin'] as $slug) { + $job = fakeNightlyJob($site, $slug); + (new RunPluginUpdate($job->id))->handle(app(ActionLogger::class)); + } + } + + $site->refresh(); + expect($site->auto_updates_paused)->toBeTrue() + ->and($site->auto_updates_paused_reason)->toStartWith('Filesystem not writable'); + + // No per-plugin pauses, so nothing client-visible either. + expect(PluginUpdateIgnore::where('site_id', $site->id)->count())->toBe(0) + ->and(ActionLog::where('site_id', $site->id)->where('action_type', ActionLog::TYPE_PLUGIN_UPDATE_AUTO_IGNORED)->count())->toBe(0); + + foreach (['alpha-plugin', 'beta-plugin'] as $slug) { + $streak = PluginUpdateFailureStreak::where('site_id', $site->id)->where('target_slug', $slug)->first(); + expect($streak->consecutive_failures)->toBe(5) + ->and($streak->reason_code)->toBe('fs_permission') + ->and($streak->ignored_at)->toBeNull() + ->and($streak->last_error)->toContain("Could not copy file. {$slug}/includes/class-main.php") + ->and($streak->last_error)->not->toContain(STREAK_FAKE_LICENSE); + } + + $pauseLogs = ActionLog::where('site_id', $site->id)->where('action_type', ActionLog::TYPE_AUTO_UPDATES_TOGGLED)->get(); + expect($pauseLogs)->toHaveCount(1) + ->and($pauseLogs[0]->actor)->toBe('auto') + ->and($pauseLogs[0]->target)->toBe('paused') + ->and($pauseLogs[0]->details['reason_code'])->toBe('fs_permission') + ->and($pauseLogs[0]->details['now'])->toBeTrue() + ->and((string) json_encode($pauseLogs[0]->details))->not->toContain(STREAK_FAKE_LICENSE); + + // A paused site drops out of the nightly loop's candidate query. + Queue::fake(); + $site->forceFill([ + 'companion_installed' => true, + 'companion_snapshot' => [ + 'plugins' => ['plugins' => [ + ['slug' => 'alpha-plugin', 'name' => 'Alpha', 'version' => '1.0.0', 'new_version' => '1.1.0', 'update_available' => true], + ]], + 'themes' => ['items' => []], + ], + ])->save(); + $this->artisan('clockwork:run-nightly-plugin-updates')->assertSuccessful(); + Queue::assertNothingPushed(); + }); + + it('keeps the per-plugin auto-ignore when another update on the site succeeded during the streak', function () { + $site = makeExampleSite(); + + Http::fake([ + "https://{$site->domain}/wp-json/clockwork/v1/plugins/update*" => Http::response( + stalledWithUpgraderError(copyFailedMessages('alpha-plugin')), + 200, + ), + ]); + + $this->mock(ChatNotifier::class, function ($mock) { + $mock->shouldReceive('pluginUpdateFailed')->andReturn(true); + $mock->shouldReceive('pluginUpdateAutoIgnored')->once()->andReturn(true); + $mock->shouldReceive('send')->never(); + }); + + for ($night = 1; $night <= 5; $night++) { + $job = fakeNightlyJob($site, 'alpha-plugin'); + (new RunPluginUpdate($job->id))->handle(app(ActionLogger::class)); + + if ($night === 1) { + // Another plugin on the same site updated fine: the site's + // files are writable, only this plugin's are not. + PluginUpdateJob::factory()->for($site)->complete()->create([ + 'batch_id' => 'nightly-'.now()->format('Y-m-d'), + 'target_slug' => 'healthy-plugin', + 'before_version' => '2.0.0', + 'after_version' => '2.1.0', + 'target_version' => '2.1.0', + ]); + } + } + + expect($site->refresh()->auto_updates_paused)->toBeFalse(); + + $ignore = PluginUpdateIgnore::where('site_id', $site->id)->where('target_slug', 'alpha-plugin')->first(); + expect($ignore)->not->toBeNull() + ->and($ignore->source)->toBe(PluginUpdateIgnore::SOURCE_AUTO_FAILURE); + }); + + it('leaves a site that is already paused alone: same reason, no log, no alert, no per-plugin ignore', function () { + $site = makeExampleSite([ + 'auto_updates_paused' => true, + 'auto_updates_paused_reason' => 'Paused by an operator during a redesign', + ]); + + $this->mock(ChatNotifier::class, function ($mock) { + $mock->shouldReceive('send')->never(); + $mock->shouldReceive('pluginUpdateAutoIgnored')->never(); + }); + + $error = 'Could not copy file. alpha-plugin/includes/class-main.php — Error: Could not copy file.'; + $job = fakeNightlyJob($site, 'alpha-plugin', [ + 'status' => PluginUpdateJob::STATUS_FAILED, + 'error' => $error, + 'reason_code' => 'fs_permission', + ]); + + $recorder = app(UpdateFailureStreakRecorder::class); + for ($i = 1; $i <= 5; $i++) { + $recorder->record($job, ['ok' => true, 'error' => $error, 'reason_code' => 'fs_permission']); + } + + $site->refresh(); + expect($site->auto_updates_paused)->toBeTrue() + ->and($site->auto_updates_paused_reason)->toBe('Paused by an operator during a redesign') + ->and(ActionLog::where('site_id', $site->id)->where('action_type', ActionLog::TYPE_AUTO_UPDATES_TOGGLED)->count())->toBe(0) + ->and(PluginUpdateIgnore::where('site_id', $site->id)->count())->toBe(0); + }); + + it('honors the auto-pause kill switch for the site-level pause too', function () { + $site = makeExampleSite(); + app(Settings::class)->put('updates.auto_ignore_enabled', false); + + $this->mock(ChatNotifier::class, function ($mock) { + $mock->shouldReceive('send')->never(); + }); + + $error = 'Could not create directory. /var/www/html/wp-content/upgrade/alpha-plugin_1.1.0-AbC12d — Error: Could not create directory.'; + $job = fakeNightlyJob($site, 'alpha-plugin', [ + 'status' => PluginUpdateJob::STATUS_FAILED, + 'error' => $error, + 'reason_code' => 'fs_permission', + ]); + + $recorder = app(UpdateFailureStreakRecorder::class); + for ($i = 1; $i <= 6; $i++) { + $recorder->record($job, ['ok' => true, 'error' => $error, 'reason_code' => 'fs_permission']); + } + + expect($site->refresh()->auto_updates_paused)->toBeFalse() + ->and(PluginUpdateFailureStreak::where('site_id', $site->id)->value('consecutive_failures'))->toBe(6); + }); +}); diff --git a/tests/Unit/Support/UpgraderOutputTest.php b/tests/Unit/Support/UpgraderOutputTest.php new file mode 100644 index 00000000..d52b3b61 --- /dev/null +++ b/tests/Unit/Support/UpgraderOutputTest.php @@ -0,0 +1,235 @@ +…" with +| the vendor's (often licensed or signed) package URL, then the outcome, and +| on failure a detail line followed by "Error: …". +*/ + +const FAKE_LICENSE = 'FAKE-LICENSE-0000-1111-2222'; +const FAKE_JWT = 'eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJsaWNlbnNlIjoiZmFrZSJ9.ZmFrZXNpZ25hdHVyZQ'; +const FAKE_AWS_KEY = 'AKIAFAKEFAKEFAKE0000'; +const FAKE_SIGNATURE = 'ZmFrZXNpZ25hdHVyZTEyMw%3D%3D'; + +function fakeLicensedDownloadLine(): string +{ + return 'Downloading update from https://updates.example.com/?action=package_download&license='.FAKE_LICENSE.'&domain=client.example.org&version=2.0.0…'; +} + +function fakeJwtPathDownloadLine(): string +{ + return 'Downloading update from https://vendor.example.net/api/v2/plugins/download/'.FAKE_JWT.'/sample_pro…'; +} + +function fakeSignedDownloadLine(): string +{ + return 'Downloading update from https://downloads.example.org/packages/sample-plugin/sample-plugin_2.0.0.zip?AWSAccessKeyId='.FAKE_AWS_KEY.'&Expires=1790000000&Signature='.FAKE_SIGNATURE.'…'; +} + +describe('UpgraderOutput::redactText', function () { + it('cuts a licensed download URL down to its origin and keeps the ellipsis', function () { + $out = UpgraderOutput::redactText(fakeLicensedDownloadLine()); + + expect($out)->toBe('Downloading update from https://updates.example.com/[redacted]…') + ->not->toContain(FAKE_LICENSE) + ->not->toContain('client.example.org'); + }); + + it('drops a JWT carried in the download path', function () { + $out = UpgraderOutput::redactText(fakeJwtPathDownloadLine()); + + expect($out)->toBe('Downloading update from https://vendor.example.net/[redacted]…') + ->not->toContain('eyJ'); + }); + + it('drops the signature and access key of a signed download URL', function () { + $out = UpgraderOutput::redactText(fakeSignedDownloadLine()); + + expect($out)->not->toContain(FAKE_AWS_KEY) + ->not->toContain('ZmFrZXNpZ25hdHVyZTEyMw') + ->toContain('https://downloads.example.org/[redacted]'); + }); + + it('handles a download URL after the ellipsis entity has been decoded', function () { + $decoded = html_entity_decode(fakeLicensedDownloadLine(), ENT_QUOTES | ENT_HTML5, 'UTF-8'); + + expect(UpgraderOutput::redactText($decoded)) + ->toBe('Downloading update from https://updates.example.com/[redacted]…'); + }); + + it('leaves a download line without a URL untouched', function () { + expect(UpgraderOutput::redactText('Downloading update from 1…')) + ->toBe('Downloading update from 1…'); + }); + + it('keeps the path of other URLs but strips credentials, query string and fragment', function () { + $out = UpgraderOutput::redactText('See https://admin:hunter2@example.com/wp-admin/update-core.php?_wpnonce=abc123&key=SECRETVALUE#top for details.'); + + expect($out)->toBe('See https://example.com/wp-admin/update-core.php for details.'); + }); + + it('masks token-like path segments in other URLs but keeps ordinary slugs', function () { + $out = UpgraderOutput::redactText('Fetched https://example.com/dl/9f86d081884c7d659a2feaa0c55ad015a3bf4f1b/plugin-name/readme.txt'); + + expect($out)->toBe('Fetched https://example.com/dl/[redacted]/plugin-name/readme.txt'); + }); + + it('masks secret-named key=value pairs, JWTs and bearer tokens outside URLs', function () { + $out = UpgraderOutput::redactText('retrying with license_key=ABCD1234EFGH&purchase_code=0000-1111 token '.FAKE_JWT.' header Bearer abcdef0123456789'); + + expect($out)->not->toContain('ABCD1234EFGH') + ->not->toContain('0000-1111') + ->not->toContain('eyJ') + ->not->toContain('abcdef0123456789') + ->toContain('license_key=[redacted]') + ->toContain('purchase_code=[redacted]') + ->toContain('Bearer [redacted]'); + }); + + it('leaves ordinary upgrader lines alone', function () { + foreach ([ + 'Unpacking the update…', + 'Could not copy file. sample-plugin/includes/class-admin.php', + 'Error: Could not copy file.', + 'Plugin updated successfully.', + 'The package could not be installed. PCLZIP_ERR_BAD_FORMAT (-10) : Unable to find End of Central Dir Record signature', + ] as $line) { + expect(UpgraderOutput::redactText($line))->toBe($line); + } + }); +}); + +describe('UpgraderOutput::redact', function () { + it('redacts every string in a nested update result, keeping keys and non-strings', function () { + $result = [ + 'ok' => true, + 'slug' => 'sample-plugin/sample-plugin.php', + 'before_version' => '1.0.0', + 'elapsed_ms' => 1200, + 'messages' => [fakeLicensedDownloadLine(), 'Unpacking the update…'], + 'nested' => ['url' => 'https://example.com/hook?token=SECRETVALUE'], + ]; + + $out = UpgraderOutput::redact($result); + + expect($out['ok'])->toBeTrue() + ->and($out['elapsed_ms'])->toBe(1200) + ->and($out['slug'])->toBe('sample-plugin/sample-plugin.php') + ->and($out['messages'][0])->toBe('Downloading update from https://updates.example.com/[redacted]…') + ->and($out['messages'][1])->toBe('Unpacking the update…') + ->and($out['nested']['url'])->toBe('https://example.com/hook') + ->and(json_encode($out))->not->toContain(FAKE_LICENSE) + ->and(json_encode($out))->not->toContain('SECRETVALUE'); + }); + + it('passes null through', function () { + expect(UpgraderOutput::redact(null))->toBeNull(); + }); +}); + +describe('UpgraderOutput::errorLines', function () { + it('returns the last Error: line and the detail line before it, decoded', function () { + $lines = UpgraderOutput::errorLines([ + fakeSignedDownloadLine(), + 'Unpacking the update…', + 'Could not copy file. sample-plugin/includes/class-admin.php', + 'Error: Could not copy file.', + ]); + + expect($lines)->toBe([ + 'Could not copy file. sample-plugin/includes/class-admin.php', + 'Error: Could not copy file.', + ]); + }); + + it('never returns the download line, even when it directly precedes the error', function () { + $lines = UpgraderOutput::errorLines([ + fakeLicensedDownloadLine(), + 'Error: Download failed.', + ]); + + expect($lines)->toBe(['Error: Download failed.']) + ->and(implode("\n", $lines))->not->toContain(FAKE_LICENSE); + }); + + it('picks the last Error: line when there are several', function () { + $lines = UpgraderOutput::errorLines([ + 'Error: First problem.', + 'Retrying…', + 'Update package not available.', + 'Error: Update package not available.', + ]); + + expect($lines)->toBe(['Update package not available.', 'Error: Update package not available.']); + }); + + it('returns nothing when the upgrader logged no error', function () { + expect(UpgraderOutput::errorLines(['Plugin is already up to date.']))->toBe([]) + ->and(UpgraderOutput::errorLines([]))->toBe([]) + ->and(UpgraderOutput::errorLines(null))->toBe([]); + }); + + it('skips non-string entries', function () { + expect(UpgraderOutput::errorLines([['nested'], 'Download failed. Forbidden', 'Error: Download failed.'])) + ->toBe(['Download failed. Forbidden', 'Error: Download failed.']); + }); +}); + +describe('UpgraderOutput::classify', function () { + it('maps upgrader errors to reason codes', function (array $messages, string $expected) { + expect(UpgraderOutput::classify(UpgraderOutput::errorLines($messages)))->toBe($expected); + })->with([ + 'copy failed' => [[ + fakeSignedDownloadLine(), + 'Unpacking the update…', + 'Could not copy file. sample-plugin/includes/class-admin.php', + 'Error: Could not copy file.', + ], UpgraderOutput::REASON_FS_PERMISSION], + 'upgrade dir not creatable' => [[ + fakeSignedDownloadLine(), + 'Unpacking the update…', + 'Could not create directory. /var/www/html/wp-content/upgrade/sample-plugin_2.0.0-AbC12d-x9Yz', + 'Error: Could not create directory.', + ], UpgraderOutput::REASON_FS_PERMISSION], + 'licence refused' => [[ + fakeJwtPathDownloadLine(), + 'Download failed. Unauthorized', + 'Error: Download failed.', + ], UpgraderOutput::REASON_DOWNLOAD_UNAUTHORIZED], + 'forbidden' => [[ + fakeLicensedDownloadLine(), + 'Download failed. Forbidden', + 'Error: Download failed.', + ], UpgraderOutput::REASON_DOWNLOAD_UNAUTHORIZED], + 'not a zip' => [[ + fakeLicensedDownloadLine(), + 'Unpacking the update…', + 'The package could not be installed. PCLZIP_ERR_BAD_FORMAT (-10) : Unable to find End of Central Dir Record signature', + 'Error: The package could not be installed.', + ], UpgraderOutput::REASON_BAD_PACKAGE], + 'no download URL' => [[ + 'Downloading update from 1…', + 'Download failed. A valid URL was not provided.', + 'Error: Download failed.', + ], UpgraderOutput::REASON_NO_PACKAGE], + 'no package' => [[ + 'Update package not available.', + 'Error: Update package not available.', + ], UpgraderOutput::REASON_NO_PACKAGE], + 'anything else' => [[ + fakeSignedDownloadLine(), + 'Download failed. cURL error 28: Operation timed out after 30001 milliseconds', + 'Error: Download failed.', + ], UpgraderOutput::REASON_UPGRADER_ERROR], + ]); +}); From 6d5dfe0b0505a9a2cb3b345d71c9cf0834766db2 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 3 Nov 2026 20:26:42 +0000 Subject: [PATCH 07/64] fix(backup-relay): require backup-create, keep per-site failures, drop stale hashes A site whose Companion build predates the backup-create capability could have off-site relay switched on (enrolment turned it on for every standalone site), and then failed every night with an error nobody could see: the direct-only pass only logs and chats its failures, and its output is not stored. - CompanionBackupRelayAdapter refuses the direct upload before calling the plugin when backup-create is missing, with an actionable message ("Companion lacks the backup-create capability; update Companion or switch to Renegade"). The run counts the site as failed instead of skipping it silently. - Relay can no longer be switched on for such a site: the per-site toggle returns 422 / a warning, the settings single toggle refuses it, the bulk form enables the other sites and lists the refused ones, and enrolment only turns relay on when /health lists backup-create. A site that is already on stays on so its failures stay visible. - New sites columns backup_relay_last_attempt_at, backup_relay_last_error and backup_relay_failing_since. ArchiveSiteBackupJob records every failure (also for queued runs), RunBackupRelayNow records failures in its per-site catch, and a successful archive clears them. The site Backups card shows the error and the start of the failure streak, and Backup Now stops polling as soon as a failure is recorded. - markArchived now sets backup_relay_last_sha256 to null when an archive is recorded without a hash (an object adopted after a timeout or gateway error), so a previous archive's hash is never paired with a newer key and one-click restore cannot fail the checksum on it. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Q7rExNMgxzTNb3pagVUZrg --- .../BackupRelaySettingsController.php | 51 ++++- app/Http/Controllers/SitesController.php | 34 +++- app/Models/Site.php | 8 + ...kup_relay_attempt_state_to_sites_table.php | 40 ++++ .../src/Console/RunBackupRelayNow.php | 5 + .../src/Jobs/ArchiveSiteBackupJob.php | 25 ++- .../Services/CompanionBackupRelayAdapter.php | 54 +++++ .../src/Services/RelayAttemptRecorder.php | 53 +++++ resources/docs/features/backup-relay.md | 6 +- .../site/widgets/_widget-backups.blade.php | 56 ++++- .../views/settings/backup-relay.blade.php | 9 + .../BackupRelayAttemptStateTest.php | 192 ++++++++++++++++++ .../BackupRelay/CompanionBackupRelayTest.php | 74 +++++++ .../SiteBackupRelayScheduleTest.php | 76 +++++++ .../BackupRelaySettingsControllerTest.php | 103 ++++++++++ tests/Feature/SiteOverviewDashboardTest.php | 58 ++++++ tests/Feature/Sites/AddStandaloneSiteTest.php | 38 +++- 17 files changed, 861 insertions(+), 21 deletions(-) create mode 100644 database/migrations/2026_11_03_120000_add_backup_relay_attempt_state_to_sites_table.php create mode 100644 modules/BackupRelay/src/Services/RelayAttemptRecorder.php create mode 100644 tests/Feature/BackupRelay/BackupRelayAttemptStateTest.php diff --git a/app/Http/Controllers/Settings/BackupRelaySettingsController.php b/app/Http/Controllers/Settings/BackupRelaySettingsController.php index 9336046e..c03f5ce7 100644 --- a/app/Http/Controllers/Settings/BackupRelaySettingsController.php +++ b/app/Http/Controllers/Settings/BackupRelaySettingsController.php @@ -16,6 +16,7 @@ use Illuminate\Support\Facades\Auth; use Illuminate\View\View; use Modules\BackupRelay\Services\BackupArchiveEnumerator; +use Modules\BackupRelay\Services\CompanionBackupRelayAdapter; use Modules\Core\Contracts\HostingProvider; use Throwable; @@ -126,6 +127,22 @@ public function update(Request $request, Settings $settings): RedirectResponse|J ]); $site = Site::query()->findOrFail($validated['site_id']); + + if ((bool) $validated['enabled'] && ! $site->backup_relay_enabled + && ($blocked = CompanionBackupRelayAdapter::enableBlockedReason($site)) !== null) { + $msg = "Backup relay was not enabled for {$site->domain}: {$blocked}"; + + if ($request->wantsJson() || $request->ajax()) { + return response()->json([ + 'success' => false, + 'message' => $msg, + 'enabled' => false, + ], 422); + } + + return back()->with('warning', $msg); + } + $site->update([ 'backup_relay_enabled' => (bool) $validated['enabled'], ]); @@ -158,10 +175,26 @@ public function update(Request $request, Settings $settings): RedirectResponse|J 'sites.*' => 'integer|exists:sites,id', ]); - $enabledIds = $validated['sites'] ?? []; + $enabledIds = array_map('intval', $validated['sites'] ?? []); - Site::query() + // Only sites being switched on are checked. A site that is already on + // stays on, so its failing runs keep showing up instead of going quiet. + $blocked = Site::query() ->whereIn('id', $enabledIds) + ->where('backup_relay_enabled', false) + ->get() + ->map(fn (Site $site) => [ + 'site_id' => $site->id, + 'domain' => $site->domain, + 'reason' => CompanionBackupRelayAdapter::enableBlockedReason($site), + ]) + ->filter(fn (array $row) => $row['reason'] !== null) + ->values(); + + $toEnable = array_values(array_diff($enabledIds, $blocked->pluck('site_id')->all())); + + Site::query() + ->whereIn('id', $toEnable) ->update(['backup_relay_enabled' => true]); Site::query() @@ -177,17 +210,23 @@ public function update(Request $request, Settings $settings): RedirectResponse|J } } - $msg = 'Backup relay settings updated for '.count($enabledIds).' site(s).'; + $msg = 'Backup relay settings updated for '.count($toEnable).' site(s).'; + $blockedMsg = $blocked->isEmpty() ? null : 'Not enabled: '.$blocked + ->map(fn (array $row) => "{$row['domain']} ({$row['reason']})") + ->implode('; '); if ($request->wantsJson() || $request->ajax()) { return response()->json([ 'success' => true, - 'message' => $msg, - 'enabled_count' => count($enabledIds), + 'message' => $blockedMsg === null ? $msg : "{$msg} {$blockedMsg}", + 'enabled_count' => count($toEnable), + 'blocked' => $blocked->all(), ]); } - return back()->with('status', $msg); + $redirect = back()->with('status', $msg); + + return $blockedMsg === null ? $redirect : $redirect->with('warning', $blockedMsg); } public function runNow(Request $request, Settings $settings): RedirectResponse|JsonResponse diff --git a/app/Http/Controllers/SitesController.php b/app/Http/Controllers/SitesController.php index 49c11013..462c33f2 100644 --- a/app/Http/Controllers/SitesController.php +++ b/app/Http/Controllers/SitesController.php @@ -51,6 +51,7 @@ use Illuminate\Validation\Rule; use Illuminate\View\View; use Modules\BackupRelay\Services\BackupArchiveEnumerator; +use Modules\BackupRelay\Services\CompanionBackupRelayAdapter; use Modules\ClientReports\Services\ReportRecipientsSyncService; use Modules\Core\Contracts\HostingProvider; use Modules\Core\ModuleStateResolver; @@ -2198,6 +2199,9 @@ public function backupsHistory(Site $site, SpacesClient $spaces): JsonResponse 'last_archived_at' => $site->backup_relay_last_archived_at?->toIso8601String(), 'last_size_bytes' => $site->backup_relay_last_size_bytes, 'next_scheduled_at' => $site->backupRelayNextScheduledAt()?->toIso8601String(), + 'last_attempt_at' => $site->backup_relay_last_attempt_at?->toIso8601String(), + 'last_error' => $site->backup_relay_last_error, + 'failing_since' => $site->backup_relay_failing_since?->toIso8601String(), ], ]); } @@ -2349,7 +2353,9 @@ public function store(Request $request): RedirectResponse 'is_multisite' => $isMultisite, 'uptime_monitoring_enabled' => true, 'cert_source' => Site::CERT_SOURCE_LIVE_PROBE, - 'backup_relay_enabled' => true, + // Off-site backups need the plugin's /backup/create. Turning + // relay on without it only produces a failed backup every day. + 'backup_relay_enabled' => CompanionBackupRelayAdapter::supportsBackupCreate($caps), 'backup_relay_frequency' => 'daily', // Schema default is paused (opt-in flip). SpinupWP/Pressable // imports unpause on create; standalone enroll must do the same @@ -2413,7 +2419,12 @@ public function store(Request $request): RedirectResponse $this->kickEnrollmentSecurityBaseline($site); - return redirect()->route('sites.show', $site)->with('flash', "Site {$site->domain} successfully connected and enrolled!"); + $flash = "Site {$site->domain} successfully connected and enrolled!"; + if (! $site->backup_relay_enabled) { + $flash .= ' Off-site backups were not turned on: '.CompanionBackupRelayAdapter::missingBackupCreateMessage($site); + } + + return redirect()->route('sites.show', $site)->with('flash', $flash); } /** @@ -2476,6 +2487,25 @@ public function updateBackupRelay(Site $site, Request $request, ActionLogger $lo $previousEnabled = (bool) $site->backup_relay_enabled; $previousFrequency = $site->backup_relay_frequency; + // Refuse only the off -> on switch. A site that is already on keeps + // its cadence editable, and its failures stay visible. + if ((bool) $validated['enabled'] && ! $previousEnabled + && ($blocked = CompanionBackupRelayAdapter::enableBlockedReason($site)) !== null) { + $msg = "Glacier backups for {$site->domain} were not turned on: {$blocked}"; + + if ($request->expectsJson()) { + return response()->json([ + 'ok' => false, + 'message' => $msg, + 'enabled' => false, + 'frequency' => $site->backupRelayFrequency(), + 'domain' => $site->domain, + ], 422); + } + + return back()->with('warning', $msg); + } + $site->forceFill([ 'backup_relay_enabled' => (bool) $validated['enabled'], 'backup_relay_frequency' => $validated['frequency'], diff --git a/app/Models/Site.php b/app/Models/Site.php index f45777b3..50d8f1a4 100644 --- a/app/Models/Site.php +++ b/app/Models/Site.php @@ -93,6 +93,9 @@ * @property ?int $backup_relay_last_size_bytes * @property ?string $backup_relay_last_sha256 * @property ?array $backup_relay_last_manifest + * @property ?Carbon $backup_relay_last_attempt_at last archive attempt (success or failure) + * @property ?string $backup_relay_last_error last failed attempt's error; null once an archive succeeds + * @property ?Carbon $backup_relay_failing_since first failure in the current unbroken failure streak * @property ?string $bill_com_customer_id * @property ?string $bill_com_customer_name * @property ?string $bill_com_linked_via_invoice @@ -336,6 +339,9 @@ class Site extends Model 'backup_relay_last_size_bytes', 'backup_relay_last_sha256', 'backup_relay_last_manifest', + 'backup_relay_last_attempt_at', + 'backup_relay_last_error', + 'backup_relay_failing_since', 'bill_com_customer_id', 'bill_com_customer_name', 'bill_com_linked_via_invoice', @@ -442,6 +448,8 @@ protected function casts(): array 'backup_relay_last_archived_at' => 'datetime', 'backup_relay_last_size_bytes' => 'integer', 'backup_relay_last_manifest' => 'array', + 'backup_relay_last_attempt_at' => 'datetime', + 'backup_relay_failing_since' => 'datetime', 'bill_com_linked_at' => 'datetime', 'care_plan_override' => 'boolean', 'auto_updates_paused' => 'boolean', diff --git a/database/migrations/2026_11_03_120000_add_backup_relay_attempt_state_to_sites_table.php b/database/migrations/2026_11_03_120000_add_backup_relay_attempt_state_to_sites_table.php new file mode 100644 index 00000000..a475f648 --- /dev/null +++ b/database/migrations/2026_11_03_120000_add_backup_relay_attempt_state_to_sites_table.php @@ -0,0 +1,40 @@ +timestamp('backup_relay_last_attempt_at') + ->nullable() + ->after('backup_relay_last_manifest'); + $table->text('backup_relay_last_error') + ->nullable() + ->after('backup_relay_last_attempt_at'); + $table->timestamp('backup_relay_failing_since') + ->nullable() + ->after('backup_relay_last_error'); + }); + } + + public function down(): void + { + Schema::table('sites', function (Blueprint $table) { + $table->dropColumn([ + 'backup_relay_last_attempt_at', + 'backup_relay_last_error', + 'backup_relay_failing_since', + ]); + }); + } +}; diff --git a/modules/BackupRelay/src/Console/RunBackupRelayNow.php b/modules/BackupRelay/src/Console/RunBackupRelayNow.php index 5a822b26..606cdaeb 100644 --- a/modules/BackupRelay/src/Console/RunBackupRelayNow.php +++ b/modules/BackupRelay/src/Console/RunBackupRelayNow.php @@ -12,6 +12,7 @@ use Illuminate\Support\Facades\Log; use Modules\BackupRelay\Jobs\ArchiveSiteBackupJob; use Modules\BackupRelay\Services\GlacierUploader; +use Modules\BackupRelay\Services\RelayAttemptRecorder; use Modules\Core\Contracts\DirectS3BackupRelayAdapter; use Modules\Core\Contracts\HostingProvider; use Throwable; @@ -120,6 +121,10 @@ public function handle(GlacierUploader $uploader, Settings $settings): int ]; $this->error(" [fail] {$site->domain} — {$e->getMessage()}"); Log::error('backup_relay.site_failed', ['site_id' => $site->id, 'direct_only' => $directOnly, 'error' => $e->getMessage()]); + // The job records its own failures; this also covers anything + // that failed around it. Neither the log nor this command's + // output is kept anywhere an operator can see it. + RelayAttemptRecorder::recordFailure($site, $e->getMessage()); } finally { if ($locked) { Cache::forget($lockKey); diff --git a/modules/BackupRelay/src/Jobs/ArchiveSiteBackupJob.php b/modules/BackupRelay/src/Jobs/ArchiveSiteBackupJob.php index 1a840648..ba8a6d5c 100644 --- a/modules/BackupRelay/src/Jobs/ArchiveSiteBackupJob.php +++ b/modules/BackupRelay/src/Jobs/ArchiveSiteBackupJob.php @@ -13,6 +13,7 @@ use Illuminate\Support\Facades\Log; use Illuminate\Support\Facades\Storage; use Modules\BackupRelay\Services\GlacierUploader; +use Modules\BackupRelay\Services\RelayAttemptRecorder; use Modules\Core\Contracts\BackupRef; use Modules\Core\Contracts\BackupRelayAdapter; use Modules\Core\Contracts\DirectS3BackupRelayAdapter; @@ -42,6 +43,22 @@ public function handle(GlacierUploader $uploader): string { $site = $this->site instanceof Site ? $this->site : Site::query()->findOrFail($this->site); + try { + return $this->archive($site, $uploader); + } catch (Throwable $e) { + // Keep the failure on the site row so it is visible without log + // access. Queued runs never reach RunBackupRelayNow's own catch. + RelayAttemptRecorder::recordFailure($site, $e->getMessage()); + + throw $e; + } + } + + /** + * @return string 'archived' | 'skipped' + */ + private function archive(Site $site, GlacierUploader $uploader): string + { $host = $site->host(); if (! $host->supports(HostingProvider::CAP_BACKUP_RELAY)) { Log::info("BackupRelay: Provider {$host->id()} does not support CAP_BACKUP_RELAY for {$site->domain}."); @@ -216,16 +233,18 @@ private function markArchived(Site $site, array $meta = [], mixed $disk = null, { $updates = [ 'backup_relay_last_archived_at' => now(), + ...RelayAttemptRecorder::successAttributes(), ]; if (array_key_exists('size_bytes', $meta) && $meta['size_bytes'] !== null) { $updates['backup_relay_last_size_bytes'] = (int) $meta['size_bytes']; } + // The stored hash describes the newest archive only. When this archive + // came without one, clear it: keeping the previous archive's hash would + // pair it with the new key, and restore would fail the checksum. $sha = isset($meta['sha256']) ? trim((string) $meta['sha256']) : ''; - if ($sha !== '') { - $updates['backup_relay_last_sha256'] = $sha; - } + $updates['backup_relay_last_sha256'] = $sha !== '' ? $sha : null; $manifest = isset($meta['manifest']) && is_array($meta['manifest']) ? $meta['manifest'] : null; if ($manifest !== null) { diff --git a/modules/BackupRelay/src/Services/CompanionBackupRelayAdapter.php b/modules/BackupRelay/src/Services/CompanionBackupRelayAdapter.php index 41d1a518..1c728417 100644 --- a/modules/BackupRelay/src/Services/CompanionBackupRelayAdapter.php +++ b/modules/BackupRelay/src/Services/CompanionBackupRelayAdapter.php @@ -22,6 +22,53 @@ */ class CompanionBackupRelayAdapter implements BackupRelayAdapter, DirectS3BackupRelayAdapter { + /** Plugin capability that exposes POST /backup/create (the direct-to-S3 dump). */ + public const CAPABILITY_BACKUP_CREATE = 'backup-create'; + + /** + * @param array|null $capabilities as reported by the plugin's /health + */ + public static function supportsBackupCreate(?array $capabilities): bool + { + return in_array(self::CAPABILITY_BACKUP_CREATE, $capabilities ?? [], true); + } + + /** + * Operator-facing reason a site's plugin cannot take a direct backup. + */ + public static function missingBackupCreateMessage(Site $site): string + { + $plugin = $site->isRenegade() ? 'Renegade' : 'Companion'; + $version = trim((string) $site->companion_version); + $label = $version !== '' ? "{$plugin} {$version}" : $plugin; + + return $site->isRenegade() + ? "{$label} lacks the backup-create capability; update Renegade." + : "{$label} lacks the backup-create capability; update Companion or switch to Renegade."; + } + + /** + * Why backup relay must not be turned on for this site, or null when it + * can be. Only sites this adapter archives (the plugin uploads straight to + * S3) need backup-create; host-streamed providers are never blocked here. + */ + public static function enableBlockedReason(Site $site): ?string + { + try { + $adapter = $site->host()->backupRelayAdapter(); + } catch (Throwable) { + return null; + } + + if (! $adapter instanceof self) { + return null; + } + + return self::supportsBackupCreate($site->companion_capabilities) + ? null + : self::missingBackupCreateMessage($site); + } + public function latestBackupRef(Site $site): ?BackupRef { if (! $site->companion_installed || ! $site->companion_secret) { @@ -47,6 +94,13 @@ public function latestBackupRef(Site $site): ?BackupRef */ public function uploadDirectToS3(Site $site, string $destinationKey, array $presigned): array { + // Fail loudly rather than skip: relay is on, so a backup was promised, + // and the run must report this site as failed with a fix the operator + // can act on (an older plugin has no /backup/create endpoint). + if (! self::supportsBackupCreate($site->companion_capabilities)) { + throw new RuntimeException("BackupRelay: {$site->domain}: ".self::missingBackupCreateMessage($site)); + } + $client = new ClockworkCompanionClient($site, timeout: 300); try { diff --git a/modules/BackupRelay/src/Services/RelayAttemptRecorder.php b/modules/BackupRelay/src/Services/RelayAttemptRecorder.php new file mode 100644 index 00000000..301e5c69 --- /dev/null +++ b/modules/BackupRelay/src/Services/RelayAttemptRecorder.php @@ -0,0 +1,53 @@ + + */ + public static function successAttributes(): array + { + return [ + 'backup_relay_last_attempt_at' => now(), + 'backup_relay_last_error' => null, + 'backup_relay_failing_since' => null, + ]; + } + + /** + * Record a failed attempt. failing_since keeps the start of the current + * streak. Never throws: losing this bookkeeping must not mask the + * original failure. + */ + public static function recordFailure(Site $site, string $error): void + { + try { + $error = trim($error) !== '' ? trim($error) : 'Backup relay attempt failed.'; + + $site->forceFill([ + 'backup_relay_last_attempt_at' => now(), + 'backup_relay_last_error' => mb_substr($error, 0, self::MAX_ERROR_LENGTH), + 'backup_relay_failing_since' => $site->backup_relay_failing_since ?? now(), + ])->save(); + } catch (Throwable $e) { + Log::warning("BackupRelay: Could not record failed attempt for {$site->domain}: {$e->getMessage()}"); + } + } +} diff --git a/resources/docs/features/backup-relay.md b/resources/docs/features/backup-relay.md index 905c036d..86ac0c39 100644 --- a/resources/docs/features/backup-relay.md +++ b/resources/docs/features/backup-relay.md @@ -181,7 +181,9 @@ SpinupWP and Pressable already take their own backups. Glacier relay on those ho For WordPress sites we **do not host** (WP Engine, Kinsta, a client box, any `custom` enroll), Companion dumps files + database and PUTs a zip straight to agency S3 Glacier Instant Retrieval. That is the backup. -- Connecting a site at `/sites/create` turns relay **on** and sets `backup_relay_frequency = daily`. +- Connecting a site at `/sites/create` turns relay **on** and sets `backup_relay_frequency = daily`, but only when the plugin's `/health` lists the `backup-create` capability. Older Companion builds have no `/backup/create` endpoint, so the site is enrolled with relay off and the flash message says why (update Companion or switch to Renegade). +- Relay cannot be switched **on** for a direct-to-S3 site without `backup-create` — from the Backups card, the single toggle or the bulk form on `/settings/backup-relay` (the bulk form enables the rest and lists the refused sites). A site that is already on stays on, so its failures keep showing. If a run reaches such a site anyway, it fails with `Companion lacks the backup-create capability; update Companion or switch to Renegade.` without calling the plugin. +- Every failed attempt is stored on the site (`backup_relay_last_attempt_at`, `backup_relay_last_error`, `backup_relay_failing_since`), whether it came from the nightly pass, Backup Now or a queued job; the next successful archive clears them. The Backups card shows the error and the start of the failure streak, so nobody needs the application log to see why a site has no fresh archive. - The site Overview **Backups** card is the per-site control: on/off, Daily / Twice weekly / Weekly, last/next, a month calendar of archives, and **Backup Now**. - Nightly `clockwork:backup-relay-run` (04:58 in `in_repo` mode; the 04:30 `--direct-only` pass in `external_agent` mode) respects that per-site cadence. Daily uses calendar day so a 15:00 Backup Now does not skip tomorrow morning. - **Backup Now** runs `clockwork:backup-relay-run --site={id} --force` in the background and writes `archives/{domain}/{Y-m-d_H-i-s}.zip` so it never collides with the scheduled daily key. @@ -192,7 +194,7 @@ For WordPress sites we **do not host** (WP Engine, Kinsta, a client box, any `cu For custom unhosted sites, Clockwork provides a safe two-step restore flow directly from the site's Backups widget (`/sites/{id}`): 1. **Two-Step Architecture**: - - **Step 1 (Stage)**: The operator selects an archive from the historical archives table and confirms the exact domain name. Control first checks `BackupArchiveEnumerator::belongsToSite()` — the object key must sit under `{archive_prefix}/{domain}/` or `{domain}/` (no `..`, no sibling domains). Then it verifies the SHA-256 integrity hash for the archive (from the `{key}.sha256.json` sidecar created during backup, or from `sites.backup_relay_last_sha256` for the newest archive). If no hash is on record, restore is strictly refused. Control generates a 24-hour presigned S3 GET URL (`BackupArchiveEnumerator::getDownloadUrl()`, `DOWNLOAD_URL_TTL_HOURS = 24`) and triggers `POST /wp-json/clockwork/v1/backup/restore/stage` on Companion. Companion refuses the download unless the URL is public HTTPS (no private/reserved IPs, no redirects), streams the zip into `wp-content/clockwork-backups/restore-staging/`, validates the SHA-256 checksum, and unpacks the archive via `ZipArchive` (or one-shot `PclZip` fallback on hosts lacking `ext-zip`). Unsafe zip entries (`../`, absolute paths) abort extract and are skipped again on file apply. Control polls `GET /wp-json/clockwork/v1/backup/restore/status` (a GET request that does not burn the HMAC replay window) until staging finishes. + - **Step 1 (Stage)**: The operator selects an archive from the historical archives table and confirms the exact domain name. Control first checks `BackupArchiveEnumerator::belongsToSite()` — the object key must sit under `{archive_prefix}/{domain}/` or `{domain}/` (no `..`, no sibling domains). Then it verifies the SHA-256 integrity hash for the archive (from the `{key}.sha256.json` sidecar created during backup, or from `sites.backup_relay_last_sha256` for the newest archive). `backup_relay_last_sha256` is reset to null whenever an archive is recorded without a hash (an object adopted after a timeout or gateway error), so an older archive's hash is never paired with a newer key. If no hash is on record, restore is strictly refused. Control generates a 24-hour presigned S3 GET URL (`BackupArchiveEnumerator::getDownloadUrl()`, `DOWNLOAD_URL_TTL_HOURS = 24`) and triggers `POST /wp-json/clockwork/v1/backup/restore/stage` on Companion. Companion refuses the download unless the URL is public HTTPS (no private/reserved IPs, no redirects), streams the zip into `wp-content/clockwork-backups/restore-staging/`, validates the SHA-256 checksum, and unpacks the archive via `ZipArchive` (or one-shot `PclZip` fallback on hosts lacking `ext-zip`). Unsafe zip entries (`../`, absolute paths) abort extract and are skipped again on file apply. Control polls `GET /wp-json/clockwork/v1/backup/restore/status` (a GET request that does not burn the HMAC replay window) until staging finishes. - **Step 2 (Apply)**: Once staged, the operator reviews the detected staging details (database dump presence, table prefix, files archive) in the modal and clicks **Apply Restore**. Companion places WordPress into maintenance mode (`.maintenance`), imports the SQL dump using gzip streaming scoped strictly to `$wpdb->prefix` (failing closed with `prefix_mismatch` if 0 tables match), and copies the restored `wp-content/` files over the active filesystem. 2. **Fail-Closed Maintenance Mode**: diff --git a/resources/views/dashboard/site/widgets/_widget-backups.blade.php b/resources/views/dashboard/site/widgets/_widget-backups.blade.php index 4d965395..a83c7dfb 100644 --- a/resources/views/dashboard/site/widgets/_widget-backups.blade.php +++ b/resources/views/dashboard/site/widgets/_widget-backups.blade.php @@ -10,6 +10,12 @@ $hasRelayOrPressable = $isPressable || $relayEnabled || $lastArchived !== null; $spacesEligible = $isSpinupWp && $spacesConfigured; $showSnapshots = $hasRelayOrPressable || $spacesEligible || $isCustom; + $relayLastError = $site->backup_relay_last_error ?? null; + $relayFailingSince = $site->backup_relay_failing_since ?? null; + $relayLastAttemptAt = $site->backup_relay_last_attempt_at ?? null; + $relayBlockedReason = ($isCustom && ! $relayEnabled && class_exists(\Modules\BackupRelay\Services\CompanionBackupRelayAdapter::class)) + ? \Modules\BackupRelay\Services\CompanionBackupRelayAdapter::enableBlockedReason($site) + : null; @endphp
toIso8601String()), nextAt: @js($nextScheduled?->toIso8601String()), + lastError: @js($relayLastError), + failingSince: @js($relayFailingSince?->toIso8601String()), + lastAttemptAt: @js($relayLastAttemptAt?->toIso8601String()), + blockedReason: @js($relayBlockedReason), saving: false, runningNow: false, runMessage: null, @@ -223,6 +233,11 @@ if (json.schedule.frequency) this.frequency = json.schedule.frequency; this.lastAt = json.schedule.last_archived_at || this.lastAt; this.nextAt = json.schedule.next_scheduled_at || this.nextAt; + if ('last_error' in json.schedule) { + this.lastError = json.schedule.last_error; + this.failingSince = json.schedule.failing_since; + this.lastAttemptAt = json.schedule.last_attempt_at; + } } }, fetchHistory() { @@ -262,6 +277,10 @@ const data = await res.json(); if (data.ok) { this.nextAt = data.next_scheduled_at; + } else { + // Refused (e.g. the plugin lacks backup-create): show the server's state. + if ('enabled' in data) this.enabled = !!data.enabled; + this.runMessage = data.message || 'Could not save the backup schedule.'; } } catch (e) { /* keep previous UI */ } finally { this.saving = false; } @@ -271,6 +290,7 @@ this.runningNow = true; this.runMessage = null; const started = this.lastAt; + const startedAttempt = this.lastAttemptAt; try { const res = await fetch('{{ route('sites.backup-relay.run-now', $site) }}', { method: 'POST', @@ -293,6 +313,11 @@ this.runningNow = false; return; } + if (this.lastError && this.lastAttemptAt && this.lastAttemptAt !== startedAttempt) { + this.runMessage = 'Backup failed: ' + this.lastError; + this.runningNow = false; + return; + } } this.runMessage = 'Backup is still running. Refresh in a few minutes.'; } catch (e) { @@ -315,9 +340,9 @@ Backups @if ($isCustom) - + - {{ $relayEnabled ? 'Protected' : 'Disabled' }} + {{ $relayEnabled ? ($relayLastError ? 'Failing' : 'Protected') : 'Disabled' }} @elseif ($hasRelayOrPressable) @@ -355,9 +380,10 @@
- - - {{ $relayEnabled && $lastArchived ? 'Backups are successful' : ($relayEnabled ? 'Glacier relay active' : 'Backups disabled') }} + + + + {{ $relayEnabled && $relayLastError ? 'Last backup attempt failed' : ($relayEnabled && $lastArchived ? 'Backups are successful' : ($relayEnabled ? 'Glacier relay active' : 'Backups disabled')) }}
@@ -376,6 +402,22 @@
+ {{-- Last failed attempt (cleared by the next successful archive) --}} +
+
+ + + {{ $relayFailingSince ? 'Failing since '.$relayFailingSince->format('Y-m-d, H:i') : 'Last backup attempt failed' }} + +
+

{{ $relayLastError }}

+

+ {{ $relayLastAttemptAt ? 'Last attempt: '.$relayLastAttemptAt->format('Y-m-d, H:i') : '' }} +

+
+ {{-- Configuration / Cadence metadata box --}}
@@ -392,7 +434,8 @@ :aria-checked="enabled ? 'true' : 'false'" aria-label="Toggle Glacier backups for {{ $site->domain }}" @click="enabled = !enabled; saveSchedule()" - :disabled="saving" + :disabled="saving || (!enabled && blockedReason)" + :title="!enabled && blockedReason ? blockedReason : null" class="cw-switch" :class="{ 'cw-switch--on': enabled, 'cw-switch--busy': saving }"> @@ -404,6 +447,7 @@ class="cw-switch"
+

{{ $relayBlockedReason }}

Cadence: +
+
+ + +
+ + +
+ + @if ($filters !== []) + Clear + @endif +
+
+ {{ $rows->total() }} {{ \Illuminate\Support\Str::plural('client', $rows->total()) }}{{ $filters !== [] ? ' match' : '' }} + @include('client-reports::partials.strip-legend') +
+ + + @if ($rows->isEmpty()) +
+ + No clients match these filters. +
+ @else +
    + @foreach ($rows as $row) + @php($clientUrl = route('client-reports.clients.show', [$row['key'], ...$clientQuery])) +
  • + {{-- Client --}} +
    + + @if ($row['client'] === null)@endif{{ $row['name'] }} + + @if ($row['contact']) +
    {{ $row['contact'] }}
    + @endif +
    + {{ $row['counts']['sites'] }} {{ \Illuminate\Support\Str::plural('site', $row['counts']['sites']) }} + @if ($row['counts']['waiting'] > 0) + {{ $row['counts']['waiting'] }} waiting + @endif + @if ($row['counts']['missing'] > 0) + {{ $row['counts']['missing'] }} no schedule + @endif +
    + + Reports & schedules + +
    + + {{-- Sites --}} +
    + @forelse ($row['sites'] as $siteRow) + @include('client-reports::clients._site-line', ['siteRow' => $siteRow, 'context' => []]) + @empty +
    No sites assigned to this client.
    + @endforelse +
    +
    • + @endforeach +
+ @if ($rows->hasPages()) +
{{ $rows->links() }}
+ @endif + @endif +
+@endsection diff --git a/modules/ClientReports/resources/views/clients/show.blade.php b/modules/ClientReports/resources/views/clients/show.blade.php new file mode 100644 index 00000000..00e072db --- /dev/null +++ b/modules/ClientReports/resources/views/clients/show.blade.php @@ -0,0 +1,270 @@ +@extends('layouts.app') + +@section('title', $row['name'].' · Client Reports · Clockwork') + +@php + $carePlansOn = \App\Models\Site::areCarePlansEnabled(); + $stateClass = \Modules\ClientReports\Support\ReportState::class; + $modeDraft = \Modules\ClientReports\Models\ClientReportSchedule::MODE_DRAFT; + $listUrl = route('client-reports.index', $filters); + $reportContext = array_filter(['period' => 'all', 'client' => $row['key'], 'from' => 'client', 'superseded' => $showSuperseded ? '1' : null]); + // The switcher's labels must be unique to map back to a client. + $switchOptions = []; + foreach ($switcher as $option) { + $label = $option['name']; + $n = 2; + while (isset($switchOptions[$label])) { + $label = $option['name'].' ('.$n++.')'; + } + $switchOptions[$label] = route('client-reports.clients.show', [$option['key'], ...$filters]); + } + $currentLabel = array_search(route('client-reports.clients.show', [$row['key'], ...$filters]), $switchOptions, true) ?: $row['name']; +@endphp + +@section('content') +@include('client-reports::_nav') +@include('client-reports::partials.flash') + +{{-- Client navigation: back to the (filtered) list, previous/next, switcher --}} +
+ All clients +
+ @if ($previousKey !== null) + + @else + Previous + @endif + @if ($position !== null) + {{ $position }} of {{ $listCount }} + @endif + @if ($nextKey !== null) + + @else + Next + @endif + +
+
+ +{{-- Client header --}} +
+
+
+

{{ $row['name'] }}

+ @if ($row['contact']) +
{{ $row['contact'] }}
+ @endif + @if ($row['client'] === null) +

Sites with no client. Reports still go out for those on a care plan with a schedule; assign a client to keep contacts and history together.

+ @endif + @if ($row['contacts'] !== []) +
+ @foreach ($row['contacts'] as $contact) + {{ $contact }} + @endforeach +
+ @endif +
+
+ + + @if (! $carePlansOn) + Care plans off: every site gets reports + @else + {{ $row['counts']['care'] }} of {{ $row['counts']['sites'] }} {{ \Illuminate\Support\Str::plural('site', $row['counts']['sites']) }} on a care plan + @endif + + @if ($row['counts']['missing'] > 0) + {{ $row['counts']['missing'] }} without a schedule + @endif + @if ($row['counts']['waiting'] > 0) + {{ $row['counts']['waiting'] }} waiting + @endif + @if ($row['client'] && \Illuminate\Support\Facades\Route::has('clients.show')) + Client record + @endif +
+
+
+ +@forelse ($row['sites'] as $siteRow) + @php + $site = $siteRow['site']; + $schedule = $siteRow['schedule']; + $recipients = array_values(array_filter((array) ($schedule?->recipients ?? []))); + $reports = $siteRow['reports']->filter(fn ($r) => $showSuperseded || $r->status !== \Modules\ClientReports\Models\ClientReport::STATUS_SUPERSEDED)->values(); + $hiddenCopies = $siteRow['reports']->count() - $siteRow['reports']->filter(fn ($r) => $r->status !== \Modules\ClientReports\Models\ClientReport::STATUS_SUPERSEDED)->count(); + @endphp +
+
+
+

+ {{ $site->domain }} +

+
+ @if ($site->is_inactive) + Inactive + @endif + @if ($carePlansOn) + @if ($siteRow['eligible']) + Care plan + @else + No care plan — reports won't send + @endif + @endif + Site dashboard +
+
+ @include('client-reports::partials.history-strip', ['months' => $siteRow['months'], 'context' => $reportContext]) +
+
+ + {{-- Schedule --}} +
+
+ Report schedule + @if ($schedule) +
+ Edit +
+ @csrf + @method('PATCH') + + +
+
+ @elseif ($siteRow['missing']) +
+ @csrf + +
+ @endif +
+ @if ($schedule) +
+
Status
+
+ @if ($schedule->is_enabled) + Enabled + @else + Disabled + @endif + · {{ ucfirst($schedule->frequency) }}, {{ $schedule->delivery_mode === $modeDraft ? 'drafts for review' : 'sends automatically' }} +
+
Recipients
+
{{ $recipients !== [] ? implode(', ', $recipients) : 'None (drafts only)' }}
+
Next run
+
{{ $schedule->next_run_at ? $schedule->next_run_at->format('M j, Y') : '—' }}
+
Last sent
+
+ @if ($schedule->last_sent_at) + {{ $schedule->last_sent_at->format('M j, Y') }} + @else + Never — its first report waits for someone to send it + @endif +
+
+ @elseif ($siteRow['missing']) +

On a care plan but no schedule, so no report is sent. Create one with the default template, monthly{{ $row['contacts'] !== [] ? ', to the client\'s contacts' : ', as drafts for review (no contacts on file)' }}.

+ @else +

No schedule{{ $carePlansOn && ! $siteRow['eligible'] ? ': reports are only sent for care-plan sites' : '' }}.

+ @endif +
+
+ + {{-- Full report history --}} + @if ($reports->isEmpty()) +
No reports yet.
+ @else +
+ + + + + + + + + + + @foreach ($reports as $report) + @php($state = $stateClass::of($report)) + + + + + + + @endforeach + +
PeriodStatusSent toActions
+
{{ $report->period_start->format('F Y') }}
+
{{ $report->period_start->format('M j') }} – {{ $report->period_end->format('M j') }}
+ 		+ @include('client-reports::partials.state-pill', ['state' => $state]) + @include('client-reports::partials.reason-chips', ['reasons' => $stateClass::reasons($report)]) + + @if (! empty($report->sent_to)) +
{{ implode(', ', $report->sent_to) }}
+ @if ($report->sent_at) +
{{ $report->sent_at->format('M j, Y') }}
+ @endif + @elseif ($report->recipients() !== []) +
Not sent · for {{ implode(', ', $report->recipients()) }}
+ @else + Not sent · no recipient + @endif + 		+ @include('client-reports::partials.report-actions', ['report' => $report, 'context' => $reportContext]) +
+
+ @endif + @if ($hiddenCopies > 0 || $showSuperseded) +
+ + {{ $showSuperseded ? 'Hide replaced copies' : 'Show '.$hiddenCopies.' replaced '.\Illuminate\Support\Str::plural('copy', $hiddenCopies) }} + +
+ @endif +
+@empty +
+ No sites are assigned to this client yet. + @if ($row['client'] && \Illuminate\Support\Facades\Route::has('clients.show')) + Assign sites on the client record. + @endif +
+@endforelse + +

Keyboard: ← previous client · → next client

+ + +@endsection diff --git a/modules/ClientReports/resources/views/partials/flash.blade.php b/modules/ClientReports/resources/views/partials/flash.blade.php new file mode 100644 index 00000000..a27179d0 --- /dev/null +++ b/modules/ClientReports/resources/views/partials/flash.blade.php @@ -0,0 +1,13 @@ +@if (session('status')) +
+ + {{ session('status') }} +
+@endif + +@if (session('status_error')) + +@endif diff --git a/modules/ClientReports/resources/views/partials/history-strip.blade.php b/modules/ClientReports/resources/views/partials/history-strip.blade.php new file mode 100644 index 00000000..5e28b426 --- /dev/null +++ b/modules/ClientReports/resources/views/partials/history-strip.blade.php @@ -0,0 +1,25 @@ +{{-- One site's last report periods as small month cells, each linking to + that month's report. $months: ClientDirectory site row 'months'; + $context: query parameters for the report links. --}} +@php + $stateClass = \Modules\ClientReports\Support\ReportState::class; + $directoryClass = \Modules\ClientReports\Services\ClientDirectory::class; +@endphp +
+ @foreach ($months as $ym => $cell) + @php + $monthLabel = $cell['month']->format('M Y'); + $title = match ($cell['state']) { + $directoryClass::CELL_NONE => "{$monthLabel}: no report", + $directoryClass::CELL_SKIPPED => "{$monthLabel}: skipped (no care plan)", + default => "{$monthLabel}: ".$stateClass::label($cell['state']), + }; + @endphp + @if ($cell['report']) + {{ $cell['month']->format('M') }} + @else + {{ $cell['month']->format('M') }} + @endif + @endforeach +
diff --git a/modules/ClientReports/resources/views/partials/reason-chips.blade.php b/modules/ClientReports/resources/views/partials/reason-chips.blade.php new file mode 100644 index 00000000..8371d609 --- /dev/null +++ b/modules/ClientReports/resources/views/partials/reason-chips.blade.php @@ -0,0 +1,12 @@ +{{-- Short chips for why a report is held, one per category; the full + sentences are in each chip's tooltip. $reasons: list of review reasons. --}} +@php($groups = \Modules\ClientReports\Support\ReviewReasonCategories::group($reasons ?? [])) +@if ($groups !== []) +
+ @foreach ($groups as $category => $texts) + + {{ \Modules\ClientReports\Support\ReviewReasonCategories::short($category) }}@if (count($texts) > 1) ×{{ count($texts) }}@endif + + @endforeach +
+@endif diff --git a/modules/ClientReports/resources/views/partials/report-actions.blade.php b/modules/ClientReports/resources/views/partials/report-actions.blade.php new file mode 100644 index 00000000..4792844a --- /dev/null +++ b/modules/ClientReports/resources/views/partials/report-actions.blade.php @@ -0,0 +1,61 @@ +{{-- A report's actions on the app-styled pages (list, client page, email + version): View, Email version, Recompile, Send, Send test to me. + $report; $context: query parameters that keep the list the operator came + from (filters, client page); $hide: actions to leave out (e.g. 'view'). --}} +@php + $stateClass = \Modules\ClientReports\Support\ReportState::class; + $context = $context ?? []; + $hide = $hide ?? []; + $blocked = $stateClass::sendBlockedReason($report); + $hasRecipients = $report->recipients() !== []; + $operatorEmail = auth()->user()?->email; +@endphp +
+ @unless (in_array('view', $hide, true)) + + View + + @endunless + @unless (in_array('email', $hide, true)) + + Email + + @endunless + {{-- Unsent scheduled reports only: refresh from current data. Never sends. --}} + @if ($report->canRecompile()) +
+ @csrf + +
+ @endif + @if ($blocked !== null) + + @else +
+ @csrf + + +
+ @endif +
+ @csrf + +
+
diff --git a/modules/ClientReports/resources/views/partials/state-pill.blade.php b/modules/ClientReports/resources/views/partials/state-pill.blade.php new file mode 100644 index 00000000..0b6f6c86 --- /dev/null +++ b/modules/ClientReports/resources/views/partials/state-pill.blade.php @@ -0,0 +1,5 @@ +{{-- A report's state as a pill. $state: a ReportState constant. --}} +@php($stateClass = \Modules\ClientReports\Support\ReportState::class) + + {{ $stateClass::label($state) }} + diff --git a/modules/ClientReports/resources/views/partials/strip-legend.blade.php b/modules/ClientReports/resources/views/partials/strip-legend.blade.php new file mode 100644 index 00000000..664812f5 --- /dev/null +++ b/modules/ClientReports/resources/views/partials/strip-legend.blade.php @@ -0,0 +1,8 @@ +@php($stateClass = \Modules\ClientReports\Support\ReportState::class) +
+ @foreach ([$stateClass::SENT, $stateClass::NEEDS_REVIEW, $stateClass::OUTDATED, $stateClass::READY, $stateClass::PARTLY_SENT] as $legendState) + {{ $stateClass::label($legendState) }} + @endforeach + Skipped (no care plan) + No report +
diff --git a/modules/ClientReports/resources/views/partials/styles.blade.php b/modules/ClientReports/resources/views/partials/styles.blade.php new file mode 100644 index 00000000..00b1ba70 --- /dev/null +++ b/modules/ClientReports/resources/views/partials/styles.blade.php @@ -0,0 +1,60 @@ +{{-- A few module-only styles on top of the app's palette (status-* pills, + CSS variables), so they follow light/dark like everything else. In the + components layer, so Tailwind utilities and the status-* classes still + win over them. --}} + diff --git a/modules/ClientReports/resources/views/schedules/index.blade.php b/modules/ClientReports/resources/views/schedules/index.blade.php index 453314ad..973718fa 100644 --- a/modules/ClientReports/resources/views/schedules/index.blade.php +++ b/modules/ClientReports/resources/views/schedules/index.blade.php @@ -3,216 +3,216 @@ @section('title', 'Scheduled Reports · Clockwork') @section('content') -
-
-

- - Client Reports -

-

- Automated, executive-ready white-labeled maintenance and performance reports for your agency clients. -

-
-
- - New Schedule + + + + New schedule -
-
+ + @include('client-reports::_nav') +@include('client-reports::partials.flash') -@if (session('status')) -
- - {{ session('status') }} -
-@endif - -@if (session('status_error')) -
- - {{ session('status_error') }} -
-@endif +@php + $stateClass = \Modules\ClientReports\Support\ReportState::class; + $runnerClass = \Modules\ClientReports\Services\ScheduledReportRunner::class; + $modeDraft = \Modules\ClientReports\Models\ClientReportSchedule::MODE_DRAFT; + $filterLabels = \Modules\ClientReports\Http\Controllers\SchedulesController::FILTERS; +@endphp -
- {{-- Active Schedules Table --}} -
-
+{{-- The coverage gap: care-plan sites that get no report at all. --}} +@if ($missingSites->isNotEmpty()) +
+

- - Automated Reporting Schedules + + Care-plan sites without a report schedule ({{ $missingSites->count() }})

-

- Cron automated dispatches aggregating monthly and weekly site maintenance telemetry. +

+ These sites are on a care plan but get no monthly report. "Create schedule" sets one up with the default template, monthly, + to the client's contacts (or as drafts for review when there are none). Nothing is sent: each first report waits for you to check and send it.

- {{ $schedules->count() }} schedule(s) configured +
+ @csrf + +
+ +
+@endif -
- - - - - - - - - - - - - - - @forelse ($schedules as $s) - - - - - + + @empty + + + + @endforelse + +
SiteTemplateFrequencyDelivery ModeRecipientsNext DispatchStateActions
- - {{ $s->site->domain }} - - @if ($s->site->care_plan_enabled) - - Care Plan - - @elseif (! \Modules\ClientReports\Services\ScheduledReportRunner::siteIsEligible($s->site)) - - No care plan — won't send - - @endif - - @if ($s->template) - - - {{ $s->template->name }} - -
- {{ count((array) $s->template->sections) }} sections -
- @else - Default (All Sections) - @endif - 		- {{ $s->frequency }} - - @if ($s->delivery_mode === \Modules\ClientReports\Models\ClientReportSchedule::MODE_DRAFT) - - Draft Review - - @else - - Auto-Send +
+
+
+

+ + Report schedules +

+
+ +
+ + + +
+ +
+
+
+ @foreach ($filterLabels as $key => $label) + + {{ $label }} {{ $counts[$key] }} + + @endforeach +
+
+ +
+ + + + + + + + + + + + + + @forelse ($schedules as $s) + @php + $recips = array_values(array_filter((array) $s->recipients)); + $eligible = $runnerClass::siteIsEligible($s->site); + $latest = $latestReports->get($s->id); + @endphp + + - - - + + + + + + - - - @empty - - - - @endforelse - -
SiteDeliveryRecipientsNext runLatest reportStateActions
+ {{ $s->site->domain }} +
+ @if (! $eligible) + + No care plan — won't send + @elseif (\App\Models\Site::areCarePlansEnabled()) + Care plan @endif -
- @php $recips = (array) $s->recipients; @endphp - @if (!empty($recips)) -
- {{ implode(', ', $recips) }} -
- @else - None + @if ($s->site->client) + {{ $s->site->client->company_name ?: $s->site->client->name }} @endif - 		- @if ($s->next_run_at) -
{{ $s->next_run_at->format('M j, Y') }}
-
{{ $s->next_run_at->diffForHumans() }}
- @else - — - @endif - 		-
+ +
+ @if ($s->delivery_mode === $modeDraft) + Drafts for review + @else + Auto-Send + @endif +
+ {{ ucfirst($s->frequency) }} · {{ $s->template?->name ?? 'Default (all sections)' }} +
+ 		+ @if ($recips !== []) +
{{ implode(', ', $recips) }}
+ @else + No recipients + @endif + 		+ @if ($s->next_run_at) +
{{ $s->next_run_at->format('M j, Y') }}
+
{{ $s->last_sent_at ? 'Last sent '.$s->last_sent_at->format('M j') : 'Never sent yet' }}
+ @else + — + @endif + 		+ @if ($latest) + @php($latestState = $stateClass::of($latest)) + +
{{ $latest->period_start->format('M Y') }}
+ @include('client-reports::partials.state-pill', ['state' => $latestState]) + + @else + None yet + @endif + 		+ + @csrf + @method('PATCH') + + + +
+
@csrf - @method('PATCH') - +
+ + Edit + +
+ @csrf + @method('DELETE') +
-
-
- {{-- Run Now Action --}} -
- @csrf - -
- - - - - -
- @csrf - @method('DELETE') - -
-
-
- - No automated schedules created yet. Configure your first client site below! -
-
-
- - {{-- Unscheduled Care Plan Sites --}} - @if ($unscheduledSites->isNotEmpty()) -
-
-

- - Unscheduled Managed Sites ({{ $unscheduledSites->count() }}) -

- Sites ready for automated reporting -
-
- @foreach ($unscheduledSites as $us) -
-
-
- {{ $us->domain }}
- @if ($us->care_plan_enabled) - Care Plan Active - @else - Standard Monitored - @endif -
- - Schedule - -
- @endforeach -
-
- @endif +
+ + {{ $show === 'all' && $q === '' ? 'No report schedules yet.' : 'No schedules match.' }} +
+
@endsection diff --git a/modules/ClientReports/routes/web.php b/modules/ClientReports/routes/web.php index e9c143a8..35b29836 100644 --- a/modules/ClientReports/routes/web.php +++ b/modules/ClientReports/routes/web.php @@ -2,6 +2,7 @@ use Illuminate\Support\Facades\Route; use Modules\ClientReports\Http\Controllers\ClientReportsController; +use Modules\ClientReports\Http\Controllers\ClientsOverviewController; use Modules\ClientReports\Http\Controllers\SchedulesController; use Modules\ClientReports\Http\Controllers\TemplatesController; @@ -12,7 +13,15 @@ // Authenticated operator routes Route::middleware(['web', 'auth', 'active'])->group(function () { - Route::get('/client-reports', [ClientReportsController::class, 'index'])->name('client-reports.index'); + // The module's landing page: clients with their care plans, schedules and report history. + Route::get('/client-reports', [ClientsOverviewController::class, 'index'])->name('client-reports.index'); + Route::get('/client-reports/clients/{client}', [ClientsOverviewController::class, 'show']) + ->where('client', 'unassigned|[0-9]+') + ->name('client-reports.clients.show'); + + Route::get('/client-reports/reports', [ClientReportsController::class, 'index'])->name('client-reports.reports'); + // Recompiles a batch of unsent scheduled reports. Never sends. + Route::post('/client-reports/reports/recompile', [ClientReportsController::class, 'recompileMany'])->name('client-reports.recompile-many'); Route::post('/client-reports/generate', [ClientReportsController::class, 'generate'])->name('client-reports.generate'); // Templates CRUD Route::resource('/client-reports/templates', TemplatesController::class) @@ -20,6 +29,11 @@ ->names('client-reports.templates'); // Schedules CRUD & Actions + // One-click default schedules for care-plan sites that have none. Never send. + Route::post('/client-reports/schedules/quick/{site}', [SchedulesController::class, 'quickStore']) + ->name('client-reports.schedules.quick'); + Route::post('/client-reports/schedules/quick-all', [SchedulesController::class, 'quickStoreAll']) + ->name('client-reports.schedules.quick-all'); Route::resource('/client-reports/schedules', SchedulesController::class) ->except(['show']) ->names('client-reports.schedules'); @@ -28,9 +42,11 @@ Route::post('/client-reports/schedules/{schedule}/send-now', [SchedulesController::class, 'sendNow']) ->name('client-reports.schedules.send-now'); - Route::get('/client-reports/{report}', [ClientReportsController::class, 'show'])->name('client-reports.show'); - Route::get('/client-reports/{report}/email', [ClientReportsController::class, 'email'])->name('client-reports.email'); - Route::post('/client-reports/{report}/send', [ClientReportsController::class, 'send'])->name('client-reports.send'); + Route::get('/client-reports/{report}', [ClientReportsController::class, 'show'])->whereNumber('report')->name('client-reports.show'); + Route::get('/client-reports/{report}/email', [ClientReportsController::class, 'email'])->whereNumber('report')->name('client-reports.email'); + Route::post('/client-reports/{report}/send', [ClientReportsController::class, 'send'])->whereNumber('report')->name('client-reports.send'); + // Emails the client's exact email to the signed-in operator only. Changes nothing. + Route::post('/client-reports/{report}/send-test', [ClientReportsController::class, 'sendTest'])->whereNumber('report')->name('client-reports.send-test'); // Refreshes an unsent scheduled report from current data. Never sends. - Route::post('/client-reports/{report}/recompile', [ClientReportsController::class, 'recompile'])->name('client-reports.recompile'); + Route::post('/client-reports/{report}/recompile', [ClientReportsController::class, 'recompile'])->whereNumber('report')->name('client-reports.recompile'); }); diff --git a/modules/ClientReports/src/Http/Controllers/ClientReportsController.php b/modules/ClientReports/src/Http/Controllers/ClientReportsController.php index 94f175b0..8b48e5d3 100644 --- a/modules/ClientReports/src/Http/Controllers/ClientReportsController.php +++ b/modules/ClientReports/src/Http/Controllers/ClientReportsController.php @@ -7,29 +7,55 @@ use Illuminate\Http\RedirectResponse; use Illuminate\Http\Request; use Illuminate\Mail\Mailables\Address; +use Illuminate\Pagination\LengthAwarePaginator; use Illuminate\Support\Carbon; use Illuminate\Support\Facades\Log; +use Illuminate\Support\Facades\Mail; use Illuminate\View\View; use InvalidArgumentException; use Modules\ClientReports\Mail\ClientReportMail; +use Modules\ClientReports\Mail\ClientReportTestMail; use Modules\ClientReports\Models\ClientReport; use Modules\ClientReports\Models\ClientReportTemplate; use Modules\ClientReports\Services\ClientReportCompiler; use Modules\ClientReports\Services\ClientReportSender; +use Modules\ClientReports\Services\ReportBrowser; use Modules\ClientReports\Services\ScheduledReportRunner; +use Modules\ClientReports\Support\ReportListFilters; +use Modules\ClientReports\Support\ReportState; use Throwable; class ClientReportsController extends Controller { + /** Most reports one "Recompile" press works through, so the request finishes. */ + public const RECOMPILE_BATCH = 100; + /** - * Display reports dashboard. + * The reports list: one period at a time (the latest by default), + * filtered by state, hold reason, client or search, with a summary of + * the period that doubles as filter links. Filters live in the query + * string so they survive paging and a trip into a report and back. */ - public function index(): View + public function index(Request $request, ReportBrowser $browser): View { - $reports = ClientReport::query() - ->with(['site', 'template']) - ->orderByDesc('created_at') - ->paginate(20); + $periods = $browser->periods(); + $filters = ReportListFilters::fromRequest($request, array_key_first($periods) ?? ReportListFilters::ALL); + $reports = $browser->list($filters); + + $page = max(1, (int) $request->query('page', 1)); + $paginated = new LengthAwarePaginator( + $reports->forPage($page, ReportBrowser::PER_PAGE)->values(), + $reports->count(), + ReportBrowser::PER_PAGE, + $page, + ['path' => route('client-reports.reports'), 'query' => $filters->toQuery()], + ); + + // The summary strip counts the whole period, whatever else is filtered. + $periodReports = $browser->list(new ReportListFilters(period: $filters->period)); + $outdatedCount = $periodReports + ->filter(fn (ClientReport $r) => $r->canRecompile() && ReportState::of($r) === ReportState::OUTDATED) + ->count(); $sites = Site::query() ->where('is_inactive', false) @@ -41,7 +67,15 @@ public function index(): View ->orderBy('name') ->get(); - return view('client-reports::index', compact('reports', 'sites', 'templates')); + return view('client-reports::index', [ + 'reports' => $paginated, + 'filters' => $filters, + 'periods' => $periods, + 'summary' => $browser->summarize($periodReports), + 'outdatedCount' => $outdatedCount, + 'sites' => $sites, + 'templates' => $templates, + ]); } /** @@ -96,18 +130,22 @@ public function generate(Request $request, ClientReportCompiler $compiler): Redi /** * Preview report for operator. */ - public function show(ClientReport $report): View + public function show(ClientReport $report, Request $request, ReportBrowser $browser): View { - $report->load('site', 'schedule'); + $report->load('site.client', 'schedule'); - return view('client-reports::reports.show', ['report' => $report, 'isOperatorPreview' => true]); + return view('client-reports::reports.show', [ + 'report' => $report, + 'isOperatorPreview' => true, + 'nav' => $this->navigation($report, $request, $browser), + ]); } /** * The email exactly as the client gets it. The preview page lays some * sections out differently, so this is what to check before sending. */ - public function email(ClientReport $report): View + public function email(ClientReport $report, Request $request, ReportBrowser $browser): View { $report->load('site.client', 'schedule'); $mail = new ClientReportMail($report); @@ -123,6 +161,7 @@ public function email(ClientReport $report): View 'html' => $mail->render(), // Plain-text clients get this part instead. 'text' => view($content->text, $content->with)->render(), + 'nav' => $this->navigation($report, $request, $browser), ]); } @@ -169,9 +208,15 @@ public function recompile(ClientReport $report, ScheduledReportRunner $runner): $message = 'Recompiled from current data for '.$fresh->period_start->format('M j').' – '.$fresh->period_end->format('M j, Y').'. Nothing was sent.'; $message .= $run->held() ? ' It is held for review: '.implode(' ', $run->heldReasons) - : ' Nothing needs review; send it from the history table when ready.'; + : ' Nothing needs review; send it when ready.'; + + // Keep the list context the operator came from (filters, client page). + $context = array_filter( + (array) request()->only(['period', 'status', 'category', 'q', 'client', 'from', 'superseded']), + fn ($value) => is_string($value) && $value !== '', + ); - return redirect()->route('client-reports.show', $fresh)->with('status', $message); + return redirect()->route('client-reports.show', [$fresh, ...$context])->with('status', $message); } /** @@ -232,4 +277,140 @@ public function send(ClientReport $report, Request $request, ClientReportSender return back()->with('status', 'Report emailed to '.implode(', ', $delivered).'.'); } + + /** + * Email the client's exact email to the signed-in operator, and only to + * them, with "[TEST]" in the subject. Nothing about the report or its + * schedule changes: it isn't marked sent, nobody is added to sent_to and + * the schedule's last send stays as it was. + */ + public function sendTest(ClientReport $report, Request $request): RedirectResponse + { + $to = (string) ($request->user()?->email ?? ''); + if (! filter_var($to, FILTER_VALIDATE_EMAIL)) { + return back()->with('status_error', 'Your account has no valid email address to send a test copy to.'); + } + + try { + Mail::to($to)->send(new ClientReportTestMail($report)); + } catch (Throwable $e) { + Log::error("Test copy of client report #{$report->id} failed to send: {$e->getMessage()}", ['exception' => $e]); + + return back()->with('status_error', "The test copy to {$to} failed: {$e->getMessage()}"); + } + + return back()->with('status', "Test copy sent to {$to} only, with [TEST] in the subject. The client got nothing and the report is unchanged."); + } + + /** + * Recompile a batch of unsent scheduled reports (every outdated one of a + * period, or the ones ticked on the list) through the same + * ScheduledReportRunner::recompile() as the single Recompile button. It + * never sends anything and never moves a schedule on. + */ + public function recompileMany(Request $request, ReportBrowser $browser, ScheduledReportRunner $runner): RedirectResponse + { + $filters = ReportListFilters::fromRequest($request, ReportListFilters::ALL); + $back = redirect()->route('client-reports.reports', $filters->toQuery()); + + if ($request->input('scope') === 'selected') { + $ids = collect((array) $request->input('ids', [])) + ->filter(fn ($id) => is_scalar($id) && ctype_digit((string) $id)) + ->map(fn ($id) => (int) $id) + ->unique() + ->values(); + if ($ids->isEmpty()) { + return $back->with('status_error', 'Tick the reports to recompile first.'); + } + } else { + $ids = $browser->list(new ReportListFilters(period: $filters->period, status: ReportState::OUTDATED)) + ->filter(fn (ClientReport $r) => $r->canRecompile()) + ->pluck('id') + ->values(); + if ($ids->isEmpty()) { + return $back->with('status', 'No outdated reports to recompile in this period.'); + } + } + + @set_time_limit(600); + $remaining = max(0, $ids->count() - self::RECOMPILE_BATCH); + $recompiled = $held = $skipped = 0; + $failed = []; + + foreach ($ids->take(self::RECOMPILE_BATCH) as $id) { + $report = ClientReport::query()->find($id); + if ($report === null || ! $report->canRecompile()) { + $skipped++; + + continue; + } + + try { + $run = $runner->recompile($report); + $recompiled++; + $held += $run->held() ? 1 : 0; + } catch (Throwable $e) { + $failed[] = $report->site?->domain ?? "#{$report->id}"; + Log::error("Recompiling client report #{$report->id} failed: {$e->getMessage()}", ['exception' => $e]); + } + } + + $message = "Recompiled {$recompiled} report(s) from current data, {$held} of them held for review."; + if ($skipped > 0) { + $message .= " Skipped {$skipped} that can't be recompiled (sent, replaced or not scheduled)."; + } + if ($remaining > 0) { + $message .= " {$remaining} more are left: press Recompile again to continue."; + } + $message .= ' Nothing was sent.'; + + return $failed === [] + ? $back->with('status', $message) + : $back->with('status_error', $message.' Failed: '.implode(', ', $failed).' (see the log).'); + } + + /** + * Where the preview and email pages lead: back to the list (or client + * page) they were opened from with its filters, and to the previous and + * next report of that same filtered list. Opened on its own, a report + * steps through its own period. + * + * @return array{query: array, previous: ?int, next: ?int, position: ?int, count: int, back_url: string, back_label: string} + */ + protected function navigation(ClientReport $report, Request $request, ReportBrowser $browser): array + { + $hasContext = collect(['period', 'status', 'category', 'q', 'client', 'superseded']) + ->contains(fn (string $key) => $request->query->has($key)); + $filters = $hasContext + ? ReportListFilters::fromRequest($request, ReportListFilters::ALL) + : new ReportListFilters( + period: $report->period_start->format('Y-m'), + showSuperseded: $report->status === ClientReport::STATUS_SUPERSEDED, + ); + + $ids = $browser->list($filters)->pluck('id')->values(); + $position = $ids->search($report->id, true); + + if ($filters->from === 'client' && $filters->client !== null) { + $backUrl = route('client-reports.clients.show', $filters->client); + $backLabel = 'Back to client'; + } else { + $query = $filters->toQuery(); + if ($position !== false && $position >= ReportBrowser::PER_PAGE) { + $query['page'] = (string) (intdiv($position, ReportBrowser::PER_PAGE) + 1); + } + $backUrl = route('client-reports.reports', $query); + $backLabel = 'Back to list'; + } + + return [ + 'query' => $filters->toQuery(), + 'previous' => $position !== false && $position > 0 ? $ids[$position - 1] : null, + 'next' => $position !== false && $position < $ids->count() - 1 ? $ids[$position + 1] : null, + 'position' => $position === false ? null : $position + 1, + 'count' => $ids->count(), + 'back_url' => $backUrl, + 'back_label' => $backLabel, + ]; + } } diff --git a/modules/ClientReports/src/Http/Controllers/ClientsOverviewController.php b/modules/ClientReports/src/Http/Controllers/ClientsOverviewController.php new file mode 100644 index 00000000..570a8345 --- /dev/null +++ b/modules/ClientReports/src/Http/Controllers/ClientsOverviewController.php @@ -0,0 +1,78 @@ +query()); + $all = $directory->rows(); + $rows = $directory->filter($all, $filters); + + $page = max(1, (int) $request->query('page', 1)); + $paginator = new LengthAwarePaginator( + $rows->forPage($page, self::PER_PAGE)->values(), + $rows->count(), + self::PER_PAGE, + $page, + ['path' => route('client-reports.index'), 'query' => $filters], + ); + + $totals = [ + 'clients' => $all->filter(fn (array $row) => $row['client'] !== null)->count(), + 'care' => $all->sum(fn (array $row) => $row['counts']['care']), + 'missing' => $all->sum(fn (array $row) => $row['counts']['missing']), + 'waiting' => $all->sum(fn (array $row) => $row['counts']['waiting']), + 'unassigned' => (int) ($all->firstWhere('key', ReportListFilters::UNASSIGNED)['counts']['sites'] ?? 0), + ]; + + return view('client-reports::clients.index', [ + 'rows' => $paginator, + 'filters' => $filters, + 'months' => $directory->months(), + 'totals' => $totals, + ]); + } + + public function show(Request $request, string $client, ClientDirectory $directory): View + { + abort_unless($client === ReportListFilters::UNASSIGNED || ctype_digit($client), 404); + + $row = $directory->find($client); + abort_if($row === null, 404); + + // Previous/next follow the list the operator came from (same filters, + // same alphabetical order); the switcher offers every client. + $filters = ClientDirectory::filtersFrom($request->query()); + $all = $directory->rows(); + $list = $directory->filter($all, $filters)->pluck('key')->values(); + $position = $list->search($row['key'], true); + + return view('client-reports::clients.show', [ + 'row' => $row, + 'filters' => $filters, + 'months' => $directory->months(), + 'previousKey' => $position !== false && $position > 0 ? $list[$position - 1] : null, + 'nextKey' => $position !== false && $position < $list->count() - 1 ? $list[$position + 1] : null, + 'position' => $position === false ? null : $position + 1, + 'listCount' => $list->count(), + 'switcher' => $all->map(fn (array $r) => ['key' => $r['key'], 'name' => $r['name'], 'sites' => $r['counts']['sites']])->values(), + 'showSuperseded' => $request->boolean('superseded'), + ]); + } +} diff --git a/modules/ClientReports/src/Http/Controllers/SchedulesController.php b/modules/ClientReports/src/Http/Controllers/SchedulesController.php index df87a44d..2ec05029 100644 --- a/modules/ClientReports/src/Http/Controllers/SchedulesController.php +++ b/modules/ClientReports/src/Http/Controllers/SchedulesController.php @@ -7,32 +7,83 @@ use Illuminate\Http\RedirectResponse; use Illuminate\Http\Request; use Illuminate\Support\Facades\Log; +use Illuminate\Support\Str; use Illuminate\View\View; +use Modules\ClientReports\Models\ClientReport; use Modules\ClientReports\Models\ClientReportSchedule; use Modules\ClientReports\Models\ClientReportTemplate; +use Modules\ClientReports\Services\ReportBrowser; use Modules\ClientReports\Services\ReportRecipientsSyncService; use Modules\ClientReports\Services\ScheduledReportRunner; use Throwable; class SchedulesController extends Controller { - public function index(): View + /** Schedule list filters: who won't get a report, and why. */ + public const FILTERS = [ + 'all' => 'All', + 'wont_send' => 'Won\'t send: no care plan', + 'no_recipients' => 'No recipients', + 'disabled' => 'Disabled', + 'draft' => 'Drafts for review', + ]; + + public function index(Request $request, ReportBrowser $browser): View { - $schedules = ClientReportSchedule::query() - ->with(['site', 'template']) + // A schedule outlives its site's archiving; those sites are left out. + $all = ClientReportSchedule::query() + ->with(['site.client:id,name,company_name', 'template']) ->get() - ->sortBy(fn ($s) => $s->site->domain); - - $scheduledSiteIds = $schedules->pluck('site_id')->all(); - - $unscheduledSites = Site::query() + ->filter(fn (ClientReportSchedule $s) => $s->site !== null) + ->sortBy(fn (ClientReportSchedule $s) => Str::lower($s->site->domain)) + ->values(); + + $matches = [ + 'all' => fn (ClientReportSchedule $s) => true, + 'wont_send' => fn (ClientReportSchedule $s) => ! ScheduledReportRunner::siteIsEligible($s->site), + 'no_recipients' => fn (ClientReportSchedule $s) => array_filter((array) $s->recipients) === [], + 'disabled' => fn (ClientReportSchedule $s) => ! $s->is_enabled, + 'draft' => fn (ClientReportSchedule $s) => $s->delivery_mode === ClientReportSchedule::MODE_DRAFT, + ]; + $counts = array_map(fn (callable $match) => $all->filter($match)->count(), $matches); + + $show = array_key_exists((string) $request->query('show'), self::FILTERS) ? (string) $request->query('show') : 'all'; + $q = trim(mb_substr((string) $request->query('q', ''), 0, 100)); + $needle = Str::lower($q); + $schedules = $all->filter($matches[$show]) + ->filter(fn (ClientReportSchedule $s) => $needle === '' || Str::contains(Str::lower(implode(' ', [ + $s->site->domain, $s->site->client?->name, $s->site->client?->company_name, ...array_filter((array) $s->recipients), + ])), $needle)) + ->values(); + + // Each schedule's latest report (newest period, newest copy), so its + // state shows next to it. + $latestReports = $browser->get($browser->query() + ->whereIn('schedule_id', $all->pluck('id')) + ->where('status', '!=', ClientReport::STATUS_SUPERSEDED)) + ->sortByDesc(fn (ClientReport $r) => $r->period_start->format('Y-m-d').'-'.str_pad((string) $r->id, 12, '0', STR_PAD_LEFT)) + ->unique('schedule_id') + ->keyBy('schedule_id'); + + // Care-plan sites that get no report at all: the coverage gap. + $sync = app(ReportRecipientsSyncService::class); + $missingSites = Site::query() ->where('is_inactive', false) - ->whereNotIn('id', $scheduledSiteIds) - ->orderByDesc('care_plan_enabled') + ->carePlanEligible() + ->whereNotIn('id', ClientReportSchedule::query()->select('site_id')) + ->with('client') ->orderBy('domain') - ->get(['id', 'domain', 'care_plan_enabled']); - - return view('client-reports::schedules.index', compact('schedules', 'unscheduledSites')); + ->get() + ->map(fn (Site $site) => ['site' => $site, 'recipients' => $sync->resolvePayloadForSite($site)['report_recipients']]); + + return view('client-reports::schedules.index', [ + 'schedules' => $schedules, + 'counts' => $counts, + 'show' => $show, + 'q' => $q, + 'latestReports' => $latestReports, + 'missingSites' => $missingSites, + ]); } public function create(Request $request): View|RedirectResponse @@ -203,7 +254,7 @@ public function destroy(ClientReportSchedule $schedule): RedirectResponse ->with('status', "Reporting schedule for {$domain} deleted."); } - public function toggle(ClientReportSchedule $schedule): RedirectResponse + public function toggle(Request $request, ClientReportSchedule $schedule): RedirectResponse { $schedule->update([ 'is_enabled' => ! $schedule->is_enabled, @@ -215,8 +266,90 @@ public function toggle(ClientReportSchedule $schedule): RedirectResponse $state = $schedule->is_enabled ? 'enabled' : 'paused'; - return redirect()->route('client-reports.schedules.index') - ->with('status', "Schedule for {$schedule->site->domain} has been {$state}."); + // The client page toggles in place and stays where it was. + $redirect = $request->input('return') === 'back' + ? back() + : redirect()->route('client-reports.schedules.index'); + + return $redirect->with('status', "Schedule for {$schedule->site->domain} has been {$state}."); + } + + /** + * One-click schedule for a care-plan site that has none: the default + * template, monthly, sent to the client's contacts (the same fallback + * WordPress is shown) or, with no contacts on file, as drafts for + * review. Its first report always waits for someone to send it. + */ + public function quickStore(Site $site): RedirectResponse + { + if ($site->is_inactive) { + return back()->with('status_error', "{$site->domain} is inactive, so it gets no reports."); + } + + if (! ScheduledReportRunner::siteIsEligible($site)) { + return back()->with('status_error', "{$site->domain} has no care plan. Client reports are only sent for care-plan sites."); + } + + if (ClientReportSchedule::query()->where('site_id', $site->id)->exists()) { + return back()->with('status', "{$site->domain} already has a report schedule."); + } + + $schedule = $this->createDefaultSchedule($site); + $recipients = (array) $schedule->recipients; + + $message = $schedule->delivery_mode === ClientReportSchedule::MODE_AUTO + ? "Created a monthly report schedule for {$site->domain}, to ".implode(', ', $recipients).'. Its first report ('.$schedule->next_run_at->format('M j').') waits for you to check and send it.' + : "Created a monthly report schedule for {$site->domain} as drafts for review: the client has no contacts on file. Add recipients to send it."; + + return back()->with('status', $message); + } + + /** + * The same one-click schedule for every active care-plan site that has + * none. Nothing is sent: each schedule's first report waits for review. + */ + public function quickStoreAll(): RedirectResponse + { + $sites = Site::query() + ->where('is_inactive', false) + ->carePlanEligible() + ->whereNotIn('id', ClientReportSchedule::query()->select('site_id')) + ->orderBy('domain') + ->get(); + + if ($sites->isEmpty()) { + return back()->with('status', 'Every care-plan site already has a report schedule.'); + } + + $drafts = $sites->map(fn (Site $site) => $this->createDefaultSchedule($site)) + ->filter(fn (ClientReportSchedule $s) => $s->delivery_mode === ClientReportSchedule::MODE_DRAFT) + ->count(); + + return back()->with('status', "Created {$sites->count()} monthly report schedule(s)" + .($drafts > 0 ? ", {$drafts} of them as drafts for review because the client has no contacts on file" : '') + .'. Each first report waits for you to check and send it. Nothing was sent.'); + } + + /** Default template, monthly, client contacts (auto) or none (drafts). */ + protected function createDefaultSchedule(Site $site): ClientReportSchedule + { + $sync = app(ReportRecipientsSyncService::class); + $recipients = $sync->resolvePayloadForSite($site)['report_recipients']; + $template = ClientReportTemplate::query()->orderByDesc('is_default')->orderBy('name')->first(); + + $schedule = ClientReportSchedule::create([ + 'site_id' => $site->id, + 'template_id' => $template?->id, + 'frequency' => ClientReportSchedule::FREQUENCY_MONTHLY, + 'delivery_mode' => $recipients !== [] ? ClientReportSchedule::MODE_AUTO : ClientReportSchedule::MODE_DRAFT, + 'recipients' => $recipients, + 'is_enabled' => true, + 'next_run_at' => (new ClientReportSchedule(['frequency' => ClientReportSchedule::FREQUENCY_MONTHLY]))->computeNextRun(), + ]); + + $sync->pushToSite($site, $schedule); + + return $schedule; } public function sendNow(ClientReportSchedule $schedule, ScheduledReportRunner $runner): RedirectResponse diff --git a/modules/ClientReports/src/Mail/ClientReportTestMail.php b/modules/ClientReports/src/Mail/ClientReportTestMail.php new file mode 100644 index 00000000..70e7e9a5 --- /dev/null +++ b/modules/ClientReports/src/Mail/ClientReportTestMail.php @@ -0,0 +1,23 @@ +subject(self::SUBJECT_PREFIX.$envelope->subject); + } +} diff --git a/modules/ClientReports/src/Services/ClientDirectory.php b/modules/ClientReports/src/Services/ClientDirectory.php new file mode 100644 index 00000000..a419f98b --- /dev/null +++ b/modules/ClientReports/src/Services/ClientDirectory.php @@ -0,0 +1,266 @@ + cell, last six + * report periods: report, state). + */ +class ClientDirectory +{ + public const MONTHS = 6; + + /** A history-strip month with no report and no reason to expect one. */ + public const CELL_NONE = 'none'; + + /** A month the schedule was set up for but the site had no care plan, so nothing was sent. */ + public const CELL_SKIPPED = 'skipped'; + + public function __construct(protected ReportBrowser $browser) {} + + /** + * The last report periods, oldest first, ending with last month. + * + * @return list + */ + public function months(int $count = self::MONTHS): array + { + $last = now()->subMonthNoOverflow()->startOfMonth(); + + return array_map(fn (int $i) => $last->copy()->subMonthsNoOverflow($i), range($count - 1, 0)); + } + + /** + * Every client (and the Unassigned group, when it has sites), sorted by name. + * + * @param bool $withHistory load every report of every site (the client page) instead of only the last months + * @return Collection> + */ + public function rows(bool $withHistory = false, ?string $only = null): Collection + { + $clients = Client::query() + ->when($only !== null && $only !== ReportListFilters::UNASSIGNED, fn ($q) => $q->whereKey((int) $only)) + ->when($only === ReportListFilters::UNASSIGNED, fn ($q) => $q->whereRaw('1 = 0')) + ->get(['id', 'name', 'company_name', 'email', 'additional_emails']); + + $sites = Site::query() + ->when($only !== null, fn ($q) => $only === ReportListFilters::UNASSIGNED + ? $q->whereNull('client_id') + : $q->where('client_id', (int) $only)) + ->orderBy('domain') + ->get(['id', 'domain', 'client_id', 'care_plan_enabled', 'is_inactive', 'created_at']); + + $schedules = ClientReportSchedule::query() + ->whereIn('site_id', $sites->pluck('id')) + ->get() + ->keyBy('site_id'); + + $months = $this->months(); + $reportQuery = $this->browser->query()->whereIn('site_id', $sites->pluck('id')); + if (! $withHistory) { + $reportQuery->where('period_start', '>=', $months[0]->toDateString()) + ->where('status', '!=', ClientReport::STATUS_SUPERSEDED); + } + $reportsBySite = $this->browser->get($reportQuery) + ->sortByDesc(fn (ClientReport $r) => $r->period_start->format('Y-m-d').'-'.str_pad((string) $r->id, 12, '0', STR_PAD_LEFT)) + ->groupBy('site_id'); + + $siteRows = $sites->map(fn (Site $site) => $this->siteRow($site, $schedules->get($site->id), $reportsBySite->get($site->id, collect()), $months)) + ->groupBy(fn (array $row) => $row['site']->client_id === null ? ReportListFilters::UNASSIGNED : (string) $row['site']->client_id); + + $rows = $clients->map(fn (Client $client) => $this->row( + (string) $client->id, + $client, + $siteRows->get((string) $client->id, collect())->values()->all(), + )); + + $unassigned = $siteRows->get(ReportListFilters::UNASSIGNED); + if ($unassigned !== null && $unassigned->isNotEmpty()) { + $rows->push($this->row(ReportListFilters::UNASSIGNED, null, $unassigned->values()->all())); + } + + // Alphabetical, the Unassigned group last. + return $rows + ->sortBy(fn (array $row) => ($row['client'] === null ? '1' : '0').Str::lower($row['name'])) + ->values(); + } + + /** @return array|null */ + public function find(string $key, bool $withHistory = true): ?array + { + return $this->rows($withHistory, $key)->first(); + } + + /** + * Narrow the rows to the list's filters: q (client, contact, site or + * recipient), care (yes: any site on a care plan, no: none), waiting + * (a report waiting for review or sending) and missing (a care-plan site + * with no schedule). + * + * @param Collection> $rows + * @param array $filters + * @return Collection> + */ + public function filter(Collection $rows, array $filters): Collection + { + $q = Str::lower(trim((string) ($filters['q'] ?? ''))); + $care = $filters['care'] ?? null; + + return $rows->filter(function (array $row) use ($q, $care, $filters) { + if ($q !== '' && ! Str::contains($this->haystack($row), $q)) { + return false; + } + if ($care === 'yes' && $row['counts']['care'] === 0) { + return false; + } + if ($care === 'no' && $row['counts']['care'] > 0) { + return false; + } + if (! empty($filters['waiting']) && $row['counts']['waiting'] === 0) { + return false; + } + + return empty($filters['missing']) || $row['counts']['missing'] > 0; + })->values(); + } + + /** + * The list filters a request asks for, in the shape filter() takes, for + * the query string of every link that should keep them. + * + * @param array $input + * @return array + */ + public static function filtersFrom(array $input): array + { + return array_filter([ + 'q' => trim(mb_substr((string) ($input['q'] ?? ''), 0, 100)), + 'care' => in_array($input['care'] ?? null, ['yes', 'no'], true) ? $input['care'] : null, + 'waiting' => ! empty($input['waiting']) ? '1' : null, + 'missing' => ! empty($input['missing']) ? '1' : null, + ], fn ($value) => $value !== null && $value !== ''); + } + + /** + * @param list> $sites + * @return array + */ + protected function row(string $key, ?Client $client, array $sites): array + { + $company = trim((string) $client?->company_name); + $person = trim((string) $client?->name); + $name = $client === null ? 'Unassigned sites' : ($company !== '' ? $company : $person); + + $active = array_filter($sites, fn (array $s) => ! $s['site']->is_inactive); + + return [ + 'key' => $key, + 'client' => $client, + 'name' => $name, + 'contact' => $client !== null && $person !== '' && $person !== $name ? $person : null, + 'contacts' => $client?->allRecipients() ?? [], + 'sites' => $sites, + 'counts' => [ + 'sites' => count($sites), + 'care' => count(array_filter($active, fn (array $s) => $s['eligible'])), + 'scheduled' => count(array_filter($sites, fn (array $s) => $s['schedule_state'] === 'scheduled')), + 'missing' => count(array_filter($sites, fn (array $s) => $s['missing'])), + 'waiting' => array_sum(array_map(fn (array $s) => $s['waiting'], $sites)), + 'review' => array_sum(array_map(fn (array $s) => $s['review'], $sites)), + ], + ]; + } + + /** + * @param Collection $reports newest period first + * @param list $months + * @return array + */ + protected function siteRow(Site $site, ?ClientReportSchedule $schedule, Collection $reports, array $months): array + { + $eligible = ScheduledReportRunner::siteIsEligible($site); + $live = $reports->filter(fn (ClientReport $r) => $r->status !== ClientReport::STATUS_SUPERSEDED); + + $cells = []; + foreach ($months as $month) { + $key = $month->format('Y-m'); + $report = $live->first(fn (ClientReport $r) => $r->period_start->format('Y-m') === $key); + $state = match (true) { + $report !== null => ReportState::of($report), + // The schedule existed by the run on the 1st of the next month + // but the site has no care plan, so that run skipped it. + $schedule !== null && ! $eligible && $schedule->created_at !== null + && $schedule->created_at->lte($month->copy()->addMonthNoOverflow()->endOfDay()) => self::CELL_SKIPPED, + default => self::CELL_NONE, + }; + $cells[$key] = ['month' => $month, 'report' => $report, 'state' => $state]; + } + + // Waiting/review count each site's newest live report only: an older + // unsent period that a newer one followed isn't something to chase. + $latestState = ($latest = $live->first()) !== null ? ReportState::of($latest) : null; + + return [ + 'site' => $site, + 'eligible' => $eligible, + 'schedule' => $schedule, + 'schedule_state' => match (true) { + $schedule === null => 'none', + $schedule->is_enabled => 'scheduled', + default => 'disabled', + }, + 'missing' => $schedule === null && $eligible && ! $site->is_inactive, + 'latest' => $latest, + 'latest_state' => $latestState, + 'waiting' => in_array($latestState, ReportState::WAITING, true) ? 1 : 0, + 'review' => in_array($latestState, [ReportState::NEEDS_REVIEW, ReportState::OUTDATED], true) ? 1 : 0, + 'reports' => $reports->values(), + 'months' => $cells, + ]; + } + + /** @param array $row */ + protected function haystack(array $row): string + { + $parts = [$row['name'], $row['contact'], ...$row['contacts']]; + foreach ($row['sites'] as $site) { + $parts[] = $site['site']->domain; + foreach ((array) ($site['schedule']?->recipients ?? []) as $recipient) { + $parts[] = $recipient; + } + } + + return Str::lower(implode(' ', array_filter($parts))); + } +} diff --git a/modules/ClientReports/src/Services/ReportBrowser.php b/modules/ClientReports/src/Services/ReportBrowser.php new file mode 100644 index 00000000..77acbe18 --- /dev/null +++ b/modules/ClientReports/src/Services/ReportBrowser.php @@ -0,0 +1,160 @@ + */ + public function query(): Builder + { + return ClientReport::query() + ->select(self::LIGHT_COLUMNS) + ->addSelect('sections_data->meta as report_meta') + ->with([ + // A report outlives its site's archiving; keep the domain readable. + 'site' => fn ($q) => $q->withoutGlobalScopes()->select(['id', 'domain', 'client_id', 'care_plan_enabled', 'is_inactive', 'archived_at']), + 'site.client:id,name,company_name', + 'schedule:id,site_id,delivery_mode,last_sent_at,is_enabled,recipients', + ]); + } + + /** + * @param Builder $query + * @return EloquentCollection + */ + public function get(Builder $query): EloquentCollection + { + $reports = $query->get(); + foreach ($reports as $report) { + $meta = json_decode((string) $report->getAttribute('report_meta'), true); + $attributes = Arr::except($report->getAttributes(), 'report_meta'); + $attributes['sections_data'] = json_encode(['meta' => is_array($meta) ? $meta : []]); + $report->setRawAttributes($attributes, true); + } + + return $reports; + } + + /** + * Months that have reports, newest first ("2026-10" => count of live reports). + * + * @return array + */ + public function periods(): array + { + $counts = []; + ClientReport::query() + ->where('status', '!=', ClientReport::STATUS_SUPERSEDED) + ->pluck('period_start') + ->each(function ($start) use (&$counts) { + $key = Carbon::parse($start)->format('Y-m'); + $counts[$key] = ($counts[$key] ?? 0) + 1; + }); + krsort($counts); + + return $counts; + } + + public function latestPeriod(): ?string + { + return array_key_first($this->periods()); + } + + /** + * Every report the filters match, in list order: newest period first, + * then by site. + * + * @return Collection + */ + public function list(ReportListFilters $filters): Collection + { + $query = $this->query(); + + if ($filters->period !== null && $filters->period !== ReportListFilters::ALL) { + $month = Carbon::createFromFormat('!Y-m', $filters->period); + $query->where('period_start', '>=', $month->toDateString()) + ->where('period_start', '<', $month->copy()->addMonthNoOverflow()->toDateString()); + } + + if (! $filters->showSuperseded && $filters->status !== ReportState::SUPERSEDED) { + $query->where('status', '!=', ClientReport::STATUS_SUPERSEDED); + } + + if ($filters->client === ReportListFilters::UNASSIGNED) { + $query->whereHas('site', fn ($q) => $q->withoutGlobalScopes()->whereNull('client_id')); + } elseif ($filters->client !== null) { + $query->whereHas('site', fn ($q) => $q->withoutGlobalScopes()->where('client_id', (int) $filters->client)); + } + + $reports = $this->get($query)->toBase(); + + if ($filters->q !== '') { + $needle = Str::lower($filters->q); + $reports = $reports->filter(fn (ClientReport $r) => Str::contains(Str::lower(implode(' ', array_filter([ + $r->site?->domain, $r->title, $r->client_name, $r->client_email, + $r->site?->client?->name, $r->site?->client?->company_name, + ]))), $needle)); + } + + if ($filters->status !== null && $filters->status !== ReportListFilters::ALL) { + $reports = $reports->filter(fn (ClientReport $r) => $filters->status === ReportListFilters::WAITING + ? in_array(ReportState::of($r), ReportState::WAITING, true) + : ReportState::of($r) === $filters->status); + } + + if ($filters->category !== null) { + $reports = $reports->filter(fn (ClientReport $r) => in_array($filters->category, ReviewReasonCategories::forReasons(ReportState::reasons($r)), true)); + } + + return $reports + ->sort(fn (ClientReport $a, ClientReport $b) => [$b->period_start->format('Y-m-d'), Str::lower((string) $a->site?->domain), $b->id] + <=> [$a->period_start->format('Y-m-d'), Str::lower((string) $b->site?->domain), $a->id]) + ->values(); + } + + /** + * How many reports are in each state, for the summary strip. + * + * @param Collection $reports + * @return array + */ + public function summarize(Collection $reports): array + { + $counts = array_fill_keys(array_keys(ReportState::options()), 0); + foreach ($reports as $report) { + $state = ReportState::of($report); + $counts[$state] = ($counts[$state] ?? 0) + 1; + } + $counts['total'] = $reports->count(); + + return $counts; + } +} diff --git a/modules/ClientReports/src/Support/ModuleNavCounts.php b/modules/ClientReports/src/Support/ModuleNavCounts.php new file mode 100644 index 00000000..a3865775 --- /dev/null +++ b/modules/ClientReports/src/Support/ModuleNavCounts.php @@ -0,0 +1,59 @@ +latestPeriod(); + $waiting = $review = 0; + if ($period !== null) { + foreach ($browser->list(new ReportListFilters(period: $period)) as $report) { + $state = ReportState::of($report); + $waiting += in_array($state, ReportState::WAITING, true) ? 1 : 0; + $review += in_array($state, [ReportState::NEEDS_REVIEW, ReportState::OUTDATED], true) ? 1 : 0; + } + } + + return [ + 'period' => $period, + 'period_label' => $period !== null ? Carbon::createFromFormat('!Y-m', $period)->format('M Y') : null, + 'waiting' => $waiting, + 'review' => $review, + 'missing' => self::sitesMissingSchedule(), + ]; + } catch (Throwable $e) { + Log::warning('client_reports.nav_counts_failed', ['error' => $e->getMessage()]); + + return ['period' => null, 'period_label' => null, 'waiting' => 0, 'review' => 0, 'missing' => 0]; + } + }); + } + + /** Active care-plan sites (all active sites when care plans are off fleet-wide) with no report schedule. */ + public static function sitesMissingSchedule(): int + { + return Site::query() + ->where('is_inactive', false) + ->carePlanEligible() + ->whereNotIn('id', ClientReportSchedule::query()->select('site_id')) + ->count(); + } +} diff --git a/modules/ClientReports/src/Support/ReportListFilters.php b/modules/ClientReports/src/Support/ReportListFilters.php new file mode 100644 index 00000000..f820501c --- /dev/null +++ b/modules/ClientReports/src/Support/ReportListFilters.php @@ -0,0 +1,88 @@ +query('period', ''); + $period = $period === self::ALL || preg_match('/^\d{4}-(0[1-9]|1[0-2])$/', $period) === 1 + ? $period + : $defaultPeriod; + + $status = (string) $request->query('status', ''); + $status = $status === self::WAITING || ReportState::isKnown($status) ? $status : null; + + $category = (string) $request->query('category', ''); + $category = ReviewReasonCategories::isKnown($category) ? $category : null; + + $client = (string) $request->query('client', ''); + $client = $client === self::UNASSIGNED || ctype_digit($client) ? $client : null; + + return new self( + period: $period, + status: $status, + category: $category, + q: trim(mb_substr((string) $request->query('q', ''), 0, 100)), + showSuperseded: $request->boolean('superseded'), + client: $client, + from: $client !== null && $request->query('from') === 'client' ? 'client' : null, + ); + } + + /** + * The non-empty filters as query parameters. + * + * @return array + */ + public function toQuery(): array + { + return array_filter([ + 'period' => $this->period, + 'status' => $this->status, + 'category' => $this->category, + 'q' => $this->q, + 'superseded' => $this->showSuperseded ? '1' : null, + 'client' => $this->client, + 'from' => $this->from, + ], fn ($value) => $value !== null && $value !== ''); + } + + /** @param array $changes */ + public function with(array $changes): array + { + return array_filter(array_merge($this->toQuery(), $changes), fn ($value) => $value !== null && $value !== ''); + } + + public function isFiltered(): bool + { + return $this->status !== null || $this->category !== null || $this->q !== '' || $this->client !== null; + } +} diff --git a/modules/ClientReports/src/Support/ReportState.php b/modules/ClientReports/src/Support/ReportState.php new file mode 100644 index 00000000..53a5fdb1 --- /dev/null +++ b/modules/ClientReports/src/Support/ReportState.php @@ -0,0 +1,169 @@ + */ + private const META = [ + self::NEEDS_REVIEW => ['label' => 'Needs review', 'pill' => 'status-orange', 'icon' => 'fa-hand', 'hint' => 'Held: something in it needs checking or fixing before it goes out.'], + self::OUTDATED => ['label' => 'Outdated', 'pill' => 'cr-pill-purple', 'icon' => 'fa-clock-rotate-left', 'hint' => 'Compiled before the latest report fixes. Recompile it before sending.'], + self::READY => ['label' => 'Ready to send', 'pill' => 'cr-pill-blue', 'icon' => 'fa-paper-plane', 'hint' => 'Nothing to fix: waiting for someone to send it (a schedule\'s first report always waits).'], + self::PARTLY_SENT => ['label' => 'Partly sent', 'pill' => 'status-yellow', 'icon' => 'fa-circle-half-stroke', 'hint' => 'Some recipients have it; sending again only emails the rest.'], + self::SENT => ['label' => 'Sent', 'pill' => 'status-green', 'icon' => 'fa-circle-check', 'hint' => 'Everyone on it has it.'], + self::SUPERSEDED => ['label' => 'Replaced', 'pill' => 'status-unknown', 'icon' => 'fa-clone', 'hint' => 'A newer copy for this period replaced it. It can\'t be sent.'], + ]; + + public static function of(ClientReport $report): string + { + return self::classify( + (string) $report->status, + (array) ($report->sent_to ?? []), + $report->isStale(), + self::reasons($report), + ); + } + + /** + * @param list $sentTo + * @param iterable $reasons + */ + public static function classify(string $status, array $sentTo, bool $stale, iterable $reasons): string + { + return match (true) { + $status === ClientReport::STATUS_SUPERSEDED => self::SUPERSEDED, + $status === ClientReport::STATUS_SENT => self::SENT, + $sentTo !== [] => self::PARTLY_SENT, + $stale => self::OUTDATED, + ReviewReasonCategories::forReasons($reasons) !== [] && ! ReviewReasonCategories::onlyFirstReport($reasons) => self::NEEDS_REVIEW, + default => self::READY, + }; + } + + /** + * What the operator should check before this report goes out: the stored + * review reasons, or nothing once it has been sent. + * + * @return list + */ + public static function reasons(ClientReport $report): array + { + if ($report->status === ClientReport::STATUS_SENT) { + return []; + } + + $data = is_array($report->sections_data) ? $report->sections_data : []; + + return array_values(array_filter( + (array) ($data['meta']['review_reasons'] ?? []), + fn ($reason) => is_string($reason) && trim($reason) !== '', + )); + } + + /** + * Why the Send button can't be used right now, or null when it can. + */ + public static function sendBlockedReason(ClientReport $report): ?string + { + return match (true) { + $report->status === ClientReport::STATUS_SUPERSEDED => 'A newer copy for this period replaced this one. Send that one instead.', + empty($report->sent_to) && $report->isStale() => $report->canRecompile() + ? 'Compiled before the latest report fixes: recompile it, check the new copy, then send that one.' + : 'Compiled before the latest report fixes: generate a new report for this period instead.', + $report->recipients() !== [] && $report->recipientsToSend() === [] => 'Everyone on this report already has it.', + default => null, + }; + } + + /** + * The confirmation shown before a real send: who gets it, and what the + * send changes. Sending a schedule's first report by hand is what lets + * that schedule email its later clean reports by itself. + */ + public static function sendConfirmation(ClientReport $report, string $glue = ' '): string + { + $to = $report->recipientsToSend() ?: $report->recipients(); + $lines = [$to === [] + ? 'Email this report? You will be asked for the address to send it to.' + : 'Email report to '.implode(', ', $to).'?']; + $lines[] = 'They get it straight away and the report is marked as sent.'; + + $schedule = $report->schedule; + if ($schedule !== null && $schedule->last_sent_at === null && $schedule->delivery_mode === ClientReportSchedule::MODE_AUTO) { + $lines[] = 'This is the first report from this schedule: once it has been sent, the schedule emails its clean reports automatically every month (anything that needs a look is still held for review).'; + } elseif ($schedule !== null && $schedule->last_sent_at === null) { + $lines[] = 'This is the first report sent from this schedule.'; + } + + return implode($glue, $lines); + } + + /** The confirmation shown before a test copy goes to the operator. */ + public static function testConfirmation(?string $operatorEmail): string + { + return 'Send a test copy to '.($operatorEmail ?: 'your own address').'? Only you get it, with [TEST] in the subject. The client gets nothing, and the report and its schedule don\'t change.'; + } + + /** @return array state => label, in the order the screens list them */ + public static function options(): array + { + return array_map(fn (array $meta) => $meta['label'], self::META); + } + + public static function isKnown(?string $state): bool + { + return $state !== null && isset(self::META[$state]); + } + + public static function label(string $state): string + { + return self::META[$state]['label'] ?? ucfirst(str_replace('_', ' ', $state)); + } + + /** CSS class of the status pill (the app's status-* palette, dark-mode aware). */ + public static function pill(string $state): string + { + return self::META[$state]['pill'] ?? 'status-unknown'; + } + + public static function icon(string $state): string + { + return self::META[$state]['icon'] ?? 'fa-circle'; + } + + public static function hint(string $state): string + { + return self::META[$state]['hint'] ?? ''; + } +} diff --git a/modules/ClientReports/src/Support/ReviewReasonCategories.php b/modules/ClientReports/src/Support/ReviewReasonCategories.php new file mode 100644 index 00000000..893572fd --- /dev/null +++ b/modules/ClientReports/src/Support/ReviewReasonCategories.php @@ -0,0 +1,169 @@ + */ + private const META = [ + self::FIRST_REPORT => ['label' => 'First report', 'short' => 'First report', 'icon' => 'fa-flag', 'tab' => null], + self::MALWARE => ['label' => 'Malware & core files', 'short' => 'Malware', 'icon' => 'fa-bug', 'tab' => 'security'], + self::VULNERABILITIES => ['label' => 'Vulnerable plugins', 'short' => 'Vulnerability', 'icon' => 'fa-shield-halved', 'tab' => 'updates'], + self::UPDATES => ['label' => 'Updates', 'short' => 'Updates', 'icon' => 'fa-arrows-rotate', 'tab' => 'updates'], + self::BACKUPS => ['label' => 'Backups', 'short' => 'Backups', 'icon' => 'fa-database', 'tab' => 'overview'], + self::UPTIME => ['label' => 'Uptime', 'short' => 'Uptime', 'icon' => 'fa-heart-pulse', 'tab' => 'overview'], + self::TRAFFIC => ['label' => 'Traffic', 'short' => 'Traffic', 'icon' => 'fa-chart-line', 'tab' => 'traffic'], + self::FORMS => ['label' => 'Contact forms', 'short' => 'Forms', 'icon' => 'fa-envelope-open-text', 'tab' => 'forms'], + self::PARTIAL_MONTH => ['label' => 'Partial month', 'short' => 'Partial month', 'icon' => 'fa-calendar-minus', 'tab' => 'overview'], + self::OTHER => ['label' => 'Other', 'short' => 'Other', 'icon' => 'fa-circle-info', 'tab' => 'overview'], + ]; + + /** + * Keyword rules, most specific first: the first match wins. A reason + * about the Companion plugin's stale snapshot mentions vulnerabilities + * in passing, so it is matched before them; "Uptime monitoring began" + * is an uptime gap, so uptime comes before the partial-month rule. + * + * @var list + */ + private const RULES = [ + ['/first report from this schedule/i', self::FIRST_REPORT], + ['/companion .*(reported|snapshot)/i', self::UPDATES], + ['/vulnerab/i', self::VULNERABILITIES], + ['/malware|blocklist|core[- ]file/i', self::MALWARE], + ['/backup|backed up/i', self::BACKUPS], + ['/uptime|still down|monitor was blocked|http 429/i', self::UPTIME], + ['/traffic|requests came from/i', self::TRAFFIC], + ['/contact[- ]form|\bforms?\b/i', self::FORMS], + ['/monitoring began|part of the month|covers none/i', self::PARTIAL_MONTH], + ['/update/i', self::UPDATES], + ]; + + public static function categorize(string $reason): string + { + foreach (self::RULES as [$pattern, $category]) { + if (preg_match($pattern, $reason) === 1) { + return $category; + } + } + + return self::OTHER; + } + + /** + * The categories a report's reasons fall into, in the order they first appear. + * + * @param iterable $reasons + * @return list + */ + public static function forReasons(iterable $reasons): array + { + return array_keys(self::group($reasons)); + } + + /** + * The reasons grouped by category, each group in its stored order. + * + * @param iterable $reasons + * @return array> + */ + public static function group(iterable $reasons): array + { + $groups = []; + foreach ($reasons as $reason) { + if (! is_string($reason) || trim($reason) === '') { + continue; + } + $groups[self::categorize($reason)][] = $reason; + } + + return $groups; + } + + /** + * Whether the only thing holding a report is that it is its schedule's + * first: nothing in it needs fixing, it just waits for a person to send it. + * + * @param iterable $reasons + */ + public static function onlyFirstReport(iterable $reasons): bool + { + $categories = self::forReasons($reasons); + + return $categories === [self::FIRST_REPORT]; + } + + /** + * The categories an operator can filter the reports list by. + * + * @return array + */ + public static function filterOptions(): array + { + $options = []; + foreach (self::META as $key => $meta) { + if ($key !== self::FIRST_REPORT) { + $options[$key] = $meta['label']; + } + } + + return $options; + } + + public static function isKnown(?string $category): bool + { + return $category !== null && isset(self::META[$category]); + } + + public static function label(string $category): string + { + return self::META[$category]['label'] ?? self::META[self::OTHER]['label']; + } + + public static function short(string $category): string + { + return self::META[$category]['short'] ?? self::META[self::OTHER]['short']; + } + + public static function icon(string $category): string + { + return self::META[$category]['icon'] ?? self::META[self::OTHER]['icon']; + } + + /** The site dashboard tab where the problem can be looked at, if there is one. */ + public static function siteTab(string $category): ?string + { + return self::META[$category]['tab'] ?? null; + } +} diff --git a/tests/Feature/Modules/ClientReportClientsTest.php b/tests/Feature/Modules/ClientReportClientsTest.php new file mode 100644 index 00000000..fe7f890e --- /dev/null +++ b/tests/Feature/Modules/ClientReportClientsTest.php @@ -0,0 +1,257 @@ +mockIssueCounterZero(); + $this->user = User::factory()->create(); +}); + +afterEach(fn () => Carbon::setTestNow()); + +function clientsSite(?Client $client, string $domain, bool $carePlan = true, array $attributes = []): Site +{ + return Site::factory()->create(array_merge([ + 'domain' => $domain, + 'client_id' => $client?->id, + 'care_plan_enabled' => $carePlan, + ], $attributes)); +} + +function clientsSchedule(Site $site, array $attributes = []): ClientReportSchedule +{ + return ClientReportSchedule::create(array_merge([ + 'site_id' => $site->id, + 'frequency' => ClientReportSchedule::FREQUENCY_MONTHLY, + 'delivery_mode' => ClientReportSchedule::MODE_AUTO, + 'recipients' => ['owner@example.com'], + 'is_enabled' => true, + 'next_run_at' => Carbon::parse('2026-12-01'), + ], $attributes)); +} + +function clientsReport(Site $site, string $month, array $attributes = [], array $meta = []): ClientReport +{ + $start = Carbon::parse($month.'-01'); + + return ClientReport::create(array_merge([ + 'site_id' => $site->id, + 'title' => "Report {$site->domain} {$month}", + 'period_start' => $start, + 'period_end' => $start->copy()->endOfMonth(), + 'sections_data' => ['meta' => array_merge(['schema_version' => ClientReportCompiler::SCHEMA_VERSION], $meta)], + 'client_email' => 'owner@example.com', + 'status' => ClientReport::STATUS_GENERATED, + ], $attributes)); +} + +describe('Clients tab', function () { + it('is the module landing page and needs a login', function () { + auth()->logout(); + $this->get(route('client-reports.index'))->assertRedirect(route('login')); + $this->get(route('client-reports.clients.show', 'unassigned'))->assertRedirect(route('login')); + }); + + it('lists each client with its sites\' care plan, schedule, recipients and a month-by-month history linking to each report', function () { + $acme = Client::create(['name' => 'Jane Example', 'company_name' => 'Acme Example Co', 'email' => 'jane@example.com']); + $covered = clientsSite($acme, 'covered.example.com'); + $schedule = clientsSchedule($covered, ['recipients' => ['jane@example.com', 'ops@example.com']]); + $sent = clientsReport($covered, '2026-09', ['schedule_id' => $schedule->id, 'status' => ClientReport::STATUS_SENT, 'sent_to' => ['jane@example.com']]); + $held = clientsReport($covered, '2026-10', ['schedule_id' => $schedule->id], ['review_reasons' => ['No backups were found this month.']]); + clientsSite($acme, 'nocare.example.com', carePlan: false); + clientsSite($acme, 'gap.example.com'); + + $html = $this->actingAs($this->user)->get(route('client-reports.index'))->assertOk()->getContent(); + + expect($html) + ->toContain('Acme Example Co') + ->toContain('Jane Example') + ->toContain('covered.example.com') + ->toContain('jane@example.com, ops@example.com') + ->toContain('Scheduled · auto') + ->toContain('No care plan — reports won\'t send') + ->toContain('action="'.route('client-reports.schedules.quick', Site::where('domain', 'gap.example.com')->first()).'"') + // History strip: a link per month that has a report, titled with its state. + ->toContain('href="'.route('client-reports.show', $sent).'"') + ->toContain('Sep 2026: Sent') + ->toContain('href="'.route('client-reports.show', $held).'"') + ->toContain('Oct 2026: Needs review') + ->toContain('Aug 2026: no report'); + }); + + it('marks months a no-care-plan site\'s schedule skipped', function () { + $client = Client::create(['name' => 'Skip Example', 'email' => 'skip@example.com']); + $site = clientsSite($client, 'skipped.example.com', carePlan: false); + $schedule = clientsSchedule($site); + $schedule->forceFill(['created_at' => Carbon::parse('2026-09-15')])->save(); + + $this->actingAs($this->user)->get(route('client-reports.index')) + ->assertOk() + ->assertSee('Oct 2026: skipped (no care plan)') + ->assertSee('Sep 2026: skipped (no care plan)') + ->assertSee('Aug 2026: no report'); + }); + + it('gathers sites without a client under Unassigned, after the clients', function () { + Client::create(['name' => 'Zed Example', 'email' => 'zed@example.com']); + clientsSite(null, 'orphan.example.com'); + + $html = $this->actingAs($this->user)->get(route('client-reports.index'))->assertOk()->getContent(); + + expect($html)->toContain('Unassigned sites')->toContain('orphan.example.com') + ->and(strpos($html, 'Zed Example'))->toBeLessThan(strpos($html, 'Unassigned sites')); + }); + + it('filters by search, care plan, waiting reports and missing schedules', function () { + $care = Client::create(['name' => 'Care Example', 'email' => 'care@example.com']); + $careSite = clientsSite($care, 'care.example.com'); + clientsReport($careSite, '2026-10', ['schedule_id' => clientsSchedule($careSite)->id], ['review_reasons' => ['No backups were found this month.']]); + + $none = Client::create(['name' => 'Nocare Example', 'email' => 'nocare@example.com']); + clientsSite($none, 'nocare.example.com', carePlan: false); + + $gap = Client::create(['name' => 'Gap Example', 'email' => 'gap@example.com']); + clientsSite($gap, 'gap.example.com'); + + $this->actingAs($this->user); + + $this->get(route('client-reports.index', ['q' => 'nocare.example']))->assertSee('Nocare Example')->assertDontSee('Gap Example'); + $this->get(route('client-reports.index', ['care' => 'no']))->assertSee('Nocare Example')->assertDontSee('Care Example'); + $this->get(route('client-reports.index', ['care' => 'yes']))->assertSee('Care Example')->assertSee('Gap Example')->assertDontSee('Nocare Example'); + $this->get(route('client-reports.index', ['waiting' => 1]))->assertSee('Care Example')->assertDontSee('Gap Example')->assertDontSee('Nocare Example'); + $this->get(route('client-reports.index', ['missing' => 1]))->assertSee('Gap Example')->assertDontSee('Care Example')->assertDontSee('Nocare Example'); + }); + + it('does not count an inactive care-plan site as missing a schedule', function () { + $client = Client::create(['name' => 'Inactive Example', 'email' => 'inactive@example.com']); + clientsSite($client, 'paused.example.com', attributes: ['is_inactive' => true]); + + $this->actingAs($this->user)->get(route('client-reports.index', ['missing' => 1])) + ->assertOk()->assertDontSee('Inactive Example'); + }); +}); + +describe('client reports page', function () { + it('shows the client\'s contacts, each site\'s schedule and full report history with every action', function () { + $client = Client::create(['name' => 'Pat Example', 'company_name' => 'Page Example Ltd', 'email' => 'pat@example.com', 'additional_emails' => ['billing@example.com']]); + $site = clientsSite($client, 'page.example.com'); + $schedule = clientsSchedule($site, ['recipients' => ['pat@example.com']]); + $old = clientsReport($site, '2026-06', ['schedule_id' => $schedule->id, 'status' => ClientReport::STATUS_SENT, 'sent_to' => ['pat@example.com']]); + $held = clientsReport($site, '2026-10', ['schedule_id' => $schedule->id], ['review_reasons' => [ScheduledReportRunner::FIRST_REPORT_REASON, 'No backups were found this month.']]); + $replaced = clientsReport($site, '2026-10', ['schedule_id' => $schedule->id, 'status' => ClientReport::STATUS_SUPERSEDED]); + + $html = $this->actingAs($this->user)->get(route('client-reports.clients.show', $client->id))->assertOk()->getContent(); + + expect($html) + ->toContain('Page Example Ltd') + ->toContain('billing@example.com') + ->toContain('1 of 1 site on a care plan') + ->toContain('Next run')->toContain('Dec 1, 2026') + ->toContain(route('client-reports.schedules.edit', $schedule)) + // Every report of the site, oldest included, with its state and reason chips. + ->toContain('June 2026')->toContain('October 2026') + ->toContain(ReportState::label(ReportState::NEEDS_REVIEW)) + ->toContain('Backups') + ->toContain(e(route('client-reports.email', [$held, 'period' => 'all', 'client' => (string) $client->id, 'from' => 'client']))) + ->toContain('action="'.e(route('client-reports.recompile', [$held, 'period' => 'all', 'client' => (string) $client->id, 'from' => 'client'])).'"') + ->toContain('action="'.route('client-reports.send', $held).'"') + ->toContain('action="'.route('client-reports.send-test', $held).'"') + ->toContain('action="'.route('client-reports.send-test', $old).'"') + // The replaced copy stays out of the way until asked for. + ->not->toContain(e(route('client-reports.show', [$replaced, 'period' => 'all', 'client' => (string) $client->id, 'from' => 'client']))) + ->toContain('Show 1 replaced copy'); + + $this->get(route('client-reports.clients.show', [$client->id, 'superseded' => 1])) + ->assertSee(e(route('client-reports.show', [$replaced, 'period' => 'all', 'client' => (string) $client->id, 'from' => 'client', 'superseded' => '1'])), false); + }); + + it('explains the consequences before a real send: who gets it and that the schedule then sends by itself', function () { + $client = Client::create(['name' => 'Confirm Example', 'email' => 'confirm@example.com']); + $site = clientsSite($client, 'confirm.example.com'); + $schedule = clientsSchedule($site, ['recipients' => ['confirm@example.com']]); + clientsReport($site, '2026-10', ['schedule_id' => $schedule->id, 'client_email' => 'confirm@example.com'], ['review_reasons' => [ScheduledReportRunner::FIRST_REPORT_REASON]]); + + $this->actingAs($this->user)->get(route('client-reports.clients.show', $client->id)) + ->assertOk() + ->assertSee('Email report to confirm@example.com?') + ->assertSee('once it has been sent, the schedule emails its clean reports automatically every month') + ->assertSee('Only you get it, with [TEST] in the subject'); + }); + + it('steps to the previous and next client of the filtered list, alphabetically, and offers every client in the switcher', function () { + $alpha = Client::create(['name' => 'Alpha Example', 'email' => 'a@example.com']); + $bravo = Client::create(['name' => 'Bravo Example', 'email' => 'b@example.com']); + $charlie = Client::create(['name' => 'Charlie Example', 'email' => 'c@example.com']); + clientsSite($alpha, 'alpha.example.com'); + clientsSite($bravo, 'bravo.example.com', carePlan: false); + clientsSite($charlie, 'charlie.example.com'); + + $this->actingAs($this->user); + + $this->get(route('client-reports.clients.show', $bravo->id)) + ->assertOk() + ->assertSee('href="'.route('client-reports.clients.show', $alpha->id).'" class="cr-btn" rel="prev"', false) + ->assertSee('href="'.route('client-reports.clients.show', $charlie->id).'" class="cr-btn" rel="next"', false) + ->assertSee('2 of 3'); + + // With "care plan: yes", Bravo drops out: Alpha's next is Charlie. + $this->get(route('client-reports.clients.show', [$alpha->id, 'care' => 'yes'])) + ->assertSee('href="'.route('client-reports.clients.show', [$charlie->id, 'care' => 'yes']).'" class="cr-btn" rel="next"', false) + ->assertSee('