fix(iac): fire delayed purchases by default; collect AWS recommendations at 02:00 UTC

CHANGELOG.md

@@ -42,6 +42,29 @@ are the full record until the next release.
   actions, `ec2:DescribeRegions`, `rds:DescribeDBInstances`, and the
   new-style `es:*ReservedInstance*` OpenSearch actions (replacing the legacy
   `es:*ReservedElasticsearch*` names).
+- **Bastion accounts whose target role trusts the CUDly host now get
+  AccessDenied.** CUDly assumes a bastion account's target role with the
+  bastion's own credentials and no longer falls back to the CUDly host's.
+  A target role whose trust policy names the CUDly host instead of the
+  bastion fails at AssumeRole until the bastion is added as the trusted
+  principal in that role's trust policy
+  ([#1556](https://github.com/LeanerCloud/cloud-commitments-platform/pull/1556)).
+
+### Changed
+
+- **The fire_scheduled_purchases sweep is now on by default on every
+  Terraform platform** (Lambda, Fargate, Cloud Run, GKE, Container Apps and
+  AKS), every 15 minutes. It executes delayed auto-approved purchases once
+  their scheduled time passes; until now they never ran unless the operator
+  turned the sweep on. The server refuses (and sends the purchase-failed
+  alert for) any purchase due more than 24h ago, so the first run cannot buy
+  a stale backlog. Set `enable_fire_scheduled_purchases_schedule = false` to
+  opt out. The process_scheduled_purchases sweep stays opt-in (#712)
+- **AWS recommendation collection moves to 02:00 UTC daily**
+  (`recommendation_schedule = "cron(0 2 * * ? *)"`, was `rate(1 day)`), the
+  same time as GCP and Azure. With a fixed time the recommendations freshness
+  endpoint can report `next_collection_at` on AWS too. Deployments that set
+  their own `recommendation_schedule` are unaffected (#756)
 
 ### Removed
 

internal/purchase/due_age_cap.go

@@ -15,11 +15,12 @@ import (
 const DefaultMaxDueRowAge = 24 * time.Hour
 
 // ErrDueRowTooOld marks a row the fire or process sweep refused because it was
-// due more than the due-row age cap ago (issue #712). The sweeps are opt-in on
-// most platforms, so the first tick after one is enabled can find a long
-// backlog of overdue rows whose recommendations and prices are stale; buying
-// them all at once is the outcome this cap prevents. The row ends failed with
-// no_submission set, so it can be retried deliberately from History.
+// due more than the due-row age cap ago (issue #712). The first tick after a
+// sweep is enabled (the fire sweep by default since #712, the process sweep
+// on opt-in) can find a long backlog of overdue rows whose recommendations
+// and prices are stale; buying them all at once is the outcome this cap
+// prevents. The row ends failed with no_submission set, so it can be retried
+// deliberately from History.
 var ErrDueRowTooOld = errors.New("scheduled purchase skipped: it was due too long ago")
 
 // dueRowMaxAge is the sweeps' age cap, DefaultMaxDueRowAge when unset.

specs/recommendations-cache.md

@@ -54,7 +54,8 @@ entry point. Flow:
 
 - **Scheduled cron** (AWS EventBridge / GCP Cloud Scheduler / Azure
   Logic App) — runs on all runtimes at `var.recommendation_schedule`,
-  default `rate(1 day)`.
+  default daily at 02:00 UTC (`cron(0 2 * * ? *)` on AWS, `0 2 * * *` on
+  GCP and Azure).
 - **Manual `POST /api/recommendations/refresh`** — user-driven via the
   Refresh button on the dashboard. All runtimes.
 - **In-request stale-while-revalidate** — GCP Cloud Run and Azure

terraform/environments/aws/dev.tfvars.example

@@ -112,7 +112,7 @@ frontend_price_class  = "PriceClass_100"  # US, Canada, Europe
 # ==============================================
 
 enable_scheduled_tasks  = true
-recommendation_schedule = "rate(24 hours)"
+recommendation_schedule = "cron(0 2 * * ? *)" # daily at 02:00 UTC
 
 # ==============================================
 # Docker Build

terraform/environments/aws/github-dev.tfvars

@@ -81,7 +81,7 @@ create_subdomain_zone = false
 # ==============================================
 
 enable_scheduled_tasks  = true
-recommendation_schedule = "rate(1 day)"
+recommendation_schedule = "cron(0 2 * * ? *)" # daily at 02:00 UTC
 
 # ==============================================
 # Multi-Account Cross-Account Access

terraform/environments/aws/github-prod.tfvars

@@ -88,7 +88,7 @@ create_subdomain_zone = false
 # ==============================================
 
 enable_scheduled_tasks  = true
-recommendation_schedule = "rate(1 day)"
+recommendation_schedule = "cron(0 2 * * ? *)" # daily at 02:00 UTC
 
 # ==============================================
 # Multi-Account Cross-Account Access

terraform/environments/aws/github-staging.tfvars

@@ -78,7 +78,7 @@ create_subdomain_zone = false
 # ==============================================
 
 enable_scheduled_tasks  = true
-recommendation_schedule = "rate(1 day)"
+recommendation_schedule = "cron(0 2 * * ? *)" # daily at 02:00 UTC
 
 # ==============================================
 # Multi-Account Cross-Account Access

terraform/environments/aws/variables.tf

@@ -365,9 +365,9 @@ variable "enable_scheduled_tasks" {
 }
 
 variable "recommendation_schedule" {
-  description = "Schedule expression for recommendation collection"
+  description = "EventBridge schedule expression for recommendation collection. Default: daily at 02:00 UTC, the same time as GCP and Azure. A cron() schedule lets the API report next_collection_at; a rate() schedule leaves it null because its phase is unknown (#756)."
   type        = string
-  default     = "rate(1 day)"
+  default     = "cron(0 2 * * ? *)"
 }
 
 variable "enable_ri_exchange_schedule" {
@@ -395,9 +395,9 @@ variable "ladder_run_schedule" {
 }
 
 variable "enable_fire_scheduled_purchases_schedule" {
-  description = "Enable the scheduled fire_scheduled_purchases sweep. Default false until auto-approve (L9) is promoted to GA."
+  description = "Schedule the fire_scheduled_purchases sweep, which executes delayed auto-approved purchases once their scheduled time has passed. On by default: the server refuses, and alerts on, any purchase due more than 24h ago (SCHEDULED_PURCHASE_MAX_OVERDUE, #1549), so turning it on cannot buy a stale backlog. Set false to opt out."
   type        = bool
-  default     = false
+  default     = true
 }
 
 variable "fire_scheduled_purchases_schedule" {

terraform/environments/azure/variables.tf

@@ -444,9 +444,9 @@ variable "analytics_collect_schedule" {
 }
 
 variable "enable_fire_scheduled_purchases_schedule" {
-  description = "Schedule the fire_scheduled_purchases sweep, which fires delayed auto-approved purchases. Opt-in: default false."
+  description = "Schedule the fire_scheduled_purchases sweep, which executes delayed auto-approved purchases once their scheduled time has passed. On by default: the server refuses, and alerts on, any purchase due more than 24h ago (SCHEDULED_PURCHASE_MAX_OVERDUE, #1549), so turning it on cannot buy a stale backlog. Set false to opt out."
   type        = bool
-  default     = false
+  default     = true
 }
 
 variable "fire_scheduled_purchases_schedule" {

terraform/environments/gcp/variables.tf

@@ -370,9 +370,9 @@ variable "analytics_collect_schedule" {
 }
 
 variable "enable_fire_scheduled_purchases_schedule" {
-  description = "Schedule the fire_scheduled_purchases sweep, which fires delayed auto-approved purchases. Opt-in: default false."
+  description = "Schedule the fire_scheduled_purchases sweep, which executes delayed auto-approved purchases once their scheduled time has passed. On by default: the server refuses, and alerts on, any purchase due more than 24h ago (SCHEDULED_PURCHASE_MAX_OVERDUE, #1549), so turning it on cannot buy a stale backlog. Set false to opt out."
   type        = bool
-  default     = false
+  default     = true
 }
 
 variable "fire_scheduled_purchases_schedule" {

terraform/modules/compute/aws/fargate/tests/sweeps.tftest.hcl

@@ -36,8 +36,16 @@ run "defaults" {
     error_message = "Each active sweep needs a target, and the sweeps need their invoker role and policy."
   }
   assert {
-    condition     = length(aws_cloudwatch_event_rule.fire_scheduled_purchases) == 0 && length(aws_cloudwatch_event_rule.ladder_run) == 0 && length(aws_cloudwatch_event_rule.ri_exchange) == 0
-    error_message = "fire, ladder_run and ri_exchange stay opt-in."
+    condition     = length(aws_cloudwatch_event_rule.fire_scheduled_purchases) == 1 && length(aws_cloudwatch_event_target.fire_scheduled_purchases) == 1
+    error_message = "The fire sweep is on by default (#712), with its target."
+  }
+  assert {
+    condition     = aws_cloudwatch_event_rule.fire_scheduled_purchases[0].schedule_expression == "rate(15 minutes)"
+    error_message = "The fire sweep runs every 15 minutes by default, as on Lambda."
+  }
+  assert {
+    condition     = length(aws_cloudwatch_event_rule.ladder_run) == 0 && length(aws_cloudwatch_event_rule.ri_exchange) == 0
+    error_message = "ladder_run and ri_exchange stay opt-in."
   }
   assert {
     condition = alltrue([
@@ -100,13 +108,18 @@ run "all_off" {
   command = plan
 
   variables {
-    enable_reap_stuck_purchases_schedule = false
-    enable_analytics_collect_schedule    = false
-    enable_finalize_revocations_schedule = false
+    enable_reap_stuck_purchases_schedule     = false
+    enable_analytics_collect_schedule        = false
+    enable_finalize_revocations_schedule     = false
+    enable_fire_scheduled_purchases_schedule = false
   }
 
   assert {
     condition     = length(aws_cloudwatch_event_rule.sweep) == 0 && length(aws_iam_role.eventbridge_sweeps) == 0 && length(aws_iam_role_policy.eventbridge_sweeps) == 0
     error_message = "With every sweep off, no rule and no invoker role may be created."
   }
+  assert {
+    condition     = length(aws_cloudwatch_event_rule.fire_scheduled_purchases) == 0 && length(aws_cloudwatch_event_target.fire_scheduled_purchases) == 0
+    error_message = "Setting enable_fire_scheduled_purchases_schedule = false must remove the fire rule and target."
+  }
 }

terraform/modules/compute/aws/fargate/variables.tf

@@ -196,9 +196,9 @@ variable "enable_scheduled_tasks" {
 }
 
 variable "recommendation_schedule" {
-  description = "Schedule expression for recommendation collection"
+  description = "EventBridge schedule expression for recommendation collection. Default: daily at 02:00 UTC, the same time as GCP and Azure. A cron() schedule lets the API report next_collection_at; a rate() schedule leaves it null because its phase is unknown (#756)."
   type        = string
-  default     = "rate(1 day)"
+  default     = "cron(0 2 * * ? *)"
 }
 
 variable "enable_ri_exchange_schedule" {
@@ -226,9 +226,9 @@ variable "ladder_run_schedule" {
 }
 
 variable "enable_fire_scheduled_purchases_schedule" {
-  description = "Enable the scheduled fire_scheduled_purchases sweep. Default false until auto-approve (L9) is promoted to GA."
+  description = "Schedule the fire_scheduled_purchases sweep, which executes delayed auto-approved purchases once their scheduled time has passed. On by default: the server refuses, and alerts on, any purchase due more than 24h ago (SCHEDULED_PURCHASE_MAX_OVERDUE, #1549), so turning it on cannot buy a stale backlog. Set false to opt out."
   type        = bool
-  default     = false
+  default     = true
 }
 
 variable "fire_scheduled_purchases_schedule" {

terraform/modules/compute/aws/lambda/main.tf

@@ -777,8 +777,9 @@ resource "aws_lambda_permission" "eventbridge_ladder_run" {
 # scheduled_execution_at is in the past and fires each one. This
 # sweep is the runtime mechanism that makes auto-approve delayed
 # purchases actually execute; without it, scheduled tranches would
-# never fire between daily ladder runs. Default-off until the
-# auto-approve feature (L9) is promoted to GA.
+# never fire between daily ladder runs. On by default (owner decision
+# for #712): the server refuses any row due more than 24h ago (#1549),
+# so a first tick after enabling cannot buy a stale backlog.
 
 resource "aws_cloudwatch_event_rule" "fire_scheduled_purchases" {
   count = var.enable_fire_scheduled_purchases_schedule ? 1 : 0

terraform/modules/compute/aws/lambda/tests/sweeps.tftest.hcl

@@ -34,8 +34,20 @@ run "defaults" {
     error_message = "reap and analytics_collect need their target and permission too."
   }
   assert {
-    condition     = length(aws_cloudwatch_event_rule.fire_scheduled_purchases) == 0 && length(aws_cloudwatch_event_rule.ladder_run) == 0 && length(aws_cloudwatch_event_rule.ri_exchange) == 0
-    error_message = "fire, ladder_run and ri_exchange stay opt-in."
+    condition     = length(aws_cloudwatch_event_rule.fire_scheduled_purchases) == 1 && length(aws_cloudwatch_event_target.fire_scheduled_purchases) == 1 && length(aws_lambda_permission.eventbridge_fire_scheduled_purchases) == 1
+    error_message = "The fire sweep is on by default (#712), with its target and permission."
+  }
+  assert {
+    condition     = aws_cloudwatch_event_rule.fire_scheduled_purchases[0].schedule_expression == "rate(15 minutes)"
+    error_message = "The fire sweep runs every 15 minutes by default."
+  }
+  assert {
+    condition     = length(aws_cloudwatch_event_rule.ladder_run) == 0 && length(aws_cloudwatch_event_rule.ri_exchange) == 0
+    error_message = "ladder_run and ri_exchange stay opt-in."
+  }
+  assert {
+    condition     = aws_cloudwatch_event_rule.scheduled_recommendations[0].schedule_expression == "cron(0 2 * * ? *)"
+    error_message = "Recommendations are collected daily at 02:00 UTC by default, on a cron the API can report next_collection_at for (#756)."
   }
   assert {
     condition = alltrue([
@@ -109,3 +121,16 @@ run "opt_in" {
     error_message = "Each target input must name its task."
   }
 }
+
+run "fire_opt_out" {
+  command = plan
+
+  variables {
+    enable_fire_scheduled_purchases_schedule = false
+  }
+
+  assert {
+    condition     = length(aws_cloudwatch_event_rule.fire_scheduled_purchases) == 0 && length(aws_cloudwatch_event_target.fire_scheduled_purchases) == 0 && length(aws_lambda_permission.eventbridge_fire_scheduled_purchases) == 0
+    error_message = "Setting enable_fire_scheduled_purchases_schedule = false must remove the fire rule, target and permission."
+  }
+}

terraform/modules/compute/aws/lambda/variables.tf

@@ -192,9 +192,9 @@ variable "enable_scheduled_tasks" {
 }
 
 variable "recommendation_schedule" {
-  description = "EventBridge schedule expression for recommendations"
+  description = "EventBridge schedule expression for recommendation collection. Default: daily at 02:00 UTC, the same time as GCP and Azure. A cron() schedule lets the API report next_collection_at; a rate() schedule leaves it null because its phase is unknown (#756)."
   type        = string
-  default     = "rate(1 day)"
+  default     = "cron(0 2 * * ? *)"
 }
 
 variable "enable_ri_exchange_schedule" {
@@ -246,9 +246,9 @@ variable "ladder_run_schedule" {
 }
 
 variable "enable_fire_scheduled_purchases_schedule" {
-  description = "Enable the scheduled fire_scheduled_purchases sweep. When true, EventBridge periodically fires purchase_executions in status=scheduled whose scheduled_execution_at is past (auto-approve delayed-fire path). Default false until auto-approve (L9) is promoted to GA."
+  description = "Schedule the fire_scheduled_purchases sweep, which executes delayed auto-approved purchases once their scheduled time has passed. On by default: the server refuses, and alerts on, any purchase due more than 24h ago (SCHEDULED_PURCHASE_MAX_OVERDUE, #1549), so turning it on cannot buy a stale backlog. Set false to opt out."
   type        = bool
-  default     = false
+  default     = true
 }
 
 variable "fire_scheduled_purchases_schedule" {

terraform/modules/compute/azure/aks/scheduled-tasks.tf

@@ -15,7 +15,7 @@
 
 locals {
   # Keyed by server task name. `job` is the CronJob name suffix. Default-on
-  # tasks match the other platforms; fire_scheduled_purchases,
+  # tasks match the other platforms, including fire_scheduled_purchases;
   # process_scheduled_purchases, send_notifications and cleanup are opt-in
   # (default off, as decided for #712). ri_exchange_reshape (as on Container
   # Apps, Cloud Run and Fargate) and ladder_run (as on Fargate and Lambda) are

terraform/modules/compute/azure/aks/tests/scheduled_tasks.tftest.hcl

@@ -54,8 +54,8 @@ run "defaults" {
   }
 
   assert {
-    condition     = toset(keys(kubernetes_cron_job_v1.sweep)) == toset(["analytics_collect", "collect_recommendations", "finalize_revocations", "reap_stuck_purchases"])
-    error_message = "Default-on tasks are collect_recommendations, reap_stuck_purchases, analytics_collect and finalize_revocations; fire, process, send_notifications and cleanup are opt-in."
+    condition     = toset(keys(kubernetes_cron_job_v1.sweep)) == toset(["analytics_collect", "collect_recommendations", "finalize_revocations", "fire_scheduled_purchases", "reap_stuck_purchases"])
+    error_message = "Default-on tasks are collect_recommendations, reap_stuck_purchases, analytics_collect, finalize_revocations and fire_scheduled_purchases; process, send_notifications and cleanup are opt-in."
   }
   assert {
     condition = alltrue([
@@ -67,12 +67,12 @@ run "defaults" {
   assert {
     condition = alltrue([
       for k, j in kubernetes_cron_job_v1.sweep :
-      j.spec[0].schedule == { collect_recommendations = var.recommendation_schedule, reap_stuck_purchases = var.reap_stuck_purchases_schedule, analytics_collect = var.analytics_collect_schedule, finalize_revocations = var.finalize_revocations_schedule }[k]
+      j.spec[0].schedule == { collect_recommendations = var.recommendation_schedule, reap_stuck_purchases = var.reap_stuck_purchases_schedule, analytics_collect = var.analytics_collect_schedule, finalize_revocations = var.finalize_revocations_schedule, fire_scheduled_purchases = var.fire_scheduled_purchases_schedule }[k]
     ])
     error_message = "Each schedule must come from its variable."
   }
   assert {
-    condition     = toset([for j in kubernetes_cron_job_v1.sweep : j.metadata[0].name]) == toset(["cudly-task-collect", "cudly-task-reap", "cudly-task-analytics", "cudly-task-finalize"])
+    condition     = toset([for j in kubernetes_cron_job_v1.sweep : j.metadata[0].name]) == toset(["cudly-task-collect", "cudly-task-reap", "cudly-task-analytics", "cudly-task-finalize", "cudly-task-fire"])
     error_message = "CronJob names must be distinct <project>-task-<job> names."
   }
   assert {
@@ -191,10 +191,11 @@ run "all_off" {
   command = plan
 
   variables {
-    enable_scheduled_tasks               = false
-    enable_reap_stuck_purchases_schedule = false
-    enable_analytics_collect_schedule    = false
-    enable_finalize_revocations_schedule = false
+    enable_scheduled_tasks                   = false
+    enable_reap_stuck_purchases_schedule     = false
+    enable_analytics_collect_schedule        = false
+    enable_finalize_revocations_schedule     = false
+    enable_fire_scheduled_purchases_schedule = false
   }
 
   plan_options {

terraform/modules/compute/azure/aks/variables.tf

@@ -277,9 +277,9 @@ variable "analytics_collect_schedule" {
 }
 
 variable "enable_fire_scheduled_purchases_schedule" {
-  description = "Schedule the fire_scheduled_purchases sweep, which fires delayed auto-approved purchases. Opt-in: default false."
+  description = "Schedule the fire_scheduled_purchases sweep, which executes delayed auto-approved purchases once their scheduled time has passed. On by default: the server refuses, and alerts on, any purchase due more than 24h ago (SCHEDULED_PURCHASE_MAX_OVERDUE, #1549), so turning it on cannot buy a stale backlog. Set false to opt out."
   type        = bool
-  default     = false
+  default     = true
 }
 
 variable "fire_scheduled_purchases_schedule" {

terraform/modules/compute/azure/container-apps/scheduled-sweeps.tf

@@ -14,7 +14,7 @@
 
 locals {
   # Keyed by server task name. `job` is the workflow name suffix. Default-on
-  # tasks match the other platforms; fire_scheduled_purchases,
+  # tasks match the other platforms, including fire_scheduled_purchases;
   # process_scheduled_purchases and send_notifications are opt-in (default
   # off, as decided for #712). cleanup has its own workflow in
   # scheduled-tasks.tf.