fix(stack): forward every selected namespace from every compartment

.github/workflows/test.yml

@@ -25,3 +25,4 @@ jobs:
       - run: terraform fmt -check
       - run: terraform init -backend=false
       - run: terraform validate
+      - run: terraform test

README.md

@@ -10,7 +10,7 @@ Better Stack never connects to your tenancy. Data leaves your tenancy only throu
 
 ## Quick start
 
-Before you start, sign in to the **home region** of your tenancy. Use a user who can create compartments and policies in the root compartment.
+Before you start, sign in to the **home region** of your tenancy. Use a user who can create compartments and policies in the root compartment, and read service limits (`inspect limits in tenancy`). Administrators can do both.
 
 1. [Create an Oracle Cloud source in Better Stack](https://telemetry.betterstack.com/team/0/sources/new?platform=oracle_cloud).
 2. On the **Ingest** tab of the source, click **Deploy to Oracle Cloud**.
@@ -26,10 +26,18 @@ The stack sends metrics and logs from the home region only.
 
 - A `betterstack` compartment for all the resources below.
 - A Function application for the forwarder. It runs in a VCN with a private subnet. A NAT gateway lets the Function reach Better Stack.
-- One Connector Hub for logs, and one or more Connector Hubs for metrics. Each Connector Hub takes up to 5 compartments and 50 metric namespaces. A large tenancy can need a higher Connector Hub service limit.
+- One Connector Hub for logs, and one or more Connector Hubs for metrics. Each Connector Hub takes up to 5 compartments and 50 metric namespaces, so it covers 2 compartments with the 25 default namespaces, and 5 with 10 or fewer. If your region's Connector Hub limit is too low, Plan stops before anything changes, but it doesn't count Connector Hubs you already use elsewhere in the region. Deselect namespaces, list the compartments to forward metrics from, or request a limit increase.
 - A policy that lets the Connector Hubs read metrics and logs, and call the Function.
 
-When the stack runs, it finds the compartments that have metrics in the selected namespaces. After you add compartments or use new services, apply the stack again.
+Every selected namespace is forwarded from each compartment: the ones you list, or every compartment that exists when you apply. Services you start using later are forwarded without running Apply again. After you add compartments, apply the stack again.
+
+## Upgrade
+
+1. Go to **Developer Services → Resource Manager → Stacks** and select the Better Stack stack.
+2. Click **Edit**, upload `betterstack-oci.zip` from the latest release, and save.
+3. Click **Apply**.
+
+Since v0.1.3, each compartment takes a place on a Connector Hub, even when it has no metrics yet. An upgrade from v0.1.2 can need more Connector Hubs than before. For example, the root compartment and 2 empty subcompartments need 3 on a limit of 2. If Plan stops, deselect namespaces to 10 or fewer, list the compartments to forward metrics from, or request a limit increase.
 
 ## Uninstall
 

main.tf

@@ -62,6 +62,15 @@ resource "oci_identity_policy" "connectors" {
     "Allow any-user to use fn-function in compartment id ${oci_identity_compartment.betterstack.id} ${local.connector_principal}",
     "Allow any-user to use fn-invocation in compartment id ${oci_identity_compartment.betterstack.id} ${local.connector_principal}",
   ])
+
+  # A failed precondition stops Plan, so nothing changes.
+  lifecycle {
+    # ponytail: compares against the whole regional limit and ignores Connector Hubs the tenancy already uses elsewhere; subtract the used count (oci_limits_resource_availability) if that bites.
+    precondition {
+      condition     = local.connector_count <= local.connector_limit
+      error_message = "The stack needs ${local.connector_count} Connector Hubs, and your region allows ${local.connector_limit}. Deselect metric namespaces, list the compartments to forward metrics from, or request a Connector Hub limit increase."
+    }
+  }
 }
 
 # The Function has no public IP, so it reaches Better Stack through a NAT gateway.
@@ -175,7 +184,7 @@ resource "oci_functions_function" "forwarder" {
 }
 
 # A monitoring source only covers its own compartment, not its children, so
-# every compartment with metrics in a selected namespace becomes a source.
+# every compartment becomes a source. Compartments added later need Apply again.
 data "oci_identity_compartments" "all" {
   compartment_id            = var.tenancy_ocid
   compartment_id_in_subtree = true
@@ -188,33 +197,24 @@ locals {
     [var.tenancy_ocid],
     [for compartment in data.oci_identity_compartments.all.compartments : compartment.id if compartment.name != "betterstack"],
   )
-}
-
-data "oci_monitoring_metrics" "namespaces" {
-  for_each = toset(local.metrics_compartment_ids)
 
-  compartment_id = each.value
-  group_by       = ["namespace"]
+  # Every compartment gets every selected namespace, so a service first used
+  # later is forwarded without Apply. A connector takes at most 5 compartments
+  # and 50 namespaces across them.
+  metric_connectors = length(var.metrics_namespaces) == 0 ? [] : chunklist(
+    local.metrics_compartment_ids,
+    min(5, floor(50 / length(var.metrics_namespaces))),
+  )
+  connector_count = length(local.metric_connectors) + (var.forward_audit_logs || length(local.log_group_ids) > 0 ? 1 : 0)
+  # With no limit returned, the check compares the count with itself and passes.
+  connector_limit = coalesce(try(data.oci_limits_limit_values.connectors.limit_values[0].value, null), local.connector_count)
 }
 
-locals {
-  metric_sources = [
-    for id in local.metrics_compartment_ids : {
-      compartment_id = id
-      namespaces     = sort(setintersection(var.metrics_namespaces, data.oci_monitoring_metrics.namespaces[id].metrics[*].namespace))
-    } if length(setintersection(var.metrics_namespaces, data.oci_monitoring_metrics.namespaces[id].metrics[*].namespace)) > 0
-  ]
-
-  # A connector takes at most 5 compartments and 50 namespaces across them, so
-  # compartments are packed with others that select as many namespaces: one
-  # compartment with many namespaces doesn't shrink every other connector.
-  metric_source_sizes = sort(distinct([for source in local.metric_sources : format("%02d", length(source.namespaces))]))
-  metric_connectors = concat([], [
-    for size in local.metric_source_sizes : chunklist(
-      [for source in local.metric_sources : source if length(source.namespaces) == tonumber(size)],
-      min(5, floor(50 / tonumber(size))),
-    )
-  ]...)
+data "oci_limits_limit_values" "connectors" {
+  compartment_id = var.tenancy_ocid
+  service_name   = "service-connector-hub"
+  name           = "service-connector-count"
+  scope_type     = "REGION"
 }
 
 resource "oci_sch_service_connector" "metrics" {
@@ -230,13 +230,13 @@ resource "oci_sch_service_connector" "metrics" {
       for_each = local.metric_connectors[count.index]
 
       content {
-        compartment_id = monitoring_sources.value.compartment_id
+        compartment_id = monitoring_sources.value
 
         namespace_details {
           kind = "selected"
 
           dynamic "namespaces" {
-            for_each = monitoring_sources.value.namespaces
+            for_each = var.metrics_namespaces
 
             content {
               namespace = namespaces.value

schema.yaml

@@ -5,7 +5,7 @@ informationalText: Run this stack in the home region of your tenancy. It creates
 # The console only loads images from data: URIs and Oracle domains, so the logo is inline (logo.png).
 logoUrl: data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAIIAAACCCAYAAACKAxD9AAAABmJLR0QA/wD/AP+gvaeTAAAOSElEQVR4nO2de3hU5Z3HP++ZyYTJSQBz45Y7t4CNXEThYasGWiJiIIESQFxo9ako6IJWVFzU1orZR1FKBWlLUZDLs9iwuLSiYtmHqmv3jwooYpGLsOViDYQYkkxCCTPv/kHCJpkzl8ycmZMh7+cvnnPe8/t9Q755z3veqyACOJ3XZWmaNg5NfAcP+QgGANcBCUBSJHJeg9QBDcC3SI6h8SUe+bnHI/c0NlafMjuZMCuQrqcPQ/PMlZISAf3NiqvwRsIxIdiBR9vocp09YEbMcI1g1/XUmQgWA8PNEKToMPuRvORyVb0JuEMNErIRdD31LiFYJiEv1BgK85BwTEM8VV9/7s1Qnu+wEZKSUgd7JGuA8aEkVESc3Zrgwbq6qiMdecjWkcK6njJXSvF7BIM7pk0RRfIk/DguPuFc06WGvcE+FGyN4NCT0tYi5Q9DFKewAslrLlfVfKApUNHARujVS9cb3NuQTDRDmyLq/JezG1Orqqrq/BXya4S0tLTExkb+KJFjzNWmiCYS+ecG3V5EZaXLVxnNz/NxDRdlhTJB7CMQY3WXewcMiPdVxmdjUU9Kex3J9MhIU1hAniPuYp+mpobfG900NEJCUtq9QspnIqtLEXUEIx1xzhNNTY2fed9qR3M/wT6ujAsorjUkLptNjKitPXe09WWvNkJzZ5EywbWKQHd7eLX95TZG0PXU2agewy6AnJCYmDaj9ZXWrwabnph6CBgYXVEKK5DwVUN9VT5wGVrVCLqechfKBF0GAf0TE1OvfhX+/6tBiEctUaSwDAmPtfxbA9D1tOGo+QRdkZG6nn4DtNQImpxjqRyFhXj+GZqNICUl1opRWIZgCoBwOq/L0my2v1mtR2EdHrcnU9M0rdBqIQpr0TRRqKGJG6wWorAaUaDhId9qGQprEYJ8DaHWIHR1JAzQuLICSdG1uU5Dkmi1CoW1SEjSEGrIuasjQNcwcf2jImYR/iavKroQyggKQBlB0YwyggJQRlA0o4ygAJQRFM0oIygAZQRFM8oIBkyfPpU/vv82f//6OGdOH+OdnW8xZfKdVsuKKEJPTJVWi+hMvPrqSn44927De2vWrOXxJ5ZGWVF0UDVCK+bOme3TBAALFsxj5sxrc6cAVSM0061bPAc++wt9+/bxW+7s2XPcMOxm6uvro6QsOqgaoZn77/9xQBMApKensWjRg1FQFF1UjQDoCQkcPLiXtLTUoMo3Nl5kxMgxnD59JsLKooeqEYD58+cFbQIAp7MbTz21JIKKok/M1wh6QgJz5s5mzOibcbvd7Nv/KZs2/Tu1tbXBPa/rfHFwL6mpKR3K6/F4uK2wiP37vXahiUli2gh5ebnsfHs7mZkZba6fPHmKqdNmcfhw4F1on1yymKVLnwgp/0cffcwdk0pDerazEdNG+OjD3YwYMczw3v79n3FbYREej8fn8z169OCLg5/Qs2fPkDWUld3Nu++9H/LznYWYbSOMHTvGpwkARowYxoQJ/ncBWrhwQVgmACgvf5a4uLiwYnQGYtYIkybdHrDMzBm+O3+Sk5NZMP++sHUMHDiAH/0o9ncViFkjFE34XsAyxcWT0HXd8N4jjzxEUpI5pwot/dfH6d69uymxrCImjdCvX1+GDh0SsFxCgpPJkyd5XU9JSWbeffeapic1NYXFixeZFs8KYtIItxd9P+iyRq8Hu92OzWY3UxIPPfgAubk5psaMJjFphAlFgV8LLYwbdyvp6WltrlVWnmXrmxWmanI4HDzz9JOmxowmMWcEh8PBuMJbgy5vt9uZNs17Z6CVK1f7/bQMhenTpzJ69E2mxowWMWeEsWPHkJjYsXW7RkPHx459xdtvv2OWLACEEJSXP4sQsbeKMOaMEMzXQntuGnUjAwZ4bwPx8opXzJDUhtE338SUKbE3m6nTGKFHjx6MunFkwM+wog60D1ozo2ya17W9e/fz4Yf/HVI8fyxb9jPi4x2mx40kncIIk4snceTwZ/zpT7s4euQAd04yPj4qI6Mf+fmhHTA3a1aZYZW94hfm1wq5OdncZ+LnaTToFEZ44YVlVzt+dF3nxeXlhuUm3j4h5Bx5ebmMHOm9uezu3Xv49FNTTtVtw5InHiU5Odn0uJHCciMkJyeTlZXZ5lp2VqbhbKGOfDYa4avL+ZeveB1fEDY9e/bkicd/YnrcSGG5EbKyMgyvDxnSdrM3h8NB4W23hJWrrGwadrt3R9L27Ts4fvxEWLGNmDfvXvr3j40Tky03QmaGsRGGDm1rhO9+d6zPcYNgSUtLpbDQ20xut5tVq38dVmwj4uLieO7nT5seNxJYb4Qga4RQPhuN8PV62LRpC5WVZ03J0ZopU4oZO7bzn5houRGyMjMNr7evES5cuGBKviklxegJ3vuHXbz4D9aufc2UHO1Z/mI5mmb5f7VfLFfns42QP7jN5976DZtoagp4xHFA9IQE7vAxl+E3a1+PyHqFYcMKmD59qulxzcRyI2T6qBF0XSe71dfEN99UsnPne6bk9LVaqaamhvXrN5mSoz3P/fwZEhKcEYltBp3ACMY1Ani3E367br0pOSd8f7zP6eurVv+KS5cumZKnNf369WX+A/NMj2sWlhohIcFJSorvTpf2k08++OAjDh36Muy8drud0tIphve+/vrvvPm7/wg7hxGPPfYwvXqlRyR2uFhqhIyMDL8jdUOGeHcnr99gTtXtbzHrihWvmD5EDZCYmMiTSxabHtcMLDWCr4ZiC+1fDQCbN2/F1dAQdu7RN48iNyfb8N7Ro8d4591dYecw4p575hr+XFZjrRF8NBRbGDx4EDZb23PMa2trqajYHnZuIQRlBiOSLby0fGXYOYyw2Ww891znO3/d2leDn4YiXFmqnpub43XdrO/9WbPKfN77ZO8+Pv74f0zJ056Jt09g/PjCiMQOFYtrBP9GAO8GI8CBAwf5yyd7w84/aNBAhg/3fZJRJIaoW3h+2c+8ajsrsbiN4P/VADDUoMEIsG7dBlM0+FsEs2vXbg4cOGhKnvYUFFzP3bNnRiR2KFhqBH99CC34alht3/6fVFdXh62hrGya37/MV1atCTuHL55++knD7m4rsMwIdrudPn16ByzXfsyhhcbGi2zevDVsHb179+KWW/7J5/1t297ibydPhZ3HiD59erPo4YciErujWGaEnj17GM4NaM+AAf1xOIzn//123XpTvvf99SlcvnyZVRGsFR55+F/IyOgXsfjBYpkRLl1qQsrAK/Lj4uIMZyADnDjxv+zZ80HYWkpLinE6u/m8v3HjFqqqzoedxwins1vI+zOYiWVGqK2tDbq72NfrAcxpNCYlJTFxYpHP+w0NjREboga4e/ZMv0v8o4GljcVfrFwdVDl/PXHvvLuLkya8wwPtn/jr37xmSo+mEZqm8ZNHFkYkdtAarEy+dWtFUGMH/moEt9vNxo1bwtZSNOF7fmcdV1dX88aGzWHn8cWkSRN9toWigaVGkFKycOGjlP/bcr/lhgbomzdj0orD4aC0pNhvmVWrf2XK5Bgj4uMddO9uzn4NoWD5fAQpJeXlL3LPvfdz8eI/DMvk5GT7bcxVVp7lD38Ifx1jSQAjnDp12pRxDiNqamqorv42IrGDwXIjtFBRsZ2S0jLDTiKbzUavdP/j+Ote2xC2huuvHxqwzMsRGqJ+440tEYkbLJ1uV7W8vFy2VWxh0KCBV6+dPn2G7xSM4vLlyz6fE0Jw06gbmVJyJyUlk30OMfvjiy8OMXpM4CX3ubk55ORkk52VSVZ2FtlZmWRnZ5GdnUXv3r06vBp6375PubN4GnV1dR3WbBadzghwZUFsefmzjB93G8ePn+Cxx5fy178e6lCMYcMKKCmZTGlJcRtT+eOnP13Gyyt+GYrkq8THO8jMvGKMrMyMq0bJys4ip9koLdTU1LB581aWPf+C5Zt8d0ojmE1+/mBKS4opKZlMQcH1hmX27PmAaT+4K2KNwRa6dYunV3o6mqZx6vQZv7VcNOkSRmhNXl4upSWTGTfuVnJysvnmm0oqtr3F66+/0Wl+KVbQ5YygMKbTfDUorEUZQQEoIyiaUUZQAMoIimaUERSAMoKiGWUEBaCMoGhGGUEBXDGC6mJWSA1JZGZkKmIGCS4NwbV12rWiwwio0wDrJsopOgvfakiOWa1CYS0CjmpoHLZaiMJapOSwhkeav0e9IsaQnwunMzlTs2knrZaisA6PW2ZojY3VpySqndCF+bKx8fwZDUAIdlitRmERUuyA5i5mId0bLBWjsBBtCzQbob7+24PAfkv1KKKPYK/LVfk5tB50krxkmSCFJQgpri5Db71Iz6Ynph4CglsfpohpJBxrqK/KB9zQdhjaLRCxcQCRImw0tCU0mwDa1ggA6Emp7yIxPoFTcY0g3nfVn2tzjI3XxBQNFiFxRU+UIppIqLfbxIL2172MUFdXdQS4LyqqFFFHSLngwoWzX7W/brj3bFNTw0FHnDMLIUZEXpoiWghY63Kdf97HPZ/E6UmpO5DcESFdimgi2elyVZUChmv//U1ebXIl2Mok8s+RUaaIFhI+drkcM/BhAgg0i7my0tWg24sQhL9lmcIqdid04w742u/c1MAnR7hcTU2XGioccQl9EYw0TZ4i4ghY66qvmt3Q0HAxiLLBo+spc6QQawQkhi5PEWkk1AvJAy5XVdBb0nZogYvLdX6TXRMjQLzfcXmKqCB4L86mDe+ICa48FiKJiWllEvk8amyis3BYoC2trz8b0umlIRuhGZuup85AsBhU+8ESBHuFFMvr689VACFv3RquEa6i670KwD0HQQkwyKy4CkMOI9khhHtT81ySsDHNCK1xOlP6aRrjQRQIwWB55fXRkyuNTOu2II8t6oB6oEbAESk5DPJzj4c9jY3nz5id7P8AzPKwqM8xHzQAAAAASUVORK5CYII=
 schemaVersion: 1.1.0
-version: "20261110"
+version: "20261112"
 locale: en
 
 variableGroups:
@@ -43,14 +43,14 @@ variables:
 
   metrics_compartment_ids:
     title: Compartments
-    description: Compartment OCIDs to forward metrics from, separated by commas or new lines. Leave empty for every compartment.
+    description: Compartment OCIDs to forward metrics from, separated by commas or new lines. Leave empty for every compartment that exists when you apply. Apply again after you add compartments.
     type: text
     multiline: true
     required: false
 
   metrics_namespaces:
     title: Metric namespaces
-    description: Clear the list to forward logs only.
+    description: Forwarded from each compartment above, including services you start using later. Each Connector Hub covers 2 compartments with the 25 default namespaces, and 5 with 10 or fewer. Clear the list to forward logs only.
     type: enum
     additionalProps:
       allowMultiple: true