From 4fe7b46c8266a9d55aaaf716be85a83071c8e0b8 Mon Sep 17 00:00:00 2001 From: Arsen Shkrumelyak Date: Tue, 27 Oct 2026 16:02:44 +0000 Subject: [PATCH 01/15] feat(forwarder): add function that posts connector hub batches --- .gitignore | 5 +++ forwarder/Dockerfile | 12 +++++++ forwarder/func.py | 23 +++++++++++++ forwarder/requirements.txt | 1 + forwarder/test_func.py | 69 ++++++++++++++++++++++++++++++++++++++ 5 files changed, 110 insertions(+) create mode 100644 .gitignore create mode 100644 forwarder/Dockerfile create mode 100644 forwarder/func.py create mode 100644 forwarder/requirements.txt create mode 100644 forwarder/test_func.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..c58cd9f --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +__pycache__/ +.terraform/ +.terraform.lock.hcl +*.tfstate* +*.zip diff --git a/forwarder/Dockerfile b/forwarder/Dockerfile new file mode 100644 index 0000000..3bf3cee --- /dev/null +++ b/forwarder/Dockerfile @@ -0,0 +1,12 @@ +FROM fnproject/python:3.11-dev AS build +WORKDIR /function +COPY requirements.txt . +RUN pip3 install --target /python/ --no-cache-dir -r requirements.txt +COPY func.py . + +FROM fnproject/python:3.11 +WORKDIR /function +COPY --from=build /python /python +COPY --from=build /function /function +ENV PYTHONPATH=/function:/python +ENTRYPOINT ["/python/bin/fdk", "/function/func.py", "handler"] diff --git a/forwarder/func.py b/forwarder/func.py new file mode 100644 index 0000000..eb96d61 --- /dev/null +++ b/forwarder/func.py @@ -0,0 +1,23 @@ +import gzip +import io +import os +import urllib.request + +from fdk import response + + +def handler(ctx, data: io.BytesIO = None): + # Connector Hub retries a batch when the function fails, so a non-2xx + # response from Better Stack is left to raise. + request = urllib.request.Request( + f"https://{os.environ['BETTERSTACK_INGESTING_HOST']}/oci", + data=gzip.compress(data.getvalue()), + headers={ + "Authorization": f"Bearer {os.environ['BETTERSTACK_SOURCE_TOKEN']}", + "Content-Type": "application/json", + "Content-Encoding": "gzip", + }, + method="POST", + ) + urllib.request.urlopen(request, timeout=60) + return response.Response(ctx, response_data="") diff --git a/forwarder/requirements.txt b/forwarder/requirements.txt new file mode 100644 index 0000000..b36c7bf --- /dev/null +++ b/forwarder/requirements.txt @@ -0,0 +1 @@ +fdk==0.1.128 diff --git a/forwarder/test_func.py b/forwarder/test_func.py new file mode 100644 index 0000000..715b082 --- /dev/null +++ b/forwarder/test_func.py @@ -0,0 +1,69 @@ +import gzip +import io +import os +import sys +import threading +import types +import unittest +import urllib.error +from http.server import BaseHTTPRequestHandler, HTTPServer +from unittest import mock + +sys.modules["fdk"] = types.SimpleNamespace(response=types.SimpleNamespace(Response=lambda ctx, response_data: response_data)) + +import func # noqa: E402 + + +class Ingesting(BaseHTTPRequestHandler): + status = 202 + received = [] + + def do_POST(self): + body = self.rfile.read(int(self.headers["Content-Length"])) + Ingesting.received.append((self.path, dict(self.headers), body)) + self.send_response(Ingesting.status) + self.end_headers() + + def log_message(self, *args): + pass + + +class HandlerTest(unittest.TestCase): + def setUp(self): + Ingesting.received.clear() + Ingesting.status = 202 + self.server = HTTPServer(("127.0.0.1", 0), Ingesting) + threading.Thread(target=self.server.serve_forever, daemon=True).start() + host = f"127.0.0.1:{self.server.server_port}" + # func.py always speaks HTTPS; the test server is plain HTTP. + real_request = urllib.request.Request + self.patches = [ + mock.patch.dict(os.environ, {"BETTERSTACK_INGESTING_HOST": host, "BETTERSTACK_SOURCE_TOKEN": "token"}), + mock.patch("urllib.request.Request", lambda url, **kw: real_request(url.replace("https://", "http://"), **kw)), + ] + for patch in self.patches: + patch.start() + + def tearDown(self): + for patch in self.patches: + patch.stop() + self.server.shutdown() + + def test_posts_the_batch_gzipped_with_the_source_token(self): + func.handler(None, io.BytesIO(b'[{"namespace":"oci_faas"}]')) + + path, headers, body = Ingesting.received[0] + self.assertEqual("/oci", path) + self.assertEqual("Bearer token", headers["Authorization"]) + self.assertEqual("gzip", headers["Content-Encoding"]) + self.assertEqual(b'[{"namespace":"oci_faas"}]', gzip.decompress(body)) + + def test_raises_on_a_rejected_batch_so_connector_hub_retries(self): + Ingesting.status = 500 + + with self.assertRaises(urllib.error.HTTPError): + func.handler(None, io.BytesIO(b"[]")) + + +if __name__ == "__main__": + unittest.main() From 11f20c3d7614c318154deda5927406a2a187560c Mon Sep 17 00:00:00 2001 From: Arsen Shkrumelyak Date: Tue, 27 Oct 2026 16:04:59 +0000 Subject: [PATCH 02/15] feat(stack): add resource manager stack for connectors and forwarder --- main.tf | 275 +++++++++++++++++++++++++++++++++++++++++++++++++++ schema.yaml | 99 +++++++++++++++++++ variables.tf | 71 +++++++++++++ 3 files changed, 445 insertions(+) create mode 100644 main.tf create mode 100644 schema.yaml create mode 100644 variables.tf diff --git a/main.tf b/main.tf new file mode 100644 index 0000000..df7ee0a --- /dev/null +++ b/main.tf @@ -0,0 +1,275 @@ +terraform { + required_version = ">= 1.5" + + required_providers { + oci = { + source = "oracle/oci" + version = ">= 6.0.0" + } + } +} + +provider "oci" { + region = var.region +} + +locals { + forwarder_version = "0.1.0" + + # Better Stack publishes the forwarder to a public repository in every region it supports. + forwarder_image = coalesce(var.forwarder_image, "${lower(data.oci_identity_regions.current.regions[0].key)}.ocir.io/lr3km3hvmrwp/betterstack/oci-forwarder:${local.forwarder_version}") + + connector_principal = "where all {request.principal.type = 'serviceconnector', request.principal.compartment.id = '${oci_identity_compartment.betterstack.id}'}" +} + +data "oci_identity_regions" "current" { + filter { + name = "name" + values = [var.region] + } +} + +data "oci_identity_region_subscriptions" "home" { + tenancy_id = var.tenancy_ocid + + filter { + name = "is_home_region" + values = [true] + } +} + +resource "oci_identity_compartment" "betterstack" { + compartment_id = var.tenancy_ocid + name = "betterstack" + description = "Better Stack metric and log forwarding" + enable_delete = true + + lifecycle { + precondition { + condition = var.region == data.oci_identity_region_subscriptions.home.region_subscriptions[0].region_name + error_message = "Run the stack in your home region. It creates a compartment and policies, which OCI only accepts there." + } + } +} + +resource "oci_identity_policy" "connectors" { + compartment_id = var.tenancy_ocid + name = "betterstack-connectors" + description = "Lets the Better Stack connectors read metrics and logs and invoke the forwarder" + statements = [ + "Allow any-user to read metrics in tenancy ${local.connector_principal}", + "Allow any-user to read log-content in tenancy ${local.connector_principal}", + "Allow any-user to use fn-function in compartment id ${oci_identity_compartment.betterstack.id} ${local.connector_principal}", + "Allow any-user to use fn-invocation in compartment id ${oci_identity_compartment.betterstack.id} ${local.connector_principal}", + ] +} + +# The Function has no public IP, so it reaches Better Stack through a NAT gateway. +resource "oci_core_vcn" "betterstack" { + compartment_id = oci_identity_compartment.betterstack.id + display_name = "betterstack" + cidr_blocks = ["10.0.0.0/16"] + dns_label = "betterstack" +} + +resource "oci_core_nat_gateway" "betterstack" { + compartment_id = oci_identity_compartment.betterstack.id + vcn_id = oci_core_vcn.betterstack.id + display_name = "betterstack" +} + +resource "oci_core_route_table" "forwarder" { + compartment_id = oci_identity_compartment.betterstack.id + vcn_id = oci_core_vcn.betterstack.id + display_name = "forwarder" + + route_rules { + destination = "0.0.0.0/0" + network_entity_id = oci_core_nat_gateway.betterstack.id + } +} + +resource "oci_core_security_list" "forwarder" { + compartment_id = oci_identity_compartment.betterstack.id + vcn_id = oci_core_vcn.betterstack.id + display_name = "forwarder" + + egress_security_rules { + destination = "0.0.0.0/0" + protocol = "6" + + tcp_options { + min = 443 + max = 443 + } + } + + egress_security_rules { + destination = "0.0.0.0/0" + protocol = "17" + + udp_options { + min = 53 + max = 53 + } + } +} + +resource "oci_core_subnet" "forwarder" { + compartment_id = oci_identity_compartment.betterstack.id + vcn_id = oci_core_vcn.betterstack.id + display_name = "forwarder" + cidr_block = "10.0.0.0/24" + dns_label = "forwarder" + prohibit_public_ip_on_vnic = true + route_table_id = oci_core_route_table.forwarder.id + security_list_ids = [oci_core_security_list.forwarder.id] +} + +resource "oci_functions_application" "betterstack" { + compartment_id = oci_identity_compartment.betterstack.id + display_name = "betterstack" + subnet_ids = [oci_core_subnet.forwarder.id] + shape = "GENERIC_X86_ARM" +} + +resource "oci_functions_function" "forwarder" { + application_id = oci_functions_application.betterstack.id + display_name = "forwarder" + image = local.forwarder_image + memory_in_mbs = 256 + timeout_in_seconds = 300 + + config = { + BETTERSTACK_INGESTING_HOST = var.betterstack_ingesting_host + BETTERSTACK_SOURCE_TOKEN = var.betterstack_source_token + } +} + +# A monitoring source only covers its own compartment, not its children, so +# every compartment with metrics in a selected namespace becomes a source. +data "oci_identity_compartments" "all" { + compartment_id = var.tenancy_ocid + compartment_id_in_subtree = true + access_level = "ANY" + state = "ACTIVE" +} + +locals { + metrics_compartment_ids = var.metrics_compartment_ids != "" ? [for id in split(",", var.metrics_compartment_ids) : trimspace(id)] : concat( + [var.tenancy_ocid], + [for compartment in data.oci_identity_compartments.all.compartments : compartment.id if compartment.name != "betterstack"], + ) +} + +data "oci_monitoring_metrics" "namespaces" { + for_each = toset(local.metrics_compartment_ids) + + compartment_id = each.value + group_by = ["namespace"] +} + +locals { + metric_sources = [ + for id in local.metrics_compartment_ids : { + compartment_id = id + namespaces = sort(setintersection(var.metrics_namespaces, data.oci_monitoring_metrics.namespaces[id].metrics[*].namespace)) + } if length(setintersection(var.metrics_namespaces, data.oci_monitoring_metrics.namespaces[id].metrics[*].namespace)) > 0 + ] + + # A connector takes at most 5 compartments and 50 namespaces across them. + metric_sources_per_connector = min(5, floor(50 / max(1, [for source in local.metric_sources : length(source.namespaces)]...))) + metric_connectors = chunklist(local.metric_sources, local.metric_sources_per_connector) +} + +resource "oci_sch_service_connector" "metrics" { + count = length(local.metric_connectors) + + compartment_id = oci_identity_compartment.betterstack.id + display_name = "betterstack-metrics-${count.index}" + + source { + kind = "monitoring" + + dynamic "monitoring_sources" { + for_each = local.metric_connectors[count.index] + + content { + compartment_id = monitoring_sources.value.compartment_id + + namespace_details { + kind = "selected" + + dynamic "namespaces" { + for_each = monitoring_sources.value.namespaces + + content { + namespace = namespaces.value + + metrics { + kind = "all" + } + } + } + } + } + } + } + + target { + kind = "functions" + function_id = oci_functions_function.forwarder.id + batch_size_in_kbs = 5000 + batch_time_in_sec = 60 + } + + depends_on = [oci_identity_policy.connectors] +} + +locals { + log_group_ids = var.log_group_ids != "" ? [for id in split(",", var.log_group_ids) : trimspace(id)] : [] +} + +data "oci_logging_log_group" "selected" { + for_each = toset(local.log_group_ids) + + log_group_id = each.value +} + +resource "oci_sch_service_connector" "logs" { + count = var.forward_audit_logs || length(local.log_group_ids) > 0 ? 1 : 0 + + compartment_id = oci_identity_compartment.betterstack.id + display_name = "betterstack-logs" + + source { + kind = "logging" + + dynamic "log_sources" { + for_each = var.forward_audit_logs ? [var.tenancy_ocid] : [] + + content { + compartment_id = log_sources.value + log_group_id = "_Audit_Include_Subcompartment" + } + } + + dynamic "log_sources" { + for_each = data.oci_logging_log_group.selected + + content { + compartment_id = log_sources.value.compartment_id + log_group_id = log_sources.key + } + } + } + + target { + kind = "functions" + function_id = oci_functions_function.forwarder.id + batch_size_in_kbs = 5000 + batch_time_in_sec = 60 + } + + depends_on = [oci_identity_policy.connectors] +} diff --git a/schema.yaml b/schema.yaml new file mode 100644 index 0000000..4aac754 --- /dev/null +++ b/schema.yaml @@ -0,0 +1,99 @@ +title: Better Stack +description: Forward Oracle Cloud metrics and logs to Better Stack. +schemaVersion: 1.1.0 +version: "20261027" +locale: en + +variableGroups: + - title: Better Stack source + variables: + - ${betterstack_ingesting_host} + - ${betterstack_source_token} + - title: Metrics + variables: + - ${metrics_compartment_ids} + - ${metrics_namespaces} + - title: Logs + variables: + - ${forward_audit_logs} + - ${log_group_ids} + - title: Advanced + variables: + - ${forwarder_image} + - title: Hidden + visible: false + variables: + - ${tenancy_ocid} + - ${region} + +variables: + betterstack_ingesting_host: + title: Ingesting host + description: Shown on your source's Configure page, for example s1234.eu-nbg-2.betterstackdata.com. + type: string + required: true + + betterstack_source_token: + title: Source token + type: password + required: true + sensitive: true + + metrics_compartment_ids: + title: Compartments + description: Comma-separated compartment OCIDs to forward metrics from. Leave empty for every compartment. + type: text + multiline: true + required: false + + metrics_namespaces: + title: Metric namespaces + type: enum + additionalProps: + allowMultiple: true + required: true + enum: + - oci_autonomous_database + - oci_blockstore + - oci_compute_infrastructure_health + - oci_computeagent + - oci_computecontainerinstance + - oci_database + - oci_database_cluster + - oci_dynamic_routing_gateway + - oci_faas + - oci_fastconnect + - oci_filestorage + - oci_gpu_infrastructure_health + - oci_lbaas + - oci_mysql_database + - oci_nat_gateway + - oci_nlb + - oci_objectstorage + - oci_oke + - oci_queue + - oci_rdma_infrastructure_health + - oci_service_connector_hub + - oci_service_gateway + - oci_vcn + - oci_vpn + - oci_waf + - oracle_oci_database + + forward_audit_logs: + title: Forward audit logs + description: Audit events from every compartment in the tenancy. + type: boolean + + log_group_ids: + title: Log groups + description: Comma-separated log group OCIDs to forward. + type: text + multiline: true + required: false + + forwarder_image: + title: Forwarder image + description: Leave empty to use the image Better Stack publishes in your region. + type: string + required: false diff --git a/variables.tf b/variables.tf new file mode 100644 index 0000000..253743e --- /dev/null +++ b/variables.tf @@ -0,0 +1,71 @@ +variable "tenancy_ocid" { + type = string +} + +variable "region" { + type = string +} + +variable "betterstack_ingesting_host" { + type = string + description = "Ingesting host of your Better Stack source, without https://." +} + +variable "betterstack_source_token" { + type = string + sensitive = true +} + +variable "metrics_compartment_ids" { + type = string + default = "" + description = "Comma-separated compartment OCIDs to forward metrics from. Leave empty for every compartment in the tenancy." +} + +variable "metrics_namespaces" { + type = list(string) + default = [ + "oci_autonomous_database", + "oci_blockstore", + "oci_compute_infrastructure_health", + "oci_computeagent", + "oci_computecontainerinstance", + "oci_database", + "oci_database_cluster", + "oci_dynamic_routing_gateway", + "oci_faas", + "oci_fastconnect", + "oci_filestorage", + "oci_gpu_infrastructure_health", + "oci_lbaas", + "oci_mysql_database", + "oci_nat_gateway", + "oci_nlb", + "oci_objectstorage", + "oci_oke", + "oci_queue", + "oci_rdma_infrastructure_health", + "oci_service_gateway", + "oci_vcn", + "oci_vpn", + "oci_waf", + "oracle_oci_database", + ] +} + +variable "forward_audit_logs" { + type = bool + default = true +} + +variable "log_group_ids" { + type = string + default = "" + description = "Comma-separated log group OCIDs to forward logs from." +} + +variable "forwarder_image" { + type = string + default = "" + description = "Overrides the forwarder image Better Stack publishes in your region." +} From 37c1f9edf508e2a569d5185e194ddcf57379d46d Mon Sep 17 00:00:00 2001 From: Arsen Shkrumelyak Date: Tue, 27 Oct 2026 16:05:07 +0000 Subject: [PATCH 03/15] docs: describe the stack as built --- README.md | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index ec87599..ae09e76 100644 --- a/README.md +++ b/README.md @@ -12,13 +12,16 @@ Better Stack never connects to your tenancy. Data only leaves through the Functi [Create a new Oracle Cloud source in Better Stack](https://telemetry.betterstack.com/team/t0/sources/new?platform=oci). The source-creation flow hands you a **Deploy to Oracle Cloud** link with every stack variable pre-filled. Click it, review the variables, and click **Create**. -Sign in to your tenancy's home region as a user who can create dynamic groups and policies in the root compartment. +Sign in to your tenancy's **home region** as a user who can create compartments and policies in the root compartment. The stack forwards metrics and logs from the home region. ## What gets deployed - A `betterstack` compartment that holds every resource below. -- In each region you pick, a Function application running the forwarder, a VCN with a private subnet and a NAT gateway so the Function can reach Better Stack, and Connector Hubs for metrics and logs. -- Dynamic groups and policies that let Connector Hub read metrics and logs, and invoke the Function. +- A Function application running the forwarder, in a VCN with a private subnet and a NAT gateway so the Function can reach Better Stack. +- A Connector Hub for logs, and one Connector Hub for metrics per batch of up to 5 compartments. +- A policy that lets the connectors read metrics and logs, and invoke the Function. + +The stack looks up which compartments have metrics in the selected namespaces when it runs. Apply the stack again after you add compartments or start using new services. ## Uninstall From ab0adaf9defcc2850ab111b93f9efae12ee55ff9 Mon Sep 17 00:00:00 2001 From: Arsen Shkrumelyak Date: Tue, 27 Oct 2026 16:05:42 +0000 Subject: [PATCH 04/15] chore: add test and release workflows --- .github/workflows/release.yml | 45 +++++++++++++++++++++++++++++++++++ .github/workflows/test.yml | 23 ++++++++++++++++++ 2 files changed, 68 insertions(+) create mode 100644 .github/workflows/release.yml create mode 100644 .github/workflows/test.yml diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..6762c7f --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,45 @@ +name: Release + +on: + push: + tags: ["v*"] + +jobs: + image: + runs-on: ubuntu-latest + strategy: + matrix: + region_key: [lhr] + steps: + - uses: actions/checkout@v4 + - id: version + run: echo "version=${GITHUB_REF_NAME#v}" >> "$GITHUB_OUTPUT" + - name: Check the stack pins this version + run: grep -q 'forwarder_version = "${{ steps.version.outputs.version }}"' main.tf + - uses: docker/setup-qemu-action@v3 + - uses: docker/setup-buildx-action@v3 + - uses: docker/login-action@v3 + with: + registry: ${{ matrix.region_key }}.ocir.io + username: ${{ secrets.OCIR_USERNAME }} + password: ${{ secrets.OCIR_TOKEN }} + - uses: docker/build-push-action@v6 + with: + context: forwarder + platforms: linux/amd64,linux/arm64 + provenance: false + sbom: false + push: true + tags: ${{ matrix.region_key }}.ocir.io/lr3km3hvmrwp/betterstack/oci-forwarder:${{ steps.version.outputs.version }} + + stack: + needs: image + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - uses: actions/checkout@v4 + - run: zip betterstack-oci.zip main.tf variables.tf schema.yaml + - run: gh release create "$GITHUB_REF_NAME" betterstack-oci.zip --generate-notes + env: + GH_TOKEN: ${{ github.token }} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..8f5b304 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,23 @@ +name: Test + +on: + pull_request: + push: + branches: [main] + +jobs: + forwarder: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - run: python3 -m unittest test_func + working-directory: forwarder + + stack: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: hashicorp/setup-terraform@v3 + - run: terraform fmt -check + - run: terraform init -backend=false + - run: terraform validate From 2754d1d6095b82fde0683c4905466d0b6e6ae893 Mon Sep 17 00:00:00 2001 From: Arsen Shkrumelyak Date: Thu, 29 Oct 2026 12:55:22 +0000 Subject: [PATCH 05/15] fix(stack): prefill metric namespaces in the resource manager form --- schema.yaml | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/schema.yaml b/schema.yaml index 4aac754..2ca74b7 100644 --- a/schema.yaml +++ b/schema.yaml @@ -52,6 +52,32 @@ variables: additionalProps: allowMultiple: true required: true + default: + - oci_autonomous_database + - oci_blockstore + - oci_compute_infrastructure_health + - oci_computeagent + - oci_computecontainerinstance + - oci_database + - oci_database_cluster + - oci_dynamic_routing_gateway + - oci_faas + - oci_fastconnect + - oci_filestorage + - oci_gpu_infrastructure_health + - oci_lbaas + - oci_mysql_database + - oci_nat_gateway + - oci_nlb + - oci_objectstorage + - oci_oke + - oci_queue + - oci_rdma_infrastructure_health + - oci_service_gateway + - oci_vcn + - oci_vpn + - oci_waf + - oracle_oci_database enum: - oci_autonomous_database - oci_blockstore From dc6045b117c13094275298246e0ee2c9583e1935 Mon Sep 17 00:00:00 2001 From: Arsen Shkrumelyak Date: Thu, 5 Nov 2026 12:20:06 +0000 Subject: [PATCH 06/15] docs: describe the deploy flow as it works, link the oracle_cloud platform --- README.md | 2 +- schema.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index ae09e76..ca4daf8 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ Better Stack never connects to your tenancy. Data only leaves through the Functi ## Quick start -[Create a new Oracle Cloud source in Better Stack](https://telemetry.betterstack.com/team/t0/sources/new?platform=oci). The source-creation flow hands you a **Deploy to Oracle Cloud** link with every stack variable pre-filled. Click it, review the variables, and click **Create**. +[Create a new Oracle Cloud source in Better Stack](https://telemetry.betterstack.com/team/0/sources/new?platform=oracle_cloud). Its **Ingest** tab has a **Deploy to Oracle Cloud** button that opens this stack in Resource Manager. Paste the source's **Ingesting host** and **Source token** into the stack form, make sure **Run apply** is selected, and click **Create**. Sign in to your tenancy's **home region** as a user who can create compartments and policies in the root compartment. The stack forwards metrics and logs from the home region. diff --git a/schema.yaml b/schema.yaml index 2ca74b7..04feb8c 100644 --- a/schema.yaml +++ b/schema.yaml @@ -29,7 +29,7 @@ variableGroups: variables: betterstack_ingesting_host: title: Ingesting host - description: Shown on your source's Configure page, for example s1234.eu-nbg-2.betterstackdata.com. + description: Shown on your source's Ingest tab, for example s1234.eu-nbg-2.betterstackdata.com. type: string required: true From 95fb41f01c5da8cf618b42f20c715916e3b76025 Mon Sep 17 00:00:00 2001 From: Arsen Shkrumelyak Date: Mon, 9 Nov 2026 14:53:55 +0000 Subject: [PATCH 07/15] fix(stack): accept pasted hosts and listed ocids, pack metric connectors tighter --- main.tf | 43 +++++++++++++++++++++++++++++-------------- schema.yaml | 10 ++++++---- variables.tf | 14 ++++++++++++-- 3 files changed, 47 insertions(+), 20 deletions(-) diff --git a/main.tf b/main.tf index df7ee0a..0c8cd33 100644 --- a/main.tf +++ b/main.tf @@ -4,7 +4,7 @@ terraform { required_providers { oci = { source = "oracle/oci" - version = ">= 6.0.0" + version = "~> 9.0" } } } @@ -20,6 +20,18 @@ locals { forwarder_image = coalesce(var.forwarder_image, "${lower(data.oci_identity_regions.current.regions[0].key)}.ocir.io/lr3km3hvmrwp/betterstack/oci-forwarder:${local.forwarder_version}") connector_principal = "where all {request.principal.type = 'serviceconnector', request.principal.compartment.id = '${oci_identity_compartment.betterstack.id}'}" + + # The form asks for the host without a scheme; a pasted URL still works. + ingesting_host = trimsuffix(trimprefix(trimprefix(trimspace(var.betterstack_ingesting_host), "https://"), "http://"), "/") + + # OCID fields accept commas, spaces or one OCID per line. + selected_compartment_ids = distinct(regexall("[^,\\s]+", var.metrics_compartment_ids)) + log_group_ids = distinct(regexall("[^,\\s]+", var.log_group_ids)) + + # Metrics are read tenancy-wide only when no compartments are chosen. + metrics_statements = length(local.selected_compartment_ids) > 0 ? [ + for id in local.selected_compartment_ids : "Allow any-user to read metrics in compartment id ${id} ${local.connector_principal}" + ] : ["Allow any-user to read metrics in tenancy ${local.connector_principal}"] } data "oci_identity_regions" "current" { @@ -56,12 +68,11 @@ resource "oci_identity_policy" "connectors" { compartment_id = var.tenancy_ocid name = "betterstack-connectors" description = "Lets the Better Stack connectors read metrics and logs and invoke the forwarder" - statements = [ - "Allow any-user to read metrics in tenancy ${local.connector_principal}", + statements = concat(local.metrics_statements, [ "Allow any-user to read log-content in tenancy ${local.connector_principal}", "Allow any-user to use fn-function in compartment id ${oci_identity_compartment.betterstack.id} ${local.connector_principal}", "Allow any-user to use fn-invocation in compartment id ${oci_identity_compartment.betterstack.id} ${local.connector_principal}", - ] + ]) } # The Function has no public IP, so it reaches Better Stack through a NAT gateway. @@ -104,8 +115,9 @@ resource "oci_core_security_list" "forwarder" { } } + # DNS goes to the VCN resolver only. egress_security_rules { - destination = "0.0.0.0/0" + destination = "169.254.169.254/32" protocol = "17" udp_options { @@ -141,7 +153,7 @@ resource "oci_functions_function" "forwarder" { timeout_in_seconds = 300 config = { - BETTERSTACK_INGESTING_HOST = var.betterstack_ingesting_host + BETTERSTACK_INGESTING_HOST = local.ingesting_host BETTERSTACK_SOURCE_TOKEN = var.betterstack_source_token } } @@ -156,7 +168,7 @@ data "oci_identity_compartments" "all" { } locals { - metrics_compartment_ids = var.metrics_compartment_ids != "" ? [for id in split(",", var.metrics_compartment_ids) : trimspace(id)] : concat( + metrics_compartment_ids = length(local.selected_compartment_ids) > 0 ? local.selected_compartment_ids : concat( [var.tenancy_ocid], [for compartment in data.oci_identity_compartments.all.compartments : compartment.id if compartment.name != "betterstack"], ) @@ -177,9 +189,16 @@ locals { } if length(setintersection(var.metrics_namespaces, data.oci_monitoring_metrics.namespaces[id].metrics[*].namespace)) > 0 ] - # A connector takes at most 5 compartments and 50 namespaces across them. - metric_sources_per_connector = min(5, floor(50 / max(1, [for source in local.metric_sources : length(source.namespaces)]...))) - metric_connectors = chunklist(local.metric_sources, local.metric_sources_per_connector) + # A connector takes at most 5 compartments and 50 namespaces across them, so + # compartments are packed with others that select as many namespaces: one + # compartment with many namespaces doesn't shrink every other connector. + metric_source_sizes = sort(distinct([for source in local.metric_sources : format("%02d", length(source.namespaces))])) + metric_connectors = concat([], [ + for size in local.metric_source_sizes : chunklist( + [for source in local.metric_sources : source if length(source.namespaces) == tonumber(size)], + min(5, floor(50 / tonumber(size))), + ) + ]...) } resource "oci_sch_service_connector" "metrics" { @@ -226,10 +245,6 @@ resource "oci_sch_service_connector" "metrics" { depends_on = [oci_identity_policy.connectors] } -locals { - log_group_ids = var.log_group_ids != "" ? [for id in split(",", var.log_group_ids) : trimspace(id)] : [] -} - data "oci_logging_log_group" "selected" { for_each = toset(local.log_group_ids) diff --git a/schema.yaml b/schema.yaml index 04feb8c..57fddf3 100644 --- a/schema.yaml +++ b/schema.yaml @@ -1,7 +1,7 @@ title: Better Stack description: Forward Oracle Cloud metrics and logs to Better Stack. schemaVersion: 1.1.0 -version: "20261027" +version: "20261109" locale: en variableGroups: @@ -32,6 +32,7 @@ variables: description: Shown on your source's Ingest tab, for example s1234.eu-nbg-2.betterstackdata.com. type: string required: true + pattern: "^\\s*(https?://)?[A-Za-z0-9.-]+(:[0-9]+)?/?\\s*$" betterstack_source_token: title: Source token @@ -41,17 +42,18 @@ variables: metrics_compartment_ids: title: Compartments - description: Comma-separated compartment OCIDs to forward metrics from. Leave empty for every compartment. + description: Compartment OCIDs to forward metrics from, separated by commas or new lines. Leave empty for every compartment. type: text multiline: true required: false metrics_namespaces: title: Metric namespaces + description: Clear the list to forward logs only. type: enum additionalProps: allowMultiple: true - required: true + required: false default: - oci_autonomous_database - oci_blockstore @@ -113,7 +115,7 @@ variables: log_group_ids: title: Log groups - description: Comma-separated log group OCIDs to forward. + description: Log group OCIDs to forward, separated by commas or new lines. type: text multiline: true required: false diff --git a/variables.tf b/variables.tf index 253743e..9533cb2 100644 --- a/variables.tf +++ b/variables.tf @@ -9,6 +9,11 @@ variable "region" { variable "betterstack_ingesting_host" { type = string description = "Ingesting host of your Better Stack source, without https://." + + validation { + condition = can(regex("^\\s*(https?://)?[A-Za-z0-9.-]+(:[0-9]+)?/?\\s*$", var.betterstack_ingesting_host)) + error_message = "Use the ingesting host shown on your source's Ingest tab, for example s1234.eu-nbg-2.betterstackdata.com." + } } variable "betterstack_source_token" { @@ -19,7 +24,7 @@ variable "betterstack_source_token" { variable "metrics_compartment_ids" { type = string default = "" - description = "Comma-separated compartment OCIDs to forward metrics from. Leave empty for every compartment in the tenancy." + description = "Compartment OCIDs to forward metrics from, separated by commas or new lines. Leave empty for every compartment in the tenancy." } variable "metrics_namespaces" { @@ -51,6 +56,11 @@ variable "metrics_namespaces" { "oci_waf", "oracle_oci_database", ] + + validation { + condition = length(var.metrics_namespaces) <= 50 + error_message = "A connector takes at most 50 metric namespaces." + } } variable "forward_audit_logs" { @@ -61,7 +71,7 @@ variable "forward_audit_logs" { variable "log_group_ids" { type = string default = "" - description = "Comma-separated log group OCIDs to forward logs from." + description = "Log group OCIDs to forward logs from, separated by commas or new lines." } variable "forwarder_image" { From 3eba106f906b13689916e4c56cb674f65f7559e3 Mon Sep 17 00:00:00 2001 From: Arsen Shkrumelyak Date: Mon, 9 Nov 2026 14:53:55 +0000 Subject: [PATCH 08/15] fix(forwarder): fail on redirects instead of following them --- forwarder/func.py | 12 +++++++++++- forwarder/test_func.py | 14 ++++++++++++++ 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/forwarder/func.py b/forwarder/func.py index eb96d61..ed96210 100644 --- a/forwarder/func.py +++ b/forwarder/func.py @@ -6,6 +6,16 @@ from fdk import response +class _NoRedirects(urllib.request.HTTPRedirectHandler): + # Following a redirect turns the POST into a bodyless GET and sends the + # token to another host, so a redirect fails the batch instead. + def redirect_request(self, *args, **kwargs): + return None + + +_opener = urllib.request.build_opener(_NoRedirects) + + def handler(ctx, data: io.BytesIO = None): # Connector Hub retries a batch when the function fails, so a non-2xx # response from Better Stack is left to raise. @@ -19,5 +29,5 @@ def handler(ctx, data: io.BytesIO = None): }, method="POST", ) - urllib.request.urlopen(request, timeout=60) + _opener.open(request, timeout=60) return response.Response(ctx, response_data="") diff --git a/forwarder/test_func.py b/forwarder/test_func.py index 715b082..f9a6678 100644 --- a/forwarder/test_func.py +++ b/forwarder/test_func.py @@ -22,6 +22,13 @@ def do_POST(self): body = self.rfile.read(int(self.headers["Content-Length"])) Ingesting.received.append((self.path, dict(self.headers), body)) self.send_response(Ingesting.status) + if 300 <= Ingesting.status < 400: + self.send_header("Location", "/elsewhere") + self.end_headers() + + def do_GET(self): + Ingesting.received.append((self.path, dict(self.headers), b"")) + self.send_response(200) self.end_headers() def log_message(self, *args): @@ -58,6 +65,13 @@ def test_posts_the_batch_gzipped_with_the_source_token(self): self.assertEqual("gzip", headers["Content-Encoding"]) self.assertEqual(b'[{"namespace":"oci_faas"}]', gzip.decompress(body)) + def test_raises_on_a_redirect_instead_of_following_it(self): + Ingesting.status = 301 + + with self.assertRaises(urllib.error.HTTPError): + func.handler(None, io.BytesIO(b"[]")) + self.assertEqual(1, len(Ingesting.received)) + def test_raises_on_a_rejected_batch_so_connector_hub_retries(self): Ingesting.status = 500 From 8198773672170d355d2ddadc485ac2e4630e052e Mon Sep 17 00:00:00 2001 From: Arsen Shkrumelyak Date: Mon, 9 Nov 2026 14:53:55 +0000 Subject: [PATCH 09/15] build(forwarder): pin dependencies and base images --- forwarder/Dockerfile | 4 ++-- forwarder/requirements.txt | 13 +++++++++++++ 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/forwarder/Dockerfile b/forwarder/Dockerfile index 3bf3cee..8e54bf8 100644 --- a/forwarder/Dockerfile +++ b/forwarder/Dockerfile @@ -1,10 +1,10 @@ -FROM fnproject/python:3.11-dev AS build +FROM fnproject/python:3.11-dev@sha256:86e385f8fa59bd5a38edf8486778cf457379d6352713ccbb2eb3221e2d39cc93 AS build WORKDIR /function COPY requirements.txt . RUN pip3 install --target /python/ --no-cache-dir -r requirements.txt COPY func.py . -FROM fnproject/python:3.11 +FROM fnproject/python:3.11@sha256:28857e94288268d4dcae7a06ffa2c8c93b180f9bbfa642a5c165f965cbec2775 WORKDIR /function COPY --from=build /python /python COPY --from=build /function /function diff --git a/forwarder/requirements.txt b/forwarder/requirements.txt index b36c7bf..7ba559d 100644 --- a/forwarder/requirements.txt +++ b/forwarder/requirements.txt @@ -1 +1,14 @@ +# fdk and everything it installs, pinned so a release image is reproducible. fdk==0.1.128 +cython==3.3.1 +httptools==0.9.0 +iniconfig==2.3.1 +iso8601==0.1.12 +packaging==26.3 +pbr==7.1.3 +pluggy==1.7.0 +pygments==2.21.0 +pytest==9.2.0 +pytest-asyncio==1.4.1.post1 +setuptools==84.2.1 +wheel==0.48.0 From 3908d0ce4f8be8ce1e623808285d3b72edc0da8e Mon Sep 17 00:00:00 2001 From: Arsen Shkrumelyak Date: Mon, 9 Nov 2026 14:53:55 +0000 Subject: [PATCH 10/15] ci: pin actions to commits and scope workflow permissions --- .github/workflows/release.yml | 16 +++++++++------- .github/workflows/test.yml | 9 ++++++--- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 6762c7f..d56162b 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -7,23 +7,25 @@ on: jobs: image: runs-on: ubuntu-latest + permissions: + contents: read strategy: matrix: region_key: [lhr] steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 - id: version run: echo "version=${GITHUB_REF_NAME#v}" >> "$GITHUB_OUTPUT" - name: Check the stack pins this version - run: grep -q 'forwarder_version = "${{ steps.version.outputs.version }}"' main.tf - - uses: docker/setup-qemu-action@v3 - - uses: docker/setup-buildx-action@v3 - - uses: docker/login-action@v3 + run: grep -qF 'forwarder_version = "${{ steps.version.outputs.version }}"' main.tf + - uses: docker/setup-qemu-action@c7c53464625b32c7a7e944ae62b3e17d2b600130 # v3 + - uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3 + - uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3 with: registry: ${{ matrix.region_key }}.ocir.io username: ${{ secrets.OCIR_USERNAME }} password: ${{ secrets.OCIR_TOKEN }} - - uses: docker/build-push-action@v6 + - uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6 with: context: forwarder platforms: linux/amd64,linux/arm64 @@ -38,7 +40,7 @@ jobs: permissions: contents: write steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 - run: zip betterstack-oci.zip main.tf variables.tf schema.yaml - run: gh release create "$GITHUB_REF_NAME" betterstack-oci.zip --generate-notes env: diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 8f5b304..6db5787 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -5,19 +5,22 @@ on: push: branches: [main] +permissions: + contents: read + jobs: forwarder: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 - run: python3 -m unittest test_func working-directory: forwarder stack: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - - uses: hashicorp/setup-terraform@v3 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + - uses: hashicorp/setup-terraform@b9cd54a3c349d3f38e8881555d616ced269862dd # v3 - run: terraform fmt -check - run: terraform init -backend=false - run: terraform validate From 9558ad90222005e9480cdc6934db073295490ab9 Mon Sep 17 00:00:00 2001 From: Arsen Shkrumelyak Date: Mon, 9 Nov 2026 14:53:55 +0000 Subject: [PATCH 11/15] docs: explain batching, token storage and connector limits --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index ca4daf8..da07503 100644 --- a/README.md +++ b/README.md @@ -4,21 +4,21 @@ Terraform stack that connects an Oracle Cloud Infrastructure tenancy to [Better ## How it works -The stack runs in OCI Resource Manager inside your tenancy. Connector Hub reads **metrics** from OCI Monitoring and **logs** from OCI Logging, then passes them to a forwarder Function. The Function batches them and sends them to Better Stack over HTTPS. +The stack runs in OCI Resource Manager inside your tenancy. Connector Hub reads **metrics** from OCI Monitoring and **logs** from OCI Logging, batches them and passes each batch to a forwarder Function, which compresses it and sends it to Better Stack over HTTPS. -Better Stack never connects to your tenancy. Data only leaves through the Function, and the only secret stored in your tenancy is the Better Stack source token. +Better Stack never connects to your tenancy. Data only leaves through the Function, and the only secret stored in your tenancy is the Better Stack source token, kept in the Function's configuration. ## Quick start [Create a new Oracle Cloud source in Better Stack](https://telemetry.betterstack.com/team/0/sources/new?platform=oracle_cloud). Its **Ingest** tab has a **Deploy to Oracle Cloud** button that opens this stack in Resource Manager. Paste the source's **Ingesting host** and **Source token** into the stack form, make sure **Run apply** is selected, and click **Create**. -Sign in to your tenancy's **home region** as a user who can create compartments and policies in the root compartment. The stack forwards metrics and logs from the home region. +Sign in to your tenancy's **home region** as a user who can create compartments and policies in the root compartment. The stack forwards metrics and logs from the home region. It creates a compartment named `betterstack`, so remove or rename any existing compartment with that name first. ## What gets deployed - A `betterstack` compartment that holds every resource below. - A Function application running the forwarder, in a VCN with a private subnet and a NAT gateway so the Function can reach Better Stack. -- A Connector Hub for logs, and one Connector Hub for metrics per batch of up to 5 compartments. +- A Connector Hub for logs, and one or more Connector Hubs for metrics. Each takes up to 5 compartments and 50 metric namespaces, so a large tenancy may need a higher Connector Hub service limit. - A policy that lets the connectors read metrics and logs, and invoke the Function. The stack looks up which compartments have metrics in the selected namespaces when it runs. Apply the stack again after you add compartments or start using new services. From 02f2c7a3c284c61248e03c913d64da16ca6e6a4c Mon Sep 17 00:00:00 2001 From: Arsen Shkrumelyak Date: Mon, 9 Nov 2026 15:15:27 +0000 Subject: [PATCH 12/15] docs: say how long uninstalling takes --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index da07503..8f6deeb 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,7 @@ The stack looks up which compartments have metrics in the selected namespaces wh ## Uninstall -Go to **Developer Services → Resource Manager → Stacks**, select the Better Stack stack, and click **Destroy**. +Go to **Developer Services → Resource Manager → Stacks**, select the Better Stack stack, and click **Destroy**. It takes about 15 minutes, most of it deleting the `betterstack` compartment. ## License From edb85d07e1a4a5cf36126cdfba1ba8bc9d0976f0 Mon Sep 17 00:00:00 2001 From: Arsen Shkrumelyak Date: Mon, 9 Nov 2026 15:46:08 +0000 Subject: [PATCH 13/15] feat(stack): run the forwarder as a code-only function --- forwarder/Dockerfile | 12 ------------ forwarder/requirements.txt | 14 ------------- main.tf | 40 ++++++++++++++++++++++++++------------ schema.yaml | 9 --------- variables.tf | 6 ------ 5 files changed, 28 insertions(+), 53 deletions(-) delete mode 100644 forwarder/Dockerfile delete mode 100644 forwarder/requirements.txt diff --git a/forwarder/Dockerfile b/forwarder/Dockerfile deleted file mode 100644 index 8e54bf8..0000000 --- a/forwarder/Dockerfile +++ /dev/null @@ -1,12 +0,0 @@ -FROM fnproject/python:3.11-dev@sha256:86e385f8fa59bd5a38edf8486778cf457379d6352713ccbb2eb3221e2d39cc93 AS build -WORKDIR /function -COPY requirements.txt . -RUN pip3 install --target /python/ --no-cache-dir -r requirements.txt -COPY func.py . - -FROM fnproject/python:3.11@sha256:28857e94288268d4dcae7a06ffa2c8c93b180f9bbfa642a5c165f965cbec2775 -WORKDIR /function -COPY --from=build /python /python -COPY --from=build /function /function -ENV PYTHONPATH=/function:/python -ENTRYPOINT ["/python/bin/fdk", "/function/func.py", "handler"] diff --git a/forwarder/requirements.txt b/forwarder/requirements.txt deleted file mode 100644 index 7ba559d..0000000 --- a/forwarder/requirements.txt +++ /dev/null @@ -1,14 +0,0 @@ -# fdk and everything it installs, pinned so a release image is reproducible. -fdk==0.1.128 -cython==3.3.1 -httptools==0.9.0 -iniconfig==2.3.1 -iso8601==0.1.12 -packaging==26.3 -pbr==7.1.3 -pluggy==1.7.0 -pygments==2.21.0 -pytest==9.2.0 -pytest-asyncio==1.4.1.post1 -setuptools==84.2.1 -wheel==0.48.0 diff --git a/main.tf b/main.tf index 0c8cd33..6c91bcc 100644 --- a/main.tf +++ b/main.tf @@ -14,10 +14,6 @@ provider "oci" { } locals { - forwarder_version = "0.1.0" - - # Better Stack publishes the forwarder to a public repository in every region it supports. - forwarder_image = coalesce(var.forwarder_image, "${lower(data.oci_identity_regions.current.regions[0].key)}.ocir.io/lr3km3hvmrwp/betterstack/oci-forwarder:${local.forwarder_version}") connector_principal = "where all {request.principal.type = 'serviceconnector', request.principal.compartment.id = '${oci_identity_compartment.betterstack.id}'}" @@ -34,13 +30,6 @@ locals { ] : ["Allow any-user to read metrics in tenancy ${local.connector_principal}"] } -data "oci_identity_regions" "current" { - filter { - name = "name" - values = [var.region] - } -} - data "oci_identity_region_subscriptions" "home" { tenancy_id = var.tenancy_ocid @@ -145,13 +134,40 @@ resource "oci_functions_application" "betterstack" { shape = "GENERIC_X86_ARM" } +# Applying the stack again moves the forwarder to the runtime's current version. +data "oci_functions_functions_runtimes" "python" { + name = "python312.ol9" +} + +data "oci_functions_functions_runtime_versions" "python" { + functions_runtime_id = data.oci_functions_functions_runtimes.python.functions_runtime_collection[0].items[0].id + is_current_version = true +} + resource "oci_functions_function" "forwarder" { application_id = oci_functions_application.betterstack.id display_name = "forwarder" - image = local.forwarder_image memory_in_mbs = 256 timeout_in_seconds = 300 + # A code-only function: the managed Python runtime ships the Fn SDK, so the + # archive holds only function/func.py and no container image is involved. + source_details { + source_type = "ARCHIVE" + handler = "func.handler" + + archive_source_details { + archive_source_type = "DIRECT_ARCHIVE" + archive_file = filebase64("${path.module}/forwarder.zip") + } + + runtime_config { + runtime_config_type = "MANUAL" + functions_runtime_name = "python312.ol9" + functions_runtime_version_id = data.oci_functions_functions_runtime_versions.python.functions_runtime_version_collection[0].items[0].id + } + } + config = { BETTERSTACK_INGESTING_HOST = local.ingesting_host BETTERSTACK_SOURCE_TOKEN = var.betterstack_source_token diff --git a/schema.yaml b/schema.yaml index 57fddf3..5c983af 100644 --- a/schema.yaml +++ b/schema.yaml @@ -17,9 +17,6 @@ variableGroups: variables: - ${forward_audit_logs} - ${log_group_ids} - - title: Advanced - variables: - - ${forwarder_image} - title: Hidden visible: false variables: @@ -119,9 +116,3 @@ variables: type: text multiline: true required: false - - forwarder_image: - title: Forwarder image - description: Leave empty to use the image Better Stack publishes in your region. - type: string - required: false diff --git a/variables.tf b/variables.tf index 9533cb2..bd74f70 100644 --- a/variables.tf +++ b/variables.tf @@ -73,9 +73,3 @@ variable "log_group_ids" { default = "" description = "Log group OCIDs to forward logs from, separated by commas or new lines." } - -variable "forwarder_image" { - type = string - default = "" - description = "Overrides the forwarder image Better Stack publishes in your region." -} From 7abf7026c4c1a0da7fcd719333418bcabd5ce9b2 Mon Sep 17 00:00:00 2001 From: Arsen Shkrumelyak Date: Mon, 9 Nov 2026 15:46:09 +0000 Subject: [PATCH 14/15] build(forwarder): package the function as a zip instead of an image --- .github/workflows/release.yml | 33 ++------------------------------- .github/workflows/test.yml | 1 + forwarder/build.sh | 11 +++++++++++ 3 files changed, 14 insertions(+), 31 deletions(-) create mode 100755 forwarder/build.sh diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index d56162b..a8da370 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -5,43 +5,14 @@ on: tags: ["v*"] jobs: - image: - runs-on: ubuntu-latest - permissions: - contents: read - strategy: - matrix: - region_key: [lhr] - steps: - - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 - - id: version - run: echo "version=${GITHUB_REF_NAME#v}" >> "$GITHUB_OUTPUT" - - name: Check the stack pins this version - run: grep -qF 'forwarder_version = "${{ steps.version.outputs.version }}"' main.tf - - uses: docker/setup-qemu-action@c7c53464625b32c7a7e944ae62b3e17d2b600130 # v3 - - uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3 - - uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3 - with: - registry: ${{ matrix.region_key }}.ocir.io - username: ${{ secrets.OCIR_USERNAME }} - password: ${{ secrets.OCIR_TOKEN }} - - uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6 - with: - context: forwarder - platforms: linux/amd64,linux/arm64 - provenance: false - sbom: false - push: true - tags: ${{ matrix.region_key }}.ocir.io/lr3km3hvmrwp/betterstack/oci-forwarder:${{ steps.version.outputs.version }} - stack: - needs: image runs-on: ubuntu-latest permissions: contents: write steps: - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 - - run: zip betterstack-oci.zip main.tf variables.tf schema.yaml + - run: forwarder/build.sh + - run: zip betterstack-oci.zip main.tf variables.tf schema.yaml forwarder.zip - run: gh release create "$GITHUB_REF_NAME" betterstack-oci.zip --generate-notes env: GH_TOKEN: ${{ github.token }} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 6db5787..6e23180 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -21,6 +21,7 @@ jobs: steps: - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 - uses: hashicorp/setup-terraform@b9cd54a3c349d3f38e8881555d616ced269862dd # v3 + - run: forwarder/build.sh - run: terraform fmt -check - run: terraform init -backend=false - run: terraform validate diff --git a/forwarder/build.sh b/forwarder/build.sh new file mode 100755 index 0000000..0919c44 --- /dev/null +++ b/forwarder/build.sh @@ -0,0 +1,11 @@ +#!/bin/sh +# Packs the forwarder for a code-only Function: the managed Python runtime +# expects function/func.py at the archive root and ships the Fn SDK itself. +set -eu +cd "$(dirname "$0")/.." +dir=$(mktemp -d) +mkdir "$dir/function" +cp forwarder/func.py "$dir/function/" +rm -f forwarder.zip +(cd "$dir" && zip -qX "$OLDPWD/forwarder.zip" function/func.py) +rm -rf "$dir" From cfed540b5044c785f5ce6827ea5baf496af90c90 Mon Sep 17 00:00:00 2001 From: Arsen Shkrumelyak Date: Tue, 10 Nov 2026 14:20:23 +0000 Subject: [PATCH 15/15] docs: simplify the readme and match the deploy steps --- README.md | 35 +++++++++++++++++++++++------------ 1 file changed, 23 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 8f6deeb..3e41aa4 100644 --- a/README.md +++ b/README.md @@ -1,31 +1,42 @@ # Better Stack Oracle Cloud integration -Terraform stack that connects an Oracle Cloud Infrastructure tenancy to [Better Stack](https://betterstack.com) for metric and log forwarding. +This Terraform stack sends metrics and logs from an Oracle Cloud Infrastructure tenancy to [Better Stack](https://betterstack.com). ## How it works -The stack runs in OCI Resource Manager inside your tenancy. Connector Hub reads **metrics** from OCI Monitoring and **logs** from OCI Logging, batches them and passes each batch to a forwarder Function, which compresses it and sends it to Better Stack over HTTPS. +The stack runs in OCI Resource Manager inside your tenancy. Connector Hub reads **metrics** from OCI Monitoring and **logs** from OCI Logging. It sends them in batches to a forwarder Function. The Function compresses each batch and sends it to Better Stack over HTTPS. -Better Stack never connects to your tenancy. Data only leaves through the Function, and the only secret stored in your tenancy is the Better Stack source token, kept in the Function's configuration. +Better Stack never connects to your tenancy. Data leaves your tenancy only through the Function. The only secret that the stack keeps in your tenancy is the Better Stack source token, in the configuration of the Function. ## Quick start -[Create a new Oracle Cloud source in Better Stack](https://telemetry.betterstack.com/team/0/sources/new?platform=oracle_cloud). Its **Ingest** tab has a **Deploy to Oracle Cloud** button that opens this stack in Resource Manager. Paste the source's **Ingesting host** and **Source token** into the stack form, make sure **Run apply** is selected, and click **Create**. +Before you start, sign in to the **home region** of your tenancy. Use a user who can create compartments and policies in the root compartment. -Sign in to your tenancy's **home region** as a user who can create compartments and policies in the root compartment. The stack forwards metrics and logs from the home region. It creates a compartment named `betterstack`, so remove or rename any existing compartment with that name first. +1. [Create an Oracle Cloud source in Better Stack](https://telemetry.betterstack.com/team/0/sources/new?platform=oracle_cloud). +2. On the **Ingest** tab of the source, click **Deploy to Oracle Cloud**. +3. Accept the Oracle Terms of Use and click **Next**. +4. Paste the **Source token** of your source and click **Next**. The **Ingesting host** is already filled in. +5. Make sure that **Run apply** is selected, then click **Create**. -## What gets deployed +The stack creates a compartment with the name `betterstack`. If your tenancy already has a compartment with this name, remove or rename it first. -- A `betterstack` compartment that holds every resource below. -- A Function application running the forwarder, in a VCN with a private subnet and a NAT gateway so the Function can reach Better Stack. -- A Connector Hub for logs, and one or more Connector Hubs for metrics. Each takes up to 5 compartments and 50 metric namespaces, so a large tenancy may need a higher Connector Hub service limit. -- A policy that lets the connectors read metrics and logs, and invoke the Function. +The stack sends metrics and logs from the home region only. -The stack looks up which compartments have metrics in the selected namespaces when it runs. Apply the stack again after you add compartments or start using new services. +## What the stack creates + +- A `betterstack` compartment for all the resources below. +- A Function application for the forwarder. It runs in a VCN with a private subnet. A NAT gateway lets the Function reach Better Stack. +- One Connector Hub for logs, and one or more Connector Hubs for metrics. Each Connector Hub takes up to 5 compartments and 50 metric namespaces. A large tenancy can need a higher Connector Hub service limit. +- A policy that lets the Connector Hubs read metrics and logs, and call the Function. + +When the stack runs, it finds the compartments that have metrics in the selected namespaces. After you add compartments or use new services, apply the stack again. ## Uninstall -Go to **Developer Services → Resource Manager → Stacks**, select the Better Stack stack, and click **Destroy**. It takes about 15 minutes, most of it deleting the `betterstack` compartment. +1. Go to **Developer Services → Resource Manager → Stacks** and select the Better Stack stack. +2. Click **Destroy**. + +This takes about 15 minutes. Most of this time goes to removing the `betterstack` compartment. ## License