From 5799e705338341cff8739fb719cab1670fb9c273 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Wed, 18 Feb 2026 10:54:05 +0100 Subject: [PATCH 1/8] Add clickhouse write url to dev --- tf/environments/dev/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tf/environments/dev/main.tf b/tf/environments/dev/main.tf index 1261c64d..8609844d 100644 --- a/tf/environments/dev/main.tf +++ b/tf/environments/dev/main.tf @@ -241,7 +241,7 @@ data "aws_ssm_parameter" "clickhouse_readonly_test_url" { } data "aws_ssm_parameter" "clickhouse_write_url" { - name = "/oonidevops/secrets/clickhouse_write_password" + name = "/oonidevops/secrets/clickhouse_write_url" } data "aws_ssm_parameter" "account_id_hashing_key" { From c27debac08100b165a9d9231c9ac12da3ed43dea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Wed, 18 Feb 2026 11:30:45 +0100 Subject: [PATCH 2/8] Allow write access to clickhouse in ooniprobe --- tf/environments/prod/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tf/environments/prod/main.tf b/tf/environments/prod/main.tf index 693b54de..0c3827be 100644 --- a/tf/environments/prod/main.tf +++ b/tf/environments/prod/main.tf @@ -807,7 +807,7 @@ module "ooniapi_ooniprobe" { POSTGRESQL_URL = data.aws_ssm_parameter.oonipg_url.arn JWT_ENCRYPTION_KEY = data.aws_ssm_parameter.jwt_secret.arn PROMETHEUS_METRICS_PASSWORD = data.aws_ssm_parameter.prometheus_metrics_password.arn - CLICKHOUSE_URL = data.aws_ssm_parameter.clickhouse_readonly_url.arn + CLICKHOUSE_URL = data.aws_ssm_parameter.clickhouse_write_url.arn ANONC_SECRET_KEY = data.aws_ssm_parameter.anonc_secret_key.arn } From 99ccce25040b703a909d86293ba2fcabab97999e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Wed, 18 Feb 2026 12:52:26 +0100 Subject: [PATCH 3/8] Set the new schema for the faulty measurements table --- scripts/cluster-migration/schema.sql | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/cluster-migration/schema.sql b/scripts/cluster-migration/schema.sql index 7f99844a..52e2b4ce 100644 --- a/scripts/cluster-migration/schema.sql +++ b/scripts/cluster-migration/schema.sql @@ -220,6 +220,7 @@ CREATE TABLE IF NOT EXISTS faulty_measurements ON CLUSTER oonidata_cluster ( `ts` DateTime64(3, 'UTC'), `type` String, + `measurement_uid` String, -- geoip lookup result for the probe IP `probe_cc` String, `probe_asn` UInt32, @@ -230,4 +231,4 @@ ENGINE = ReplicatedReplacingMergeTree ( '/clickhouse/{cluster}/tables/ooni/faulty_measurements/{shard}', '{replica}' ) -ORDER BY (ts, type, probe_cc, probe_asn); +ORDER BY (ts, type, probe_cc, probe_asn, measurement_uid); From 2bdf8d307c1fd6391f5cfb1ec359a0d673cad13c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Wed, 18 Feb 2026 13:04:55 +0100 Subject: [PATCH 4/8] Change measurement_uid to a unique uid --- scripts/cluster-migration/schema.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/cluster-migration/schema.sql b/scripts/cluster-migration/schema.sql index 52e2b4ce..292fd13b 100644 --- a/scripts/cluster-migration/schema.sql +++ b/scripts/cluster-migration/schema.sql @@ -220,7 +220,7 @@ CREATE TABLE IF NOT EXISTS faulty_measurements ON CLUSTER oonidata_cluster ( `ts` DateTime64(3, 'UTC'), `type` String, - `measurement_uid` String, + `uid` UUID64 DEFAULT toUUID64(generateUUIDv4()), -- geoip lookup result for the probe IP `probe_cc` String, `probe_asn` UInt32, From 5619174d285d919b0f1e88041cbcf5184a9f1694 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Wed, 18 Feb 2026 13:10:43 +0100 Subject: [PATCH 5/8] Set uid as part of sorting key --- scripts/cluster-migration/schema.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/cluster-migration/schema.sql b/scripts/cluster-migration/schema.sql index 292fd13b..7dc5a637 100644 --- a/scripts/cluster-migration/schema.sql +++ b/scripts/cluster-migration/schema.sql @@ -231,4 +231,4 @@ ENGINE = ReplicatedReplacingMergeTree ( '/clickhouse/{cluster}/tables/ooni/faulty_measurements/{shard}', '{replica}' ) -ORDER BY (ts, type, probe_cc, probe_asn, measurement_uid); +ORDER BY (ts, type, probe_cc, probe_asn, uid); From fa410fd550c731dde63b02def713d83b8163aa05 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Wed, 18 Feb 2026 13:15:14 +0100 Subject: [PATCH 6/8] use UUID type for uid field --- scripts/cluster-migration/schema.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/cluster-migration/schema.sql b/scripts/cluster-migration/schema.sql index 7dc5a637..cb6d815b 100644 --- a/scripts/cluster-migration/schema.sql +++ b/scripts/cluster-migration/schema.sql @@ -220,7 +220,7 @@ CREATE TABLE IF NOT EXISTS faulty_measurements ON CLUSTER oonidata_cluster ( `ts` DateTime64(3, 'UTC'), `type` String, - `uid` UUID64 DEFAULT toUUID64(generateUUIDv4()), + `uid` UUID DEFAULT generateUUIDv4(), -- geoip lookup result for the probe IP `probe_cc` String, `probe_asn` UInt32, From cb4dea0d8f029a0484f3005c9d9f24a662fc44a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Wed, 18 Feb 2026 13:51:39 +0100 Subject: [PATCH 7/8] Add default timestamping to faulty_measurements table --- scripts/cluster-migration/schema.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/cluster-migration/schema.sql b/scripts/cluster-migration/schema.sql index cb6d815b..7ff59c8d 100644 --- a/scripts/cluster-migration/schema.sql +++ b/scripts/cluster-migration/schema.sql @@ -218,7 +218,7 @@ ALTER TABLE event_detector_changepoints ON CLUSTER oonidata_cluster ADD COLUMN ` -- faulty measurements CREATE TABLE IF NOT EXISTS faulty_measurements ON CLUSTER oonidata_cluster ( - `ts` DateTime64(3, 'UTC'), + `ts` DateTime64(3, 'UTC') DEFAULT now64(), `type` String, `uid` UUID DEFAULT generateUUIDv4(), -- geoip lookup result for the probe IP From 1a5cf5ebffaffedf5814070b04648fe7ecaa097f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Wed, 18 Feb 2026 16:26:04 +0100 Subject: [PATCH 8/8] point ooniprobe to master --- tf/environments/dev/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tf/environments/dev/main.tf b/tf/environments/dev/main.tf index 8609844d..196248e7 100644 --- a/tf/environments/dev/main.tf +++ b/tf/environments/dev/main.tf @@ -554,7 +554,7 @@ module "ooniapi_ooniprobe_deployer" { service_name = "ooniprobe" repo = "ooni/backend" - branch_name = "geoip-heuristic" + branch_name = "master" trigger_path = "ooniapi/services/ooniprobe/**" buildspec_path = "ooniapi/services/ooniprobe/buildspec.yml" codestar_connection_arn = aws_codestarconnections_connection.oonidevops.arn