diff --git a/README.md b/README.md index fa9b8c7a..45bb0076 100644 --- a/README.md +++ b/README.md @@ -7,6 +7,8 @@ flowchart TB apiorg([api.ooni.org])-->alb apiio([api.ooni.io])-->backend ecs[Backend API ECS]<-->ch[(Clickhouse Cluster)] + cz[Citizenlab API EC2]<-->ch + subgraph Hetzner backend[OONI Backend Monolith]<-->ch monitoring[Monitoring host] @@ -15,6 +17,7 @@ flowchart TB subgraph AWS alb[API Load Balancer]<-->ecs alb-->backend + alb-->cz ecs<-->s3[(OONI S3 Buckets)] s3<-->backend end diff --git a/ansible/deploy-testlists.yml b/ansible/deploy-testlists.yml new file mode 100644 index 00000000..ccee26ed --- /dev/null +++ b/ansible/deploy-testlists.yml @@ -0,0 +1,21 @@ +--- +- name: Deploy testlists + hosts: + - testl.dev.ooni.io + - testl.prod.ooni.io + become: true + roles: + - role: bootstrap + - role: prometheus_node_exporter + vars: + use_https: false + use_nginx: false + node_exporter_port: 9100 + node_exporter_host: "0.0.0.0" + node_exporter_options: "" + - role: geerlingguy.docker + docker_users: + - testlists + - ubuntu + docker_package_state: latest + - role: testlists diff --git a/ansible/deploy-tier2.yml b/ansible/deploy-tier2.yml index cff8d695..ae76ab04 100644 --- a/ansible/deploy-tier2.yml +++ b/ansible/deploy-tier2.yml @@ -9,6 +9,9 @@ - name: Include notebook playbook ansible.builtin.import_playbook: deploy-notebook.yml +- name: Include testlists playbook + ansible.builtin.import_playbook: deploy-testlists.yml + # commented out due to the fact it requires manual config of ~/.ssh/config #- name: Setup codesign box # hosts: codesign-box diff --git a/ansible/host_vars/testl.dev.ooni.io/vars.yml b/ansible/host_vars/testl.dev.ooni.io/vars.yml new file mode 100644 index 00000000..7ab99273 --- /dev/null +++ b/ansible/host_vars/testl.dev.ooni.io/vars.yml @@ -0,0 +1,3 @@ +jwt_encryption_key: "{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/ooni_services/jwt_secret', profile='oonidevops_user_dev') }}" +github_token: "{{ lookup('amazon.aws.aws_secret', 'oonidevops/ooni_services/testlists_github_token', profile='oonidevops_user_dev') }}" +log_level: "debug" diff --git a/ansible/host_vars/testl.prod.ooni.io/vars.yml b/ansible/host_vars/testl.prod.ooni.io/vars.yml new file mode 100644 index 00000000..4a8d32d5 --- /dev/null +++ b/ansible/host_vars/testl.prod.ooni.io/vars.yml @@ -0,0 +1,3 @@ +jwt_encryption_key: "{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/ooni_services/jwt_secret', profile='oonidevops_user_prod') }}" +github_token: "{{ lookup('amazon.aws.aws_secret', 'oonidevops/ooni_services/testlists_github_token', profile='oonidevops_user_prod') }}" +log_level: "info" diff --git a/ansible/inventory b/ansible/inventory index 7437f1af..a270ee4b 100644 --- a/ansible/inventory +++ b/ansible/inventory @@ -49,3 +49,5 @@ fastpath.prod.ooni.io anonc.dev.ooni.io jumphost.dev.ooni.io jumphost.prod.ooni.io +testl.dev.ooni.io +testl.prod.ooni.io diff --git a/ansible/roles/ooni-backend/templates/api.conf b/ansible/roles/ooni-backend/templates/api.conf index 0ff0ca01..86ba04c2 100644 --- a/ansible/roles/ooni-backend/templates/api.conf +++ b/ansible/roles/ooni-backend/templates/api.conf @@ -48,7 +48,7 @@ MAIL_PASSWORD = "{{ mail_smtp_password }}" MAIL_SOURCE_ADDRESS = "contact@ooni.org" LOGIN_BASE_URL = "{{ login_base_url }}" -GITHUB_WORKDIR = "/var/lib/ooniapi/citizenlab" +GITHUB_WORKDIR = "/var/lib/ooniapi/testlists" GITHUB_TOKEN = "{{ github_token }}" GITHUB_USER = "ooni-bot" GITHUB_ORIGIN_REPO = "{{ github_origin_repo }}" diff --git a/ansible/roles/prometheus/templates/prometheus.yml b/ansible/roles/prometheus/templates/prometheus.yml index d1e5b90e..ebe5cb71 100755 --- a/ansible/roles/prometheus/templates/prometheus.yml +++ b/ansible/roles/prometheus/templates/prometheus.yml @@ -369,4 +369,50 @@ scrape_configs: replacement: "/$2" target_label: "__metrics_path__" action: "replace" + + - job_name: "testlists" + static_configs: + - targets: + - testlists.dev.ooni.io:9102 + - testlists.prod.ooni.io:9102 + scrape_interval: 5s + scheme: https + relabel_configs: # Change the host to the proxy host with relabeling + # Store ip in ecs_host + - source_labels: [__address__] + regex: "([a-z\\.]+):([0-9]+)" # :" + replacement: "$1" + target_label: "ec2_host" + action: "replace" + # Extract environment from address + - source_labels: [__address__] + regex: ".*(dev|prod).*" + replacement: "$1" + target_label: "env" + action: "replace" + # Store the full adress with path in proxy_host + - source_labels: [__address__] + regex: "([a-z\\.]+):([0-9]+)" # : + replacement: "{{monitoring_proxy_host}}:9200/${1}/${2}/metrics" # proxy.org:9200///metrics + target_label: "__proxy_host" + action: "replace" + # Change the environment part in proxy host + - source_labels: [__proxy_host, env] + separator: ";" + regex: "([^;]*)ENV([^;]*);(.*)" # __proxy_host;env + replacement: "$1$3$2" + target_label: "__proxy_host" + action: "replace" + # Change the address where to send the scrape request to + - source_labels: [__proxy_host] + regex: "([^/]*)/(.*)" + replacement: "$1" + target_label: "__address__" + action: "replace" + # Change the metrics path to include ip address and /metrics path + - source_labels: [__proxy_host] + regex: "([^/]*)/(.*)" + replacement: "/$2" + target_label: "__metrics_path__" + action: "replace" ... diff --git a/ansible/roles/prometheus_node_exporter/defaults/main.yml b/ansible/roles/prometheus_node_exporter/defaults/main.yml index 3433498f..9066e94c 100644 --- a/ansible/roles/prometheus_node_exporter/defaults/main.yml +++ b/ansible/roles/prometheus_node_exporter/defaults/main.yml @@ -2,6 +2,9 @@ prometheus_nginx_proxy_config: - location: /metrics/node_exporter proxy_pass: http://127.0.0.1:8100/metrics +prometheus_user: 'node_exporter' +prometheus_group: 'node_exporter' + node_exporter_version: '1.8.2' node_exporter_arch: 'amd64' node_exporter_download_url: https://github.com/prometheus/node_exporter/releases/download/v{{ node_exporter_version }}/node_exporter-{{ node_exporter_version }}.linux-{{ node_exporter_arch }}.tar.gz @@ -9,6 +12,7 @@ node_exporter_download_url: https://github.com/prometheus/node_exporter/releases node_exporter_bin_path: /usr/local/bin/node_exporter node_exporter_host: 'localhost' node_exporter_port: 8100 +node_exporter_htpasswd: '/etc/ooni/prometheus_passwd' node_exporter_options: '' node_exporter_state: started diff --git a/ansible/roles/prometheus_node_exporter/tasks/install.yml b/ansible/roles/prometheus_node_exporter/tasks/install.yml index 2ad7ccd7..47ed566e 100644 --- a/ansible/roles/prometheus_node_exporter/tasks/install.yml +++ b/ansible/roles/prometheus_node_exporter/tasks/install.yml @@ -27,12 +27,31 @@ node_exporter_download_check is changed or node_exporter_version_check.stdout | length == 0 +- name: Create node_exporter group. + group: + name: "{{ prometheus_group }}" + state: present + - name: Create node_exporter user. user: - name: node_exporter + name: "{{ prometheus_user }}" shell: /sbin/nologin + group: "{{ prometheus_group }}" state: present +- name: Add a user to a password file and ensure permissions are set + community.general.htpasswd: + path: "{{ node_exporter_htpasswd }}" + name: prom + password: "{{ prometheus_metrics_password }}" + owner: root + group: "{{ 'nginx' if use_nginx else prometheus_group }}" + mode: 0640 + tags: + - monitoring + - node_exporter + - config + - name: Copy the node_exporter systemd unit file. template: src: node_exporter.service.j2 diff --git a/ansible/roles/prometheus_node_exporter/tasks/main.yml b/ansible/roles/prometheus_node_exporter/tasks/main.yml index c79a618e..698b34ac 100644 --- a/ansible/roles/prometheus_node_exporter/tasks/main.yml +++ b/ansible/roles/prometheus_node_exporter/tasks/main.yml @@ -3,17 +3,7 @@ tags: - nginx - node_exporter - -- ansible.builtin.include_role: - name: dehydrated - tags: - - oonidata - - dehydrated - vars: - ssl_domains: - - "{{ inventory_hostname }}" - -- include_tasks: install.yml + when: use_nginx - name: create ooni configuration directory ansible.builtin.file: @@ -25,19 +15,6 @@ - node_exporter - config -- name: Add a user to a password file and ensure permissions are set - community.general.htpasswd: - path: /etc/ooni/prometheus_passwd - name: prom - password: "{{ prometheus_metrics_password }}" - owner: root - group: nginx - mode: 0640 - tags: - - monitoring - - node_exporter - - config - - name: Setup prometheus nginx config ansible.builtin.template: src: "nginx-prometheus.j2" @@ -49,3 +26,6 @@ - monitoring - node_exporter - config + when: use_nginx + +- include_tasks: install.yml diff --git a/ansible/roles/prometheus_node_exporter/vars/main.yml b/ansible/roles/prometheus_node_exporter/vars/main.yml index 1cf0521e..4be67fb4 100644 --- a/ansible/roles/prometheus_node_exporter/vars/main.yml +++ b/ansible/roles/prometheus_node_exporter/vars/main.yml @@ -1 +1,2 @@ -use_https: true \ No newline at end of file +use_https: true +use_nginx: true diff --git a/ansible/roles/testlists/defaults/main.yml b/ansible/roles/testlists/defaults/main.yml new file mode 100644 index 00000000..b58d9932 --- /dev/null +++ b/ansible/roles/testlists/defaults/main.yml @@ -0,0 +1,6 @@ +# testlists user +testlists_user: testlists +testlists_home: "/opt/{{ testlists_user }}" + +# testlists settings +clickhouse_url: "clickhouse://write:{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/clickhouse_write_password', profile='oonidevops_user_prod') }}@clickhouseproxy.dev.ooni.io/oonitest" diff --git a/ansible/roles/testlists/handlers/main.yml b/ansible/roles/testlists/handlers/main.yml new file mode 100644 index 00000000..d44c00f4 --- /dev/null +++ b/ansible/roles/testlists/handlers/main.yml @@ -0,0 +1,11 @@ +- name: reload nftables + tags: nftables + ansible.builtin.systemd_service: + name: nftables + state: reloaded + +- name: restart docker + tags: docker + ansible.builtin.systemd_service: + name: docker + state: restarted diff --git a/ansible/roles/testlists/tasks/main.yml b/ansible/roles/testlists/tasks/main.yml new file mode 100644 index 00000000..b05efbd2 --- /dev/null +++ b/ansible/roles/testlists/tasks/main.yml @@ -0,0 +1,116 @@ +--- +# For prometheus scrape requests +- name: Flush all handlers + meta: flush_handlers + +- name: Allow traffic on port 9100 + become: true + tags: prometheus-proxy + blockinfile: + path: /etc/ooni/nftables/tcp/9100.nft + create: yes + block: | + add rule inet filter input tcp dport 9100 counter accept comment "node exporter" + notify: + - reload nftables + +# For incoming testlists traffic +- name: Allow traffic on port 80 + become: true + tags: testlists + blockinfile: + path: /etc/ooni/nftables/tcp/80.nft + create: yes + block: | + add rule inet filter input tcp dport 80 counter accept comment "testlists" + notify: + - reload nftables + +### Create testlists user +- name: Ensure the testlists group exists + ansible.builtin.group: + name: "{{ testlists_user }}" + state: present + become: yes + +- name: Create the testlists user + ansible.builtin.user: + name: "{{ testlists_user }}" + home: "{{ testlists_home }}" + shell: "/bin/bash" + group: "{{ testlists_user }}" + create_home: yes + system: yes + become: yes + +- name: Set ownership of the testlists directory + ansible.builtin.file: + path: "{{ testlists_home }}" + owner: "{{ testlists_user }}" + group: "{{ testlists_user }}" + state: directory + mode: "0700" + become: yes + +### Run testlists + +- name: Ensure ooniapi directory existence + ansible.builtin.file: + path: /var/lib/ooniapi + state: directory + mode: "0711" + owner: "{{testlists_user}}" + group: "{{testlists_user}}" + +- name: Ensure testlists users var dir exists + ansible.builtin.file: + path: /var/lib/ooniapi/testlists/users + state: directory + mode: "0700" + owner: "{{testlists_user}}" + group: "{{testlists_user}}" + +- name: Get UID of a specific user + command: id -u {{testlists_user}} + register: user_uid + changed_when: false + +- name: Get GID of a specific user + command: id -g {{testlists_user}} + register: user_gid + changed_when: false + +- name: Make containerd use nftables backend + ansible.builtin.lineinfile: + path: /etc/containerd/config.toml + line: 'firewall-backend="nftables"' + state: present + notify: + - restart docker + +- name: Ensure testlists is running + community.docker.docker_container: + env: + CLICKHOUSE_URL: "{{ clickhouse_url }}" + JWT_ENCRYPTION_KEY: "{{ jwt_encryption_key }}" + WORKING_DIR: "/var/lib/ooniapi/testlists" + GITHUB_TOKEN: "{{ github_token }}" + GITHUB_USER: "ooni-bot" + ORIGIN_REPO: "citizenlab/test-lists" + PUSH_REPO: "ooni/test-lists" + LOG_LEVEL: "{{ log_level }}" + USER: "ooniapi" + EMAIL: "ooniapi@test-lists.ooni.org" + + name: testlists + image: ooni/api-testlists:latest + state: started + user: "{{user_uid.stdout}}:{{user_gid.stdout}}" + # use network mode = host to allow traffic from testlists to the statsd exporter without + # creating a network with redirection rules to match the ports + ports: + - "80:80" + volumes: + - /var/lib/ooniapi:/var/lib/ooniapi + tags: + - testlists diff --git a/tf/environments/dev/main.tf b/tf/environments/dev/main.tf index 099b5c5f..28005460 100644 --- a/tf/environments/dev/main.tf +++ b/tf/environments/dev/main.tf @@ -706,7 +706,8 @@ module "ooni_clickhouse_proxy" { from_port = 9000, to_port = 9000, protocol = "tcp", - cidr_blocks = concat(module.network.vpc_subnet_private[*].cidr_block, ["${module.ooni_fastpath.aws_instance_private_ip}/32", "${module.ooni_fastpath.aws_instance_public_ip}/32"]), + cidr_blocks = concat(module.network.vpc_subnet_private[*].cidr_block, ["${module.ooni_fastpath.aws_instance_private_ip}/32", "${module.ooni_fastpath.aws_instance_public_ip}/32"], + ["${module.ooniapi_testlists.aws_instance_private_ip}/32", "${module.ooniapi_testlists.aws_instance_public_ip}/32"]), }, { // For the prometheus proxy: from_port = 9200, @@ -1150,6 +1151,99 @@ module "ooniapi_oonimeasurements" { ) } +### Tier2 testlists service +module "ooniapi_testlists" { + source = "../../modules/ec2" + + stage = local.environment + + vpc_id = module.network.vpc_id + subnet_id = module.network.vpc_subnet_public[0].id + private_subnet_cidr = module.network.vpc_subnet_private[*].cidr_block + dns_zone_ooni_io = local.dns_zone_ooni_io + + key_name = module.adm_iam_roles.oonidevops_key_name + instance_type = "t3a.micro" + + name = "oonitestlists" + ingress_rules = [{ + from_port = 22, + to_port = 22, + protocol = "tcp", + cidr_blocks = ["0.0.0.0/0"], + }, { + from_port = 80, + to_port = 80, + protocol = "tcp", + cidr_blocks = ["0.0.0.0/0"], + }, { + from_port = 443, + to_port = 443, + protocol = "tcp", + cidr_blocks = ["0.0.0.0/0"], + }, { + // For the prometheus proxy: + from_port = 9200, + to_port = 9200, + protocol = "tcp" + cidr_blocks = [for ip in flatten(data.dns_a_record_set.monitoring_host.*.addrs) : "${tostring(ip)}/32"] + }, { + from_port = 9100, + to_port = 9100, + protocol = "tcp" + cidr_blocks = ["${module.ooni_monitoring_proxy.aws_instance_private_ip}/32"] + }] + + egress_rules = [{ + from_port = 0, + to_port = 0, + protocol = "-1", + cidr_blocks = ["0.0.0.0/0"], + }, { + from_port = 0, + to_port = 0, + protocol = "-1", + ipv6_cidr_blocks = ["::/0"] + }] + + sg_prefix = "oonitestl" + tg_prefix = "tstl" + + disk_size = 20 + + tags = merge( + local.tags, + { Name = "ooni-tier2-testlists" } + ) +} + +resource "aws_route53_record" "testlists_alias" { + zone_id = local.dns_zone_ooni_io + name = "testl.${local.environment}.ooni.io" + type = "CNAME" + ttl = 300 + + records = [ + module.ooniapi_testlists.aws_instance_public_dns + ] +} + +module "testlists_builder" { + source = "../../modules/ooni_docker_build" + trigger_tag = "" + + service_name = "testlists" + repo = "ooni/backend" + branch_name = "add_testlists_url_management" + buildspec_path = "ooniapi/services/testlists/buildspec.yml" + trigger_path = "ooniapi/services/testlists/**" + codestar_connection_arn = aws_codestarconnections_connection.oonidevops.arn + + codepipeline_bucket = aws_s3_bucket.ooniapi_codepipeline_bucket.bucket + + ecs_cluster_name = module.ooniapi_cluster.cluster_name +} + #### OONI Tier0 API Frontend module "ooniapi_frontend" { @@ -1164,6 +1258,7 @@ module "ooniapi_frontend" { ooniapi_ooniprobe_target_group_arn = module.ooniapi_ooniprobe.alb_target_group_id ooniapi_oonifindings_target_group_arn = module.ooniapi_oonifindings.alb_target_group_id ooniapi_oonimeasurements_target_group_arn = module.ooniapi_oonimeasurements.alb_target_group_id + ooniapi_testlists_target_group_arn = module.ooniapi_testlists.alb_target_group_id ooniapi_service_security_groups = [ module.ooniapi_cluster.web_security_group_id, @@ -1190,6 +1285,7 @@ locals { "oonirun.${local.environment}.ooni.io" : local.dns_zone_ooni_io, "oonimeasurements.${local.environment}.ooni.io" : local.dns_zone_ooni_io, "8.th.dev.ooni.io" : local.dns_zone_ooni_io, + "testlists.${local.environment}.ooni.io" : local.dns_zone_ooni_io, } ooniapi_frontend_main_domain_name = "api.${local.environment}.ooni.io" ooniapi_frontend_main_domain_name_zone_id = local.dns_zone_ooni_io diff --git a/tf/modules/ec2/outputs.tf b/tf/modules/ec2/outputs.tf index a09c3362..4aa12946 100644 --- a/tf/modules/ec2/outputs.tf +++ b/tf/modules/ec2/outputs.tf @@ -16,4 +16,8 @@ output "aws_instance_private_ip" { output "aws_instance_public_ip" { value = aws_instance.ooni_ec2.public_ip -} \ No newline at end of file +} + +output "alb_target_group_id" { + value = aws_alb_target_group.ooni_ec2.id +} diff --git a/tf/modules/ooniapi_frontend/main.tf b/tf/modules/ooniapi_frontend/main.tf index d8ba5b87..4b8ff069 100644 --- a/tf/modules/ooniapi_frontend/main.tf +++ b/tf/modules/ooniapi_frontend/main.tf @@ -477,3 +477,24 @@ resource "aws_lb_listener_rule" "ooniapi_oonimeasurements_rule_2" { } } } + +resource "aws_lb_listener_rule" "ooniapi_testlists_rule" { + listener_arn = aws_alb_listener.ooniapi_listener_https.arn + priority = 143 + + action { + type = "forward" + target_group_arn = var.ooniapi_testlists_target_group_arn + } + condition { + path_pattern { + values = [ + "/api/_/url-submission/test-list/*", + "/api/_/url-priorities/list", + "/api/_/url-priorities/update", + "/api/v1/url-submission/submit", + "/api/v1/url-submission/update-url", + ] + } + } +} diff --git a/tf/modules/ooniapi_frontend/variables.tf b/tf/modules/ooniapi_frontend/variables.tf index d4ec3dd0..58b227ff 100644 --- a/tf/modules/ooniapi_frontend/variables.tf +++ b/tf/modules/ooniapi_frontend/variables.tf @@ -37,6 +37,10 @@ variable "ooniapi_oonimeasurements_target_group_arn" { default = null } +variable "ooniapi_testlists_target_group_arn" { + description = "arn for the target group of the testlists service" +} + variable "dns_zone_ooni_io" { description = "id of the DNS zone for ooni_io" }