Add test lists service without nginx

README.md

@@ -7,6 +7,8 @@ flowchart TB
     apiorg([api.ooni.org])-->alb
     apiio([api.ooni.io])-->backend
     ecs[Backend API ECS]<-->ch[(Clickhouse Cluster)]
+    cz[Citizenlab API EC2]<-->ch
+
     subgraph Hetzner
         backend[OONI Backend Monolith]<-->ch
         monitoring[Monitoring host]
@@ -15,6 +17,7 @@ flowchart TB
     subgraph AWS
     alb[API Load Balancer]<-->ecs
     alb-->backend
+    alb-->cz
     ecs<-->s3[(OONI S3 Buckets)]
     s3<-->backend
     end

ansible/deploy-testlists.yml

@@ -0,0 +1,21 @@
+---
+- name: Deploy testlists
+  hosts:
+    - testl.dev.ooni.io
+    - testl.prod.ooni.io
+  become: true
+  roles:
+    - role: bootstrap
+    - role: prometheus_node_exporter
+      vars:
+        use_https: false
+        use_nginx: false
+        node_exporter_port: 9100
+        node_exporter_host: "0.0.0.0"
+        node_exporter_options: ""
+    - role: geerlingguy.docker
+      docker_users:
+        - testlists
+        - ubuntu
+      docker_package_state: latest
+    - role: testlists

ansible/deploy-tier2.yml

@@ -9,6 +9,9 @@
 - name: Include notebook playbook
   ansible.builtin.import_playbook: deploy-notebook.yml
 
+- name: Include testlists playbook
+  ansible.builtin.import_playbook: deploy-testlists.yml
+
 # commented out due to the fact it requires manual config of ~/.ssh/config
 #- name: Setup codesign box
 #  hosts: codesign-box

ansible/host_vars/testl.dev.ooni.io/vars.yml

@@ -0,0 +1,3 @@
+jwt_encryption_key: "{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/ooni_services/jwt_secret', profile='oonidevops_user_dev') }}"
+github_token: "{{ lookup('amazon.aws.aws_secret', 'oonidevops/ooni_services/testlists_github_token', profile='oonidevops_user_dev') }}"
+log_level: "debug"

ansible/host_vars/testl.prod.ooni.io/vars.yml

@@ -0,0 +1,3 @@
+jwt_encryption_key: "{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/ooni_services/jwt_secret', profile='oonidevops_user_prod') }}"
+github_token: "{{ lookup('amazon.aws.aws_secret', 'oonidevops/ooni_services/testlists_github_token', profile='oonidevops_user_prod') }}"
+log_level: "info"

ansible/inventory

@@ -49,3 +49,5 @@ fastpath.prod.ooni.io
 anonc.dev.ooni.io
 jumphost.dev.ooni.io
 jumphost.prod.ooni.io
+testl.dev.ooni.io
+testl.prod.ooni.io

ansible/roles/ooni-backend/templates/api.conf

@@ -48,7 +48,7 @@ MAIL_PASSWORD = "{{ mail_smtp_password }}"
 MAIL_SOURCE_ADDRESS = "contact@ooni.org"
 LOGIN_BASE_URL = "{{ login_base_url }}"
 
-GITHUB_WORKDIR = "/var/lib/ooniapi/citizenlab"
+GITHUB_WORKDIR = "/var/lib/ooniapi/testlists"
 GITHUB_TOKEN = "{{ github_token }}"
 GITHUB_USER = "ooni-bot"
 GITHUB_ORIGIN_REPO = "{{ github_origin_repo }}"

ansible/roles/prometheus/templates/prometheus.yml

@@ -369,4 +369,50 @@ scrape_configs:
         replacement: "/$2"
         target_label: "__metrics_path__"
         action: "replace"
+
+  - job_name: "testlists"
+    static_configs:
+    - targets:
+      - testlists.dev.ooni.io:9102
+      - testlists.prod.ooni.io:9102
+    scrape_interval: 5s
+    scheme: https
+    relabel_configs: # Change the host to the proxy host with relabeling
+      # Store ip in ecs_host
+      - source_labels: [__address__]
+        regex: "([a-z\\.]+):([0-9]+)" # <host>:<port>"
+        replacement: "$1"
+        target_label: "ec2_host"
+        action: "replace"
+      # Extract environment from address
+      - source_labels: [__address__]
+        regex: ".*(dev|prod).*"
+        replacement: "$1"
+        target_label: "env"
+        action: "replace"
+      # Store the full adress with path in proxy_host
+      - source_labels: [__address__]
+        regex: "([a-z\\.]+):([0-9]+)" # <host>:<port>
+        replacement: "{{monitoring_proxy_host}}:9200/${1}/${2}/metrics" # proxy.org:9200/<hostname>/<port>/metrics
+        target_label: "__proxy_host"
+        action: "replace"
+      # Change the environment part in proxy host
+      - source_labels: [__proxy_host, env]
+        separator: ";"
+        regex: "([^;]*)ENV([^;]*);(.*)" # __proxy_host;env
+        replacement: "$1$3$2"
+        target_label: "__proxy_host"
+        action: "replace"
+      # Change the address where to send the scrape request to
+      - source_labels: [__proxy_host]
+        regex: "([^/]*)/(.*)"
+        replacement: "$1"
+        target_label: "__address__"
+        action: "replace"
+      # Change the metrics path to include ip address and /metrics path
+      - source_labels: [__proxy_host]
+        regex: "([^/]*)/(.*)"
+        replacement: "/$2"
+        target_label: "__metrics_path__"
+        action: "replace"
 ...

ansible/roles/prometheus_node_exporter/defaults/main.yml

@@ -2,13 +2,17 @@ prometheus_nginx_proxy_config:
   - location: /metrics/node_exporter
     proxy_pass: http://127.0.0.1:8100/metrics
 
+prometheus_user: 'node_exporter'
+prometheus_group: 'node_exporter'
+
 node_exporter_version: '1.8.2'
 node_exporter_arch: 'amd64'
 node_exporter_download_url: https://github.com/prometheus/node_exporter/releases/download/v{{ node_exporter_version }}/node_exporter-{{ node_exporter_version }}.linux-{{ node_exporter_arch }}.tar.gz
 
 node_exporter_bin_path: /usr/local/bin/node_exporter
 node_exporter_host: 'localhost'
 node_exporter_port: 8100
+node_exporter_htpasswd: '/etc/ooni/prometheus_passwd'
 node_exporter_options: ''
 
 node_exporter_state: started

ansible/roles/prometheus_node_exporter/tasks/install.yml

@@ -27,12 +27,31 @@
     node_exporter_download_check is changed
     or node_exporter_version_check.stdout | length == 0
 
+- name: Create node_exporter group.
+  group:
+    name: "{{ prometheus_group }}"
+    state: present
+
 - name: Create node_exporter user.
   user:
-    name: node_exporter
+    name: "{{ prometheus_user }}"
     shell: /sbin/nologin
+    group: "{{ prometheus_group }}"
     state: present
 
+- name: Add a user to a password file and ensure permissions are set
+  community.general.htpasswd:
+    path: "{{ node_exporter_htpasswd }}"
+    name: prom
+    password: "{{ prometheus_metrics_password }}"
+    owner: root
+    group: "{{ 'nginx' if use_nginx else prometheus_group }}"
+    mode: 0640
+  tags:
+    - monitoring
+    - node_exporter
+    - config
+
 - name: Copy the node_exporter systemd unit file.
   template:
     src: node_exporter.service.j2

ansible/roles/prometheus_node_exporter/tasks/main.yml

@@ -3,17 +3,7 @@
   tags:
     - nginx
     - node_exporter
-
-- ansible.builtin.include_role:
-    name: dehydrated
-  tags:
-    - oonidata
-    - dehydrated
-  vars:
-    ssl_domains:
-      - "{{ inventory_hostname }}"
-
-- include_tasks: install.yml
+  when: use_nginx
 
 - name: create ooni configuration directory
   ansible.builtin.file:
@@ -25,19 +15,6 @@
     - node_exporter
     - config
 
-- name: Add a user to a password file and ensure permissions are set
-  community.general.htpasswd:
-    path: /etc/ooni/prometheus_passwd
-    name: prom
-    password: "{{ prometheus_metrics_password }}"
-    owner: root
-    group: nginx
-    mode: 0640
-  tags:
-    - monitoring
-    - node_exporter
-    - config
-
 - name: Setup prometheus nginx config
   ansible.builtin.template:
     src: "nginx-prometheus.j2"
@@ -49,3 +26,6 @@
     - monitoring
     - node_exporter
     - config
+  when: use_nginx
+
+- include_tasks: install.yml

ansible/roles/prometheus_node_exporter/vars/main.yml

@@ -1 +1,2 @@
-use_https: true
+use_https: true
+use_nginx: true

ansible/roles/testlists/defaults/main.yml

@@ -0,0 +1,6 @@
+# testlists user
+testlists_user: testlists
+testlists_home: "/opt/{{ testlists_user }}"
+
+# testlists settings
+clickhouse_url: "clickhouse://write:{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/clickhouse_write_password', profile='oonidevops_user_prod') }}@clickhouseproxy.dev.ooni.io/oonitest"

ansible/roles/testlists/handlers/main.yml

@@ -0,0 +1,11 @@
+- name: reload nftables
+  tags: nftables
+  ansible.builtin.systemd_service:
+    name: nftables
+    state: reloaded
+
+- name: restart docker
+  tags: docker
+  ansible.builtin.systemd_service:
+    name: docker
+    state: restarted

ansible/roles/testlists/tasks/main.yml

@@ -0,0 +1,116 @@
+---
+# For prometheus scrape requests
+- name: Flush all handlers
+  meta: flush_handlers
+
+- name: Allow traffic on port 9100
+  become: true
+  tags: prometheus-proxy
+  blockinfile:
+    path: /etc/ooni/nftables/tcp/9100.nft
+    create: yes
+    block: |
+      add rule inet filter input tcp dport 9100 counter accept comment "node exporter"
+  notify:
+   - reload nftables
+
+# For incoming testlists traffic
+- name: Allow traffic on port 80
+  become: true
+  tags: testlists
+  blockinfile:
+    path: /etc/ooni/nftables/tcp/80.nft
+    create: yes
+    block: |
+      add rule inet filter input tcp dport 80 counter accept comment "testlists"
+  notify:
+   - reload nftables
+
+### Create testlists user
+- name: Ensure the testlists group exists
+  ansible.builtin.group:
+    name: "{{ testlists_user }}"
+    state: present
+  become: yes
+
+- name: Create the testlists user
+  ansible.builtin.user:
+    name: "{{ testlists_user }}"
+    home: "{{ testlists_home }}"
+    shell: "/bin/bash"
+    group: "{{ testlists_user }}"
+    create_home: yes
+    system: yes
+  become: yes
+
+- name: Set ownership of the testlists directory
+  ansible.builtin.file:
+    path: "{{ testlists_home }}"
+    owner: "{{ testlists_user }}"
+    group: "{{ testlists_user }}"
+    state: directory
+    mode: "0700"
+  become: yes
+
+### Run testlists
+
+- name: Ensure ooniapi directory existence
+  ansible.builtin.file:
+    path: /var/lib/ooniapi
+    state: directory
+    mode: "0711"
+    owner: "{{testlists_user}}"
+    group: "{{testlists_user}}"
+
+- name: Ensure testlists users var dir exists
+  ansible.builtin.file:
+    path: /var/lib/ooniapi/testlists/users
+    state: directory
+    mode: "0700"
+    owner: "{{testlists_user}}"
+    group: "{{testlists_user}}"
+
+- name: Get UID of a specific user
+  command: id -u {{testlists_user}}
+  register: user_uid
+  changed_when: false
+
+- name: Get GID of a specific user
+  command: id -g {{testlists_user}}
+  register: user_gid
+  changed_when: false
+
+- name: Make containerd use nftables backend
+  ansible.builtin.lineinfile:
+    path: /etc/containerd/config.toml
+    line: 'firewall-backend="nftables"'
+    state: present
+  notify:
+   - restart docker
+
+- name: Ensure testlists is running
+  community.docker.docker_container:
+    env:
+      CLICKHOUSE_URL: "{{ clickhouse_url }}"
+      JWT_ENCRYPTION_KEY: "{{ jwt_encryption_key }}"
+      WORKING_DIR: "/var/lib/ooniapi/testlists"
+      GITHUB_TOKEN: "{{ github_token }}"
+      GITHUB_USER: "ooni-bot"
+      ORIGIN_REPO: "citizenlab/test-lists"
+      PUSH_REPO: "ooni/test-lists"
+      LOG_LEVEL: "{{ log_level }}"
+      USER: "ooniapi"
+      EMAIL: "ooniapi@test-lists.ooni.org"
+
+    name: testlists
+    image: ooni/api-testlists:latest
+    state: started
+    user: "{{user_uid.stdout}}:{{user_gid.stdout}}"
+    # use network mode = host to allow traffic from testlists to the statsd exporter without
+    # creating a network with redirection rules to match the ports
+    ports:
+      - "80:80"
+    volumes:
+      - /var/lib/ooniapi:/var/lib/ooniapi
+  tags:
+    - testlists