support host-specific global proxy settings: (#2995)

- fixes #2994
- requires crawler 0.7.0 or higher
- if 'matchHosts' use mapping from hosts->named proxies
- build proxy mapping json in proxies helm chart, bump to 0.2.0
- check if 'has-proxy-match-hosts' configmap is defined, and if so, map
proxy secrets to volume
- enable proxy volume mapping if either main proxy id or
'has-proxy-match-hosts' is defined

Author: ikreymer

backend/btrixcloud/operator/crawls.py

@@ -338,6 +338,7 @@ async def sync_crawls(self, data: MCSyncData):
         if pull_policy:
             params["crawler_image_pull_policy"] = pull_policy
 
+        proxy = None
         if crawl.proxy_id and not crawl.is_qa:
             proxy = self.crawl_config_ops.get_crawler_proxy(crawl.proxy_id)
             if proxy:
@@ -346,6 +347,10 @@ async def sync_crawls(self, data: MCSyncData):
                 params["proxy_ssh_private_key"] = proxy.has_private_key
                 params["proxy_ssh_host_public_key"] = proxy.has_host_public_key
 
+        params["add_proxies"] = proxy or (
+            not crawl.is_qa and data.related[CMAP].get("has-proxy-match-hosts")
+        )
+
         params["storage_filename"] = spec["storage_filename"]
         params["restart_time"] = spec.get("restartTime")
 
@@ -741,7 +746,14 @@ async def set_state(
 
     def get_related(self, data: MCBaseRequest):
         """return objects related to crawl pods"""
-        related_resources = []
+        related_resources = [
+            {
+                "apiVersion": "v1",
+                "resource": "configmaps",
+                "labelSelector": {"matchLabels": {"role": "has-proxy-match-hosts"}},
+            }
+        ]
+
         if self.k8s.enable_auto_resize:
             spec = data.parent.get("spec", {})
             crawl_id = spec["id"]

backend/test/test_run_crawl.py

@@ -429,9 +429,7 @@ def test_verify_wacz():
         ("all-crawls"),
     ],
 )
-def test_download_wacz_crawls(
-    admin_auth_headers, default_org_id, type_path
-):
+def test_download_wacz_crawls(admin_auth_headers, default_org_id, type_path):
     with TemporaryFile() as fh:
         with requests.get(
             f"{API_PREFIX}/orgs/{default_org_id}/{type_path}/{curr_admin_crawl_id}/download",

chart/Chart.lock

@@ -10,6 +10,6 @@ dependencies:
   version: 4.11.11
 - name: btrix-proxies
   repository: file://./proxies/
-  version: 0.1.0
-digest: sha256:2fd9472f857e9e3eacdcc616a3cffac5bb2951411cc2d34aea84253092225ecf
-generated: "2024-08-15T11:19:17.884682494+02:00"
+  version: 0.2.0
+digest: sha256:7c0ea8ce57470fe27977bb1d6b88dda6da836f829484de55f9d41ee81351b272
+generated: "2025-05-11T12:23:32.959101-07:00"

chart/Chart.yaml

@@ -19,6 +19,6 @@ dependencies:
     version: 4.11.11
     repository: "oci://ghcr.io/metacontroller"
   - name: btrix-proxies
-    version: 0.1.0
+    version: 0.2.0
     condition: btrix-proxies.enabled
     repository: file://./proxies/

chart/app-templates/crawler.yaml

@@ -76,7 +76,7 @@ spec:
     {% endif %}
     - name: tmpdir
       emptyDir: {}
-    {% if proxy_id %}
+    {% if add_proxies %}
     - name: proxies
       secret:
         secretName: proxies
@@ -147,17 +147,21 @@ spec:
         - --saveProfile
       {% endif %}
       {% endif %}
-      {% if proxy_id %}
+      {% if add_proxies %}
+      {% if proxy_url %}
         - --proxyServer
         - "{{ proxy_url }}"
-      {% if proxy_ssh_private_key %}
+      {% endif %}
+      {% if proxy_id and proxy_ssh_private_key %}
         - --sshProxyPrivateKeyFile
-        - /tmp/ssh-proxy/private-key
+        - /tmp/proxies/{{ proxy_id }}-private-key
       {% endif %}
-      {% if proxy_ssh_host_public_key %}
+      {% if proxy_id and proxy_ssh_host_public_key %}
         - --sshProxyKnownHostsFile
-        - /tmp/ssh-proxy/known-hosts
+        - /tmp/proxies/{{ proxy_id }}-known-hosts
       {% endif %}
+        - --proxyServerConfig
+        - /tmp/proxies/host-proxies.json
       {% endif %}
       volumeMounts:
         - name: crawl-config
@@ -169,19 +173,10 @@ spec:
           mountPath: /tmp/qa/
           readOnly: True
       {% endif %}
-      {% if proxy_id %}
-      {% if proxy_ssh_private_key %}
-        - name: proxies
-          mountPath: /tmp/ssh-proxy/private-key
-          subPath: {{ proxy_id }}-private-key
-          readOnly: true
-      {% endif %}
-      {% if proxy_ssh_host_public_key %}
+      {% if add_proxies %}
         - name: proxies
-          mountPath: /tmp/ssh-proxy/known-hosts
-          subPath: {{ proxy_id }}-known-hosts
+          mountPath: /tmp/proxies/
           readOnly: true
-      {% endif %}
         - name: force-user-and-group-name
           mountPath: /etc/passwd
           subPath: passwd
@@ -190,7 +185,7 @@ spec:
           mountPath: /etc/group
           subPath: group
           readOnly: true
-       {% endif %}
+      {% endif %}
         - name: crawl-data
           mountPath: /crawls
 

chart/charts/btrix-proxies-0.1.0.tgz

@@ -7,9 +7,9 @@ icon: https://webrecorder.net/assets/icon.png
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.1.0
+version: 0.2.0
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to
 # follow Semantic Versioning. They should reflect the version the application is using.
-appVersion: 0.1.0
+appVersion: 0.2.0

chart/charts/btrix-proxies-0.2.0.tgz

@@ -7,19 +7,60 @@ metadata:
   namespace: {{ .Values.crawler_namespace | default "crawlers" }}
 type: Opaque
 stringData:
+
+{{ $proxyDict := dict }}
+{{ $hasMatchHosts := false }}
+
 {{- range .Values.proxies }}
 
+{{ $proxyEntry := dict "url" .url }}
+
 {{- if .ssh_private_key }}
   {{ .id }}-private-key: |
 {{ .ssh_private_key | indent 4 }}
+{{- $_ := set $proxyEntry "privateKeyFile" (printf "/tmp/proxies/%s-private-key" .id) }}
 {{- end }}
 
 {{- if .ssh_host_public_key }}
   {{ .id }}-known-hosts: |
 {{ .ssh_host_public_key | indent 4 }}
+{{- $_ := set $proxyEntry "publicHostsFile" (printf "/tmp/proxies/%s-known-hosts" .id) }}
+{{- end }}
+
+{{- $_ := set $proxyDict .id $proxyEntry }}
+
 {{- end }}
 
+{{- if .Values.matchHosts }}
+  {{- $proxies := dict }}
+
+  {{- range $hostrx, $name := .Values.matchHosts }}
+    {{- $proxyEntry := get $proxyDict $name }}
+    {{- if not $proxyEntry }}
+      {{- fail (cat "Invalid proxy: 'matchHosts' referencing unknown proxy:" $name) }}
+    {{- end }}
+    {{- $_ := set $proxies $name $proxyEntry }}
+    {{- $hasMatchHosts = true }}
+  {{- end }}
+
+{{- if $hasMatchHosts }}
+data:
+  host-proxies.json: {{ dict "matchHosts" .Values.matchHosts "proxies" $proxies | toJson | b64enc | quote }}
 {{- end }}
+
+{{- end }}
+
+{{- if $hasMatchHosts }}
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: has-proxy-match-hosts
+  namespace: {{ .Values.crawler_namespace | default "crawlers" }}
+  labels:
+    role: has-proxy-match-hosts
+{{- end }}
+
 ---
 apiVersion: v1
 kind: Secret
@@ -31,4 +72,5 @@ type: Opaque
 data:
   crawler_proxies_last_update: {{ now | unixEpoch | toString | b64enc | quote }}
   crawler_proxies.json: {{ .Values.proxies | toJson | b64enc | quote }}
+
 {{- end }}

chart/proxies/Chart.yaml

@@ -1,2 +1,5 @@
+#matchHosts: # optional setting to always match certain hosts to certain proxies
+#  example.com/.*: my-proxy
+
 proxies: [] # see proxies description in main helm chart
 crawler_namespace: crawlers # namespace to deploy ssh keys to