Add option to distribute jobs to clusters based on cluster size

deploy/infrabox/templates/api/deployment.yaml

@@ -75,6 +75,11 @@ spec:
                 -
                     name: INFRABOX_GERRIT_ENABLED
                     value: {{ .Values.gerrit.enabled | quote }}
+{{ if .Values.job.weighted_cluster_assignment }}
+                -
+                    name: INFRABOX_WEIGHTED_CLUSTER_ASSIGNMENT
+                    value: {{ .Values.job.weighted_cluster_assignment | quote }}
+{{ end }}
             volumes:
                 {{ include "volumes_database" . | indent 16 }}
                 {{ include "volumes_rsa" . | indent 16 }}

deploy/infrabox/values.yaml

@@ -351,6 +351,8 @@ job:
 
     storage_driver: overlay
 
+    weighted_cluster_assignment: false
+
 monitoring:
     # Enable InfraBox Monitoring. Requires prometheus-operator to be available.
     enabled: false

src/api/handlers/job_api.py

@@ -664,6 +664,8 @@ def find_leaf_jobs(jobs):
 @api.route("/api/job/create_jobs", doc=False)
 class CreateJobs(Resource):
     def get_target_cluster(self, clusters, cluster_selector):
+        possible_clusters = []
+        total_weight = 0
         for c in clusters:
             matches = True
             for s in cluster_selector:
@@ -672,13 +674,24 @@ def get_target_cluster(self, clusters, cluster_selector):
                     break
 
             if matches:
+                if os.environ.get('INFRABOX_WEIGHTED_CLUSTER_ASSIGNMENT', 'true') != 'true':
+                    return c['name']
+                possible_clusters.append(c)
+                total_weight += c['weight']
+
+        r = random.randrange(0, total_weight)
+        l = n = 0
+        for c in possible_clusters:
+            n = l + c['weight']
+            if r >= l and r < n:
                 return c['name']
+            l = n
 
         return None
 
     def assign_cluster(self, jobs):
         clusters = g.db.execute_many_dict('''
-            SELECT name, labels
+            SELECT name, labels, weight
             FROM cluster
             WHERE active = true
             AND enabled = true

src/db/migrations/00033.sql

@@ -0,0 +1 @@
+ALTER TABLE cluster ADD COLUMN weight INTEGER NOT NULL DEFAULT 1;

src/scheduler/kubernetes/scheduler.py

@@ -1171,7 +1171,7 @@ def handle_orphaned_jobs(self):
     def get_default_cluster(self):
         cursor = self.conn.cursor()
         cursor.execute("""
-            SELECT name, labels
+            SELECT name, labels, weight
             FROM cluster
             WHERE active = true AND
                   enabled = true
@@ -1181,13 +1181,26 @@ def get_default_cluster(self):
 
         random.shuffle(result)
 
+        possible_clusters = []
+        total_weight = 0
         for row in result:
             cluster_name = row[0]
             labels = row[1]
 
             for l in labels:
                 if l == 'default':
-                    return cluster_name
+                    if os.environ.get('INFRABOX_WEIGHTED_CLUSTER_ASSIGNMENT', 'true') != 'true':
+                        return cluster_name
+                    possible_clusters.append(row)
+                    total_weight += row['weight']
+
+        r = random.randrange(0, total_weight)
+        l = n = 0
+        for c in possible_clusters:
+            n = l + c['weight']
+            if r >= l and r < n:
+                return c['name']
+            l = n
 
         return None