From 24dfa499f9e4fd9938782abf93fb7d4834d67459 Mon Sep 17 00:00:00 2001 From: Karsten Amrhein Date: Wed, 25 Sep 2019 11:02:47 +0200 Subject: [PATCH 1/2] Add option to distribute jobs to clusters based on cluster size --- deploy/infrabox/templates/api/deployment.yaml | 5 +++++ deploy/infrabox/values.yaml | 2 ++ src/api/handlers/job_api.py | 15 ++++++++++++++- 3 files changed, 21 insertions(+), 1 deletion(-) diff --git a/deploy/infrabox/templates/api/deployment.yaml b/deploy/infrabox/templates/api/deployment.yaml index 077329626..2136d95d9 100644 --- a/deploy/infrabox/templates/api/deployment.yaml +++ b/deploy/infrabox/templates/api/deployment.yaml @@ -75,6 +75,11 @@ spec: - name: INFRABOX_GERRIT_ENABLED value: {{ .Values.gerrit.enabled | quote }} +{{ if .Values.job.weighted_cluster_assignment }} + - + name: INFRABOX_WEIGHTED_CLUSTER_ASSIGNMENT + value: {{ .Values.job.weighted_cluster_assignment | quote }} +{{ end }} volumes: {{ include "volumes_database" . | indent 16 }} {{ include "volumes_rsa" . | indent 16 }} diff --git a/deploy/infrabox/values.yaml b/deploy/infrabox/values.yaml index 783345b9b..04df6b09b 100644 --- a/deploy/infrabox/values.yaml +++ b/deploy/infrabox/values.yaml @@ -351,6 +351,8 @@ job: storage_driver: overlay + weighted_cluster_assignment: false + monitoring: # Enable InfraBox Monitoring. Requires prometheus-operator to be available. enabled: false diff --git a/src/api/handlers/job_api.py b/src/api/handlers/job_api.py index a9387ce68..4547cedd3 100644 --- a/src/api/handlers/job_api.py +++ b/src/api/handlers/job_api.py @@ -664,6 +664,8 @@ def find_leaf_jobs(jobs): @api.route("/api/job/create_jobs", doc=False) class CreateJobs(Resource): def get_target_cluster(self, clusters, cluster_selector): + possible_clusters = [] + total_nodes = 0 for c in clusters: matches = True for s in cluster_selector: @@ -672,13 +674,24 @@ def get_target_cluster(self, clusters, cluster_selector): break if matches: + if os.environ.get('INFRABOX_WEIGHTED_CLUSTER_ASSIGNMENT', 'true') != 'true': + return c['name'] + possible_clusters.append(c) + total_nodes += c['nodes'] + + r = random.randrange(0, total_nodes) + l = n = 0 + for c in possible_clusters: + n = l + c['nodes'] + if r >= l and r < n: return c['name'] + l = n return None def assign_cluster(self, jobs): clusters = g.db.execute_many_dict(''' - SELECT name, labels + SELECT name, labels, nodes FROM cluster WHERE active = true AND enabled = true From 724a78d5e607cb3bb09df8b0dd097d13df1f8c0c Mon Sep 17 00:00:00 2001 From: Karsten Amrhein Date: Mon, 7 Oct 2019 14:26:40 +0200 Subject: [PATCH 2/2] Distribute jobs based on configured node weight --- src/api/handlers/job_api.py | 10 +++++----- src/db/migrations/00033.sql | 1 + src/scheduler/kubernetes/scheduler.py | 17 +++++++++++++++-- 3 files changed, 21 insertions(+), 7 deletions(-) create mode 100644 src/db/migrations/00033.sql diff --git a/src/api/handlers/job_api.py b/src/api/handlers/job_api.py index 4547cedd3..05eefad4d 100644 --- a/src/api/handlers/job_api.py +++ b/src/api/handlers/job_api.py @@ -665,7 +665,7 @@ def find_leaf_jobs(jobs): class CreateJobs(Resource): def get_target_cluster(self, clusters, cluster_selector): possible_clusters = [] - total_nodes = 0 + total_weight = 0 for c in clusters: matches = True for s in cluster_selector: @@ -677,12 +677,12 @@ def get_target_cluster(self, clusters, cluster_selector): if os.environ.get('INFRABOX_WEIGHTED_CLUSTER_ASSIGNMENT', 'true') != 'true': return c['name'] possible_clusters.append(c) - total_nodes += c['nodes'] + total_weight += c['weight'] - r = random.randrange(0, total_nodes) + r = random.randrange(0, total_weight) l = n = 0 for c in possible_clusters: - n = l + c['nodes'] + n = l + c['weight'] if r >= l and r < n: return c['name'] l = n @@ -691,7 +691,7 @@ def get_target_cluster(self, clusters, cluster_selector): def assign_cluster(self, jobs): clusters = g.db.execute_many_dict(''' - SELECT name, labels, nodes + SELECT name, labels, weight FROM cluster WHERE active = true AND enabled = true diff --git a/src/db/migrations/00033.sql b/src/db/migrations/00033.sql new file mode 100644 index 000000000..fe524d47e --- /dev/null +++ b/src/db/migrations/00033.sql @@ -0,0 +1 @@ +ALTER TABLE cluster ADD COLUMN weight INTEGER NOT NULL DEFAULT 1; \ No newline at end of file diff --git a/src/scheduler/kubernetes/scheduler.py b/src/scheduler/kubernetes/scheduler.py index e974889fe..5b8cd1d7f 100644 --- a/src/scheduler/kubernetes/scheduler.py +++ b/src/scheduler/kubernetes/scheduler.py @@ -1171,7 +1171,7 @@ def handle_orphaned_jobs(self): def get_default_cluster(self): cursor = self.conn.cursor() cursor.execute(""" - SELECT name, labels + SELECT name, labels, weight FROM cluster WHERE active = true AND enabled = true @@ -1181,13 +1181,26 @@ def get_default_cluster(self): random.shuffle(result) + possible_clusters = [] + total_weight = 0 for row in result: cluster_name = row[0] labels = row[1] for l in labels: if l == 'default': - return cluster_name + if os.environ.get('INFRABOX_WEIGHTED_CLUSTER_ASSIGNMENT', 'true') != 'true': + return cluster_name + possible_clusters.append(row) + total_weight += row['weight'] + + r = random.randrange(0, total_weight) + l = n = 0 + for c in possible_clusters: + n = l + c['weight'] + if r >= l and r < n: + return c['name'] + l = n return None