dockerize, use celery beat for automatic scheduling

Author: tedivm

Makefile

@@ -10,7 +10,11 @@ fresh: clean dependencies
 
 fulluninstall: uninstall clean
 
-install:
+testenv: clean_testenv
+	docker-compose up --build
+
+clean_testenv:
+	docker-compose down
 
 dependencies:
 	if [ ! -d $(ROOT_DIR)/venv ]; then python3 -m venv $(ROOT_DIR)/venv; fi

docker-compose.yaml

@@ -0,0 +1,19 @@
+version: "3"
+
+services:
+  worker:
+    build:
+      context: .
+      dockerfile: dockerfile
+    volumes:
+      - ./gitconsensusservice:/app/gitconsensusservice
+      - ./github_app.private-key.pem:/app/github_app.private-key.pem
+    environment:
+      - 'CELERY_BROKER=pyamqp://guest@rabbitmq//'
+      - 'DEBUG=true'
+      - 'GITHUB_APP_ID=9231'
+    depends_on:
+      - rabbitmq
+
+  rabbitmq:
+    image: rabbitmq

docker/start_worker.sh

@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+
+if [[ ! -v 'MINIMUM_WORKERS' ]]; then
+  MINIMUM_WORKERS=2
+fi
+
+if [[ ! -v 'MAXIMUM_WORKERS' ]]; then
+  MAXIMUM_WORKERS=10
+fi
+
+if [ "$DISABLE_BEAT" = "true" ]
+then
+  echo 'Launching celery worker without beat'
+  celery -A gitconsensusservice.worker.celery worker --loglevel=info --autoscale=$MAXIMUM_WORKERS,$MINIMUM_WORKERS
+else
+  echo 'Launching celery worker with beat enabled'
+  rm -f ~/celerybeat-schedule
+  celery -A gitconsensusservice.worker.celery worker --loglevel=info --autoscale=$MAXIMUM_WORKERS,$MINIMUM_WORKERS -B -s ~/celerybeat-schedule
+fi

dockerfile

@@ -0,0 +1,17 @@
+FROM python:3.6
+
+ADD requirements.txt /app/requirements.txt
+WORKDIR /app/
+RUN pip install -r requirements.txt
+
+ENV SETTINGS /app/settings.yaml
+ENV GITHUB_PRIVATE_KEY /app/github_app.private-key.pem
+
+ADD ./gitconsensusservice/ /app/gitconsensusservice
+
+RUN useradd -ms /bin/bash gitconsensusservice
+USER gitconsensusservice
+
+ADD ./docker/start_worker.sh /home/gitconsensusservice/start_worker.sh
+
+ENTRYPOINT /home/gitconsensusservice/start_worker.sh

gitconsensusservice/__init__.py

@@ -11,6 +11,13 @@
         with open(os.environ['SETTINGS'], 'r') as infile:
             app.config.update(yaml.load(infile.read()))
 
+
+SETTINGS = ['DEBUG', 'GITHUB_PRIVATE_KEY', 'GITHUB_APP_ID', 'GITHUB_WEBHOOK_SECRET', 'CELERY_BROKER']
+for setting in SETTINGS:
+    if setting in os.environ:
+        app.config[setting] = os.environ[setting]
+
+
 if 'CELERY_BROKER' in app.config:
     celery = Celery('gitconsensus', broker=app.config['CELERY_BROKER'])
 else:

gitconsensusservice/cli.py

@@ -41,14 +41,17 @@ def install_repos(install_id):
 def list_repos():
     installs = gh.get_installations()
     for install_id in installs:
-        click.echo('Install %s:' % install_id)
-        installation = get_githubapp_install(install_id)
-        repos = installation.get_repositories()
-        for repo in repos:
-            user, repo = repo.split('/')
-            repository = installation.get_repository(user, repo)
-            if repository.rules:
-                click.echo('\t%s/%s' % (user, repo))
+        try:
+            click.echo('Install %s:' % install_id)
+            installation = get_githubapp_install(install_id)
+            repos = installation.get_repositories()
+            for repo in repos:
+                user, repo = repo.split('/')
+                repository = installation.get_repository(user, repo)
+                if repository.rules:
+                    click.echo('\t%s/%s' % (user, repo))
+        except:
+            click.echo('Unable to process install %s' % install_id)
 
 
 @cli.command(short_help="List details about the current application.")

gitconsensusservice/worker.py

@@ -1,2 +1,11 @@
-import gitconsensusservice.jobs.consensus
+from celery.schedules import crontab
+from gitconsensusservice.jobs import consensus
 from gitconsensusservice import celery
+
+
+@celery.on_after_finalize.connect
+def setup_periodic_tasks(sender, **kwargs):
+    sender.add_periodic_task(
+        5 * 60.0,
+        consensus.process_installs.s(),
+        name='Root Task - Schedule Installation Jobs')