From bee7c9ec30aea95ff6bae759be3afc5b91a16f27 Mon Sep 17 00:00:00 2001 From: bus Date: Fri, 10 Jul 2020 18:32:40 +0200 Subject: [PATCH 01/13] combined code from multiple repositories --- buildbot-buildmaster/README.md | 30 ++ buildbot-buildmaster/buildbot/README.md | 96 ++++++ buildbot-buildmaster/buildbot/db.env | 6 + .../buildbot/docker-compose.yml | 80 +++++ buildbot-buildmaster/buildbot/master.cfg | 278 ++++++++++++++++++ buildbot-buildmaster/buildbot/multiarch.env | 2 + .../buildbot/multiarch_dockerfile | 37 +++ .../buildbot/secrets/README.md | 7 + .../buildbot/secrets/githubToken | 0 .../buildbot/secrets/htpasswd | 0 buildbot-buildmaster/raspberry-pi/README.md | 6 + .../raspberry-pi/arm64v8/arm64v8.env | 2 + .../raspberry-pi/armv6/armv6.env | 2 + .../raspberry-pi/armv7/armv7.env | 2 + .../raspberry-pi/armv8/armv8.env | 2 + buildbot-buildmaster/traefik/README.md | 45 +++ .../traefik/certs/DigiCertCA.crt | 1 + buildbot-buildmaster/traefik/certs/README.md | 8 + .../certs/mendel_fh-salzburg_ac_at.crt | 1 + .../certs/mendel_fh-salzburg_ac_at.key | 1 + .../traefik/docker-compose.yml | 37 +++ .../traefik/dynamic/redirect-to-https.yml | 7 + buildbot-buildmaster/traefik/dynamic/tls.yml | 26 ++ buildbot-buildmaster/traefik/traefik.yml | 20 ++ buildbot-buildmaster/windows/README.md | 2 + buildbot-buildmaster/windows/windows.env | 2 + raspberry-pi-workers/README.md | 15 + raspberry-pi-workers/arm64v8/arm64v8.env | 2 + .../arm64v8/arm64v8_dockerfile | 10 + .../arm64v8/docker-compose.yml | 10 + raspberry-pi-workers/armv6/armv6.env | 2 + raspberry-pi-workers/armv6/armv6_dockerfile | 10 + raspberry-pi-workers/armv6/docker-compose.yml | 10 + raspberry-pi-workers/armv7/armv7.env | 2 + raspberry-pi-workers/armv7/armv7_dockerfile | 10 + raspberry-pi-workers/armv7/docker-compose.yml | 10 + raspberry-pi-workers/armv8/armv8.env | 2 + raspberry-pi-workers/armv8/armv8_dockerfile | 10 + raspberry-pi-workers/armv8/docker-compose.yml | 10 + raspberry-pi-workers/buildmaster.env | 2 + windows-worker/README.md | 23 ++ windows-worker/buildbot-worker-start.cmd | 2 + windows-worker/buildmaster.env | 2 + windows-worker/setupPythonEnv.sh | 35 +++ windows-worker/windows.env | 2 + 45 files changed, 869 insertions(+) create mode 100644 buildbot-buildmaster/README.md create mode 100644 buildbot-buildmaster/buildbot/README.md create mode 100644 buildbot-buildmaster/buildbot/db.env create mode 100644 buildbot-buildmaster/buildbot/docker-compose.yml create mode 100644 buildbot-buildmaster/buildbot/master.cfg create mode 100644 buildbot-buildmaster/buildbot/multiarch.env create mode 100644 buildbot-buildmaster/buildbot/multiarch_dockerfile create mode 100644 buildbot-buildmaster/buildbot/secrets/README.md create mode 100644 buildbot-buildmaster/buildbot/secrets/githubToken create mode 100644 buildbot-buildmaster/buildbot/secrets/htpasswd create mode 100644 buildbot-buildmaster/raspberry-pi/README.md create mode 100644 buildbot-buildmaster/raspberry-pi/arm64v8/arm64v8.env create mode 100644 buildbot-buildmaster/raspberry-pi/armv6/armv6.env create mode 100644 buildbot-buildmaster/raspberry-pi/armv7/armv7.env create mode 100644 buildbot-buildmaster/raspberry-pi/armv8/armv8.env create mode 100644 buildbot-buildmaster/traefik/README.md create mode 100644 buildbot-buildmaster/traefik/certs/DigiCertCA.crt create mode 100644 buildbot-buildmaster/traefik/certs/README.md create mode 100644 buildbot-buildmaster/traefik/certs/mendel_fh-salzburg_ac_at.crt create mode 100644 buildbot-buildmaster/traefik/certs/mendel_fh-salzburg_ac_at.key create mode 100644 buildbot-buildmaster/traefik/docker-compose.yml create mode 100644 buildbot-buildmaster/traefik/dynamic/redirect-to-https.yml create mode 100644 buildbot-buildmaster/traefik/dynamic/tls.yml create mode 100644 buildbot-buildmaster/traefik/traefik.yml create mode 100644 buildbot-buildmaster/windows/README.md create mode 100644 buildbot-buildmaster/windows/windows.env create mode 100644 raspberry-pi-workers/README.md create mode 100644 raspberry-pi-workers/arm64v8/arm64v8.env create mode 100644 raspberry-pi-workers/arm64v8/arm64v8_dockerfile create mode 100644 raspberry-pi-workers/arm64v8/docker-compose.yml create mode 100644 raspberry-pi-workers/armv6/armv6.env create mode 100644 raspberry-pi-workers/armv6/armv6_dockerfile create mode 100644 raspberry-pi-workers/armv6/docker-compose.yml create mode 100644 raspberry-pi-workers/armv7/armv7.env create mode 100644 raspberry-pi-workers/armv7/armv7_dockerfile create mode 100644 raspberry-pi-workers/armv7/docker-compose.yml create mode 100644 raspberry-pi-workers/armv8/armv8.env create mode 100644 raspberry-pi-workers/armv8/armv8_dockerfile create mode 100644 raspberry-pi-workers/armv8/docker-compose.yml create mode 100644 raspberry-pi-workers/buildmaster.env create mode 100644 windows-worker/README.md create mode 100644 windows-worker/buildbot-worker-start.cmd create mode 100644 windows-worker/buildmaster.env create mode 100755 windows-worker/setupPythonEnv.sh create mode 100644 windows-worker/windows.env diff --git a/buildbot-buildmaster/README.md b/buildbot-buildmaster/README.md new file mode 100644 index 0000000..10e18db --- /dev/null +++ b/buildbot-buildmaster/README.md @@ -0,0 +1,30 @@ +# buildbot-buildmaster +This project contains dockerized services, serving a buildbot buildmaster to the public web over tls. +It is intended to be used on a linux server. +Currently, it runs under Debian Buster. +## Setup +- clone this repository to your server. +- install docker and docker-compose. +- create a dedicated, unprivileged user for this project +- add the user to the docker group +- clone this repository to your server. You only need the folder containing this README. + - optional: create a branch for your secrets (git checkout -b secrets) +- follow the setup instructions in the subfolders. Start with traefik and then buildbot. + +This project comes mostly preconfigured. + +What you have to do - an overview (refer to the README files in the subfolders): +- changing passwords / specifying credentials (see buildbot/README.md) +- providing a certificate for tls connections (traefik/certs) +- adjusting the web url (buildbot/docker-compose.yml) +- creating a persistent data directory +- adjusting file permissions + +## Usage +- cd traefik +- docker-compose up -d +- cd ../buildbot +- docker-compose up -d + +You might have to restart traefik again, when workers can't connect to the buildmaster. +Typically the raspberry-pis have this problem. diff --git a/buildbot-buildmaster/buildbot/README.md b/buildbot-buildmaster/buildbot/README.md new file mode 100644 index 0000000..b3a3984 --- /dev/null +++ b/buildbot-buildmaster/buildbot/README.md @@ -0,0 +1,96 @@ +# buildbot-buildmaster +This directory contains the configuration of the buildbot buildmaster. + +The buildmaster listens to changes on the following repositories: +- https://github.com/CenterForSecureEnergyInformatics/data-compressor (pull requests and branch master) +- https://github.com/brhat/checkBitSize.git (branch master) +- https://github.com/brhat/data-compressor-tests.git (branch master) + +When changes are detected (or the force button in the web ui is pressed), the project is built and tested. +## Usage +This project is dockerized and uses docker-compose. +The file docker-compose.yml tells docker-compose what to do, so you have to change into the directory containing the file, before executing any of these commands! +### Starting +- docker-compose up -d + +or to restart +- docker-compose restart + +Note: If the Raspberry Pis can't connect, simply go to ../traefik and run docker-compose restart from there, too. +### Stopping +- docker-compose down +### Updating +Run the following steps in this order: +- docker-compose down +- docker-compose pull +- docker-compose build +- docker image prune +- docker-compose up -d +### Debugging +To view logs in realtime, run +- docker-compose logs -f + +Exit with CRTL+C +## How it works +Buildbot-workers are defined in master.cfg. +The workers are connected via port 9989 to the buildmaster. +Workers on different platforms are implemented in this setup: +- windows 10. Unfortunately, tls seems not to work on windows with buildbot, so the vm running the buildmaster has to share a subnet with the windows-vm (which it does in our setup). +- Raspberri Pi (1, 2b and 3b+), dockerized. They use a tls connection handled by the reverse proxy traefik. +- Linux, dockerized. See multiarch_dockerfile and docker-compose.yml for the definition. Runs on the same host as the buildmaster. + +Buildfactories define which steps are to be executed on a build. +For each platform/architecture a scheduler is defined in this configuration. + +Builders are assigned to jobs (defined by factories). +Finally, schedulers trigger builds on the actual workers. + +For a detailed overview, please refer to http://docs.buildbot.net/latest/manual/introduction.html + +## Setup / Configuration +The file master.cfg is the main configuration file of buildbot, docker-compose.yml handles all services involved. +### master.cfg +Everything is pre-configured in this setup and does not require much changes. +Please refer to http://docs.buildbot.net/latest/manual/configuration/ for details, as a detailed explanation is beyond the scope of this README. +#### Credentials for the Web Ui +To force builds, you need to be logged in. +In master.cfg, fill in your e-mail address(es) under util.RolesFromEmails(admins=["you@email.provider"]) +Make sure, that: +- your credentials are filled in secrets/htpasswd (only clear text, so keep permissions (600) in mind). +- the e-mail address is from a contributor in the github-project. +#### Workers +DO NOT CHANGE master.cfg IN THIS STEP! + +All workers are pre-configured. +Each of them has a name (do not change!) and a password (please change!), specified in .env files: +- ../windows/windows.env +- the .env files in the subfolders of ../rasperry-pi/ +- multiarch.env + +Make sure to set these passwords on the corresponding workers as well! +On the workers, the files are located in the same folders and files to make this step easier. + +#### GitHubPullRequestPoller +Important Note: for the GithubPullrequestPoller to work, the owner and repository name (NOT the URL) has to be provided. +In this configuration, the following is used: +- compressorRepoName = 'data-compressor' +- compressorRepoOwner = 'CenterForSecureEnergyInformatics' + +### docker-compose.yml +The following services are specified here: +- buildbot-buildmaster +- db: a database for the buildmaster +- worker: a buildbot worker running linux (used to crosscompile), defined in multiarch_dockerfile + +#### Persistent Data Storage +You have to create a directory for persistent storage for the database service. +- mkdir -p /data/buildbot/db + +If you are unhappy with this location, you can specify another one in docker-compose.yml. +To do so, modify the volume of the service "db" accordingly. + +#### Web URL +If you aren't running this service under mendel.fh-salzburg.ac.at, you have to specify a different url in docker-compose.yml. +You'll find this option in the labels of the service buildbot-buildmaster +### db.env +Specify a database password. diff --git a/buildbot-buildmaster/buildbot/db.env b/buildbot-buildmaster/buildbot/db.env new file mode 100644 index 0000000..9b512ca --- /dev/null +++ b/buildbot-buildmaster/buildbot/db.env @@ -0,0 +1,6 @@ +# database parameters are shared between containers +POSTGRES_PASSWORD=changeme! +POSTGRES_USER=buildbot +POSTGRES_DB=buildbot +# in master.cfg, this variable is str.format()ed with the environment variables +BUILDBOT_DB_URL=postgresql+psycopg2://{POSTGRES_USER}:{POSTGRES_PASSWORD}@db/{POSTGRES_DB} diff --git a/buildbot-buildmaster/buildbot/docker-compose.yml b/buildbot-buildmaster/buildbot/docker-compose.yml new file mode 100644 index 0000000..4640940 --- /dev/null +++ b/buildbot-buildmaster/buildbot/docker-compose.yml @@ -0,0 +1,80 @@ +version: '3' +services: + buildbot-buildmaster: + image: buildbot/buildbot-master:master + restart: unless-stopped + env_file: + - ./db.env + - ./multiarch.env + - ../windows/windows.env + - ../raspberry-pi/armv6/armv6.env + - ../raspberry-pi/armv7/armv7.env + - ../raspberry-pi/armv8/armv8.env + - ../raspberry-pi/arm64v8/arm64v8.env + environment: + - BUILDBOT_CONFIG_DIR=config + - BUILDBOT_WORKER_PORT=9989 + - BUILDBOT_WEB_URL=https://mendel.fh-salzburg.ac.at/ + - BUILDBOT_WEB_PORT=tcp:port=80 + - BUILDBOT_TITLE=datacompressor testing + - BUILDBOT_TITLE_URL=https://github.com/CenterForSecureEnergyInformatics + volumes: + - ./secrets/:/var/lib/buildbot/secrets + - ./master.cfg:/var/lib/buildbot/master.cfg + networks: + - proxy + - buildbot + ports: + - 10.10.41.44:9988:9989 + labels: + - "traefik.enable=true" + - "traefik.docker.network=proxy" + + - "traefik.http.routers.buildbot.rule=Host(`mendel.fh-salzburg.ac.at`)" + - "traefik.http.routers.buildbot.entrypoints=http" + - "traefik.http.routers.buildbot.middlewares=redirect-to-https@file" + + - "traefik.http.routers.buildbot-secure.rule=Host(`mendel.fh-salzburg.ac.at`)" + - "traefik.http.routers.buildbot-secure.entrypoints=https" + - "traefik.http.routers.buildbot-secure.service=buildbot" + - "traefik.http.routers.buildbot-secure.tls=true" + - "traefik.http.routers.buildbot-secure.tls.certresolver=http" + - "traefik.http.routers.buildbot-secure.middlewares=hsts-header@file" + + - "traefik.http.services.buildbot.loadbalancer.server.port=80" + + - "traefik.tcp.routers.buildbot.rule=HostSNI(`mendel.fh-salzburg.ac.at`)" + - "traefik.tcp.routers.buildbot.entrypoints=buildbot" + - "traefik.tcp.routers.buildbot.service=buildbot" + - "traefik.tcp.routers.buildbot.tls=true" + + - "traefik.tcp.services.buildbot.loadbalancer.server.port=9989" + db: + env_file: + - db.env + image: "postgres:9.4" + restart: unless-stopped + volumes: + - /data/buildbot/db:/var/lib/postgresql/data + networks: + - buildbot + + worker: + build: + context: . + dockerfile: multiarch_dockerfile + restart: unless-stopped + env_file: + - ./multiarch.env + environment: + BUILDMASTER: buildbot-buildmaster + BUILDMASTER_PORT: 9989 + networks: + - buildbot + +networks: + proxy: + external: true + buildbot: + external: false + diff --git a/buildbot-buildmaster/buildbot/master.cfg b/buildbot-buildmaster/buildbot/master.cfg new file mode 100644 index 0000000..10ae590 --- /dev/null +++ b/buildbot-buildmaster/buildbot/master.cfg @@ -0,0 +1,278 @@ +# -*- python -*- +# ex: set filetype=python: + +import os + +from buildbot.plugins import * +# This is a sample buildmaster config file. It must be installed as +# 'master.cfg' in your buildmaster's base directory. + +# This is the dictionary that the buildmaster pays attention to. We also use +# a shorter alias to save typing. +c = BuildmasterConfig = {} + +secretsdir = os.environ.get("SECRETSDIR", "secrets") +c['secretsProviders'] = [secrets.SecretInAFile(dirname=secretsdir)] + +####### WORKERS + +# The 'workers' list defines the set of recognized workers. Each element is +# a Worker object, specifying a unique worker name and password. The same +# worker name and password must be configured on the worker. +multiarch_worker = os.environ.get("MULTIARCH_NAME",'multiarch') +armv6_worker = os.environ.get("ARMV6_NAME",'rpi_armv6') +armv7_worker = os.environ.get("ARMV7_NAME",'rpi_armv7') +armv8_worker = os.environ.get("ARMV8_NAME",'rpi_armv8') +arm64v8_worker = os.environ.get("ARM64V8_NAME",'rpi_arm64v8') +windows_worker = os.environ.get("WINDOWS_NAME",'windows') +c['workers'] = [ + worker.Worker(multiarch_worker, os.environ.get("MULTIARCH_PASS",'pass')), + worker.Worker(armv6_worker, os.environ.get("ARMV6_PASS",'pass')), + worker.Worker(armv7_worker, os.environ.get("ARMV7_PASS",'pass')), + worker.Worker(armv8_worker, os.environ.get("ARMV8_PASS",'pass')), + worker.Worker(arm64v8_worker, os.environ.get("ARM64V8_PASS",'pass')), + worker.Worker(windows_worker, os.environ.get("WINDOWS_PASS",'pass')), + ] + +if 'BUILDBOT_MQ_URL' in os.environ: + c['mq'] = { + 'type' : 'wamp', + 'router_url': os.environ['BUILDBOT_MQ_URL'], + 'realm': os.environ.get('BUILDBOT_MQ_REALM', 'buildbot').decode('utf-8'), + 'debug' : 'BUILDBOT_MQ_DEBUG' in os.environ, + 'debug_websockets' : 'BUILDBOT_MQ_DEBUG' in os.environ, + 'debug_lowlevel' : 'BUILDBOT_MQ_DEBUG' in os.environ, + } +# 'protocols' contains information about protocols which master will use for +# communicating with workers. You must define at least 'port' option that workers +# could connect to your master with this protocol. +# 'port' must match the value configured into the workers (with their +# --master option) +c['protocols'] = {'pb': {'port': os.environ.get("BUILDBOT_WORKER_PORT", 9989)}} + +####### CHANGESOURCES + +# the 'change_source' setting tells the buildmaster how it should find out +# about source code changes. Here we point to the buildbot clone of pyflakes. + +testRepoName = "data-compressor-tests" +#testRepoOwner= "CenterForSecureEnergyInformatics" +testRepoOwner= "brhat" +testRepo = "https://github.com/" + testRepoOwner + "/" + testRepoName + ".git" + +compressorRepoName = 'data-compressor' +#compressorRepoOwner = 'CenterForSecureEnergyInformatics' +compressorRepoOwner = 'brhat' +compressorRepo = "https://github.com/" + compressorRepoOwner + "/" + compressorRepoName + ".git" + +compressorRepoOwner = 'CenterForSecureEnergyInformatics' # WARNING this is duplicated! +#compressorRepo = 'https://github.com/CenterForSecureEnergyInformatics/data-compressor.git' +token = open(secretsdir + "/githubToken").read().strip() + +c['change_source'] = [] +c['change_source'].append(changes.GitPoller( + compressorRepo, + workdir='gitpoller-workdir', + branches=['master'], + pollInterval=30)) +c['change_source'].append(changes.GitHubPullrequestPoller( + owner=compressorRepoOwner, + repo=compressorRepoName, + pollInterval=30, + token=token)) + +####### SCHEDULERS + +# Configure the Schedulers, which decide how to react to incoming changes. In this +# case, just kick off a 'runtests' build + +c['schedulers'] = [] +c['schedulers'].append(schedulers.AnyBranchScheduler( + name="anyBranch", + treeStableTimer=None, + builderNames=["x86_64","i386","armv6_native","armv7_native","armv8_native","arm64v8_native","arm","win_x64","win32"])) +c['schedulers'].append(schedulers.ForceScheduler( + name="force", + builderNames=["x86_64","i386","armv6_native","armv7_native","armv8_native","arm64v8_native","arm","win_x64","win32"])) + +####### FACTORIES +testWorkdirName = "tests" +testdir = "../" + testRepoName + "/" + testWorkdirName + "/" + +desc = "building and testing" +descDone = "build and test" + +windowsShell = "C:\\Program Files\\Git\\bin\\bash.exe" + +factoryLinux_x86_64 = util.BuildFactory() +factoryLinux_x86_64.addStep(steps.Git(name="git - data-compressor", repourl=compressorRepo, workdir='data-compressor', mode='full', alwaysUseLatest='true')) +factoryLinux_x86_64.addStep(steps.ShellCommand(description=desc, descriptionDone=descDone, command=[testdir + "checkBits.sh", "x86_64"])) +factoryLinux_x86_64.addStep(steps.ShellCommand(description=desc, descriptionDone=descDone, command=[testdir + "buildAndTest.sh", "x86_64", "8"])) +factoryLinux_x86_64.addStep(steps.ShellCommand(description=desc, descriptionDone=descDone, command=[testdir + "buildAndTest.sh", "x86_64", "16"])) +factoryLinux_x86_64.addStep(steps.ShellCommand(description=desc, descriptionDone=descDone, command=[testdir + "buildAndTest.sh", "x86_64", "32"])) +factoryLinux_x86_64.addStep(steps.ShellCommand(description=desc, descriptionDone=descDone, command=[testdir + "buildAndTest.sh", "x86_64", "64"])) + +factoryLinux_i386 = util.BuildFactory() +factoryLinux_i386.addStep(steps.Git(name="git - tests", repourl=testRepo, workdir=testWorkdirName, mode='full', alwaysUseLatest='true')) +factoryLinux_i386.addStep(steps.Git(name="git - data-compressor", repourl=compressorRepo, workdir='data-compressor', mode='full', alwaysUseLatest='true')) +factoryLinux_i386.addStep(steps.ShellCommand(description=desc, descriptionDone=descDone, command=[testdir + "checkBits.sh", "i386"])) +factoryLinux_i386.addStep(steps.ShellCommand(description=desc, descriptionDone=descDone, command=[testdir + "buildAndTest.sh", "i386", "8"])) +factoryLinux_i386.addStep(steps.ShellCommand(description=desc, descriptionDone=descDone, command=[testdir + "buildAndTest.sh", "i386", "16"])) +factoryLinux_i386.addStep(steps.ShellCommand(description=desc, descriptionDone=descDone, command=[testdir + "buildAndTest.sh", "i386", "32"])) + +factoryLinux_armhf = util.BuildFactory() +factoryLinux_armhf.addStep(steps.Git(name="git - tests", repourl=testRepo, workdir=testWorkdirName, mode='full', alwaysUseLatest='true')) +factoryLinux_armhf.addStep(steps.Git(name="git - data-compressor", repourl=compressorRepo, workdir='data-compressor', mode='full', alwaysUseLatest='true')) +factoryLinux_armhf.addStep(steps.ShellCommand(description=desc, descriptionDone=descDone, command=[testdir + "checkBits.sh", "armhf"])) +factoryLinux_armhf.addStep(steps.ShellCommand(description=desc, descriptionDone=descDone, command=[testdir + "buildAndTest.sh", "armhf", "8"])) +factoryLinux_armhf.addStep(steps.ShellCommand(description=desc, descriptionDone=descDone, command=[testdir + "buildAndTest.sh", "armhf", "16"])) +factoryLinux_armhf.addStep(steps.ShellCommand(description=desc, descriptionDone=descDone, command=[testdir + "buildAndTest.sh", "armhf", "32"])) + +factory_arm= util.BuildFactory() +factory_arm.addStep(steps.Git(name="git - tests", repourl=testRepo, workdir=testWorkdirName, mode='full', alwaysUseLatest='true')) +factory_arm.addStep(steps.Git(name="git - data-compressor", repourl=compressorRepo, workdir='data-compressor', mode='full', alwaysUseLatest='true')) +factory_arm.addStep(steps.ShellCommand(description=desc, descriptionDone=descDone, command=[testdir + "checkBits.sh", "arm"])) +factory_arm.addStep(steps.ShellCommand(description=desc, descriptionDone=descDone, command=[testdir + "buildAndTest.sh", "arm", "8"])) +factory_arm.addStep(steps.ShellCommand(description=desc, descriptionDone=descDone, command=[testdir + "buildAndTest.sh", "arm", "16"])) +factory_arm.addStep(steps.ShellCommand(description=desc, descriptionDone=descDone, command=[testdir + "buildAndTest.sh", "arm", "32"])) + +factory_aarch64= util.BuildFactory() +factory_aarch64.addStep(steps.Git(name="git - tests", repourl=testRepo, workdir=testWorkdirName, mode='full', alwaysUseLatest='true')) +factory_aarch64.addStep(steps.Git(name="git - data-compressor", repourl=compressorRepo, workdir='data-compressor', mode='full', alwaysUseLatest='true')) +factory_aarch64.addStep(steps.ShellCommand(description=desc, descriptionDone=descDone, command=[testdir + "checkBits.sh", "arm"])) +factory_aarch64.addStep(steps.ShellCommand(description=desc, descriptionDone=descDone, command=[testdir + "buildAndTest.sh", "arm", "8"])) +factory_aarch64.addStep(steps.ShellCommand(description=desc, descriptionDone=descDone, command=[testdir + "buildAndTest.sh", "arm", "16"])) +factory_aarch64.addStep(steps.ShellCommand(description=desc, descriptionDone=descDone, command=[testdir + "buildAndTest.sh", "arm", "32"])) +factory_aarch64.addStep(steps.ShellCommand(description=desc, descriptionDone=descDone, command=[testdir + "buildAndTest.sh", "arm", "64"])) + +factoryWin_x64 = util.BuildFactory() +factoryWin_x64.addStep(steps.Git(name="git - tests", repourl=testRepo, workdir=testWorkdirName, mode='full', alwaysUseLatest='true')) +factoryWin_x64.addStep(steps.Git(name="git - data-compressor", repourl=compressorRepo, workdir='data-compressor', mode='full', alwaysUseLatest='true')) +factoryWin_x64.addStep(steps.ShellCommand(description=desc, descriptionDone=descDone, command=[windowsShell, testdir + "checkBits.sh", "x64"])) +factoryWin_x64.addStep(steps.ShellCommand(description=desc, descriptionDone=descDone, command=[windowsShell, testdir + "buildAndTest.sh", "x64", "8"])) +factoryWin_x64.addStep(steps.ShellCommand(description=desc, descriptionDone=descDone, command=[windowsShell, testdir + "buildAndTest.sh", "x64", "16"])) +factoryWin_x64.addStep(steps.ShellCommand(description=desc, descriptionDone=descDone, command=[windowsShell, testdir + "buildAndTest.sh", "x64", "32"])) +factoryWin_x64.addStep(steps.ShellCommand(description=desc, descriptionDone=descDone, command=[windowsShell, testdir + "buildAndTest.sh", "x64", "64"])) + +factoryWin32 = util.BuildFactory() +factoryWin32.addStep(steps.Git(name="git - tests", repourl=testRepo, workdir=testWorkdirName, mode='full', alwaysUseLatest='true')) +factoryWin32.addStep(steps.Git(name="git - data-compressor", repourl=compressorRepo, workdir='data-compressor', mode='full', alwaysUseLatest='true')) +factoryWin32.addStep(steps.ShellCommand(description=desc, descriptionDone=descDone, command=[windowsShell, testdir + "checkBits.sh", "win32"])) +factoryWin32.addStep(steps.ShellCommand(description=desc, descriptionDone=descDone, command=[windowsShell, testdir + "buildAndTest.sh", "win32", "8"])) +factoryWin32.addStep(steps.ShellCommand(description=desc, descriptionDone=descDone, command=[windowsShell, testdir + "buildAndTest.sh", "win32", "16"])) +factoryWin32.addStep(steps.ShellCommand(description=desc, descriptionDone=descDone, command=[windowsShell, testdir + "buildAndTest.sh", "win32", "32"])) + +####### BUILDERS + +# The 'builders' list defines the Builders, which tell Buildbot how to perform a build: +# what steps, and which workers can execute them. Note that any particular build will +# only take place on one worker. + +c['builders'] = [] + +c['builders'].append( + util.BuilderConfig(name="x86_64", + workernames=[multiarch_worker], + factory=factoryLinux_x86_64)) + +c['builders'].append( + util.BuilderConfig(name="i386", + workernames=[multiarch_worker], + factory=factoryLinux_i386)) + +c['builders'].append( + util.BuilderConfig(name="arm", + workernames=[multiarch_worker], + factory=factoryLinux_armhf)) + +c['builders'].append( + util.BuilderConfig(name="armv6_native", + workernames=[armv6_worker], + factory=factory_arm)) + +c['builders'].append( + util.BuilderConfig(name="armv7_native", + workernames=[armv7_worker], + factory=factory_arm)) + +c['builders'].append( + util.BuilderConfig(name="armv8_native", + workernames=[armv8_worker], + factory=factory_arm)) + +c['builders'].append( + util.BuilderConfig(name="arm64v8_native", + workernames=[arm64v8_worker], + factory=factory_aarch64)) + +c['builders'].append( + util.BuilderConfig(name="win_x64", + workernames=[windows_worker], + factory=factoryWin_x64)) + +c['builders'].append( + util.BuilderConfig(name="win32", + workernames=[windows_worker], + factory=factoryWin32)) + +################ + +gs = reporters.GitHubStatusPush(token=util.Secret("githubToken"), + context=util.Interpolate("buildbot/%(prop:buildername)s"), + startDescription='Build started.', + endDescription='Build end', + verbose=True) + +c['services'] = [] +#c['services'].append(gs) + +####### STATUS TARGETS + +# 'status' is a list of Status Targets. The results of each build will be +# pushed to these targets. buildbot/status/*.py has a variety to choose from, +# like IRC bots. + +#c['status'] = [] + +####### PROJECT IDENTITY + +# the 'title' string will appear at the top of this buildbot installation's +# home pages (linked to the 'titleURL'). + +c['title'] = os.environ.get("BUILDBOT_TITLE", "buildbot") +c['titleURL'] = os.environ.get("BUILDBOT_TITLE_URL", "https://docs.buildbot.net/latest/developer/config.html#") + +# the 'buildbotURL' string should point to the location where the buildbot's +# internal web server is visible. This typically uses the port number set in +# the 'www' entry below, but with an externally-visible host name which the +# buildbot cannot figure out without some help. + +c['buildbotURL'] = os.environ.get("BUILDBOT_WEB_URL", "http://localhost:8010/") + +# minimalistic config to activate new web UI +c['www'] = dict(port=os.environ.get("BUILDBOT_WEB_PORT", 8010), + plugins=dict(waterfall_view={}, console_view={}, grid_view={})) + + +c['www']['avatar_methods'] = [] + +authz = util.Authz( + allowRules = [ + util.AnyControlEndpointMatcher(role="admins") + ], + roleMatchers = [ + util.RolesFromEmails(admins=["you@email.provider"]) + ] +) +auth = util.HTPasswdAuth(secretsdir + "/htpasswd") +c['www']['auth'] = auth +c['www']['authz'] = authz + +####### DB URL + +c['db'] = { + # This specifies what database buildbot uses to store its state. You can leave + # this at its default for all but the largest installations. + 'db_url' : os.environ.get("BUILDBOT_DB_URL", "sqlite://").format(**os.environ), +} +c['buildbotNetUsageData'] = None diff --git a/buildbot-buildmaster/buildbot/multiarch.env b/buildbot-buildmaster/buildbot/multiarch.env new file mode 100644 index 0000000..f4c8024 --- /dev/null +++ b/buildbot-buildmaster/buildbot/multiarch.env @@ -0,0 +1,2 @@ +MULTIARCH_NAME=multiarch +MULTIARCH_PASS=changeme! diff --git a/buildbot-buildmaster/buildbot/multiarch_dockerfile b/buildbot-buildmaster/buildbot/multiarch_dockerfile new file mode 100644 index 0000000..d7df1ec --- /dev/null +++ b/buildbot-buildmaster/buildbot/multiarch_dockerfile @@ -0,0 +1,37 @@ +FROM ubuntu:bionic + +# hints related to apt and multiple architectures: +# https://stackoverflow.com/questions/37706635/in-docker-apt-get-install-fails-with-failed-to-fetch-http-archive-ubuntu-com +# https://wiki.ubuntuusers.de/sources.list/ +# https://wiki.debian.org/QemuUserEmulation +# https://wiki.debian.org/Multiarch/HOWTO + +RUN mv /etc/apt/sources.list /etc/apt/sources.list.old + +RUN printf \ +"deb [arch=amd64,i386] http://de.archive.ubuntu.com/ubuntu bionic main restricted universe multiverse\n\ +deb [arch=amd64,i386] http://de.archive.ubuntu.com/ubuntu bionic-updates main restricted universe multiverse\n\ +deb [arch=amd64,i386] http://de.archive.ubuntu.com/ubuntu bionic-security main restricted universe multiverse\n\ +deb [arch=amd64,i386] http://de.archive.ubuntu.com/ubuntu bionic-backports main restricted universe multiverse\n\ +deb [arch=armhf] http://ports.ubuntu.com/ubuntu-ports/ bionic main universe restricted multiverse\n\ +deb [arch=armhf] http://ports.ubuntu.com/ubuntu-ports/ bionic-security main universe restricted multiverse\n\ +deb [arch=armhf] http://ports.ubuntu.com/ubuntu-ports/ bionic-updates main universe restricted multiverse\n\ +deb [arch=armhf] http://ports.ubuntu.com/ubuntu-ports/ bionic-backports main universe restricted multiverse\n"\ +> /etc/apt/sources.list + + +RUN apt-get update && apt-get install -y git +RUN dpkg --add-architecture i386 && dpkg --add-architecture armhf && apt-get update +RUN apt-get install -y build-essential gcc-multilib mingw-w64 wine64 wine32 qemu binfmt-support qemu-user-static +RUN apt-get install -y crossbuild-essential-armhf libc6:armhf +# buildbot and docker: +RUN apt install -y \ + python-dev \ + python-pip +RUN pip install buildbot-worker +RUN groupadd -r buildbot && useradd -r -g buildbot buildbot +RUN mkdir /worker && chown buildbot:buildbot /worker +# Install your build-dependencies here ... +USER buildbot +WORKDIR /worker +ENTRYPOINT buildbot-worker create-worker . $BUILDMASTER $MULTIARCH_NAME $MULTIARCH_PASS && unset BUILDMASTER MULTIARCH_NAME MULTIARCH_PASS BUILDMASTER_PORT && buildbot-worker start --nodaemon diff --git a/buildbot-buildmaster/buildbot/secrets/README.md b/buildbot-buildmaster/buildbot/secrets/README.md new file mode 100644 index 0000000..17313b8 --- /dev/null +++ b/buildbot-buildmaster/buildbot/secrets/README.md @@ -0,0 +1,7 @@ +This is the place for +- htpasswd file +- github access token + +It seems that you have to provide the passwords in htpasswd as clear text for buildbot, e.g. you@email.provider:password. + +Think of proper file permissions! (chmod 600) diff --git a/buildbot-buildmaster/buildbot/secrets/githubToken b/buildbot-buildmaster/buildbot/secrets/githubToken new file mode 100644 index 0000000..e69de29 diff --git a/buildbot-buildmaster/buildbot/secrets/htpasswd b/buildbot-buildmaster/buildbot/secrets/htpasswd new file mode 100644 index 0000000..e69de29 diff --git a/buildbot-buildmaster/raspberry-pi/README.md b/buildbot-buildmaster/raspberry-pi/README.md new file mode 100644 index 0000000..111311e --- /dev/null +++ b/buildbot-buildmaster/raspberry-pi/README.md @@ -0,0 +1,6 @@ +This folder contains the credentials of the buildbot-workers (running on Raspberry Pis) on the buildbot buildmaster. + +The credentials (builder name and password) have to be placed in the .env files located in the subfolders. +Each subfolder represents an architecture, e.g. armv6. + +The .env files are required by the buildbot-buildmaster (see master.cfg). diff --git a/buildbot-buildmaster/raspberry-pi/arm64v8/arm64v8.env b/buildbot-buildmaster/raspberry-pi/arm64v8/arm64v8.env new file mode 100644 index 0000000..a32f40e --- /dev/null +++ b/buildbot-buildmaster/raspberry-pi/arm64v8/arm64v8.env @@ -0,0 +1,2 @@ +ARM64V8_PASS=changeme! +ARM64V8_NAME=rpi3b_plus_64 diff --git a/buildbot-buildmaster/raspberry-pi/armv6/armv6.env b/buildbot-buildmaster/raspberry-pi/armv6/armv6.env new file mode 100644 index 0000000..6dfc6d3 --- /dev/null +++ b/buildbot-buildmaster/raspberry-pi/armv6/armv6.env @@ -0,0 +1,2 @@ +ARMV6_PASS=changeme! +ARMV6_NAME=rpi1 diff --git a/buildbot-buildmaster/raspberry-pi/armv7/armv7.env b/buildbot-buildmaster/raspberry-pi/armv7/armv7.env new file mode 100644 index 0000000..fbcad3f --- /dev/null +++ b/buildbot-buildmaster/raspberry-pi/armv7/armv7.env @@ -0,0 +1,2 @@ +ARMV7_PASS=changeme! +ARMV7_NAME=rpi2b diff --git a/buildbot-buildmaster/raspberry-pi/armv8/armv8.env b/buildbot-buildmaster/raspberry-pi/armv8/armv8.env new file mode 100644 index 0000000..ab24618 --- /dev/null +++ b/buildbot-buildmaster/raspberry-pi/armv8/armv8.env @@ -0,0 +1,2 @@ +ARMV8_PASS=changeme! +ARMV8_NAME=rpi3b_plus_32 diff --git a/buildbot-buildmaster/traefik/README.md b/buildbot-buildmaster/traefik/README.md new file mode 100644 index 0000000..edbec9c --- /dev/null +++ b/buildbot-buildmaster/traefik/README.md @@ -0,0 +1,45 @@ +# Reverse Proxy (traefik) +This folder contains the configuration for the reverse proxy in front of the buildbot buildmaster. +It is based on the traefik project: https://containo.us/traefik/ + +The proxy handles tls connections and automatically redirects from port 80 to port 443. +Furthermore, it listens on port 9989 and provides a tls connection for buildbot-workers to the buildmaster. + +The services behind traefik reside in private networks, only the ports needed are exposed to the internet. + +Traefik needs access to the docker-socket: var/run/docker.sock. +Since this could result in vulnerabilities, the socket is made available over a socket-proxy (tecnativa/docker-socket-proxy), which runs as a service next to traefik. + +## Usage +This project is dockerized and uses docker-compose. +The file docker-compose.yml tells docker-compose what to do, so you have to change into the directory containing the file, before executing any of these commands! +### Starting +- docker-compose up -d + +or to restart +- docker-compose restart + +Note: If the Raspberry Pis can't connect to the buildmaster, simply run docker-compose restart. +### Stopping +- docker-compose down +### Updating +Run the following steps in this order: +- docker-compose down +- docker-compose pull +- docker-compose build +- docker image prune +- docker-compose up -d +### Debugging +To view logs in realtime, run +- docker-compose logs -f + +Exit with CTRL+C + +## Setup / Configuration +Traefik has a static configuration (docker-compose.yml and traefik.yml) and a dynamic configuration (folders certs and dynamic) +Everything is preconfigured, so no action is required. +The only thing to keep in mind is to renew the certificates, located in certs (please refer to the README in this folder). +Traefik could handle renewals by itself when using letsencrypt, but in this setup we specified a certificate manually. + +The services handled by traefik are configured via labels in the corresponding docker-compose.yml files, in our case ../buildbot/docker-compose.yml, so please have a look there + diff --git a/buildbot-buildmaster/traefik/certs/DigiCertCA.crt b/buildbot-buildmaster/traefik/certs/DigiCertCA.crt new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/buildbot-buildmaster/traefik/certs/DigiCertCA.crt @@ -0,0 +1 @@ + diff --git a/buildbot-buildmaster/traefik/certs/README.md b/buildbot-buildmaster/traefik/certs/README.md new file mode 100644 index 0000000..0bb679d --- /dev/null +++ b/buildbot-buildmaster/traefik/certs/README.md @@ -0,0 +1,8 @@ +Place your cert files in this folder. Keep permissions (600) in mind! + +To get the full chain, append DigiCertCA.crt to mendel_fh-salzburg_ac_at.crt + +To do so, run: + +cat DigiCertCA.crt >> mendel_fh-salzburg_ac_at.crt + diff --git a/buildbot-buildmaster/traefik/certs/mendel_fh-salzburg_ac_at.crt b/buildbot-buildmaster/traefik/certs/mendel_fh-salzburg_ac_at.crt new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/buildbot-buildmaster/traefik/certs/mendel_fh-salzburg_ac_at.crt @@ -0,0 +1 @@ + diff --git a/buildbot-buildmaster/traefik/certs/mendel_fh-salzburg_ac_at.key b/buildbot-buildmaster/traefik/certs/mendel_fh-salzburg_ac_at.key new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/buildbot-buildmaster/traefik/certs/mendel_fh-salzburg_ac_at.key @@ -0,0 +1 @@ + diff --git a/buildbot-buildmaster/traefik/docker-compose.yml b/buildbot-buildmaster/traefik/docker-compose.yml new file mode 100644 index 0000000..34aee60 --- /dev/null +++ b/buildbot-buildmaster/traefik/docker-compose.yml @@ -0,0 +1,37 @@ +version: '3' + +services: + socket-proxy: + image: tecnativa/docker-socket-proxy + container_name: socket-proxy + restart: unless-stopped + volumes: + - /var/run/docker.sock:/var/run/docker.sock:ro + environment: + CONTAINERS: 1 + networks: + - socketproxy + + traefik: + image: traefik:v2.0 + container_name: traefik + restart: unless-stopped + security_opt: + - no-new-privileges:true + networks: + - proxy + - socketproxy + ports: + - 80:80 + - 443:443 + - 9989:9989 + volumes: + - /etc/localtime:/etc/localtime:ro + - ./traefik.yml:/traefik.yml:ro + - ./dynamic/:/etc/traefik/conf/:ro + - ./certs/:/etc/traefik/certs:ro +networks: + socketproxy: + external: true + proxy: + external: true diff --git a/buildbot-buildmaster/traefik/dynamic/redirect-to-https.yml b/buildbot-buildmaster/traefik/dynamic/redirect-to-https.yml new file mode 100644 index 0000000..81000b3 --- /dev/null +++ b/buildbot-buildmaster/traefik/dynamic/redirect-to-https.yml @@ -0,0 +1,7 @@ +http: + middlewares: + redirect-to-https: + redirectScheme: + scheme: https + permanent: true + diff --git a/buildbot-buildmaster/traefik/dynamic/tls.yml b/buildbot-buildmaster/traefik/dynamic/tls.yml new file mode 100644 index 0000000..592a355 --- /dev/null +++ b/buildbot-buildmaster/traefik/dynamic/tls.yml @@ -0,0 +1,26 @@ +# tls options. derived from toml-file generated by: +# https://ssl-config.mozilla.org/#server=traefik&version=2&config=intermediate&guideline=5.4 + +http: + middlewares: + hsts-header: + headers: + customResponseHeaders: + Strict-Transport-Security: "max-age=63072000" + stsPreload: true +tls: + certificates: + - certFile: /etc/traefik/certs/mendel_fh-salzburg_ac_at.crt + keyFile: /etc/traefik/certs/mendel_fh-salzburg_ac_at.key + options: + default: + minVersion: "VersionTLS12" + sniStrict: true + cipherSuites: + - "TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384" + - "TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384" + - "TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256" + - "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256" + - "TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305" + - "TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305" + diff --git a/buildbot-buildmaster/traefik/traefik.yml b/buildbot-buildmaster/traefik/traefik.yml new file mode 100644 index 0000000..9ff77fc --- /dev/null +++ b/buildbot-buildmaster/traefik/traefik.yml @@ -0,0 +1,20 @@ +#api: +# dashboard: true + +entryPoints: + http: + address: ":80" + https: + address: ":443" + buildbot: + address: ":9989" + +providers: + docker: + endpoint: "tcp://socket-proxy:2375" + watch: true + exposedByDefault: false + + file: + directory: "/etc/traefik/conf/" + diff --git a/buildbot-buildmaster/windows/README.md b/buildbot-buildmaster/windows/README.md new file mode 100644 index 0000000..e298ec7 --- /dev/null +++ b/buildbot-buildmaster/windows/README.md @@ -0,0 +1,2 @@ +Place the credentials (worker name and password) of the buildbot-worker running on windows in the .env file. +This is required by the buildbot-buildmaster (see master.cfg). diff --git a/buildbot-buildmaster/windows/windows.env b/buildbot-buildmaster/windows/windows.env new file mode 100644 index 0000000..e2068e2 --- /dev/null +++ b/buildbot-buildmaster/windows/windows.env @@ -0,0 +1,2 @@ +WINDOWS_NAME=winworker +WINDOWS_PASS=changeme! diff --git a/raspberry-pi-workers/README.md b/raspberry-pi-workers/README.md new file mode 100644 index 0000000..36798cd --- /dev/null +++ b/raspberry-pi-workers/README.md @@ -0,0 +1,15 @@ +# zse-raspberry-pi-workers +dockerized buildbot-workers for various versions of the raspberry-pi + +## Prerequisites +- you have to have docker and docker-compose installed. +- add your unprivileged user to the docker group + +## Setup +* clone / copy this repository to a raspberry-pi. + * (Optional) Create a branch for your configuration including secrets (git checkout -b secrets) +* modify URL and port of the buildmaster in buildmaster.env. +* cd into the folder fitting your model (run uname -m if you are unsure) + * modify the passphrase and worker name in the .env file + * make sure that the buildmaster is configured with the same values +* run docker-compose up -d diff --git a/raspberry-pi-workers/arm64v8/arm64v8.env b/raspberry-pi-workers/arm64v8/arm64v8.env new file mode 100644 index 0000000..a32f40e --- /dev/null +++ b/raspberry-pi-workers/arm64v8/arm64v8.env @@ -0,0 +1,2 @@ +ARM64V8_PASS=changeme! +ARM64V8_NAME=rpi3b_plus_64 diff --git a/raspberry-pi-workers/arm64v8/arm64v8_dockerfile b/raspberry-pi-workers/arm64v8/arm64v8_dockerfile new file mode 100644 index 0000000..fe82137 --- /dev/null +++ b/raspberry-pi-workers/arm64v8/arm64v8_dockerfile @@ -0,0 +1,10 @@ +FROM arm64v8/alpine + +RUN apk add --update --no-cache libc-dev gcc make g++ openssh git file grep py3-setuptools python3-dev libffi libffi-dev openssl openssl-dev vim +RUN pip3 install --upgrade pip buildbot-worker pyopenssl service_identity +RUN addgroup -S buildbot && adduser -S buildbot -G buildbot +RUN mkdir /worker && chown buildbot:buildbot /worker +RUN apk add --update --no-cache bash +USER buildbot +WORKDIR /worker +ENTRYPOINT buildbot-worker create-worker --use-tls . $BUILDMASTER $ARM64V8_NAME $ARM64V8_PASS && unset BUILDMASTER ARM64V8_NAME ARM64V8_PASS BUILDMASTER_PORT && buildbot-worker start --nodaemon diff --git a/raspberry-pi-workers/arm64v8/docker-compose.yml b/raspberry-pi-workers/arm64v8/docker-compose.yml new file mode 100644 index 0000000..4e92a33 --- /dev/null +++ b/raspberry-pi-workers/arm64v8/docker-compose.yml @@ -0,0 +1,10 @@ +version: '3' +services: + worker: + build: + context: . + dockerfile: arm64v8_dockerfile + restart: unless-stopped + env_file: + - ../buildmaster.env + - ./arm64v8.env diff --git a/raspberry-pi-workers/armv6/armv6.env b/raspberry-pi-workers/armv6/armv6.env new file mode 100644 index 0000000..6dfc6d3 --- /dev/null +++ b/raspberry-pi-workers/armv6/armv6.env @@ -0,0 +1,2 @@ +ARMV6_PASS=changeme! +ARMV6_NAME=rpi1 diff --git a/raspberry-pi-workers/armv6/armv6_dockerfile b/raspberry-pi-workers/armv6/armv6_dockerfile new file mode 100644 index 0000000..63a5f8b --- /dev/null +++ b/raspberry-pi-workers/armv6/armv6_dockerfile @@ -0,0 +1,10 @@ +FROM arm32v6/bash + +RUN apk add --update --no-cache libc-dev gcc openssh git file py3-setuptools python3-dev libffi libffi-dev openssl openssl-dev vim +RUN pip3 install --upgrade pip buildbot-worker pyopenssl service_identity +RUN addgroup -S buildbot && adduser -S buildbot -G buildbot +RUN mkdir /worker && chown buildbot:buildbot /worker +RUN apk add --update --no-cache make g++ +USER buildbot +WORKDIR /worker +ENTRYPOINT buildbot-worker create-worker --use-tls . $BUILDMASTER $ARMV6_NAME $ARMV6_PASS && unset BUILDMASTER ARMV6_NAME ARMV6_PASS BUILDMASTER_PORT && buildbot-worker start --nodaemon diff --git a/raspberry-pi-workers/armv6/docker-compose.yml b/raspberry-pi-workers/armv6/docker-compose.yml new file mode 100644 index 0000000..8287dec --- /dev/null +++ b/raspberry-pi-workers/armv6/docker-compose.yml @@ -0,0 +1,10 @@ +version: '3' +services: + worker: + build: + context: . + dockerfile: armv6_dockerfile + restart: unless-stopped + env_file: + - ./armv6.env + - ../buildmaster.env diff --git a/raspberry-pi-workers/armv7/armv7.env b/raspberry-pi-workers/armv7/armv7.env new file mode 100644 index 0000000..fbcad3f --- /dev/null +++ b/raspberry-pi-workers/armv7/armv7.env @@ -0,0 +1,2 @@ +ARMV7_PASS=changeme! +ARMV7_NAME=rpi2b diff --git a/raspberry-pi-workers/armv7/armv7_dockerfile b/raspberry-pi-workers/armv7/armv7_dockerfile new file mode 100644 index 0000000..edbd39f --- /dev/null +++ b/raspberry-pi-workers/armv7/armv7_dockerfile @@ -0,0 +1,10 @@ +FROM arm32v7/bash + +RUN apk add --update --no-cache libc-dev gcc openssh git file grep py3-setuptools python3-dev libffi libffi-dev openssl openssl-dev vim +RUN pip3 install --upgrade pip buildbot-worker pyopenssl service_identity +RUN addgroup -S buildbot && adduser -S buildbot -G buildbot +RUN mkdir /worker && chown buildbot:buildbot /worker +RUN apk add --update --no-cache make g++ +USER buildbot +WORKDIR /worker +ENTRYPOINT buildbot-worker create-worker --maxdelay=30 --maxretries=999 --keepalive=60 --use-tls . $BUILDMASTER $ARMV7_NAME $ARMV7_PASS && unset BUILDMASTER ARMV7_NAME ARMV7_PASS BUILDMASTER_PORT && buildbot-worker start --nodaemon diff --git a/raspberry-pi-workers/armv7/docker-compose.yml b/raspberry-pi-workers/armv7/docker-compose.yml new file mode 100644 index 0000000..67939b7 --- /dev/null +++ b/raspberry-pi-workers/armv7/docker-compose.yml @@ -0,0 +1,10 @@ +version: '3' +services: + worker: + build: + context: . + dockerfile: armv7_dockerfile + restart: unless-stopped + env_file: + - ../buildmaster.env + - ./armv7.env diff --git a/raspberry-pi-workers/armv8/armv8.env b/raspberry-pi-workers/armv8/armv8.env new file mode 100644 index 0000000..ab24618 --- /dev/null +++ b/raspberry-pi-workers/armv8/armv8.env @@ -0,0 +1,2 @@ +ARMV8_PASS=changeme! +ARMV8_NAME=rpi3b_plus_32 diff --git a/raspberry-pi-workers/armv8/armv8_dockerfile b/raspberry-pi-workers/armv8/armv8_dockerfile new file mode 100644 index 0000000..b0f3aea --- /dev/null +++ b/raspberry-pi-workers/armv8/armv8_dockerfile @@ -0,0 +1,10 @@ +FROM arm32v7/bash + +RUN apk add --update --no-cache libc-dev gcc openssh git file grep py3-setuptools python3-dev libffi libffi-dev openssl openssl-dev vim +RUN pip3 install --upgrade pip buildbot-worker pyopenssl service_identity +RUN addgroup -S buildbot && adduser -S buildbot -G buildbot +RUN mkdir /worker && chown buildbot:buildbot /worker +RUN apk add --update --no-cache make g++ +USER buildbot +WORKDIR /worker +ENTRYPOINT buildbot-worker create-worker --use-tls . $BUILDMASTER $ARMV8_NAME $ARMV8_PASS && unset BUILDMASTER ARMV8_NAME ARMV8_PASS BUILDMASTER_PORT && buildbot-worker --verbose start --nodaemon diff --git a/raspberry-pi-workers/armv8/docker-compose.yml b/raspberry-pi-workers/armv8/docker-compose.yml new file mode 100644 index 0000000..e8b2652 --- /dev/null +++ b/raspberry-pi-workers/armv8/docker-compose.yml @@ -0,0 +1,10 @@ +version: '3' +services: + worker: + build: + context: . + dockerfile: armv8_dockerfile + restart: unless-stopped + env_file: + - ../buildmaster.env + - ./armv8.env diff --git a/raspberry-pi-workers/buildmaster.env b/raspberry-pi-workers/buildmaster.env new file mode 100644 index 0000000..fae223a --- /dev/null +++ b/raspberry-pi-workers/buildmaster.env @@ -0,0 +1,2 @@ +BUILDMASTER=your.buildbot.buildmaster +BUILDMASTER_PORT=9989 diff --git a/windows-worker/README.md b/windows-worker/README.md new file mode 100644 index 0000000..355638e --- /dev/null +++ b/windows-worker/README.md @@ -0,0 +1,23 @@ +# zse-windows-worker +buildbot-worker on windows 10 + +## Prerequisites +- git (you need git bash) +- Microsoft C++ Build Tools +- python3 +- NSSM - the Non-Sucking Service Manager (https://nssm.cc/) +- set a password for your user +## Warning +Some problem prevents a tls connection between the buildbot-worker on windows and the reverse proxy (traefik) in front of the buildmaster. +Because of this, tls is deactivated! +Make sure to establish the connection between worker and buildmaster in a trusted environment only +## Setup +* clone / copy this repository to your windows host + * (Optional) Create a branch for your configuration including secrets (git checkout -b secrets) +* modify URL and port of the buildmaster in buildmaster.env +* modify the passphrase and worker name in the .env file + * make sure that the buildmaster is configured with the same values +* run Git-Bash as Administrator + * run setupPythonEnv.py +* run nssm.exe as Administrator and create a new service that calls buildbot-worker-start.cmd + * use your local user and password for the service diff --git a/windows-worker/buildbot-worker-start.cmd b/windows-worker/buildbot-worker-start.cmd new file mode 100644 index 0000000..15dd340 --- /dev/null +++ b/windows-worker/buildbot-worker-start.cmd @@ -0,0 +1,2 @@ +call sandbox\Scripts\activate.bat +buildbot-worker start --nodaemon diff --git a/windows-worker/buildmaster.env b/windows-worker/buildmaster.env new file mode 100644 index 0000000..75d595a --- /dev/null +++ b/windows-worker/buildmaster.env @@ -0,0 +1,2 @@ +BUILDMASTER=your.buildbot.buildmaster +BUILDMASTER_PORT=9988 diff --git a/windows-worker/setupPythonEnv.sh b/windows-worker/setupPythonEnv.sh new file mode 100755 index 0000000..4888987 --- /dev/null +++ b/windows-worker/setupPythonEnv.sh @@ -0,0 +1,35 @@ +#!/bin/bash + +source windows.env +source buildmaster.env +function isadmin() +{ + # only for windows. + net session > /dev/null 2>&1 + if [ $? -eq 0 ]; then echo "admin" + else echo "user"; fi +} + +kernel="$(uname -s)" + +if [[ $kernel == MINGW64* ]]; then + if [[ $(isadmin) == admin ]]; then + python -m venv sandbox + source sandbox/Scripts/activate + pip install --upgrade --trusted-host pypi.org pip incremental pywin32 buildbot[bundle] pyopenssl service_identity + # this is why we ned to be admin: + python sandbox/Scripts/pywin32_postinstall.py -install + # + buildbot-worker create-worker . "${BUILDMASTER:?}" "${WINDOWS_NAME:?}" "${WINDOWS_PASS:?}" + echo "the only thing left for you to do is: get nssm and create a service using buildbot-worker-start.cmd" + echo "note: in the gui, specify the local user + password" + echo "note: after installation, the service does not run. you have to start it manually." + echo "note: it should, however, start at boot." + else + echo "you need to run this terminal as administrator" + exit 1 + fi +else + echo "Error. This scrpit is intended for windows (mingw64)" + exit 1 +fi diff --git a/windows-worker/windows.env b/windows-worker/windows.env new file mode 100644 index 0000000..e2068e2 --- /dev/null +++ b/windows-worker/windows.env @@ -0,0 +1,2 @@ +WINDOWS_NAME=winworker +WINDOWS_PASS=changeme! From 0e0024b3f440ce1078f7e4998752a7b631e513a2 Mon Sep 17 00:00:00 2001 From: bus Date: Fri, 10 Jul 2020 21:50:22 +0200 Subject: [PATCH 02/13] fixed paths --- buildbot-buildmaster/buildbot/master.cfg | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/buildbot-buildmaster/buildbot/master.cfg b/buildbot-buildmaster/buildbot/master.cfg index 10ae590..ac030b8 100644 --- a/buildbot-buildmaster/buildbot/master.cfg +++ b/buildbot-buildmaster/buildbot/master.cfg @@ -96,8 +96,8 @@ c['schedulers'].append(schedulers.ForceScheduler( builderNames=["x86_64","i386","armv6_native","armv7_native","armv8_native","arm64v8_native","arm","win_x64","win32"])) ####### FACTORIES -testWorkdirName = "tests" -testdir = "../" + testRepoName + "/" + testWorkdirName + "/" +testWorkdirName = testRepoName +testdir = "../" + testWorkdirName + "/tests/" desc = "building and testing" descDone = "build and test" @@ -105,6 +105,7 @@ descDone = "build and test" windowsShell = "C:\\Program Files\\Git\\bin\\bash.exe" factoryLinux_x86_64 = util.BuildFactory() +factoryLinux_x86_64.addStep(steps.Git(name="git - tests", repourl=testRepo, workdir=testWorkdirName, mode='full', alwaysUseLatest='true')) factoryLinux_x86_64.addStep(steps.Git(name="git - data-compressor", repourl=compressorRepo, workdir='data-compressor', mode='full', alwaysUseLatest='true')) factoryLinux_x86_64.addStep(steps.ShellCommand(description=desc, descriptionDone=descDone, command=[testdir + "checkBits.sh", "x86_64"])) factoryLinux_x86_64.addStep(steps.ShellCommand(description=desc, descriptionDone=descDone, command=[testdir + "buildAndTest.sh", "x86_64", "8"])) From 43c7c4fd7d60b9b50819b97e016a21c1cd7bdccb Mon Sep 17 00:00:00 2001 From: bus Date: Fri, 10 Jul 2020 22:02:14 +0200 Subject: [PATCH 03/13] improved top-level readme --- README.md | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index d446993..dc663aa 100644 --- a/README.md +++ b/README.md @@ -1 +1,10 @@ -This repository contains the buildbot environment for server and worker hosts. +# Buildbot setup for the DataCompressor + +This repository contains the buildbot environment used for automated builds of the DataCompressor https://github.com/CenterForSecureEnergyInformatics/data-compressor + +Subfolder contents: +- buildbot-buildmaster: the buildbot server configuration. +- raspberry-pi-workers: buildbot worker for some variants of the Raspberry Pi. +- windows-worker: buildbot worker for windows 10 + +Please refer to the README files in the subfolders for setup instructions and information. From 9cfe5c91d741d4519070ad42e2a47e0141f69f62 Mon Sep 17 00:00:00 2001 From: bus Date: Fri, 10 Jul 2020 22:10:08 +0200 Subject: [PATCH 04/13] replaced references to the forks with the main repos --- buildbot-buildmaster/buildbot/master.cfg | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/buildbot-buildmaster/buildbot/master.cfg b/buildbot-buildmaster/buildbot/master.cfg index ac030b8..71c79ff 100644 --- a/buildbot-buildmaster/buildbot/master.cfg +++ b/buildbot-buildmaster/buildbot/master.cfg @@ -56,16 +56,13 @@ c['protocols'] = {'pb': {'port': os.environ.get("BUILDBOT_WORKER_PORT", 9989)}} # about source code changes. Here we point to the buildbot clone of pyflakes. testRepoName = "data-compressor-tests" -#testRepoOwner= "CenterForSecureEnergyInformatics" -testRepoOwner= "brhat" +testRepoOwner = 'CenterForSecureEnergyInformatics' testRepo = "https://github.com/" + testRepoOwner + "/" + testRepoName + ".git" compressorRepoName = 'data-compressor' -#compressorRepoOwner = 'CenterForSecureEnergyInformatics' -compressorRepoOwner = 'brhat' +compressorRepoOwner = 'CenterForSecureEnergyInformatics' compressorRepo = "https://github.com/" + compressorRepoOwner + "/" + compressorRepoName + ".git" -compressorRepoOwner = 'CenterForSecureEnergyInformatics' # WARNING this is duplicated! #compressorRepo = 'https://github.com/CenterForSecureEnergyInformatics/data-compressor.git' token = open(secretsdir + "/githubToken").read().strip() From 5b15fe7868b2014d6144e644e807729376ad197c Mon Sep 17 00:00:00 2001 From: burs Date: Mon, 17 Aug 2020 23:20:51 +0200 Subject: [PATCH 05/13] enumeration, grammar, spelling, better hints --- buildbot-buildmaster/README.md | 14 ++++++------ buildbot-buildmaster/buildbot/README.md | 10 ++++----- .../buildbot/secrets/README.md | 21 +++++++++++++----- .../buildbot/secrets/githubToken | 1 + .../buildbot/secrets/htpasswd | 2 ++ raspberry-pi-workers/README.md | 22 +++++++++++-------- windows-worker/setupPythonEnv.sh | 6 ++--- 7 files changed, 47 insertions(+), 29 deletions(-) diff --git a/buildbot-buildmaster/README.md b/buildbot-buildmaster/README.md index 10e18db..37d453b 100644 --- a/buildbot-buildmaster/README.md +++ b/buildbot-buildmaster/README.md @@ -3,13 +3,13 @@ This project contains dockerized services, serving a buildbot buildmaster to the It is intended to be used on a linux server. Currently, it runs under Debian Buster. ## Setup -- clone this repository to your server. -- install docker and docker-compose. -- create a dedicated, unprivileged user for this project -- add the user to the docker group -- clone this repository to your server. You only need the folder containing this README. - - optional: create a branch for your secrets (git checkout -b secrets) -- follow the setup instructions in the subfolders. Start with traefik and then buildbot. +1. clone this repository to your server. +2. install docker and docker-compose. +3. create a dedicated, unprivileged user for this project +4. add the user to the docker group +5. clone this repository to your server. You only need the folder containing this README. + - optional: create a branch for your secrets (git checkout -b secrets) +6. follow the setup instructions in the subfolders. Start with traefik and then buildbot. This project comes mostly preconfigured. diff --git a/buildbot-buildmaster/buildbot/README.md b/buildbot-buildmaster/buildbot/README.md index b3a3984..69ec36b 100644 --- a/buildbot-buildmaster/buildbot/README.md +++ b/buildbot-buildmaster/buildbot/README.md @@ -37,7 +37,7 @@ The workers are connected via port 9989 to the buildmaster. Workers on different platforms are implemented in this setup: - windows 10. Unfortunately, tls seems not to work on windows with buildbot, so the vm running the buildmaster has to share a subnet with the windows-vm (which it does in our setup). - Raspberri Pi (1, 2b and 3b+), dockerized. They use a tls connection handled by the reverse proxy traefik. -- Linux, dockerized. See multiarch_dockerfile and docker-compose.yml for the definition. Runs on the same host as the buildmaster. +- Linux, dockerized. See multiarch_dockerfile and docker-compose.yml for the definition. It runs on the same host as the buildmaster. Buildfactories define which steps are to be executed on a build. For each platform/architecture a scheduler is defined in this configuration. @@ -50,7 +50,7 @@ For a detailed overview, please refer to http://docs.buildbot.net/latest/manual/ ## Setup / Configuration The file master.cfg is the main configuration file of buildbot, docker-compose.yml handles all services involved. ### master.cfg -Everything is pre-configured in this setup and does not require much changes. +Everything is pre-configured in this setup and does not require many changes. Please refer to http://docs.buildbot.net/latest/manual/configuration/ for details, as a detailed explanation is beyond the scope of this README. #### Credentials for the Web Ui To force builds, you need to be logged in. @@ -71,14 +71,14 @@ Make sure to set these passwords on the corresponding workers as well! On the workers, the files are located in the same folders and files to make this step easier. #### GitHubPullRequestPoller -Important Note: for the GithubPullrequestPoller to work, the owner and repository name (NOT the URL) has to be provided. -In this configuration, the following is used: +Important Note: for the GithubPullrequestPoller to work, the owner and repository name (NOT the URL) have to be provided. +In this configuration, the following are used: - compressorRepoName = 'data-compressor' - compressorRepoOwner = 'CenterForSecureEnergyInformatics' ### docker-compose.yml The following services are specified here: -- buildbot-buildmaster +- buildbot-buildmaster: the buildmaster and the webinterface itself - db: a database for the buildmaster - worker: a buildbot worker running linux (used to crosscompile), defined in multiarch_dockerfile diff --git a/buildbot-buildmaster/buildbot/secrets/README.md b/buildbot-buildmaster/buildbot/secrets/README.md index 17313b8..49f0b1a 100644 --- a/buildbot-buildmaster/buildbot/secrets/README.md +++ b/buildbot-buildmaster/buildbot/secrets/README.md @@ -1,7 +1,18 @@ -This is the place for -- htpasswd file -- github access token +# Secrets +This is the place for secrets. +Be sure to set the correct file permissions! (chmod 600) +## htpasswd file +This file contains the login credentials for the users defined in `../master.cfg`. +They are defined in the following form: + +`contributor@email.provider:cleartext-password` + +Note, that: +- passwords have to be set in clear text +- users have to be specified in `../master.cfg` +- the user has to be a contributor in the github-project +## github access token +The access token used by buildbot to communicate with github. +You can define it in your github account. -It seems that you have to provide the passwords in htpasswd as clear text for buildbot, e.g. you@email.provider:password. -Think of proper file permissions! (chmod 600) diff --git a/buildbot-buildmaster/buildbot/secrets/githubToken b/buildbot-buildmaster/buildbot/secrets/githubToken index e69de29..af6823c 100644 --- a/buildbot-buildmaster/buildbot/secrets/githubToken +++ b/buildbot-buildmaster/buildbot/secrets/githubToken @@ -0,0 +1 @@ +YourTokenHere diff --git a/buildbot-buildmaster/buildbot/secrets/htpasswd b/buildbot-buildmaster/buildbot/secrets/htpasswd index e69de29..9cf5bf9 100644 --- a/buildbot-buildmaster/buildbot/secrets/htpasswd +++ b/buildbot-buildmaster/buildbot/secrets/htpasswd @@ -0,0 +1,2 @@ +contributor@email.provider:cleartext-password +another.contributor@email.provider:cleartext-password diff --git a/raspberry-pi-workers/README.md b/raspberry-pi-workers/README.md index 36798cd..e73fa77 100644 --- a/raspberry-pi-workers/README.md +++ b/raspberry-pi-workers/README.md @@ -1,15 +1,19 @@ -# zse-raspberry-pi-workers -dockerized buildbot-workers for various versions of the raspberry-pi +# Raspberry Pi workers +dockerized buildbot-workers for various versions of the Raspberry Pi ## Prerequisites +- a physical Rasperry Pi + - run `uname -m` to determine the architecture - you have to have docker and docker-compose installed. - add your unprivileged user to the docker group ## Setup -* clone / copy this repository to a raspberry-pi. - * (Optional) Create a branch for your configuration including secrets (git checkout -b secrets) -* modify URL and port of the buildmaster in buildmaster.env. -* cd into the folder fitting your model (run uname -m if you are unsure) - * modify the passphrase and worker name in the .env file - * make sure that the buildmaster is configured with the same values -* run docker-compose up -d +1. clone / copy this repository to the Raspberry Pi + - (Optional) Create a branch for your configuration including secrets (git checkout -b secrets) +2. modify URL and port of the buildmaster in `buildmaster.env` +4. `cd` into the folder fitting your model (run `uname -m` if you are unsure) + - modify the passphrase and worker name in the .env file + - make sure that the buildmaster is configured with the same values +5. run docker-compose up -d + +Note that docker-compose restarts the service automatically after reboots. diff --git a/windows-worker/setupPythonEnv.sh b/windows-worker/setupPythonEnv.sh index 4888987..ce7a57e 100755 --- a/windows-worker/setupPythonEnv.sh +++ b/windows-worker/setupPythonEnv.sh @@ -21,10 +21,10 @@ if [[ $kernel == MINGW64* ]]; then python sandbox/Scripts/pywin32_postinstall.py -install # buildbot-worker create-worker . "${BUILDMASTER:?}" "${WINDOWS_NAME:?}" "${WINDOWS_PASS:?}" - echo "the only thing left for you to do is: get nssm and create a service using buildbot-worker-start.cmd" + echo "the only thing left for you to do is: get nssm and create a service which calls buildbot-worker-start.cmd" echo "note: in the gui, specify the local user + password" - echo "note: after installation, the service does not run. you have to start it manually." - echo "note: it should, however, start at boot." + echo "note: after installation, the service does not run. You have to enable and start start it manually." + echo "note: once enabled, the service starts automatically on reboot" else echo "you need to run this terminal as administrator" exit 1 From 3dc1999fff88824ab95aad49f93871b0f8b507c4 Mon Sep 17 00:00:00 2001 From: bus Date: Tue, 18 Aug 2020 02:23:22 +0200 Subject: [PATCH 06/13] automated service setup --- windows-worker/setupPythonEnv.sh | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/windows-worker/setupPythonEnv.sh b/windows-worker/setupPythonEnv.sh index ce7a57e..adf4350 100755 --- a/windows-worker/setupPythonEnv.sh +++ b/windows-worker/setupPythonEnv.sh @@ -13,18 +13,29 @@ function isadmin() kernel="$(uname -s)" if [[ $kernel == MINGW64* ]]; then + DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd -W )" + if [[ $(isadmin) == admin ]]; then python -m venv sandbox source sandbox/Scripts/activate pip install --upgrade --trusted-host pypi.org pip incremental pywin32 buildbot[bundle] pyopenssl service_identity # this is why we ned to be admin: python sandbox/Scripts/pywin32_postinstall.py -install - # + buildbot-worker create-worker . "${BUILDMASTER:?}" "${WINDOWS_NAME:?}" "${WINDOWS_PASS:?}" - echo "the only thing left for you to do is: get nssm and create a service which calls buildbot-worker-start.cmd" - echo "note: in the gui, specify the local user + password" - echo "note: after installation, the service does not run. You have to enable and start start it manually." - echo "note: once enabled, the service starts automatically on reboot" + + # service setup: + # download and unzip nssm + curl https://nssm.cc/ci/nssm-2.24-101-g897c7ad.zip -o nssm.zip + archivepath=$(zipinfo -1 nssm.zip | grep exe | grep 64) + unzip -p nssm.zip $archivepath > nssm.exe + + #install service + ./nssm.exe install $servicename $DIR/buildbot-worker-start.cmd + + #modify username (keep ".\\"!) and password + ./nssm.exe set $servicename ObjectName '.\\invalid-username' 'invalid-password' + ./nssm.exe start $servicename else echo "you need to run this terminal as administrator" exit 1 From a3b75b9a1fbb12d43f8178b1e9ac93a3cb446430 Mon Sep 17 00:00:00 2001 From: burs Date: Tue, 18 Aug 2020 12:45:15 +0200 Subject: [PATCH 07/13] typo --- windows-worker/setupPythonEnv.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/windows-worker/setupPythonEnv.sh b/windows-worker/setupPythonEnv.sh index adf4350..7f04257 100755 --- a/windows-worker/setupPythonEnv.sh +++ b/windows-worker/setupPythonEnv.sh @@ -41,6 +41,6 @@ if [[ $kernel == MINGW64* ]]; then exit 1 fi else - echo "Error. This scrpit is intended for windows (mingw64)" + echo "Error. This script is intended for windows (mingw64)" exit 1 fi From b9116a71552322245438458a15b5984249d01716 Mon Sep 17 00:00:00 2001 From: burs Date: Tue, 18 Aug 2020 13:49:43 +0200 Subject: [PATCH 08/13] builder names as vars, code reduction, title url --- buildbot-buildmaster/buildbot/master.cfg | 140 ++++++++++++++--------- 1 file changed, 83 insertions(+), 57 deletions(-) diff --git a/buildbot-buildmaster/buildbot/master.cfg b/buildbot-buildmaster/buildbot/master.cfg index 71c79ff..0aedf65 100644 --- a/buildbot-buildmaster/buildbot/master.cfg +++ b/buildbot-buildmaster/buildbot/master.cfg @@ -83,14 +83,26 @@ c['change_source'].append(changes.GitHubPullrequestPoller( # Configure the Schedulers, which decide how to react to incoming changes. In this # case, just kick off a 'runtests' build +# first, we define the builders: +x86_64 = "x86_64" +i386 = "i386" +armv6_native = "armv6_native" +armv7_native = "armv7_native" +armv8_native = "armv8_native" +arm64v8_native = "arm64v8_native" +arm = "arm" +win_x64 = "win_x64" +win32 = "win32" +# ------ + c['schedulers'] = [] c['schedulers'].append(schedulers.AnyBranchScheduler( name="anyBranch", treeStableTimer=None, - builderNames=["x86_64","i386","armv6_native","armv7_native","armv8_native","arm64v8_native","arm","win_x64","win32"])) + builderNames=[ x86_64, i386, armv6_native , armv7_native , armv8_native , arm64v8_native , arm , win_x64 , win32 ])) c['schedulers'].append(schedulers.ForceScheduler( name="force", - builderNames=["x86_64","i386","armv6_native","armv7_native","armv8_native","arm64v8_native","arm","win_x64","win32"])) + builderNames=[ x86_64, i386, armv6_native , armv7_native , armv8_native , arm64v8_native , arm , win_x64 , win32 ])) ####### FACTORIES testWorkdirName = testRepoName @@ -102,63 +114,77 @@ descDone = "build and test" windowsShell = "C:\\Program Files\\Git\\bin\\bash.exe" factoryLinux_x86_64 = util.BuildFactory() -factoryLinux_x86_64.addStep(steps.Git(name="git - tests", repourl=testRepo, workdir=testWorkdirName, mode='full', alwaysUseLatest='true')) -factoryLinux_x86_64.addStep(steps.Git(name="git - data-compressor", repourl=compressorRepo, workdir='data-compressor', mode='full', alwaysUseLatest='true')) -factoryLinux_x86_64.addStep(steps.ShellCommand(description=desc, descriptionDone=descDone, command=[testdir + "checkBits.sh", "x86_64"])) -factoryLinux_x86_64.addStep(steps.ShellCommand(description=desc, descriptionDone=descDone, command=[testdir + "buildAndTest.sh", "x86_64", "8"])) -factoryLinux_x86_64.addStep(steps.ShellCommand(description=desc, descriptionDone=descDone, command=[testdir + "buildAndTest.sh", "x86_64", "16"])) -factoryLinux_x86_64.addStep(steps.ShellCommand(description=desc, descriptionDone=descDone, command=[testdir + "buildAndTest.sh", "x86_64", "32"])) -factoryLinux_x86_64.addStep(steps.ShellCommand(description=desc, descriptionDone=descDone, command=[testdir + "buildAndTest.sh", "x86_64", "64"])) +factoryLinux_x86_64.addSteps([ + steps.Git(name="git - tests", repourl=testRepo, workdir=testWorkdirName, mode='full', alwaysUseLatest='true'), + steps.Git(name="git - data-compressor", repourl=compressorRepo, workdir='data-compressor', mode='full', alwaysUseLatest='true'), + steps.ShellCommand(description=desc, descriptionDone=descDone, command=[testdir + "checkBits.sh", "x86_64"]), + steps.ShellCommand(description=desc, descriptionDone=descDone, command=[testdir + "buildAndTest.sh", "x86_64", "8"]), + steps.ShellCommand(description=desc, descriptionDone=descDone, command=[testdir + "buildAndTest.sh", "x86_64", "16"]), + steps.ShellCommand(description=desc, descriptionDone=descDone, command=[testdir + "buildAndTest.sh", "x86_64", "32"]), + steps.ShellCommand(description=desc, descriptionDone=descDone, command=[testdir + "buildAndTest.sh", "x86_64", "64"]), +]) factoryLinux_i386 = util.BuildFactory() -factoryLinux_i386.addStep(steps.Git(name="git - tests", repourl=testRepo, workdir=testWorkdirName, mode='full', alwaysUseLatest='true')) -factoryLinux_i386.addStep(steps.Git(name="git - data-compressor", repourl=compressorRepo, workdir='data-compressor', mode='full', alwaysUseLatest='true')) -factoryLinux_i386.addStep(steps.ShellCommand(description=desc, descriptionDone=descDone, command=[testdir + "checkBits.sh", "i386"])) -factoryLinux_i386.addStep(steps.ShellCommand(description=desc, descriptionDone=descDone, command=[testdir + "buildAndTest.sh", "i386", "8"])) -factoryLinux_i386.addStep(steps.ShellCommand(description=desc, descriptionDone=descDone, command=[testdir + "buildAndTest.sh", "i386", "16"])) -factoryLinux_i386.addStep(steps.ShellCommand(description=desc, descriptionDone=descDone, command=[testdir + "buildAndTest.sh", "i386", "32"])) +factoryLinux_i386.addSteps([ + steps.Git(name="git - tests", repourl=testRepo, workdir=testWorkdirName, mode='full', alwaysUseLatest='true'), + steps.Git(name="git - data-compressor", repourl=compressorRepo, workdir='data-compressor', mode='full', alwaysUseLatest='true'), + steps.ShellCommand(description=desc, descriptionDone=descDone, command=[testdir + "checkBits.sh", "i386"]), + steps.ShellCommand(description=desc, descriptionDone=descDone, command=[testdir + "buildAndTest.sh", "i386", "8"]), + steps.ShellCommand(description=desc, descriptionDone=descDone, command=[testdir + "buildAndTest.sh", "i386", "16"]), + steps.ShellCommand(description=desc, descriptionDone=descDone, command=[testdir + "buildAndTest.sh", "i386", "32"]), +]) factoryLinux_armhf = util.BuildFactory() -factoryLinux_armhf.addStep(steps.Git(name="git - tests", repourl=testRepo, workdir=testWorkdirName, mode='full', alwaysUseLatest='true')) -factoryLinux_armhf.addStep(steps.Git(name="git - data-compressor", repourl=compressorRepo, workdir='data-compressor', mode='full', alwaysUseLatest='true')) -factoryLinux_armhf.addStep(steps.ShellCommand(description=desc, descriptionDone=descDone, command=[testdir + "checkBits.sh", "armhf"])) -factoryLinux_armhf.addStep(steps.ShellCommand(description=desc, descriptionDone=descDone, command=[testdir + "buildAndTest.sh", "armhf", "8"])) -factoryLinux_armhf.addStep(steps.ShellCommand(description=desc, descriptionDone=descDone, command=[testdir + "buildAndTest.sh", "armhf", "16"])) -factoryLinux_armhf.addStep(steps.ShellCommand(description=desc, descriptionDone=descDone, command=[testdir + "buildAndTest.sh", "armhf", "32"])) +factoryLinux_armhf.addSteps([ + steps.Git(name="git - tests", repourl=testRepo, workdir=testWorkdirName, mode='full', alwaysUseLatest='true'), + steps.Git(name="git - data-compressor", repourl=compressorRepo, workdir='data-compressor', mode='full', alwaysUseLatest='true'), + steps.ShellCommand(description=desc, descriptionDone=descDone, command=[testdir + "checkBits.sh", "armhf"]), + steps.ShellCommand(description=desc, descriptionDone=descDone, command=[testdir + "buildAndTest.sh", "armhf", "8"]), + steps.ShellCommand(description=desc, descriptionDone=descDone, command=[testdir + "buildAndTest.sh", "armhf", "16"]), + steps.ShellCommand(description=desc, descriptionDone=descDone, command=[testdir + "buildAndTest.sh", "armhf", "32"]), +]) factory_arm= util.BuildFactory() -factory_arm.addStep(steps.Git(name="git - tests", repourl=testRepo, workdir=testWorkdirName, mode='full', alwaysUseLatest='true')) -factory_arm.addStep(steps.Git(name="git - data-compressor", repourl=compressorRepo, workdir='data-compressor', mode='full', alwaysUseLatest='true')) -factory_arm.addStep(steps.ShellCommand(description=desc, descriptionDone=descDone, command=[testdir + "checkBits.sh", "arm"])) -factory_arm.addStep(steps.ShellCommand(description=desc, descriptionDone=descDone, command=[testdir + "buildAndTest.sh", "arm", "8"])) -factory_arm.addStep(steps.ShellCommand(description=desc, descriptionDone=descDone, command=[testdir + "buildAndTest.sh", "arm", "16"])) -factory_arm.addStep(steps.ShellCommand(description=desc, descriptionDone=descDone, command=[testdir + "buildAndTest.sh", "arm", "32"])) +factory_arm.addSteps([ + steps.Git(name="git - tests", repourl=testRepo, workdir=testWorkdirName, mode='full', alwaysUseLatest='true'), + steps.Git(name="git - data-compressor", repourl=compressorRepo, workdir='data-compressor', mode='full', alwaysUseLatest='true'), + steps.ShellCommand(description=desc, descriptionDone=descDone, command=[testdir + "checkBits.sh", "arm"]), + steps.ShellCommand(description=desc, descriptionDone=descDone, command=[testdir + "buildAndTest.sh", "arm", "8"]), + steps.ShellCommand(description=desc, descriptionDone=descDone, command=[testdir + "buildAndTest.sh", "arm", "16"]), + steps.ShellCommand(description=desc, descriptionDone=descDone, command=[testdir + "buildAndTest.sh", "arm", "32"]), +]) factory_aarch64= util.BuildFactory() -factory_aarch64.addStep(steps.Git(name="git - tests", repourl=testRepo, workdir=testWorkdirName, mode='full', alwaysUseLatest='true')) -factory_aarch64.addStep(steps.Git(name="git - data-compressor", repourl=compressorRepo, workdir='data-compressor', mode='full', alwaysUseLatest='true')) -factory_aarch64.addStep(steps.ShellCommand(description=desc, descriptionDone=descDone, command=[testdir + "checkBits.sh", "arm"])) -factory_aarch64.addStep(steps.ShellCommand(description=desc, descriptionDone=descDone, command=[testdir + "buildAndTest.sh", "arm", "8"])) -factory_aarch64.addStep(steps.ShellCommand(description=desc, descriptionDone=descDone, command=[testdir + "buildAndTest.sh", "arm", "16"])) -factory_aarch64.addStep(steps.ShellCommand(description=desc, descriptionDone=descDone, command=[testdir + "buildAndTest.sh", "arm", "32"])) -factory_aarch64.addStep(steps.ShellCommand(description=desc, descriptionDone=descDone, command=[testdir + "buildAndTest.sh", "arm", "64"])) +factory_aarch64.addSteps([ + steps.Git(name="git - tests", repourl=testRepo, workdir=testWorkdirName, mode='full', alwaysUseLatest='true'), + steps.Git(name="git - data-compressor", repourl=compressorRepo, workdir='data-compressor', mode='full', alwaysUseLatest='true'), + steps.ShellCommand(description=desc, descriptionDone=descDone, command=[testdir + "checkBits.sh", "arm"]), + steps.ShellCommand(description=desc, descriptionDone=descDone, command=[testdir + "buildAndTest.sh", "arm", "8"]), + steps.ShellCommand(description=desc, descriptionDone=descDone, command=[testdir + "buildAndTest.sh", "arm", "16"]), + steps.ShellCommand(description=desc, descriptionDone=descDone, command=[testdir + "buildAndTest.sh", "arm", "32"]), + steps.ShellCommand(description=desc, descriptionDone=descDone, command=[testdir + "buildAndTest.sh", "arm", "64"]), +]) factoryWin_x64 = util.BuildFactory() -factoryWin_x64.addStep(steps.Git(name="git - tests", repourl=testRepo, workdir=testWorkdirName, mode='full', alwaysUseLatest='true')) -factoryWin_x64.addStep(steps.Git(name="git - data-compressor", repourl=compressorRepo, workdir='data-compressor', mode='full', alwaysUseLatest='true')) -factoryWin_x64.addStep(steps.ShellCommand(description=desc, descriptionDone=descDone, command=[windowsShell, testdir + "checkBits.sh", "x64"])) -factoryWin_x64.addStep(steps.ShellCommand(description=desc, descriptionDone=descDone, command=[windowsShell, testdir + "buildAndTest.sh", "x64", "8"])) -factoryWin_x64.addStep(steps.ShellCommand(description=desc, descriptionDone=descDone, command=[windowsShell, testdir + "buildAndTest.sh", "x64", "16"])) -factoryWin_x64.addStep(steps.ShellCommand(description=desc, descriptionDone=descDone, command=[windowsShell, testdir + "buildAndTest.sh", "x64", "32"])) -factoryWin_x64.addStep(steps.ShellCommand(description=desc, descriptionDone=descDone, command=[windowsShell, testdir + "buildAndTest.sh", "x64", "64"])) +factoryWin_x64.addSteps([ + steps.Git(name="git - tests", repourl=testRepo, workdir=testWorkdirName, mode='full', alwaysUseLatest='true'), + steps.Git(name="git - data-compressor", repourl=compressorRepo, workdir='data-compressor', mode='full', alwaysUseLatest='true'), + steps.ShellCommand(description=desc, descriptionDone=descDone, command=[windowsShell, testdir + "checkBits.sh", "x64"]), + steps.ShellCommand(description=desc, descriptionDone=descDone, command=[windowsShell, testdir + "buildAndTest.sh", "x64", "8"]), + steps.ShellCommand(description=desc, descriptionDone=descDone, command=[windowsShell, testdir + "buildAndTest.sh", "x64", "16"]), + steps.ShellCommand(description=desc, descriptionDone=descDone, command=[windowsShell, testdir + "buildAndTest.sh", "x64", "32"]), + steps.ShellCommand(description=desc, descriptionDone=descDone, command=[windowsShell, testdir + "buildAndTest.sh", "x64", "64"]), +]) factoryWin32 = util.BuildFactory() -factoryWin32.addStep(steps.Git(name="git - tests", repourl=testRepo, workdir=testWorkdirName, mode='full', alwaysUseLatest='true')) -factoryWin32.addStep(steps.Git(name="git - data-compressor", repourl=compressorRepo, workdir='data-compressor', mode='full', alwaysUseLatest='true')) -factoryWin32.addStep(steps.ShellCommand(description=desc, descriptionDone=descDone, command=[windowsShell, testdir + "checkBits.sh", "win32"])) -factoryWin32.addStep(steps.ShellCommand(description=desc, descriptionDone=descDone, command=[windowsShell, testdir + "buildAndTest.sh", "win32", "8"])) -factoryWin32.addStep(steps.ShellCommand(description=desc, descriptionDone=descDone, command=[windowsShell, testdir + "buildAndTest.sh", "win32", "16"])) -factoryWin32.addStep(steps.ShellCommand(description=desc, descriptionDone=descDone, command=[windowsShell, testdir + "buildAndTest.sh", "win32", "32"])) +factoryWin32.addSteps([ + steps.Git(name="git - tests", repourl=testRepo, workdir=testWorkdirName, mode='full', alwaysUseLatest='true'), + steps.Git(name="git - data-compressor", repourl=compressorRepo, workdir='data-compressor', mode='full', alwaysUseLatest='true'), + steps.ShellCommand(description=desc, descriptionDone=descDone, command=[windowsShell, testdir + "checkBits.sh", "win32"]), + steps.ShellCommand(description=desc, descriptionDone=descDone, command=[windowsShell, testdir + "buildAndTest.sh", "win32", "8"]), + steps.ShellCommand(description=desc, descriptionDone=descDone, command=[windowsShell, testdir + "buildAndTest.sh", "win32", "16"]), + steps.ShellCommand(description=desc, descriptionDone=descDone, command=[windowsShell, testdir + "buildAndTest.sh", "win32", "32"]), +]) ####### BUILDERS @@ -169,47 +195,47 @@ factoryWin32.addStep(steps.ShellCommand(description=desc, descriptionDone=descDo c['builders'] = [] c['builders'].append( - util.BuilderConfig(name="x86_64", + util.BuilderConfig(name=x86_64, workernames=[multiarch_worker], factory=factoryLinux_x86_64)) c['builders'].append( - util.BuilderConfig(name="i386", + util.BuilderConfig(name=i386, workernames=[multiarch_worker], factory=factoryLinux_i386)) c['builders'].append( - util.BuilderConfig(name="arm", + util.BuilderConfig(name=arm, workernames=[multiarch_worker], factory=factoryLinux_armhf)) c['builders'].append( - util.BuilderConfig(name="armv6_native", + util.BuilderConfig(name=armv6_native, workernames=[armv6_worker], factory=factory_arm)) c['builders'].append( - util.BuilderConfig(name="armv7_native", + util.BuilderConfig(name=armv7_native, workernames=[armv7_worker], factory=factory_arm)) c['builders'].append( - util.BuilderConfig(name="armv8_native", + util.BuilderConfig(name=armv8_native, workernames=[armv8_worker], factory=factory_arm)) c['builders'].append( - util.BuilderConfig(name="arm64v8_native", + util.BuilderConfig(name=arm64v8_native, workernames=[arm64v8_worker], factory=factory_aarch64)) c['builders'].append( - util.BuilderConfig(name="win_x64", + util.BuilderConfig(name=win_x64, workernames=[windows_worker], factory=factoryWin_x64)) c['builders'].append( - util.BuilderConfig(name="win32", + util.BuilderConfig(name=win32, workernames=[windows_worker], factory=factoryWin32)) @@ -238,7 +264,7 @@ c['services'] = [] # home pages (linked to the 'titleURL'). c['title'] = os.environ.get("BUILDBOT_TITLE", "buildbot") -c['titleURL'] = os.environ.get("BUILDBOT_TITLE_URL", "https://docs.buildbot.net/latest/developer/config.html#") +c['titleURL'] = os.environ.get("BUILDBOT_TITLE_URL", "https://github.com/CenterForSecureEnergyInformatics/data-compressor") # the 'buildbotURL' string should point to the location where the buildbot's # internal web server is visible. This typically uses the port number set in From fdac43e06daeae6f8d9891e5828d85a42584b501 Mon Sep 17 00:00:00 2001 From: burs Date: Tue, 18 Aug 2020 17:33:25 +0200 Subject: [PATCH 09/13] fewer layers --- .../buildbot/multiarch_dockerfile | 24 +++++++++++-------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/buildbot-buildmaster/buildbot/multiarch_dockerfile b/buildbot-buildmaster/buildbot/multiarch_dockerfile index d7df1ec..9bf79c5 100644 --- a/buildbot-buildmaster/buildbot/multiarch_dockerfile +++ b/buildbot-buildmaster/buildbot/multiarch_dockerfile @@ -20,17 +20,21 @@ deb [arch=armhf] http://ports.ubuntu.com/ubuntu-ports/ bionic-backports main uni > /etc/apt/sources.list -RUN apt-get update && apt-get install -y git -RUN dpkg --add-architecture i386 && dpkg --add-architecture armhf && apt-get update -RUN apt-get install -y build-essential gcc-multilib mingw-w64 wine64 wine32 qemu binfmt-support qemu-user-static -RUN apt-get install -y crossbuild-essential-armhf libc6:armhf -# buildbot and docker: -RUN apt install -y \ - python-dev \ - python-pip +RUN apt-get update && \ + apt-get install -y git && \ + dpkg --add-architecture i386 && \ + dpkg --add-architecture armhf && \ + apt-get update && \ + apt-get install -y build-essential gcc-multilib mingw-w64 wine64 wine32 qemu binfmt-support qemu-user-static && \ + apt-get install -y crossbuild-essential-armhf libc6:armhf && \ + apt-get install -y python-dev python-pip + RUN pip install buildbot-worker -RUN groupadd -r buildbot && useradd -r -g buildbot buildbot -RUN mkdir /worker && chown buildbot:buildbot /worker + +RUN groupadd -r buildbot && \ + useradd -r -g buildbot buildbot && \ + mkdir /worker && \ + chown buildbot:buildbot /worker # Install your build-dependencies here ... USER buildbot WORKDIR /worker From 640775c1972726fd606ee724e05c209b7e642149 Mon Sep 17 00:00:00 2001 From: burs Date: Tue, 18 Aug 2020 18:05:37 +0200 Subject: [PATCH 10/13] links, versions and modified instructions --- windows-worker/README.md | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/windows-worker/README.md b/windows-worker/README.md index 355638e..c75a23b 100644 --- a/windows-worker/README.md +++ b/windows-worker/README.md @@ -1,23 +1,23 @@ -# zse-windows-worker -buildbot-worker on windows 10 +# Windows-worker +Buildbot-worker on Windows 10 pro ## Prerequisites -- git (you need git bash) -- Microsoft C++ Build Tools -- python3 -- NSSM - the Non-Sucking Service Manager (https://nssm.cc/) -- set a password for your user +- A working Windows 10 pro setup. + - Set a password for your user +- Git including Git Bash. Download the current version here: https://git-scm.com/download/win +- Microsoft C++ Build Toolsi 2019. Download: https://visualstudio.microsoft.com/de/downloads/ +- Python. Currently verison 3.8.2 is in use. Download: https://www.python.org/downloads/ ## Warning Some problem prevents a tls connection between the buildbot-worker on windows and the reverse proxy (traefik) in front of the buildmaster. Because of this, tls is deactivated! -Make sure to establish the connection between worker and buildmaster in a trusted environment only +Make sure to establish the connection between worker and buildmaster in a trusted environment only. +Currently, the buildmaster and the windows-vm share a subnet for this purpose. ## Setup -* clone / copy this repository to your windows host - * (Optional) Create a branch for your configuration including secrets (git checkout -b secrets) -* modify URL and port of the buildmaster in buildmaster.env -* modify the passphrase and worker name in the .env file - * make sure that the buildmaster is configured with the same values -* run Git-Bash as Administrator - * run setupPythonEnv.py -* run nssm.exe as Administrator and create a new service that calls buildbot-worker-start.cmd - * use your local user and password for the service +1. Clone / copy this repository to your windows host + * (Optional) Create a branch for your configuration including secrets (git checkout -b secrets) +2. Modify URL and port of the buildmaster in [buildmaster.env](buildmaster.env) +3. Modify the passphrase and worker name in [windows.env](windows.env) + * Make sure that the buildmaster is configured with the same values +4. Change the username and password in the file [setupPythonEnv.py](setupPythonEnv.py) +4. Run Git-Bash as Administrator and navigate to this directory + * Run `./setupPythonEnv.py` From 21e046941e40d7578388a3a2859edace344ea628 Mon Sep 17 00:00:00 2001 From: burs Date: Tue, 18 Aug 2020 21:08:23 +0200 Subject: [PATCH 11/13] modified the README files --- README.md | 8 +-- buildbot-buildmaster/README.md | 38 ++++++----- buildbot-buildmaster/buildbot/README.md | 67 ++++++++++--------- .../buildbot/secrets/README.md | 12 ++-- buildbot-buildmaster/traefik/README.md | 38 ++++++----- buildbot-buildmaster/traefik/certs/README.md | 5 +- raspberry-pi-workers/README.md | 22 +++--- 7 files changed, 98 insertions(+), 92 deletions(-) diff --git a/README.md b/README.md index dc663aa..ec3ecb2 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,10 @@ # Buildbot setup for the DataCompressor -This repository contains the buildbot environment used for automated builds of the DataCompressor https://github.com/CenterForSecureEnergyInformatics/data-compressor +This repository contains the buildbot environment used for automated builds of the [Data Compressor](https://github.com/CenterForSecureEnergyInformatics/data-compressor) Subfolder contents: -- buildbot-buildmaster: the buildbot server configuration. -- raspberry-pi-workers: buildbot worker for some variants of the Raspberry Pi. -- windows-worker: buildbot worker for windows 10 +- [buildbot-buildmaster](buildbot-buildmaster): the buildbot server configuration. +- [raspberry-pi-workers](raspberry-pi-workers): buildbot worker for some variants of the Raspberry Pi. +- [windows-worker](windows-worker): buildbot worker for Windows 10 pro Please refer to the README files in the subfolders for setup instructions and information. diff --git a/buildbot-buildmaster/README.md b/buildbot-buildmaster/README.md index 37d453b..dd6bd54 100644 --- a/buildbot-buildmaster/README.md +++ b/buildbot-buildmaster/README.md @@ -1,30 +1,32 @@ # buildbot-buildmaster -This project contains dockerized services, serving a buildbot buildmaster to the public web over tls. -It is intended to be used on a linux server. +This project contains dockerized services, serving a buildbot buildmaster to the public WEB over TLS. +It is intended to be used on a Linux server. Currently, it runs under Debian Buster. ## Setup -1. clone this repository to your server. -2. install docker and docker-compose. -3. create a dedicated, unprivileged user for this project -4. add the user to the docker group -5. clone this repository to your server. You only need the folder containing this README. - - optional: create a branch for your secrets (git checkout -b secrets) -6. follow the setup instructions in the subfolders. Start with traefik and then buildbot. +1. Clone this repository to your server. +2. Install docker and docker-compose. +3. Create a dedicated, unprivileged user for this project +4. Add the user to the docker group +5. Clone this repository to your server. You only need the folder containing this README. + - Optional: create a branch for your secrets (git checkout -b secrets) +6. Follow the setup instructions in the subfolders. Start with traefik and then buildbot. This project comes mostly preconfigured. What you have to do - an overview (refer to the README files in the subfolders): -- changing passwords / specifying credentials (see buildbot/README.md) -- providing a certificate for tls connections (traefik/certs) -- adjusting the web url (buildbot/docker-compose.yml) -- creating a persistent data directory -- adjusting file permissions +- Changing passwords / specifying credentials (see [buildbot/README.md](buildbot/README.md)) +- Providing a certificate for TLS connections ([traefik/certs](traefik/certs)) +- Adjusting the WEB URL ([buildbot/docker-compose.yml](buildbot/docker-compose.yml)) +- Creating a persistent data directory (see [buildbot/README.md](buildbot/README.md)) +- Adjusting file permissions (traefik and buildbot secrets) ## Usage -- cd traefik -- docker-compose up -d -- cd ../buildbot -- docker-compose up -d + ``` + cd traefik + docker-compose up -d + cd ../buildbot + docker-compose up -d + ``` You might have to restart traefik again, when workers can't connect to the buildmaster. Typically the raspberry-pis have this problem. diff --git a/buildbot-buildmaster/buildbot/README.md b/buildbot-buildmaster/buildbot/README.md index 69ec36b..65c5b98 100644 --- a/buildbot-buildmaster/buildbot/README.md +++ b/buildbot-buildmaster/buildbot/README.md @@ -3,41 +3,42 @@ This directory contains the configuration of the buildbot buildmaster. The buildmaster listens to changes on the following repositories: - https://github.com/CenterForSecureEnergyInformatics/data-compressor (pull requests and branch master) -- https://github.com/brhat/checkBitSize.git (branch master) -- https://github.com/brhat/data-compressor-tests.git (branch master) +- https://github.com/CenterForSecureEnergyInformatics/data-compressor-tests (branch master) When changes are detected (or the force button in the web ui is pressed), the project is built and tested. ## Usage This project is dockerized and uses docker-compose. -The file docker-compose.yml tells docker-compose what to do, so you have to change into the directory containing the file, before executing any of these commands! +The file [docker-compose.yml](docker-compose.yml) tells docker-compose what to do, so you have to change into the directory containing the file, before executing any of these commands! ### Starting -- docker-compose up -d +- `docker-compose up -d` or to restart -- docker-compose restart +- `docker-compose restart` -Note: If the Raspberry Pis can't connect, simply go to ../traefik and run docker-compose restart from there, too. +Note: If the Raspberry Pis can't connect, simply go to `../traefik` and run docker-compose restart from there, too. ### Stopping - docker-compose down ### Updating Run the following steps in this order: -- docker-compose down -- docker-compose pull -- docker-compose build -- docker image prune -- docker-compose up -d +``` +docker-compose down +docker-compose pull +docker-compose build +docker image prune +docker-compose up -d +``` ### Debugging To view logs in realtime, run -- docker-compose logs -f +`docker-compose logs -f` -Exit with CRTL+C +Exit with `CRTL+C` ## How it works -Buildbot-workers are defined in master.cfg. +Buildbot-workers are defined in [master.cfg](master.cfg). The workers are connected via port 9989 to the buildmaster. Workers on different platforms are implemented in this setup: -- windows 10. Unfortunately, tls seems not to work on windows with buildbot, so the vm running the buildmaster has to share a subnet with the windows-vm (which it does in our setup). -- Raspberri Pi (1, 2b and 3b+), dockerized. They use a tls connection handled by the reverse proxy traefik. -- Linux, dockerized. See multiarch_dockerfile and docker-compose.yml for the definition. It runs on the same host as the buildmaster. +- Windows 10. Unfortunately, tls seems not to work on Windows with buildbot, so the VM running the buildmaster has to share a subnet with the Windows-VM (which it does in our setup). +- Raspberri Pi (1, 2b and 3b+), dockerized. They use a TLS connection handled by the reverse proxy traefik. +- Linux, dockerized. See [multiarch_dockerfile](multiarch_dockerfile) and [docker-compose.yml](docker-compose.yml) for the definition. It runs on the same host as the buildmaster. Buildfactories define which steps are to be executed on a build. For each platform/architecture a scheduler is defined in this configuration. @@ -45,27 +46,27 @@ For each platform/architecture a scheduler is defined in this configuration. Builders are assigned to jobs (defined by factories). Finally, schedulers trigger builds on the actual workers. -For a detailed overview, please refer to http://docs.buildbot.net/latest/manual/introduction.html +For a detailed overview, please refer to the [Buildbot Manual](http://docs.buildbot.net/latest/manual/introduction.html). ## Setup / Configuration -The file master.cfg is the main configuration file of buildbot, docker-compose.yml handles all services involved. +The file [master.cfg](master.cfg) is the main configuration file of buildbot, [docker-compose.yml](docker-compose.yml) handles all services involved. ### master.cfg Everything is pre-configured in this setup and does not require many changes. -Please refer to http://docs.buildbot.net/latest/manual/configuration/ for details, as a detailed explanation is beyond the scope of this README. -#### Credentials for the Web Ui +Please refer to [Buildbot Manual](http://docs.buildbot.net/latest/manual/introduction.html) for details, as a detailed explanation is beyond the scope of this README. +#### Credentials for the WEB UI To force builds, you need to be logged in. -In master.cfg, fill in your e-mail address(es) under util.RolesFromEmails(admins=["you@email.provider"]) +In [master.cfg](master.cfg), fill in your e-mail address(es) under `util.RolesFromEmails(admins=["you@email.provider"])` Make sure, that: -- your credentials are filled in secrets/htpasswd (only clear text, so keep permissions (600) in mind). -- the e-mail address is from a contributor in the github-project. +- Your credentials are filled in [secrets/htpasswd](secrets/htpasswd) (only clear text, so keep permissions (600) in mind). +- The e-mail address is from a contributor in the github-project. #### Workers -DO NOT CHANGE master.cfg IN THIS STEP! +DO NOT CHANGE [master.cfg](master.cfg) IN THIS STEP! All workers are pre-configured. Each of them has a name (do not change!) and a password (please change!), specified in .env files: -- ../windows/windows.env -- the .env files in the subfolders of ../rasperry-pi/ -- multiarch.env +- [../windows/windows.env](../windows/windows.env) +- The .env files in the subfolders of [../rasperry-pi/](../raspberry-pi) +- [multiarch.env](multiarch.env) Make sure to set these passwords on the corresponding workers as well! On the workers, the files are located in the same folders and files to make this step easier. @@ -73,8 +74,8 @@ On the workers, the files are located in the same folders and files to make this #### GitHubPullRequestPoller Important Note: for the GithubPullrequestPoller to work, the owner and repository name (NOT the URL) have to be provided. In this configuration, the following are used: -- compressorRepoName = 'data-compressor' -- compressorRepoOwner = 'CenterForSecureEnergyInformatics' +- `compressorRepoName = 'data-compressor'` +- `compressorRepoOwner = 'CenterForSecureEnergyInformatics'` ### docker-compose.yml The following services are specified here: @@ -84,13 +85,13 @@ The following services are specified here: #### Persistent Data Storage You have to create a directory for persistent storage for the database service. -- mkdir -p /data/buildbot/db +- `mkdir -p /data/buildbot/db` If you are unhappy with this location, you can specify another one in docker-compose.yml. To do so, modify the volume of the service "db" accordingly. -#### Web URL -If you aren't running this service under mendel.fh-salzburg.ac.at, you have to specify a different url in docker-compose.yml. +#### WEB URL +If you aren't running this service under mendel.fh-salzburg.ac.at, you have to specify a different URL in docker-compose.yml. You'll find this option in the labels of the service buildbot-buildmaster ### db.env Specify a database password. diff --git a/buildbot-buildmaster/buildbot/secrets/README.md b/buildbot-buildmaster/buildbot/secrets/README.md index 49f0b1a..6c7e032 100644 --- a/buildbot-buildmaster/buildbot/secrets/README.md +++ b/buildbot-buildmaster/buildbot/secrets/README.md @@ -2,17 +2,17 @@ This is the place for secrets. Be sure to set the correct file permissions! (chmod 600) ## htpasswd file -This file contains the login credentials for the users defined in `../master.cfg`. +[htpasswd](htpasswd) contains the login credentials for the users defined in [../master.cfg](../master.cfg) They are defined in the following form: `contributor@email.provider:cleartext-password` Note, that: -- passwords have to be set in clear text -- users have to be specified in `../master.cfg` -- the user has to be a contributor in the github-project +- Passwords have to be set in clear text +- Users have to be specified in [../master.cfg](../master.cfg) +- The user has to be a contributor in the github-project ## github access token -The access token used by buildbot to communicate with github. -You can define it in your github account. +[githubToken](githubToken) contains the access token used by buildbot to communicate with Github. +You can define it in your Github account. diff --git a/buildbot-buildmaster/traefik/README.md b/buildbot-buildmaster/traefik/README.md index edbec9c..84bb245 100644 --- a/buildbot-buildmaster/traefik/README.md +++ b/buildbot-buildmaster/traefik/README.md @@ -2,44 +2,46 @@ This folder contains the configuration for the reverse proxy in front of the buildbot buildmaster. It is based on the traefik project: https://containo.us/traefik/ -The proxy handles tls connections and automatically redirects from port 80 to port 443. -Furthermore, it listens on port 9989 and provides a tls connection for buildbot-workers to the buildmaster. +The proxy handles TLS connections and automatically redirects from port 80 to port 443. +Furthermore, it listens on port 9989 and provides a TLS connection for buildbot-workers to the buildmaster. The services behind traefik reside in private networks, only the ports needed are exposed to the internet. -Traefik needs access to the docker-socket: var/run/docker.sock. -Since this could result in vulnerabilities, the socket is made available over a socket-proxy (tecnativa/docker-socket-proxy), which runs as a service next to traefik. +Traefik needs access to the docker-socket: `var/run/docker.sock`. +Since this could result in vulnerabilities, the socket is made available over a socket-proxy (`tecnativa/docker-socket-proxy`), which runs as a service next to traefik. ## Usage This project is dockerized and uses docker-compose. The file docker-compose.yml tells docker-compose what to do, so you have to change into the directory containing the file, before executing any of these commands! ### Starting -- docker-compose up -d +- `docker-compose up -d` or to restart -- docker-compose restart +- `docker-compose restart` -Note: If the Raspberry Pis can't connect to the buildmaster, simply run docker-compose restart. +Note: If the Raspberry Pis can't connect to the buildmaster, simply run `docker-compose` restart. ### Stopping -- docker-compose down +- `docker-compose dow`n ### Updating Run the following steps in this order: -- docker-compose down -- docker-compose pull -- docker-compose build -- docker image prune -- docker-compose up -d +``` +docker-compose down +docker-compose pull +docker-compose build +docker image prune +docker-compose up -d +``` ### Debugging To view logs in realtime, run -- docker-compose logs -f +- `docker-compose logs -f` -Exit with CTRL+C +Exit with `CTRL+C` ## Setup / Configuration -Traefik has a static configuration (docker-compose.yml and traefik.yml) and a dynamic configuration (folders certs and dynamic) +Traefik has a static configuration ([`docker-compose.yml`](docker-compose.yml) and [`traefik.yml`](traefik.yml)) and a dynamic configuration (folders [`certs`](certs) and [`dynamic`](dynamic)) Everything is preconfigured, so no action is required. -The only thing to keep in mind is to renew the certificates, located in certs (please refer to the README in this folder). +The only thing to keep in mind is to renew the certificates, located in [`certs`](certs) (please refer to the [README](certs/README.md) in this folder). Traefik could handle renewals by itself when using letsencrypt, but in this setup we specified a certificate manually. -The services handled by traefik are configured via labels in the corresponding docker-compose.yml files, in our case ../buildbot/docker-compose.yml, so please have a look there +The services handled by traefik are configured via labels in the corresponding docker-compose.yml files, in our case [`../buildbot/docker-compose.yml`](../buildbot/docker-compose.yml), so please have a look there diff --git a/buildbot-buildmaster/traefik/certs/README.md b/buildbot-buildmaster/traefik/certs/README.md index 0bb679d..7dbadea 100644 --- a/buildbot-buildmaster/traefik/certs/README.md +++ b/buildbot-buildmaster/traefik/certs/README.md @@ -1,8 +1,9 @@ Place your cert files in this folder. Keep permissions (600) in mind! -To get the full chain, append DigiCertCA.crt to mendel_fh-salzburg_ac_at.crt +To get the full chain, append `DigiCertCA.crt` to `mendel_fh-salzburg_ac_at.crt`. +Obviously, you want to change filenames if you host on a different domain. To do so, run: -cat DigiCertCA.crt >> mendel_fh-salzburg_ac_at.crt +`cat DigiCertCA.crt >> mendel_fh-salzburg_ac_at.crt` diff --git a/raspberry-pi-workers/README.md b/raspberry-pi-workers/README.md index e73fa77..b564121 100644 --- a/raspberry-pi-workers/README.md +++ b/raspberry-pi-workers/README.md @@ -1,19 +1,19 @@ # Raspberry Pi workers -dockerized buildbot-workers for various versions of the Raspberry Pi +Dockerized buildbot-workers for various versions of the Raspberry Pi. ## Prerequisites -- a physical Rasperry Pi - - run `uname -m` to determine the architecture -- you have to have docker and docker-compose installed. -- add your unprivileged user to the docker group +- A physical Rasperry Pi + - Run `uname -m` to determine the architecture +- You have to have docker and docker-compose installed. +- Add your unprivileged user to the docker group ## Setup -1. clone / copy this repository to the Raspberry Pi - - (Optional) Create a branch for your configuration including secrets (git checkout -b secrets) -2. modify URL and port of the buildmaster in `buildmaster.env` +1. Clone / copy this repository to the Raspberry Pi + - (Optional) Create a branch for your configuration including secrets (`git checkout -b secrets`) +2. Modify URL and port of the buildmaster in [buildmaster.env](buildmaster.env) 4. `cd` into the folder fitting your model (run `uname -m` if you are unsure) - - modify the passphrase and worker name in the .env file - - make sure that the buildmaster is configured with the same values -5. run docker-compose up -d + - Modify the passphrase and worker name in the `.env` file + - Make sure that the buildmaster is configured with the same values +5. Run `docker-compose up -d` Note that docker-compose restarts the service automatically after reboots. From bf99c44b0d9eb1e7f1d8e44ace30cf69c220303d Mon Sep 17 00:00:00 2001 From: burs Date: Wed, 19 Aug 2020 20:05:58 +0200 Subject: [PATCH 12/13] traefik now uses Let's Encrypt --- buildbot-buildmaster/buildbot/README.md | 9 ++++-- .../buildbot/docker-compose.yml | 5 ++- buildbot-buildmaster/traefik/README.md | 31 ++++++++++++++----- buildbot-buildmaster/traefik/acme.json | 0 .../traefik/certs/DigiCertCA.crt | 1 - buildbot-buildmaster/traefik/certs/README.md | 9 ------ .../certs/mendel_fh-salzburg_ac_at.crt | 1 - .../certs/mendel_fh-salzburg_ac_at.key | 1 - .../traefik/docker-compose.yml | 17 +++++++++- buildbot-buildmaster/traefik/dynamic/tls.yml | 3 -- buildbot-buildmaster/traefik/traefik.yml | 10 ++++-- 11 files changed, 58 insertions(+), 29 deletions(-) create mode 100644 buildbot-buildmaster/traefik/acme.json delete mode 100644 buildbot-buildmaster/traefik/certs/DigiCertCA.crt delete mode 100644 buildbot-buildmaster/traefik/certs/README.md delete mode 100644 buildbot-buildmaster/traefik/certs/mendel_fh-salzburg_ac_at.crt delete mode 100644 buildbot-buildmaster/traefik/certs/mendel_fh-salzburg_ac_at.key diff --git a/buildbot-buildmaster/buildbot/README.md b/buildbot-buildmaster/buildbot/README.md index 65c5b98..aa611ed 100644 --- a/buildbot-buildmaster/buildbot/README.md +++ b/buildbot-buildmaster/buildbot/README.md @@ -91,7 +91,12 @@ If you are unhappy with this location, you can specify another one in docker-com To do so, modify the volume of the service "db" accordingly. #### WEB URL -If you aren't running this service under mendel.fh-salzburg.ac.at, you have to specify a different URL in docker-compose.yml. -You'll find this option in the labels of the service buildbot-buildmaster +If you aren't running this service under `mendel.fh-salzburg.ac.at`, you have to replace all occurences in [docker-compose.yml](docker-compose.yml) with a different URL. +You'll find them in the labels of the service buildbot-buildmaster. +### Subnet for non-TLS communication +Windows workers currently cannot connect to the buildmaster via TLS. +For this reason, the VM hosting the buildmaster and the Windows 10 VM running the worker share a subnet to ensure secure communication. +You have to provide the IP of the Subnet in [docker-compose.yml](docker-compose.yml). +To do so, replace `` with your the server's IP in that subnet. ### db.env Specify a database password. diff --git a/buildbot-buildmaster/buildbot/docker-compose.yml b/buildbot-buildmaster/buildbot/docker-compose.yml index 4640940..92b3dff 100644 --- a/buildbot-buildmaster/buildbot/docker-compose.yml +++ b/buildbot-buildmaster/buildbot/docker-compose.yml @@ -21,11 +21,13 @@ services: volumes: - ./secrets/:/var/lib/buildbot/secrets - ./master.cfg:/var/lib/buildbot/master.cfg + - ../traefik/dump/mendel.fh-salzburg.ac.at/:/var/lib/buildbot/certificate.pem:ro + - ../traefik/dump/mendel.fh-salzburg.ac.at/:/var/lib/buildbot/privateky.pem:ro networks: - proxy - buildbot ports: - - 10.10.41.44:9988:9989 + - :9989:9989 labels: - "traefik.enable=true" - "traefik.docker.network=proxy" @@ -49,6 +51,7 @@ services: - "traefik.tcp.routers.buildbot.tls=true" - "traefik.tcp.services.buildbot.loadbalancer.server.port=9989" + - "traefik.tcp.services.buildbot.loadbalancer.terminationDelay=42" db: env_file: - db.env diff --git a/buildbot-buildmaster/traefik/README.md b/buildbot-buildmaster/traefik/README.md index 84bb245..8aa5d1a 100644 --- a/buildbot-buildmaster/traefik/README.md +++ b/buildbot-buildmaster/traefik/README.md @@ -8,8 +8,9 @@ Furthermore, it listens on port 9989 and provides a TLS connection for buildbot- The services behind traefik reside in private networks, only the ports needed are exposed to the internet. Traefik needs access to the docker-socket: `var/run/docker.sock`. -Since this could result in vulnerabilities, the socket is made available over a socket-proxy (`tecnativa/docker-socket-proxy`), which runs as a service next to traefik. +Since this could result in vulnerabilities, the socket is made available over a socket-proxy (image: `tecnativa/docker-socket-proxy`), which runs as a service next to traefik. +The automatically obtained certs are extracted from [acme.json](acme.json) by the dockerized service `cert-dumper` (image: `ldez/traefik-certs-dumper:v2.7.0`) and stored in the folder [dynamic](dynamic). ## Usage This project is dockerized and uses docker-compose. The file docker-compose.yml tells docker-compose what to do, so you have to change into the directory containing the file, before executing any of these commands! @@ -21,7 +22,7 @@ or to restart Note: If the Raspberry Pis can't connect to the buildmaster, simply run `docker-compose` restart. ### Stopping -- `docker-compose dow`n +- `docker-compose down` ### Updating Run the following steps in this order: ``` @@ -38,10 +39,26 @@ To view logs in realtime, run Exit with `CTRL+C` ## Setup / Configuration -Traefik has a static configuration ([`docker-compose.yml`](docker-compose.yml) and [`traefik.yml`](traefik.yml)) and a dynamic configuration (folders [`certs`](certs) and [`dynamic`](dynamic)) -Everything is preconfigured, so no action is required. -The only thing to keep in mind is to renew the certificates, located in [`certs`](certs) (please refer to the [README](certs/README.md) in this folder). -Traefik could handle renewals by itself when using letsencrypt, but in this setup we specified a certificate manually. +Traefik has a static configuration ([docker-compose.yml](docker-compose.yml) and [traefik.yml](traefik.yml)) and a dynamic configuration ([acme.json](acme.json) and [dynamic](dynamic)). +Trafik handles certificate renewals automatically. +The certificates are stored in [acme.json](acme.json). +The service `cert-dumper` extracts and stores the certificates in the folder [dump](dump). -The services handled by traefik are configured via labels in the corresponding docker-compose.yml files, in our case [`../buildbot/docker-compose.yml`](../buildbot/docker-compose.yml), so please have a look there +### Steps: + +You have to change the file permissions for [acme.json](acme.json) and [dynamic](dynamic). +Todo so, run: +- `chmod 600 acme.json` +- `chmod 600 acme.json` + +For the `cert-dumper` to be able to create files under a username running the services, you have to add the following to your `~/.bashrc`: + +``` +export COMPOSE_UID=$(id -u) +export COMPOSE_GID=$(id -g) +``` + +### Configuration of services behind traefik: +The services handled by traefik are configured via labels in the corresponding docker-compose.yml files, in our case [../buildbot/docker-compose.yml](../buildbot/docker-compose.yml). +Please refer to [../buildbot/README.md](../buildbot/README.md) for setup instructions. diff --git a/buildbot-buildmaster/traefik/acme.json b/buildbot-buildmaster/traefik/acme.json new file mode 100644 index 0000000..e69de29 diff --git a/buildbot-buildmaster/traefik/certs/DigiCertCA.crt b/buildbot-buildmaster/traefik/certs/DigiCertCA.crt deleted file mode 100644 index 8b13789..0000000 --- a/buildbot-buildmaster/traefik/certs/DigiCertCA.crt +++ /dev/null @@ -1 +0,0 @@ - diff --git a/buildbot-buildmaster/traefik/certs/README.md b/buildbot-buildmaster/traefik/certs/README.md deleted file mode 100644 index 7dbadea..0000000 --- a/buildbot-buildmaster/traefik/certs/README.md +++ /dev/null @@ -1,9 +0,0 @@ -Place your cert files in this folder. Keep permissions (600) in mind! - -To get the full chain, append `DigiCertCA.crt` to `mendel_fh-salzburg_ac_at.crt`. -Obviously, you want to change filenames if you host on a different domain. - -To do so, run: - -`cat DigiCertCA.crt >> mendel_fh-salzburg_ac_at.crt` - diff --git a/buildbot-buildmaster/traefik/certs/mendel_fh-salzburg_ac_at.crt b/buildbot-buildmaster/traefik/certs/mendel_fh-salzburg_ac_at.crt deleted file mode 100644 index 8b13789..0000000 --- a/buildbot-buildmaster/traefik/certs/mendel_fh-salzburg_ac_at.crt +++ /dev/null @@ -1 +0,0 @@ - diff --git a/buildbot-buildmaster/traefik/certs/mendel_fh-salzburg_ac_at.key b/buildbot-buildmaster/traefik/certs/mendel_fh-salzburg_ac_at.key deleted file mode 100644 index 8b13789..0000000 --- a/buildbot-buildmaster/traefik/certs/mendel_fh-salzburg_ac_at.key +++ /dev/null @@ -1 +0,0 @@ - diff --git a/buildbot-buildmaster/traefik/docker-compose.yml b/buildbot-buildmaster/traefik/docker-compose.yml index 34aee60..6449e3e 100644 --- a/buildbot-buildmaster/traefik/docker-compose.yml +++ b/buildbot-buildmaster/traefik/docker-compose.yml @@ -11,6 +11,21 @@ services: CONTAINERS: 1 networks: - socketproxy + cert-dumper: + image: ldez/traefik-certs-dumper:v2.7.0 + restart: unless-stopped + container_name: cert-dumper + # add the following to your ~/.bashrc: + # export COMPOSE_UID=$(id -u) + # export COMPOSE_GID=$(id -g) + # OR add IDs directly to .env + user: ${COMPOSE_UID:?Please export COMPOSE_UID; see comments in docker-compose.yml}:${COMPOSE_GID:?Please export COMPOSE_UID; see comments in docker-compose.yml} + working_dir: $HOME + entrypoint: "/usr/bin/traefik-certs-dumper file --source $HOME/acme.json --dest $HOME/dump --domain-subdir --crt-ext=.pem --key-ext=.pem --version v2 --watch" + network_mode: "none" + volumes: + - ./acme.json:$HOME/acme.json:ro + - ./dump:$HOME/dump traefik: image: traefik:v2.0 @@ -29,7 +44,7 @@ services: - /etc/localtime:/etc/localtime:ro - ./traefik.yml:/traefik.yml:ro - ./dynamic/:/etc/traefik/conf/:ro - - ./certs/:/etc/traefik/certs:ro + - ./acme.json:/acme.json networks: socketproxy: external: true diff --git a/buildbot-buildmaster/traefik/dynamic/tls.yml b/buildbot-buildmaster/traefik/dynamic/tls.yml index 592a355..5c3da65 100644 --- a/buildbot-buildmaster/traefik/dynamic/tls.yml +++ b/buildbot-buildmaster/traefik/dynamic/tls.yml @@ -9,9 +9,6 @@ http: Strict-Transport-Security: "max-age=63072000" stsPreload: true tls: - certificates: - - certFile: /etc/traefik/certs/mendel_fh-salzburg_ac_at.crt - keyFile: /etc/traefik/certs/mendel_fh-salzburg_ac_at.key options: default: minVersion: "VersionTLS12" diff --git a/buildbot-buildmaster/traefik/traefik.yml b/buildbot-buildmaster/traefik/traefik.yml index 9ff77fc..ede20f8 100644 --- a/buildbot-buildmaster/traefik/traefik.yml +++ b/buildbot-buildmaster/traefik/traefik.yml @@ -1,6 +1,3 @@ -#api: -# dashboard: true - entryPoints: http: address: ":80" @@ -18,3 +15,10 @@ providers: file: directory: "/etc/traefik/conf/" +certificatesResolvers: + http: + acme: + email: sebastian.burkhart@fh-salzburg.ac.at + storage: acme.json + httpChallenge: + entryPoint: http From e30fa062960652f9774f3d06624a9f8b02fef2c6 Mon Sep 17 00:00:00 2001 From: bus Date: Mon, 19 Oct 2020 23:15:12 +0200 Subject: [PATCH 13/13] minor fixes for pull request --- buildbot-buildmaster/buildbot/docker-compose.yml | 1 - buildbot-buildmaster/traefik/traefik.yml | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/buildbot-buildmaster/buildbot/docker-compose.yml b/buildbot-buildmaster/buildbot/docker-compose.yml index 92b3dff..ee16f6a 100644 --- a/buildbot-buildmaster/buildbot/docker-compose.yml +++ b/buildbot-buildmaster/buildbot/docker-compose.yml @@ -51,7 +51,6 @@ services: - "traefik.tcp.routers.buildbot.tls=true" - "traefik.tcp.services.buildbot.loadbalancer.server.port=9989" - - "traefik.tcp.services.buildbot.loadbalancer.terminationDelay=42" db: env_file: - db.env diff --git a/buildbot-buildmaster/traefik/traefik.yml b/buildbot-buildmaster/traefik/traefik.yml index ede20f8..b284a82 100644 --- a/buildbot-buildmaster/traefik/traefik.yml +++ b/buildbot-buildmaster/traefik/traefik.yml @@ -18,7 +18,7 @@ providers: certificatesResolvers: http: acme: - email: sebastian.burkhart@fh-salzburg.ac.at + email: admin@email.address storage: acme.json httpChallenge: entryPoint: http