Add GitHub Action to Release All Images (#514)

Author: chatton

.evergreen.yml

@@ -39,15 +39,6 @@ functions:
         permissions: public-read
         content_type: text/plain
 
-  # This is a blocker for the release process. It will *always* fail and needs to be overriden
-  # if the release needs to proceed.
-  release_blocker:
-    - command: subprocess.exec
-      type: setup
-      params:
-        working_dir: mongodb-kubernetes-operator
-        binary: scripts/ci/release_blocker
-
   setup_kubernetes_environment:
     - command: subprocess.exec
       type: setup
@@ -125,20 +116,6 @@ functions:
         binary: scripts/ci/build_and_push_image_sonar.sh
 
 
-  add_supported_release:
-    - *python_venv
-    - *python_requirements
-    - command: subprocess.exec
-      type: setup
-      params:
-        working_dir: mongodb-kubernetes-operator
-        include_expansions_in_env:
-          - atlas_connection_string
-          - atlas_password
-          - atlas_database
-          - image_name
-        command: "${workdir}/venv/bin/python scripts/ci/add_supported_release.py --image ${image_name}"
-
 task_groups:
 - name: e2e_test_group
   max_hosts: 8
@@ -336,62 +313,6 @@ tasks:
         vars:
           test: replica_set_custom_role
 
-  - name: release_blocker
-    commands:
-      - func: clone
-      - func: release_blocker
-
-  - name: release_operator
-    commands:
-      - func: clone
-      - func: setup_virtualenv
-      - func: build_and_push_image_sonar
-        vars:
-          image_name: operator-ubi
-          release: true
-
-  - name: release_version_upgrade_post_start_hook
-    commands:
-      - func: clone
-      - func: setup_virtualenv
-      - func: build_and_push_image_sonar
-        vars:
-          image_name: version-post-start-hook-init
-          release: true
-
-
-  - name: release_readiness_probe
-    commands:
-      - func: clone
-      - func: setup_virtualenv
-      - func: build_and_push_image_sonar
-        vars:
-          image_name: readiness-probe-init
-          release: true
-
-  - name: release_agent_ubuntu
-    commands:
-      - func: clone
-      - func: setup_virtualenv
-      - func: build_and_push_image_sonar
-        vars:
-          image_name: agent-ubuntu
-          release: true
-      - func: add_supported_release
-        vars:
-          image_name: mongodb-agent
-
-  - name: release_agent_ubi
-    commands:
-      - func: clone
-      - func: setup_virtualenv
-      - func: build_and_push_image_sonar
-        vars:
-          image_name: agent-ubi
-          release: true
-      - func: add_supported_release
-        vars:
-          image_name: mongodb-agent
 
 buildvariants:
   - name: e2e_tests_ubuntu
@@ -445,32 +366,3 @@ buildvariants:
       - name: build_agent_image_ubi
       - name: build_agent_image_ubuntu
       - name: build_readiness_probe_image
-
-  - name: release_blocker
-    display_name: release_blocker
-    run_on:
-      - ubuntu1604-packer  # Note: cheapest machine I found
-    tasks:
-      - name: release_blocker
-
-  - name: release_images_quay
-    display_name: release_images_quay
-    depends_on:
-      - name: release_blocker
-        variant: release_blocker
-      - name: build_operator_image
-        variant: init_test_run
-      - name: build_prehook_image
-        variant: init_test_run
-      - name: build_agent_image_ubuntu
-        variant: init_test_run
-      - name: build_agent_image_ubi
-        variant: init_test_run
-    run_on:
-      - ubuntu1804-small
-    tasks:
-    - name: release_operator
-    - name: release_version_upgrade_post_start_hook
-    - name: release_readiness_probe
-    - name: release_agent_ubuntu
-    - name: release_agent_ubi

.github/workflows/code_health.yml renamed to .github/workflows/code-health.yml

@@ -0,0 +1,92 @@
+name: Release Images
+
+on:
+  pull_request_review:
+    types: [submitted]
+
+jobs:
+  release-images:
+    runs-on: ubuntu-latest
+    if: startsWith(github.event.pull_request.title, 'Release MongoDB Kubernetes Operator') && github.event.review.state == 'approved'
+    strategy:
+      matrix:
+        include:
+          - pipeline-argument: operator-ubi
+            release-key: mongodb-kubernetes-operator
+          - pipeline-argument: version-post-start-hook-init
+            release-key: version-upgrade-hook
+          - pipeline-argument: readiness-probe-init
+            release-key: readiness-probe
+          - pipeline-argument: agent-ubi
+            release-key: mongodb-agent
+          - pipeline-argument: agent-ubuntu
+            release-key: mongodb-agent
+    steps:
+      - name: Checkout Code
+        uses: actions/checkout@v2
+      - name: Setup Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: '3.8'
+          architecture: 'x64'
+
+      - uses: actions/cache@v2
+        with:
+          path: ~/.cache/pip
+          key: ${{ hashFiles('requirements.txt') }}
+
+      - name: Install Python Dependencies
+        run: pip install -r requirements.txt
+      - name: Determine if release is needed
+        id: release_status
+        run: |
+          OUTPUT=$(scripts/ci/determine_required_releases.py ${{ matrix.release-key }})
+          echo "::set-output name=OUTPUT::$OUTPUT"
+
+      - name: Login to Quay.io
+        uses: docker/login-action@v1
+        with:
+          registry: quay.io
+          username: ${{ secrets.QUAY_USERNAME }}
+          password: ${{ secrets.QUAY_ROBOT_TOKEN }}
+
+      - name: Publish Image To Quay
+        if: steps.release_status.outputs.OUTPUT == 'unreleased'
+        run: python pipeline.py --image-name ${{ matrix.pipeline-argument }} --release true
+        env:
+          MONGODB_COMMUNITY_CONFIG: "${{ github.workspace }}/scripts/ci/config.json"
+
+      - name: Add Supported Release
+        if: steps.release_status.outputs.OUTPUT == 'unreleased'
+        run: python scripts/ci/add_supported_release.py --image-name ${{ matrix.release-key }}
+        env:
+          ATLAS_DATABASE: "${{ secrets.ATLAS_DATABASE }}"
+          ATLAS_CONNECTION_STRING: "${{ secrets.ATLAS_CONNECTION_STRING }}"
+
+  merge-release-pr-and-draft-github-release:
+    runs-on: ubuntu-latest
+    needs: [release-images]
+    steps:
+    - name: Checkout Code
+      uses: actions/checkout@v2
+    - name: Determine Release Tag
+      id: release_tag
+      run: |
+        OUTPUT=$(jq -r '."mongodb-kubernetes-operator"' < $GITHUB_WORKSPACE/release.json)
+        echo "::set-output name=OUTPUT::$OUTPUT"
+    - name: Merge Release PR
+      uses: pascalgn/automerge-action@v0.14.2
+      env:
+        GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
+        MERGE_LABELS: ""
+        MERGE_DELETE_BRANCH: true
+        MERGE_METHOD: squash
+        MERGE_COMMIT_MESSAGE: "Release MongoDB Kubernetes Operator v${{ steps.release_tag.outputs.OUTPUT }}"
+    - name: Create Github Release
+      uses: ncipollo/release-action@v1
+      with:
+        tag: ${{ steps.release_tag.outputs.OUTPUT }}
+        name: MongoDB Kubernetes Operator
+        bodyFile: "${{ github.workspace }}/docs/RELEASE_NOTES.md"
+        draft: true
+        token: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/create_release_pr.yml renamed to .github/workflows/create-release-pr.yml

@@ -1,17 +1,17 @@
 
 ## How to Release
 
-* Update any finished tickets in [kube-community-next](https://jira.mongodb.org/issues?jql=project%20%3D%20CLOUDP%20AND%20component%20%3D%20%22Kubernetes%20Community%22%20%20AND%20status%20in%20(Resolved%2C%20Closed)%20and%20fixVersion%3D%20kube-community-next%20%20ORDER%20BY%20resolved) to have the version of the release you're doing (kube-community-x.y)
+* Ensure the versions specified in [release.json](../release.json) are correct. 
+    * Prepare a PR to bump and versions as required. 
 
-* Prepare the release PR
-  1. Increment any image version changes.
-  2. Create a github draft release `./scripts/dev/create_github_release.sh`.
-  3. Commit changes.
-  
-* Create release PR
-  1. Reconfigure the Evergreen run to add the relevant release task(s).
+* Ensure that [the release notes](./RELEASE_NOTES.md) are up to date for this release.
+    * Review the [tickets for this release](https://jira.mongodb.org/issues?jql=project%20%3D%20CLOUDP%20AND%20component%20%20%3D%20"Kubernetes%20Community"%20%20AND%20status%20in%20(Resolved%2C%20Closed)%20AND%20fixVersion%20%3D%20kube-community-0.6.0%20) (ensure relevant fix version is in the jql query)
 
+* Run the `Create Release PR` GitHub Action
+    * In the GitHub UI:
+        * `Actions` > `Create Release PR` > `Run Workflow` (on master)
+        
+* Review and Approve the release PR that is created by this action.
+    * Upon approval, all new images for this release will be built and released, and a Github release draft will be created.
 
-* Unblock release task once everything is green
-
-Once the images are released, merge release PR & publish github release
+* Review and publish the new GitHub release draft.

.github/workflows/release-images.yml

@@ -1,5 +1,5 @@
 {
-  "mongodb-kubernetes-operator": "0.6.0",
+  "mongodb-kubernetes-operator": "0.6.1",
   "version-upgrade-hook": "1.0.2",
   "readiness-probe": "1.0.4",
   "mongodb-agent": {

docs/how-to-release.md

@@ -9,5 +9,6 @@ tqdm==v4.49.0
 boto3==1.16.21
 pymongo==3.11.4
 dnspython==2.0.0
+requests==2.24.0
 pyyaml==5.4.1
 rsa>=4.7 # not directly required, pinned by Snyk to avoid a vulnerability

release.json

@@ -30,10 +30,7 @@ def get_release() -> Dict[str, Any]:
 
 
 def get_atlas_connection_string() -> str:
-    password = os.environ["atlas_password"]
-    cnx_str = os.environ["atlas_connection_string"]
-
-    return cnx_str.format(password=password)
+    return os.environ["ATLAS_CONNECTION_STRING"]
 
 
 def mongo_client() -> pymongo.MongoClient:
@@ -44,7 +41,7 @@ def mongo_client() -> pymongo.MongoClient:
 def add_release_version(image: str, version: str) -> None:
     client = mongo_client()
 
-    database = os.environ["atlas_database"]
+    database = os.environ["ATLAS_DATABASE"]
     collection = client[database][image]
 
     year_from_now = datetime.datetime.now() + datetime.timedelta(days=365)
@@ -72,18 +69,23 @@ def add_release_version(image: str, version: str) -> None:
 def main() -> int:
     parser = argparse.ArgumentParser()
     parser.add_argument(
-        "--image", help="image to add a new supported version", type=str
+        "--image-name", help="image to add a new supported version", type=str
     )
     args = parser.parse_args()
 
-    if args.image not in VALID_IMAGES:
-        raise ValueError("Image {} not supported".format(args.image))
+    if args.image_name not in VALID_IMAGES:
+        print(
+            "Image {} not supported. Not adding release version.".format(
+                args.image_name
+            )
+        )
+        return 0
 
     # for now, there is just one version to add as a supported release.
-    version = get_release()[args.image]["version"]
-    logging.info("Adding new release: {} {}".format(args.image, version))
+    version = get_release()[args.image_name]["version"]
+    logging.info("Adding new release: {} {}".format(args.image_name, version))
 
-    add_release_version(args.image, version)
+    add_release_version(args.image_name, version)
 
     return 0
 

requirements.txt

@@ -0,0 +1,62 @@
+#!/usr/bin/env python3
+
+# This script accepts a key from the "release.json" file.
+# If the corresponding image of the specified version has been released,
+
+import json
+import sys
+from typing import List, Dict
+
+import requests
+
+# contains a map of the quay urls to fetch data about the corresponding images.
+QUAY_URL_MAP = {
+    "mongodb-agent": "https://quay.io/api/v1/repository/mongodb/mongodb-agent-ubi",
+    "readiness-probe": "https://quay.io/api/v1/repository/mongodb/mongodb-kubernetes-readinessprobe",
+    "version-upgrade-hook": "https://quay.io/api/v1/repository/mongodb/mongodb-kubernetes-operator-version-upgrade-post-start-hook",
+    "mongodb-kubernetes-operator": "https://quay.io/api/v1/repository/mongodb/mongodb-kubernetes-operator",
+}
+
+
+def _get_all_released_tags(image_type: str) -> List[str]:
+    url = QUAY_URL_MAP[image_type]
+    resp = requests.get(url).json()
+    tags = resp["tags"]
+    return list(tags.keys())
+
+
+def _load_image_name_to_version_map() -> Dict:
+    with open("release.json") as f:
+        release = json.loads(f.read())
+
+    # agent section is a sub object, we change the mapping so the key corresponds to the version directly.
+    release["mongodb-agent"] = release["mongodb-agent"]["version"]
+    return release
+
+
+def main() -> int:
+    if len(sys.argv) != 2:
+        raise ValueError("usage: determine_required_releases.py [image-name]")
+    image_name = sys.argv[1]
+    image_name_map = _load_image_name_to_version_map()
+
+    if image_name not in image_name_map:
+        raise ValueError(
+            "Unknown image type [{}], valid values are [{}]".format(
+                image_name, ",".join(image_name_map.keys())
+            )
+        )
+
+    if image_name not in QUAY_URL_MAP:
+        raise ValueError("No associated image url with key [{}]".format(image_name))
+
+    tags = _get_all_released_tags(image_name)
+    if image_name_map[image_name] in tags:
+        print("released")
+    else:
+        print("unreleased")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())