Add support for macos arm64 build (#1770)

umbynos · web-flow · commit 0de6c37b09a5 · 2022-06-22T16:14:02.000+02:00
* add support for macos arm64 build in the taskfile

* update workflows to handle darwin_arm64 notarization

- the gon config is now hardcoded in the workflow (it allows customization)
- the notarization step is run in parallel now
- the updated checksum is passed to the `create-release` job
- the `create-release` job handles the checksum update in `checksum.txt` file

* use working-directory, this should simplify a bit the workflow

* we cannot upload all the artifacts since the job is being parallelized

* Apply suggestions from code review

Co-authored-by: per1234 &lt;accounts@perglass.com&gt;

* add macos arm 64 download links in `installation.md`
diff --git a/.github/workflows/publish-go-nightly-task.yml b/.github/workflows/publish-go-nightly-task.yml
@@ -45,8 +45,23 @@ jobs:
           path: ${{ env.DIST_DIR }}
 
   notarize-macos:
+    name: Notarize ${{ matrix.artifact.name }}
     runs-on: macos-latest
     needs: create-nightly-artifacts
+    outputs:
+      checksum-darwin_amd64: ${{ steps.re-package.outputs.checksum-darwin_amd64 }}
+      checksum-darwin_arm64: ${{ steps.re-package.outputs.checksum-darwin_arm64 }}
+
+    env:
+      GON_CONFIG_PATH: gon.config.hcl
+
+    strategy:
+      matrix:
+        artifact:
+          - name: darwin_amd64
+            path: "macOS_64bit.tar.gz"
+          - name: darwin_arm64
+            path: "macOS_ARM64.tar.gz"
 
     steps:
       - name: Checkout repository
@@ -86,34 +101,58 @@ jobs:
           wget -q https://github.com/mitchellh/gon/releases/download/v0.2.3/gon_macos.zip
           unzip gon_macos.zip -d /usr/local/bin
 
+      - name: Write gon config to file
+        # gon does not allow env variables in config file (https://github.com/mitchellh/gon/issues/20)
+        run: |
+          cat > "${{ env.GON_CONFIG_PATH }}" <<EOF
+          # See: https://github.com/mitchellh/gon#configuration-file
+          source = ["dist/arduino-cli_osx_${{ matrix.artifact.name }}/arduino-cli"]
+          bundle_id = "cc.arduino.arduino-cli"
+
+          sign {
+            application_identity = "Developer ID Application: ARDUINO SA (7KT7ZWMCJT)"
+          }
+
+          # Ask Gon for zip output to force notarization process to take place.
+          # The CI will ignore the zip output, using the signed binary only.
+          zip {
+            output_path = "unused.zip"
+          }
+          EOF
+
       - name: Sign and notarize binary
         env:
           AC_USERNAME: ${{ secrets.AC_USERNAME }}
           AC_PASSWORD: ${{ secrets.AC_PASSWORD }}
         run: |
-          gon gon.config.hcl
+          gon "${{ env.GON_CONFIG_PATH }}"
 
-      - name: Re-package binary and update checksum
+      - name: Re-package binary and output checksum
+        id: re-package
+        working-directory: ${{ env.DIST_DIR }}
         # This step performs the following:
         # 1. Repackage the signed binary replaced in place by Gon (ignoring the output zip file)
-        # 2. Recalculate package checksum and replace it in the nnnnnn-checksums.txt file
+        # 2. Recalculate package checksum
+        # 3. Output the new checksum to include in the nnnnnn-checksums.txt file
+        #    (it cannot be done there because of workflow job parallelization)
         run: |
           # GitHub's upload/download-artifact@v2 actions don't preserve file permissions,
           # so we need to add execution permission back until the action is made to do this.
-          chmod +x "${{ env.DIST_DIR }}/${{ env.PROJECT_NAME }}_osx_darwin_amd64/${{ env.PROJECT_NAME }}"
-          PACKAGE_FILENAME="$(basename ${{ env.DIST_DIR }}/${{ env.PROJECT_NAME }}_nightly-*_macOS_64bit.tar.gz)"
-          tar -czvf "${{ env.DIST_DIR }}/$PACKAGE_FILENAME" \
-          -C "${{ env.DIST_DIR }}/${{ env.PROJECT_NAME }}_osx_darwin_amd64/" "${{ env.PROJECT_NAME }}" \
+          chmod +x "${{ env.PROJECT_NAME }}_osx_${{ matrix.artifact.name }}/${{ env.PROJECT_NAME }}"
+          PACKAGE_FILENAME="${{ env.PROJECT_NAME }}_nightly-*${{ matrix.artifact.path }}"
+          tar -czvf "$PACKAGE_FILENAME" \
+          -C "${{ env.PROJECT_NAME }}_osx_${{ matrix.artifact.name }}/" "${{ env.PROJECT_NAME }}" \
           -C ../../ LICENSE.txt
-          CHECKSUM="$(shasum -a 256 ${{ env.DIST_DIR }}/$PACKAGE_FILENAME | cut -d " " -f 1)"
-          perl -pi -w -e "s/.*${PACKAGE_FILENAME}/${CHECKSUM} ${PACKAGE_FILENAME}/g;" ${{ env.DIST_DIR }}/*-checksums.txt
+          CHECKSUM_LINE="$(shasum -a 256 $PACKAGE_FILENAME)"
+          echo "PACKAGE_FILENAME=$PACKAGE_FILENAME" >> $GITHUB_ENV
+          echo "::set-output name=checksum-${{ matrix.artifact.name }}::$CHECKSUM_LINE"
 
       - name: Upload artifacts
         uses: actions/upload-artifact@v3
         with:
           if-no-files-found: error
           name: ${{ env.ARTIFACT_NAME }}
-          path: ${{ env.DIST_DIR }}
+          path: ${{ env.DIST_DIR }}/${{ env.PACKAGE_FILENAME }}
 
   publish-nightly:
     runs-on: ubuntu-latest
@@ -126,6 +165,16 @@ jobs:
           name: ${{ env.ARTIFACT_NAME }}
           path: ${{ env.DIST_DIR }}
 
+      - name: Update checksum
+        run: |
+          declare -a checksum_lines=("${{ needs.notarize-macos.outputs.checksum-darwin_amd64 }}" "${{ needs.notarize-macos.outputs.checksum-darwin_arm64 }}")
+          for checksum_line in "${checksum_lines[@]}"
+          do
+            CHECKSUM=$(echo ${checksum_line} | cut -d " " -f 1)
+            PACKAGE_FILENAME=$(echo ${checksum_line} | cut -d " " -f 2)
+            perl -pi -w -e "s/.*${PACKAGE_FILENAME}/${CHECKSUM}  ${PACKAGE_FILENAME}/g;" ${{ env.DIST_DIR }}/*-checksums.txt
+          done
+
       - name: Upload release files on Arduino downloads servers
         uses: docker://plugins/s3
         env:
diff --git a/.github/workflows/publish-go-tester-task.yml b/.github/workflows/publish-go-tester-task.yml
@@ -108,6 +108,8 @@ jobs:
             name: Linux_ARMv7
           - path: "*macOS_64bit.tar.gz"
             name: macOS_64
+          - path: "*macOS_ARM64.tar.gz"
+            name: macOS_ARM64
           - path: "*Windows_32bit.zip"
             name: Windows_X86-32
           - path: "*Windows_64bit.zip"
diff --git a/.github/workflows/release-go-task.yml b/.github/workflows/release-go-task.yml
@@ -50,8 +50,23 @@ jobs:
           path: ${{ env.DIST_DIR }}
 
   notarize-macos:
+    name: Notarize ${{ matrix.artifact.name }}
     runs-on: macos-latest
     needs: create-release-artifacts
+    outputs:
+      checksum-darwin_amd64: ${{ steps.re-package.outputs.checksum-darwin_amd64 }}
+      checksum-darwin_arm64: ${{ steps.re-package.outputs.checksum-darwin_arm64 }}
+
+    env:
+      GON_CONFIG_PATH: gon.config.hcl
+
+    strategy:
+      matrix:
+        artifact:
+          - name: darwin_amd64
+            path: "macOS_64bit.tar.gz"
+          - name: darwin_arm64
+            path: "macOS_ARM64.tar.gz"
 
     steps:
       - name: Checkout repository
@@ -91,34 +106,59 @@ jobs:
           wget -q https://github.com/mitchellh/gon/releases/download/v0.2.3/gon_macos.zip
           unzip gon_macos.zip -d /usr/local/bin
 
+      - name: Write gon config to file
+        # gon does not allow env variables in config file (https://github.com/mitchellh/gon/issues/20)
+        run: |
+          cat > "${{ env.GON_CONFIG_PATH }}" <<EOF
+          # See: https://github.com/mitchellh/gon#configuration-file
+          source = ["dist/arduino-cli_osx_${{ matrix.artifact.name }}/arduino-cli"]
+          bundle_id = "cc.arduino.arduino-cli"
+
+          sign {
+            application_identity = "Developer ID Application: ARDUINO SA (7KT7ZWMCJT)"
+          }
+
+          # Ask Gon for zip output to force notarization process to take place.
+          # The CI will ignore the zip output, using the signed binary only.
+          zip {
+            output_path = "unused.zip"
+          }
+          EOF
+
       - name: Sign and notarize binary
         env:
           AC_USERNAME: ${{ secrets.AC_USERNAME }}
           AC_PASSWORD: ${{ secrets.AC_PASSWORD }}
         run: |
-          gon gon.config.hcl
+          gon "${{ env.GON_CONFIG_PATH }}"
 
-      - name: Re-package binary and update checksum
+      - name: Re-package binary and output checksum
+        id: re-package
+        working-directory: ${{ env.DIST_DIR }}
         # This step performs the following:
         # 1. Repackage the signed binary replaced in place by Gon (ignoring the output zip file)
-        # 2. Recalculate package checksum and replace it in the nnnnnn-checksums.txt file
+        # 2. Recalculate package checksum
+        # 3. Output the new checksum to include in the nnnnnn-checksums.txt file
+        #    (it cannot be done there because of workflow job parallelization)
         run: |
           # GitHub's upload/download-artifact@v2 actions don't preserve file permissions,
           # so we need to add execution permission back until the action is made to do this.
-          chmod +x ${{ env.DIST_DIR }}/${{ env.PROJECT_NAME }}_osx_darwin_amd64/${{ env.PROJECT_NAME }}
+          chmod +x "${{ env.PROJECT_NAME }}_osx_${{ matrix.artifact.name }}/${{ env.PROJECT_NAME }}"
           TAG="${GITHUB_REF/refs\/tags\//}"
-          tar -czvf "${{ env.DIST_DIR }}/${{ env.PROJECT_NAME }}_${TAG}_macOS_64bit.tar.gz" \
-          -C ${{ env.DIST_DIR }}/${{ env.PROJECT_NAME }}_osx_darwin_amd64/  ${{ env.PROJECT_NAME }}   \
+          PACKAGE_FILENAME="${{ env.PROJECT_NAME }}_${TAG}_${{ matrix.artifact.path }}"
+          tar -czvf "$PACKAGE_FILENAME" \
+          -C "${{ env.PROJECT_NAME }}_osx_${{ matrix.artifact.name }}/" "${{ env.PROJECT_NAME }}" \
           -C ../../ LICENSE.txt
-          CHECKSUM="$(shasum -a 256 ${{ env.DIST_DIR }}/${{ env.PROJECT_NAME }}_${TAG}_macOS_64bit.tar.gz | cut -d " " -f 1)"
-          perl -pi -w -e "s/.*${{ env.PROJECT_NAME }}_${TAG}_macOS_64bit.tar.gz/${CHECKSUM}  ${{ env.PROJECT_NAME }}_${TAG}_macOS_64bit.tar.gz/g;" ${{ env.DIST_DIR }}/*-checksums.txt
+          CHECKSUM_LINE="$(shasum -a 256 $PACKAGE_FILENAME)"
+          echo "PACKAGE_FILENAME=$PACKAGE_FILENAME" >> $GITHUB_ENV
+          echo "::set-output name=checksum-${{ matrix.artifact.name }}::$CHECKSUM_LINE"
 
       - name: Upload artifacts
         uses: actions/upload-artifact@v3
         with:
           if-no-files-found: error
           name: ${{ env.ARTIFACT_NAME }}
-          path: ${{ env.DIST_DIR }}
+          path: ${{ env.DIST_DIR }}/${{ env.PACKAGE_FILENAME }}
 
   create-release:
     runs-on: ubuntu-latest
@@ -131,6 +171,16 @@ jobs:
           name: ${{ env.ARTIFACT_NAME }}
           path: ${{ env.DIST_DIR }}
 
+      - name: Update checksum
+        run: |
+          declare -a checksum_lines=("${{ needs.notarize-macos.outputs.checksum-darwin_amd64 }}" "${{ needs.notarize-macos.outputs.checksum-darwin_arm64 }}")
+          for checksum_line in "${checksum_lines[@]}"
+          do
+            CHECKSUM=$(echo ${checksum_line} | cut -d " " -f 1)
+            PACKAGE_FILENAME=$(echo ${checksum_line} | cut -d " " -f 2)
+            perl -pi -w -e "s/.*${PACKAGE_FILENAME}/${CHECKSUM}  ${PACKAGE_FILENAME}/g;" ${{ env.DIST_DIR }}/*-checksums.txt
+          done
+
       - name: Identify Prerelease
         # This is a workaround while waiting for create-release action
         # to implement auto pre-release based on tag
diff --git a/DistTasks.yml b/DistTasks.yml
@@ -34,6 +34,7 @@ tasks:
       - task: Linux_ARMv7
       - task: Linux_ARM64
       - task: macOS_64bit
+      - task: macOS_ARM64
 
   Windows_32bit:
     desc: Builds Windows 32 bit binaries
@@ -251,3 +252,25 @@ tasks:
       CONTAINER_TAG: "{{.GO_VERSION}}-darwin-debian10"
       PACKAGE_PLATFORM: "macOS_64bit"
       PACKAGE_NAME: "{{.PROJECT_NAME}}_{{.VERSION}}_{{.PACKAGE_PLATFORM}}.tar.gz"
+
+  macOS_ARM64:
+    desc: Builds Mac OS X ARM64 binaries
+    dir: "{{.DIST_DIR}}"
+    cmds:
+      - |
+        docker run -v `pwd`/..:/home/build -w /home/build \
+        -e CGO_ENABLED=1 \
+        {{.CONTAINER}}:{{.CONTAINER_TAG}} \
+        --build-cmd "{{.BUILD_COMMAND}}" \
+        -p "{{.BUILD_PLATFORM}}"
+
+        tar cz -C {{.PLATFORM_DIR}} {{.PROJECT_NAME}} -C ../.. LICENSE.txt  -f {{.PACKAGE_NAME}}
+        sha256sum {{.PACKAGE_NAME}} >> {{.CHECKSUM_FILE}}
+
+    vars:
+      PLATFORM_DIR: "{{.PROJECT_NAME}}_osx_darwin_arm64"
+      BUILD_COMMAND: "go build -o {{.DIST_DIR}}/{{.PLATFORM_DIR}}/{{.PROJECT_NAME}} {{.LDFLAGS}}"
+      BUILD_PLATFORM: "darwin/arm64"
+      CONTAINER_TAG: "{{.GO_VERSION}}-darwin-arm64-debian10"
+      PACKAGE_PLATFORM: "macOS_ARM64"
+      PACKAGE_NAME: "{{.PROJECT_NAME}}_{{.VERSION}}_{{.PACKAGE_PLATFORM}}.tar.gz"
diff --git a/docs/installation.md b/docs/installation.md
@@ -63,15 +63,17 @@ in your [`PATH`](https://en.wikipedia.org/wiki/PATH%5F%28variable%29) or add the
 | Linux     | [32 bit][linux32]    | [64 bit][linux64]    |
 | Linux ARM | [32 bit][linuxarm32] | [64 bit][linuxarm64] |
 | Windows   | [32 bit][windows32]  | [64 bit][windows64]  |
-| macOS     |                      | [64 bit][macos]      |
+| macOS     |                      | [64 bit][macos64]    |
+| macOS ARM |                      | [64 bit][macosarm64] |
 
 [linux64]: https://downloads.arduino.cc/arduino-cli/arduino-cli_latest_Linux_64bit.tar.gz
 [linux32]: https://downloads.arduino.cc/arduino-cli/arduino-cli_latest_Linux_32bit.tar.gz
 [linuxarm64]: https://downloads.arduino.cc/arduino-cli/arduino-cli_latest_Linux_ARM64.tar.gz
 [linuxarm32]: https://downloads.arduino.cc/arduino-cli/arduino-cli_latest_Linux_ARMv7.tar.gz
 [windows64]: https://downloads.arduino.cc/arduino-cli/arduino-cli_latest_Windows_64bit.zip
 [windows32]: https://downloads.arduino.cc/arduino-cli/arduino-cli_latest_Windows_32bit.zip
-[macos]: https://downloads.arduino.cc/arduino-cli/arduino-cli_latest_macOS_64bit.tar.gz
+[macos64]: https://downloads.arduino.cc/arduino-cli/arduino-cli_latest_macOS_64bit.tar.gz
+[macosarm64]: https://downloads.arduino.cc/arduino-cli/arduino-cli_latest_macOS_ARM64.tar.gz
 
 > **Deprecation notice**: links in the form
 > `http://downloads.arduino.cc/arduino-cli/arduino-cli-latest-<platform>.tar.bz2` won’t be further updated. That URL
@@ -91,15 +93,17 @@ to get the latest nightly build available for the supported platform, use the fo
 | Linux     | [32 bit][linux32-nightly]    | [64 bit][linux64-nightly]    |
 | Linux ARM | [32 bit][linuxarm32-nightly] | [64 bit][linuxarm64-nightly] |
 | Windows   | [32 bit][windows32-nightly]  | [64 bit][windows64-nightly]  |
-| macOS     |                              | [64 bit][macos-nightly]      |
+| macOS     |                              | [64 bit][macos64-nightly]    |
+| macOS ARM |                              | [64 bit][macosarm64-nightly] |
 
 [linux64-nightly]: https://downloads.arduino.cc/arduino-cli/nightly/arduino-cli_nightly-latest_Linux_64bit.tar.gz
 [linux32-nightly]: https://downloads.arduino.cc/arduino-cli/nightly/arduino-cli_nightly-latest_Linux_32bit.tar.gz
 [linuxarm64-nightly]: https://downloads.arduino.cc/arduino-cli/nightly/arduino-cli_nightly-latest_Linux_ARM64.tar.gz
 [linuxarm32-nightly]: https://downloads.arduino.cc/arduino-cli/nightly/arduino-cli_nightly-latest_Linux_ARMv7.tar.gz
 [windows64-nightly]: https://downloads.arduino.cc/arduino-cli/nightly/arduino-cli_nightly-latest_Windows_64bit.zip
 [windows32-nightly]: https://downloads.arduino.cc/arduino-cli/nightly/arduino-cli_nightly-latest_Windows_32bit.zip
-[macos-nightly]: https://downloads.arduino.cc/arduino-cli/nightly/arduino-cli_nightly-latest_macOS_64bit.tar.gz
+[macos64-nightly]: https://downloads.arduino.cc/arduino-cli/nightly/arduino-cli_nightly-latest_macOS_64bit.tar.gz
+[macosarm64-nightly]: https://downloads.arduino.cc/arduino-cli/nightly/arduino-cli_nightly-latest_macOS_ARM64.tar.gz
 
 > These links return a `302: Found` response, redirecting to latest generated builds by replacing `latest` with the
 > latest available build date, using the format YYYYMMDD (i.e for 2019-08-06 `latest` is replaced with `20190806` )
diff --git a/gon.config.hcl b/gon.config.hcl
