Merge pull request #158 from lightninglabs/docker-arm

docker: refactor build process to support building ARM images

Author: Roasbeef

.gitattributes

@@ -0,0 +1,5 @@
+# Instruct the GitHub linguist tool to interpret the generated JS code as just
+# that, generated. This will make sure the JS (and TS types for that matter)
+# will not show up in the repository's language statistics.
+
+app/src/types/generated* linguist-generated

.github/workflows/docker.yml

@@ -38,6 +38,7 @@ jobs:
         uses: docker/build-push-action@v2
         with:
           push: true
+          platforms: linux/amd64,linux/arm/v7,linux/arm64
           tags: "${{ env.DOCKER_REPO }}/${{ env.DOCKER_IMAGE }}:${{ env.RELEASE_VERSION }}"
           build-args: checkout=${{ env.RELEASE_VERSION }}
 

.github/workflows/main.yml

@@ -8,6 +8,10 @@ on:
     branches:
       - "*"
 
+env:
+  # go needs absolute directories, using the $HOME variable doesn't work here.
+  DOWNLOAD_CACHE: /home/runner/work/download_cache
+
 jobs:
   frontend:
     name: frontend tests on ${{ matrix.os }}
@@ -118,3 +122,54 @@ jobs:
 
       - name: build backend binary
         run: make build
+
+  proto-compile-check:
+    name: RPC proto compilation check
+    runs-on: ubuntu-latest
+    steps:
+      - name: set git config
+        run: |
+          git config --global core.eol lf
+          git config --global core.autocrlf false
+
+      - name: git checkout
+        uses: actions/checkout@v2
+
+      - name: setup nodejs v${{ matrix.node_version }}
+        uses: actions/setup-node@v1
+        with:
+          node-version: 12.x
+
+      - name: download cache
+        uses: actions/cache@v1
+        with:
+          path: /home/runner/work/download_cache
+          key: lnd-${{ runner.os }}-download-${{ hashFiles('**/install_protoc.sh') }}
+          restore-keys: |
+            lnd-${{ runner.os }}-download-${{ hashFiles('**/install_protoc.sh') }}
+            lnd-${{ runner.os }}-download-
+
+      - name: install protoc and protobuf libraries
+        run: ./scripts/install_protoc.sh
+
+      - name: get yarn cache dir
+        id: yarn-cache-dir
+        run: echo "::set-output name=dir::$(yarn cache dir)"
+
+      - name: yarn cache
+        uses: actions/cache@v2
+        id: yarn-cache
+        with:
+          path: ${{ steps.yarn-cache-dir.outputs.dir }}
+          key: ${{ runner.os }}-yarn-${{ matrix.node_version }}-${{ hashFiles('**/yarn.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-yarn-${{ matrix.node_version }}-${{ hashFiles('**/yarn.lock') }}
+            ${{ runner.os }}-yarn-${{ matrix.node_version }}-
+            ${{ runner.os }}-yarn-
+
+      - name: install dependencies
+        working-directory: ./app
+        run: yarn
+
+      - name: run check
+        run: make protos-check

.gitignore

@@ -1,6 +1,3 @@
-# generated proto scripts
-/app/src/types/generated
-
 # local environment vars to ignore
 .env*.local
 

Dockerfile

@@ -1,48 +1,54 @@
-FROM golang:1.15.5-alpine as builder
-
-# Force Go to use the cgo based DNS resolver. This is required to ensure DNS
-# queries required to connect to linked containers succeed.
-ENV GODEBUG netdns=cgo
+# Start with a NodeJS base image that also contains yarn.
+FROM node:12.17.0-alpine as nodejsbuilder
 
 # Pass a tag, branch or a commit using build-arg. This allows a docker image to
 # be built from a specified Git state. The default image will use the Git tip of
 # master by default.
 ARG checkout="master"
 
-# Explicitly turn on the use of modules (until this becomes the default).
-ENV GO111MODULE on
+# There seem to be multiple problems when using yarn for a build inside of a
+# docker image:
+#   1. For building and installing node-gyp, python is required. This seems to
+#      be missing from the NodeJS base image for ARM builds (or is just required
+#      when building for ARM?).
+#   2. Because of a problem in the docker internal network on ARM, some TCP
+#      packages are being dropped and the yarn installation times out. This can
+#      be mitigated by switching to HTTP and increasing the network timeout.
+#      See https://github.com/yarnpkg/yarn/issues/5259 for more info.
+RUN apk add --no-cache --update alpine-sdk \
+    python \
+    git \
+  && git clone https://github.com/lightninglabs/lightning-terminal /go/src/github.com/lightninglabs/lightning-terminal \
+  && cd /go/src/github.com/lightninglabs/lightning-terminal \
+  && git checkout $checkout \
+  && cd app \
+  && npm config set registry "http://registry.npmjs.org" \
+  && yarn config set registry "http://registry.npmjs.org" \
+  && yarn install --frozen-lockfile --network-timeout 1000000 \
+  && yarn build
 
-ENV NODE_VERSION=v12.17.0
+# The first stage is already done and all static assets should now be generated
+# in the app/build sub directory.
+FROM golang:1.15.5-alpine as golangbuilder
 
-# We need some additional proto files with google annotations, the version
-# should match what's in lnd's scripts/install_travis_proto.sh
-ENV PROTOC_VERSION=3.4.0
-ENV PROTOC_URL="https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOC_VERSION}/protoc-${PROTOC_VERSION}-linux-x86_64.zip"
+# Instead of checking out from git again, we just copy the whole working
+# directory of the previous stage that includes the generated static assets.
+COPY --from=nodejsbuilder /go/src/github.com/lightninglabs/lightning-terminal /go/src/github.com/lightninglabs/lightning-terminal
+
+# Force Go to use the cgo based DNS resolver. This is required to ensure DNS
+# queries required to connect to linked containers succeed.
+ENV GODEBUG netdns=cgo
+
+# Explicitly turn on the use of modules (until this becomes the default).
+ENV GO111MODULE on
 
 # Install dependencies and install/build lightning-terminal.
 RUN apk add --no-cache --update alpine-sdk \
-    git \
     make \
-    curl \
-    bash \
-    binutils \
-    tar \
-    protobuf-dev \
-    zip \
-&& curl -sfSLO ${PROTOC_URL} \
-&& unzip protoc-${PROTOC_VERSION}-linux-x86_64.zip -d /usr/local \
-&& rm /usr/local/bin/protoc /usr/local/readme.txt \
-&& touch ~/.bashrc \
-&& curl -sfSLO https://unofficial-builds.nodejs.org/download/release/${NODE_VERSION}/node-${NODE_VERSION}-linux-x64-musl.tar.xz \
-&& tar -xf node-${NODE_VERSION}-linux-x64-musl.tar.xz -C /usr --strip 1 \
-&& rm node-${NODE_VERSION}-linux-x64-musl.tar.xz \
-&& curl -o- -L https://yarnpkg.com/install.sh | bash \
-&& . ~/.bashrc \
-&& git clone https://github.com/lightninglabs/lightning-terminal /go/src/github.com/lightninglabs/lightning-terminal \
-&& cd /go/src/github.com/lightninglabs/lightning-terminal \
-&& git checkout $checkout \
-&& make install \
-&& make go-install-cli
+  && cd /go/src/github.com/lightninglabs/lightning-terminal \
+  && make statik-only \
+  && make go-install \
+  && make go-install-cli
 
 # Start a new, final image to reduce size.
 FROM alpine as final
@@ -54,11 +60,11 @@ VOLUME /root/.lnd
 EXPOSE 8443 10009 9735
 
 # Copy the binaries and entrypoint from the builder image.
-COPY --from=builder /go/bin/litd /bin/
-COPY --from=builder /go/bin/lncli /bin/
-COPY --from=builder /go/bin/frcli /bin/
-COPY --from=builder /go/bin/loop /bin/
-COPY --from=builder /go/bin/pool /bin/
+COPY --from=golangbuilder /go/bin/litd /bin/
+COPY --from=golangbuilder /go/bin/lncli /bin/
+COPY --from=golangbuilder /go/bin/frcli /bin/
+COPY --from=golangbuilder /go/bin/loop /bin/
+COPY --from=golangbuilder /go/bin/pool /bin/
 
 # Add bash.
 RUN apk add --no-cache \

Makefile

@@ -97,10 +97,12 @@ yarn-install:
 # ============
 # INSTALLATION
 # ============
-statik-build: $(STATIK_BIN) app-build
+statik-only: $(STATIK_BIN)
 	@$(call print, "Building statik package.")
 	statik -src=app/build
 
+statik-build: app-build statik-only
+
 build: statik-build go-build
 install: statik-build go-install
 
@@ -192,6 +194,14 @@ list:
 		grep -v Makefile | \
 		sort
 
+protos:
+	@$(call print, "Compiling protos.")
+	cd ./app; yarn protos
+
+protos-check: protos
+	@$(call print, "Verifying compiled protos.")
+	if test -n "$$(git describe --dirty | grep dirty)"; then echo "Protos not properly formatted or not compiled with v3.4.0"; git status; git diff; exit 1; fi
+
 clean:
 	@$(call print, "Cleaning source.$(NC)")
 	$(RM) ./lightning-terminal-debug

app/package.json

@@ -12,7 +12,6 @@
     "eject": "react-scripts eject",
     "lint": "eslint --ext .ts,.tsx --ignore-path .eslintignore .",
     "tsc": "tsc --noEmit",
-    "postinstall": "yarn protos",
     "protos": "node ./scripts/build-protos.js",
     "storybook": "start-storybook -p 9009 -s public",
     "build-storybook": "build-storybook -s public"