diff --git a/.aws-architecture b/.aws-architecture new file mode 100644 index 0000000..a20334e --- /dev/null +++ b/.aws-architecture @@ -0,0 +1 @@ +linux/arm64 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 704ee9d..2fd8206 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,5 +1,12 @@ name: CI -on: push + +on: + push: + paths-ignore: + - '.github/**' + +permissions: read-all + jobs: test: uses: mitlibraries/.github/.github/workflows/python-uv-shared-test.yml@main diff --git a/.github/workflows/dev-build.yml b/.github/workflows/dev-build.yml new file mode 100644 index 0000000..a059d64 --- /dev/null +++ b/.github/workflows/dev-build.yml @@ -0,0 +1,60 @@ +### This is the Terraform-generated dev-build.yml workflow for the ### +### timdex-semantic-builder-dev app repository. ### +### If this is a Lambda repo, uncomment the FUNCTION line at the end of ### +### the document. If the container requires any additional pre-build ### +### commands, uncomment and edit the PREBUILD line at the end of the ### +### document. ### + +name: Dev Container Build and Deploy +on: + workflow_dispatch: + pull_request: + branches: + - main + paths-ignore: + - '.github/**' + +permissions: + id-token: write + contents: read + +jobs: + prep: + name: Prep for Build + runs-on: ubuntu-latest + outputs: + cpuarch: ${{ steps.setarch.outputs.cpuarch }} + steps: + - name: Checkout + uses: actions/checkout@v5 + + - name: Set CPU Architecture + id: setarch + run: | + echo "### :abacus: Architecture Selection" >> $GITHUB_STEP_SUMMARY + if [[ -f .aws-architecture ]]; then + ARCH=$(cat .aws-architecture) + echo "\`$ARCH\` was read from \`.aws-architecture\` and passed to the deploy job." >> $GITHUB_STEP_SUMMARY + else + ARCH="linux/amd64" + echo "No \`.aws-architecture\` file, so default \`$ARCH\` was passed to the deploy job." >> $GITHUB_STEP_SUMMARY + fi + if [[ "$ARCH" != "linux/arm64" && "$ARCH" != "linux/amd64" ]]; then + echo "$ARCH is INVALID architecture!" + echo "$ARCH is INVALID architecture!" >> $GITHUB_STEP_SUMMARY + exit 1 + fi + echo "cpuarch=$ARCH" >> $GITHUB_OUTPUT + + deploy: + needs: prep + name: Dev Deploy + uses: mitlibraries/.github/.github/workflows/ecr-multi-arch-deploy-dev.yml@main + secrets: inherit + with: + AWS_REGION: "us-east-1" + GHA_ROLE: "timdex-semantic-builder-gha-dev" + ECR: "timdex-semantic-builder-dev" + CPU_ARCH: ${{ needs.prep.outputs.cpuarch }} + FUNCTION: "timdex-semantic-builder-dev" + # PREBUILD: diff --git a/.github/workflows/prod-deploy.yml b/.github/workflows/prod-deploy.yml new file mode 100644 index 0000000..656ceb7 --- /dev/null +++ b/.github/workflows/prod-deploy.yml @@ -0,0 +1,57 @@ +### This is the Terraform-generated prod-promote.yml workflow for the ### +### timdex-semantic-builder-prod repository. ### +### If this is a Lambda repo, uncomment the FUNCTION line at the end of ### +### the document. ### + +name: Prod Container Promote +on: + workflow_dispatch: + release: + types: [published] + +permissions: + id-token: write + contents: read + +jobs: + prep: + name: Prep for Promote + runs-on: ubuntu-latest + outputs: + cpuarch: ${{ steps.setarch.outputs.cpuarch }} + steps: + - name: Checkout + uses: actions/checkout@v5 + + - name: Set CPU Architecture + id: setarch + run: | + echo "### :abacus: Architecture Selection" >> $GITHUB_STEP_SUMMARY + if [[ -f .aws-architecture ]]; then + ARCH=$(cat .aws-architecture) + echo "\`$ARCH\` was read from \`.aws-architecture\` and passed to the deploy job." >> $GITHUB_STEP_SUMMARY + else + ARCH="linux/amd64" + echo "No \`.aws-architecture\` file, so default \`$ARCH\` was passed to the deploy job." >> $GITHUB_STEP_SUMMARY + fi + if [[ "$ARCH" != "linux/arm64" && "$ARCH" != "linux/amd64" ]]; then + echo "$ARCH is INVALID architecture!" + echo "$ARCH is INVALID architecture!" >> $GITHUB_STEP_SUMMARY + exit 1 + fi + echo "cpuarch=$ARCH" >> $GITHUB_OUTPUT + + deploy: + needs: prep + name: Deploy + uses: mitlibraries/.github/.github/workflows/ecr-multi-arch-promote-prod.yml@main + secrets: inherit + with: + AWS_REGION: "us-east-1" + GHA_ROLE_STAGE: timdex-semantic-builder-gha-stage + GHA_ROLE_PROD: timdex-semantic-builder-gha-prod + ECR_STAGE: "timdex-semantic-builder-stage" + ECR_PROD: "timdex-semantic-builder-prod" + CPU_ARCH: ${{ needs.prep.outputs.cpuarch }} + FUNCTION: "timdex-semantic-builder-prod" + \ No newline at end of file diff --git a/.github/workflows/stage-build.yml b/.github/workflows/stage-build.yml new file mode 100644 index 0000000..c4ec7e7 --- /dev/null +++ b/.github/workflows/stage-build.yml @@ -0,0 +1,60 @@ +### This is the Terraform-generated stage-build.yml workflow for the ### +### timdex-semantic-builder-stage app repository. ### +### If this is a Lambda repo, uncomment the FUNCTION line at the end of ### +### the document. If the container requires any additional pre-build ### +### commands, uncomment and edit the PREBUILD line at the end of the ### +### document. ### + +name: Stage Container Build and Deploy +on: + workflow_dispatch: + push: + branches: + - main + paths-ignore: + - '.github/**' + +permissions: + id-token: write + contents: read + +jobs: + prep: + name: Prep for Build + runs-on: ubuntu-latest + outputs: + cpuarch: ${{ steps.setarch.outputs.cpuarch }} + steps: + - name: Checkout + uses: actions/checkout@v5 + + - name: Set CPU Architecture + id: setarch + run: | + echo "### :abacus: Architecture Selection" >> $GITHUB_STEP_SUMMARY + if [[ -f .aws-architecture ]]; then + ARCH=$(cat .aws-architecture) + echo "\`$ARCH\` was read from \`.aws-architecture\` and passed to the deploy job." >> $GITHUB_STEP_SUMMARY + else + ARCH="linux/amd64" + echo "No \`.aws-architecture\` file, so default \`$ARCH\` was passed to the deploy job." >> $GITHUB_STEP_SUMMARY + fi + if [[ "$ARCH" != "linux/arm64" && "$ARCH" != "linux/amd64" ]]; then + echo "$ARCH is INVALID architecture!" + echo "$ARCH is INVALID architecture!" >> $GITHUB_STEP_SUMMARY + exit 1 + fi + echo "cpuarch=$ARCH" >> $GITHUB_OUTPUT + + deploy: + needs: prep + name: Stage Deploy + uses: mitlibraries/.github/.github/workflows/ecr-multi-arch-deploy-stage.yml@main + secrets: inherit + with: + AWS_REGION: "us-east-1" + GHA_ROLE: "timdex-semantic-builder-gha-stage" + ECR: "timdex-semantic-builder-stage" + CPU_ARCH: ${{ needs.prep.outputs.cpuarch }} + FUNCTION: "timdex-semantic-builder-stage" + # PREBUILD: diff --git a/.gitignore b/.gitignore index e20e6f1..b3febb8 100644 --- a/.gitignore +++ b/.gitignore @@ -141,3 +141,6 @@ dmypy.json # SAM .aws-sam/ tests/sam/env.json + +# For multi-arch deploys +.arch_tag diff --git a/Makefile b/Makefile index 5a40936..f009fa4 100644 --- a/Makefile +++ b/Makefile @@ -1,12 +1,21 @@ SHELL=/bin/bash DATETIME:=$(shell date -u +%Y%m%dT%H%M%SZ) +### This is the Terraform-generated header for timdex-semantic-builder-dev. If +### this is a Lambda repo, uncomment the FUNCTION line below +### and review the other commented lines in the document. +ECR_NAME_DEV := timdex-semantic-builder-dev +ECR_URL_DEV := 222053980223.dkr.ecr.us-east-1.amazonaws.com/timdex-semantic-builder-dev +CPU_ARCH ?= $(shell cat .aws-architecture 2>/dev/null || echo "linux/amd64") +FUNCTION_DEV := timdex-semantic-builder-dev +### End of Terraform-generated header + help: # Preview Makefile commands @awk 'BEGIN { FS = ":.*#"; print "Usage: make \n\nTargets:" } \ /^[-_[:alpha:]]+:.?*#/ { printf " %-15s%s\n", $$1, $$2 }' $(MAKEFILE_LIST) # ensure OS binaries aren't called if naming conflict with Make recipes -.PHONY: help venv install update test coveralls lint black mypy ruff safety lint-apply black-apply ruff-apply +.PHONY: help venv install update test coveralls lint black mypy ruff safety lint-apply black-apply ruff-apply check-arch dist-dev publish-dev update-lambda-dev docker-clean ############################################## # Python Environment and Dependency commands @@ -79,3 +88,57 @@ sam-http-ping: # Send curl command to SAM HTTP server curl --location 'http://localhost:3000/myapp' \ --header 'Content-Type: application/json' \ --data '{"msg":"in a bottle"}' + +#################################### +# Deployment to ECR +#################################### +### Terraform-generated Developer Deploy Commands for Dev environment ### +check-arch: + @ARCH_FILE=".aws-architecture"; \ + if [[ "$(CPU_ARCH)" != "linux/amd64" && "$(CPU_ARCH)" != "linux/arm64" ]]; then \ + echo "Invalid CPU_ARCH: $(CPU_ARCH)"; exit 1; \ + fi; \ + if [[ -f $$ARCH_FILE ]]; then \ + echo "latest-$(shell echo $(CPU_ARCH) | cut -d'/' -f2)" > .arch_tag; \ + else \ + echo "latest" > .arch_tag; \ + fi + +dist-dev: check-arch ## Build docker container (intended for developer-based manual build) + @ARCH_TAG=$$(cat .arch_tag); \ + docker buildx inspect $(ECR_NAME_DEV) >/dev/null 2>&1 || docker buildx create --name $(ECR_NAME_DEV) --use; \ + docker buildx use $(ECR_NAME_DEV); \ + docker buildx build --platform $(CPU_ARCH) \ + --load \ + --tag $(ECR_URL_DEV):$$ARCH_TAG \ + --tag $(ECR_URL_DEV):make-$$ARCH_TAG \ + --tag $(ECR_URL_DEV):make-$(shell git describe --always) \ + --tag $(ECR_NAME_DEV):$$ARCH_TAG \ + . + +publish-dev: dist-dev ## Build, tag and push (intended for developer-based manual publish) + @ARCH_TAG=$$(cat .arch_tag); \ + aws ecr get-login-password --region us-east-1 | docker login --username AWS --password-stdin $(ECR_URL_DEV); \ + docker push $(ECR_URL_DEV):$$ARCH_TAG; \ + docker push $(ECR_URL_DEV):make-$$ARCH_TAG; \ + docker push $(ECR_URL_DEV):make-$(shell git describe --always); \ + echo "Cleaning up dangling Docker images..."; \ + docker image prune -f --filter "dangling=true" + +## If this is a Lambda repo, uncomment the two lines below +update-lambda-dev: ## Updates the lambda with whatever is the most recent image in the ecr (intended for developer-based manual update) + @ARCH_TAG=$$(cat .arch_tag); \ + aws lambda update-function-code \ + --region us-east-1 \ + --function-name $(FUNCTION_DEV) \ + --image-uri $(ECR_URL_DEV):make-$$ARCH_TAG + +docker-clean: ## Clean up Docker detritus generated by dist-dev and publish-dev + @ARCH_TAG=$$(cat .arch_tag); \ + echo "Cleaning up Docker leftovers (containers, images, builders)"; \ + docker rmi -f $(ECR_URL_DEV):$$ARCH_TAG; \ + docker rmi -f $(ECR_URL_DEV):make-$$ARCH_TAG; \ + docker rmi -f $(ECR_URL_DEV):make-$(shell git describe --always) || true; \ + docker rmi -f $(ECR_NAME_DEV):$$ARCH_TAG || true; \ + docker buildx rm $(ECR_NAME_DEV) || true + @rm -rf .arch_tag