Skip to content

Commit 6fc411c

Browse files
dsessler7dsessler7
committed
Adding GitHub Actions Job for E2E testing. Refactoring kubernetes-k3d Job to use new K3d action for setting up k3d. Adjusting root-cert-ownership kuttl test to work with POSIX shell used in Github Actions.
[sc-17404]
1 parent ef6b798 commit 6fc411c

File tree

5 files changed

+181
-40
lines changed

5 files changed

+181
-40
lines changed

‎.github/actions/k3d/action.yaml‎

Lines changed: 94 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,94 @@
1+
name: k3d
2+
description: Start k3s using k3d
3+
inputs:
4+
k3d-tag:
5+
default: latest
6+
required: true
7+
description: >
8+
Git tag from https://github.com/rancher/k3d/releases or "latest"
9+
k3s-channel:
10+
default: latest
11+
required: true
12+
description: >
13+
https://rancher.com/docs/k3s/latest/en/upgrades/basic/#release-channels
14+
prefetch-images:
15+
required: true
16+
description: >
17+
Each line is the name of an image to fetch onto all Kubernetes nodes
18+
prefetch-timeout:
19+
default: 90s
20+
required: true
21+
description: >
22+
Amount of time to wait for images to be fetched
23+
24+
outputs:
25+
k3d-version:
26+
value: ${{ steps.k3d.outputs.k3d }}
27+
description: >
28+
K3d version
29+
kubernetes-version:
30+
value: ${{ steps.k3s.outputs.server }}
31+
description: >
32+
Kubernetes server version, as reported by the Kubernetes API
33+
pause-image:
34+
value: ${{ steps.k3s.outputs.pause-image }}
35+
description: >
36+
Pause image for prefetch images DaemonSet
37+
38+
runs:
39+
using: composite
40+
steps:
41+
- id: k3d
42+
name: Install k3d
43+
shell: bash
44+
env:
45+
K3D_TAG: ${{ inputs.k3d-tag }}
46+
run: |
47+
curl --fail --silent https://raw.githubusercontent.com/rancher/k3d/main/install.sh |
48+
TAG="${K3D_TAG#latest}" bash
49+
k3d version | awk '{ print "${tolower($1)}=${$3}" >> $GITHUB_OUTPUT }'
50+
51+
- id: k3s
52+
name: Start k3s
53+
shell: bash
54+
run: |
55+
k3d cluster create --image '+${{ inputs.k3s-channel }}' --no-lb --timeout=2m --wait
56+
kubectl version --short | awk '{ print "${tolower($1)}=${$3}" >> $GITHUB_OUTPUT }'
57+
58+
PAUSE_IMAGE=$(docker exec $(k3d node list --output json | jq --raw-output 'first.name') \
59+
k3s agent --help | awk '$1 == "--pause-image" {
60+
match($0, /default: "[^"]*"/);
61+
print substr($0, RSTART+10, RLENGTH-11)
62+
}')
63+
echo "pause-image=${PAUSE_IMAGE}" >> $GITHUB_OUTPUT
64+
65+
- name: Prefetch container images
66+
shell: bash
67+
env:
68+
INPUT_IMAGES: ${{ inputs.prefetch-images }}
69+
INPUT_TIMEOUT: ${{ inputs.prefetch-timeout }}
70+
run: |
71+
jq <<< "$INPUT_IMAGES" --raw-input 'select(. != "")' |
72+
jq --slurp \
73+
--arg pause '${{ steps.k3s.outputs.pause-image }}' \
74+
--argjson labels '{"name":"image-prefetch"}' \
75+
--argjson name '"image-prefetch"' \
76+
'{
77+
apiVersion: "apps/v1", kind: "DaemonSet",
78+
metadata: { name: $name, labels: $labels },
79+
spec: {
80+
selector: { matchLabels: $labels },
81+
template: {
82+
metadata: { labels: $labels },
83+
spec: {
84+
initContainers: to_entries | map({
85+
name: "c\(.key)", image: .value, command: ["true"],
86+
}),
87+
containers: [{ name: "pause", image: $pause }]
88+
}
89+
}
90+
}
91+
}' |
92+
kubectl create --filename=-
93+
kubectl rollout status daemonset.apps/image-prefetch --timeout "$INPUT_TIMEOUT" ||
94+
kubectl describe daemonset.apps/image-prefetch

‎.github/workflows/test.yaml‎

Lines changed: 79 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -60,43 +60,15 @@ jobs:
6060
- uses: actions/setup-go@v3
6161
with: { go-version: 1.x }
6262

63-
- name: Install k3d
64-
# Git tag from https://github.com/rancher/k3d/releases or "latest"
65-
env: { K3D_TAG: latest }
66-
run: |
67-
curl --fail --silent https://raw.githubusercontent.com/rancher/k3d/main/install.sh |
68-
TAG="${K3D_TAG#latest}" bash && k3d version | head -n1
69-
7063
- name: Start k3s
71-
# https://rancher.com/docs/k3s/latest/en/upgrades/basic/#release-channels
72-
env: { K3S_CHANNEL: "${{ matrix.kubernetes }}" }
73-
run: k3d cluster create --image="+${K3S_CHANNEL}" --no-lb --timeout=2m --wait
64+
uses: ./.github/actions/k3d
65+
with:
66+
k3s-channel: "${{ matrix.kubernetes }}"
67+
prefetch-images: |
68+
registry.developers.crunchydata.com/crunchydata/crunchy-pgbackrest:ubi8-2.41-2
69+
registry.developers.crunchydata.com/crunchydata/crunchy-pgbouncer:ubi8-1.17-5
70+
registry.developers.crunchydata.com/crunchydata/crunchy-postgres:ubi8-13.9-2
7471
75-
- name: Prefetch container images
76-
run: |
77-
{
78-
echo '"registry.developers.crunchydata.com/crunchydata/crunchy-postgres:ubi8-13.6-1"'
79-
echo '"registry.developers.crunchydata.com/crunchydata/crunchy-pgbackrest:ubi8-2.38-0"'
80-
echo '"registry.developers.crunchydata.com/crunchydata/crunchy-pgbouncer:ubi8-1.16-2"'
81-
} |
82-
jq --slurp --arg name 'image-prefetch' --argjson labels '{"name":"image-prefetch"}' '{
83-
apiVersion: "apps/v1", kind: "DaemonSet",
84-
metadata: { name: $name, labels: $labels },
85-
spec: {
86-
selector: { matchLabels: $labels },
87-
template: {
88-
metadata: { labels: $labels },
89-
spec: {
90-
initContainers: to_entries | map({ name: "c\(.key)", command: ["true"], image: .value }),
91-
containers: [{ name: "pause", image: "k8s.gcr.io/pause:3.5" }]
92-
}
93-
}
94-
}
95-
}' |
96-
kubectl create --filename=- && {
97-
kubectl rollout status daemonset.apps/image-prefetch --timeout=90s ||
98-
kubectl describe daemonset.apps/image-prefetch
99-
}
10072
- run: make createnamespaces check-envtest-existing
10173
env:
10274
PGO_TEST_TIMEOUT_SCALE: 1.2
@@ -110,6 +82,78 @@ jobs:
11082
path: envtest-existing.coverage.gz
11183
retention-days: 1
11284

85+
kuttl-k3d:
86+
runs-on: ubuntu-20.04
87+
needs: [go-test]
88+
strategy:
89+
fail-fast: false
90+
matrix:
91+
kubernetes: [v1.25, v1.24, v1.23, v1.22, v1.21]
92+
steps:
93+
- uses: actions/checkout@v3
94+
- uses: actions/setup-go@v3
95+
with: { go-version: 1.x }
96+
97+
- name: Start k3s
98+
uses: ./.github/actions/k3d
99+
with:
100+
k3s-channel: "${{ matrix.kubernetes }}"
101+
prefetch-images: |
102+
registry.developers.crunchydata.com/crunchydata/crunchy-pgadmin4:ubi8-4.30-8
103+
registry.developers.crunchydata.com/crunchydata/crunchy-pgbackrest:ubi8-2.41-2
104+
registry.developers.crunchydata.com/crunchydata/crunchy-pgbouncer:ubi8-1.17-5
105+
registry.developers.crunchydata.com/crunchydata/crunchy-postgres-exporter:ubi8-5.3.0-0
106+
registry.developers.crunchydata.com/crunchydata/crunchy-upgrade:ubi8-5.3.0-0
107+
registry.developers.crunchydata.com/crunchydata/crunchy-postgres:ubi8-13.9-2
108+
registry.developers.crunchydata.com/crunchydata/crunchy-postgres-gis:ubi8-13.9-3.1-2
109+
registry.developers.crunchydata.com/crunchydata/crunchy-postgres:ubi8-14.6-2
110+
registry.developers.crunchydata.com/crunchydata/crunchy-postgres-gis:ubi8-14.6-3.1-2
111+
- run: go mod download
112+
- name: Build executable
113+
run: PGO_VERSION='${{ github.sha }}' make build-postgres-operator
114+
115+
# Start a Docker container with the working directory mounted.
116+
- name: Start PGO
117+
run: |
118+
kubectl apply --server-side -k ./config/namespace
119+
kubectl apply --server-side -k ./config/dev
120+
hack/create-kubeconfig.sh postgres-operator pgo
121+
docker run --detach --network host --read-only \
122+
--volume "$(pwd):/mnt" --workdir '/mnt' --env 'PATH=/mnt/bin' \
123+
--env 'KUBECONFIG=hack/.kube/postgres-operator/pgo' \
124+
--env 'RELATED_IMAGE_PGADMIN=registry.developers.crunchydata.com/crunchydata/crunchy-pgadmin4:ubi8-4.30-8' \
125+
--env 'RELATED_IMAGE_PGBACKREST=registry.developers.crunchydata.com/crunchydata/crunchy-pgbackrest:ubi8-2.41-2' \
126+
--env 'RELATED_IMAGE_PGBOUNCER=registry.developers.crunchydata.com/crunchydata/crunchy-pgbouncer:ubi8-1.17-5' \
127+
--env 'RELATED_IMAGE_PGEXPORTER=registry.developers.crunchydata.com/crunchydata/crunchy-postgres-exporter:ubi8-5.3.0-0' \
128+
--env 'RELATED_IMAGE_PGUPGRADE=registry.developers.crunchydata.com/crunchydata/crunchy-upgrade:ubi8-5.3.0-0' \
129+
--env 'RELATED_IMAGE_POSTGRES_13=registry.developers.crunchydata.com/crunchydata/crunchy-postgres:ubi8-13.9-2' \
130+
--env 'RELATED_IMAGE_POSTGRES_13_GIS_3.1=registry.developers.crunchydata.com/crunchydata/crunchy-postgres-gis:ubi8-13.9-3.1-2' \
131+
--env 'RELATED_IMAGE_POSTGRES_14=registry.developers.crunchydata.com/crunchydata/crunchy-postgres:ubi8-14.6-2' \
132+
--env 'RELATED_IMAGE_POSTGRES_14_GIS_3.1=registry.developers.crunchydata.com/crunchydata/crunchy-postgres-gis:ubi8-14.6-3.1-2' \
133+
--name 'postgres-operator' ubuntu \
134+
postgres-operator
135+
- name: Install kuttl
136+
run: |
137+
curl -Lo /usr/local/bin/kubectl-kuttl https://github.com/kudobuilder/kuttl/releases/download/v0.13.0/kubectl-kuttl_0.13.0_linux_x86_64
138+
chmod +x /usr/local/bin/kubectl-kuttl
139+
140+
- run: make generate-kuttl
141+
env:
142+
KUTTL_PG_UPGRADE_FROM_VERSION: '13'
143+
KUTTL_PG_UPGRADE_TO_VERSION: '14'
144+
KUTTL_PG_VERSION: '14'
145+
KUTTL_POSTGIS_VERSION: '3.1'
146+
KUTTL_PSQL_IMAGE: 'registry.developers.crunchydata.com/crunchydata/crunchy-postgres:ubi8-14.6-2'
147+
- run: |
148+
make check-kuttl && exit
149+
failed=$?
150+
echo '::group::PGO logs'; docker logs 'postgres-operator'; echo '::endgroup::'
151+
exit $failed
152+
env:
153+
KUTTL_TEST: kubectl-kuttl test
154+
- name: Stop PGO
155+
run: docker stop 'postgres-operator' || true
156+
113157
coverage-report:
114158
if: ${{ success() || contains(needs.*.result, 'success') }}
115159
runs-on: ubuntu-latest

‎testing/kuttl/e2e/root-cert-ownership/01--check-owners.yaml‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,8 @@ commands:
1010
CURRENT_OWNERS=$(kubectl --namespace="${NAMESPACE}" get secret \
1111
pgo-root-cacert -o jsonpath='{.metadata.ownerReferences[*].name}')
1212
# If owner1 and owner2 are both listed, exit successfully
13-
if [[ "$CURRENT_OWNERS" == *"owner1"* ]] && [[ "$CURRENT_OWNERS" == *"owner2"* ]]; then
13+
contains() { bash -ceu '[[ "$1" == *"$2"* ]]' - "$@"; }
14+
if contains "${CURRENT_OWNERS}" "owner1" && contains "${CURRENT_OWNERS}" "owner2"; then
1415
exit 0
1516
fi
1617
done

‎testing/kuttl/e2e/root-cert-ownership/03--check-owners.yaml‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,8 @@ commands:
1010
CURRENT_OWNERS=$(kubectl --namespace="${NAMESPACE}" get secret \
1111
pgo-root-cacert -o jsonpath='{.metadata.ownerReferences[*].name}')
1212
# If owner1 is removed and owner2 is still listed, exit successfully
13-
if [[ "$CURRENT_OWNERS" != *"owner1"* ]] && [[ "$CURRENT_OWNERS" == *"owner2"* ]]; then
13+
contains() { bash -ceu '[[ "$1" == *"$2"* ]]' - "$@"; }
14+
if ! contains "${CURRENT_OWNERS}" "owner1" && contains "${CURRENT_OWNERS}" "owner2"; then
1415
exit 0
1516
fi
1617
done

‎testing/kuttl/e2e/root-cert-ownership/05--check-secret.yaml‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -8,13 +8,14 @@ commands:
88
# secret should be deleted.
99
- script: |
1010
NUM_CLUSTERS=$(kubectl --namespace="${NAMESPACE}" get postgrescluster --output name | wc -l)
11-
if [[ "$NUM_CLUSTERS" != 0 ]]; then
11+
if [ "$NUM_CLUSTERS" != 0 ]; then
1212
for i in {1..5}; do
1313
sleep 1 # This sleep allows time for the owner reference list to be updated
1414
CURRENT_OWNERS=$(kubectl --namespace="${NAMESPACE}" get secret \
1515
pgo-root-cacert -o jsonpath='{.metadata.ownerReferences[*].name}')
1616
# If neither owner is listed, exit successfully
17-
if [[ "$CURRENT_OWNERS" != *"owner1"* ]] || [[ "$CURRENT_OWNERS" != *"owner2"* ]]; then
17+
contains() { bash -ceu '[[ "$1" == *"$2"* ]]' - "$@"; }
18+
if ! contains "${CURRENT_OWNERS}" "owner1" && ! contains "${CURRENT_OWNERS}" "owner2"; then
1819
exit 0
1920
fi
2021
done
@@ -25,7 +26,7 @@ commands:
2526
sleep 1 # this sleep allows time for garbage collector to delete the secret
2627
ROOT_SECRET=$(kubectl --namespace="${NAMESPACE}" get --ignore-not-found \
2728
secret pgo-root-cacert --output name | wc -l)
28-
if [[ "$ROOT_SECRET" == 0 ]]; then
29+
if [ "$ROOT_SECRET" = 0 ]; then
2930
exit 0
3031
fi
3132
done

0 commit comments

Comments
 (0)