Skip to content

Commit 4b9c22b

Browse files
horshors
authored
Merge pull request #2113 from percona/cloud-940
CLOUD-940 Use authenticated calls to check backup existence
2 parents 959c43b + a1bead7 commit 4b9c22b

File tree

9 files changed

+355
-66
lines changed
  • Jenkinsfile
  • e2e-tests
    • data-at-rest-encryption
    • demand-backup-eks-credentials-irsa
    • demand-backup-eks-credentials
    • demand-backup-sharded
    • demand-backup
    • functions
    • scheduled-backup
    • upgrade-partial-backup

9 files changed

+355
-66
lines changed

‎Jenkinsfile‎

Lines changed: 43 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -218,22 +218,24 @@ void makeReport() {
218218
}
219219

220220
void clusterRunner(String cluster) {
221-
def clusterCreated=0
221+
withCredentials([[$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: 'AMI/OVF', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY']]){
222+
def clusterCreated=0
222223

223-
for (int i=0; i<tests.size(); i++) {
224-
if (tests[i]["result"] == "skipped" && currentBuild.nextBuild == null) {
225-
tests[i]["result"] = "failure"
226-
tests[i]["cluster"] = cluster
227-
if (clusterCreated == 0) {
228-
createCluster(cluster)
229-
clusterCreated++
224+
for (int i=0; i<tests.size(); i++) {
225+
if (tests[i]["result"] == "skipped" && currentBuild.nextBuild == null) {
226+
tests[i]["result"] = "failure"
227+
tests[i]["cluster"] = cluster
228+
if (clusterCreated == 0) {
229+
createCluster(cluster)
230+
clusterCreated++
231+
}
232+
runTest(i)
230233
}
231-
runTest(i)
232234
}
233-
}
234235

235-
if (clusterCreated >= 1) {
236-
shutdownCluster(cluster)
236+
if (clusterCreated >= 1) {
237+
shutdownCluster(cluster)
238+
}
237239
}
238240
}
239241

@@ -306,6 +308,35 @@ EOF
306308
307309
curl -sL https://github.com/mitchellh/golicense/releases/latest/download/golicense_0.2.0_linux_x86_64.tar.gz | sudo tar -C /usr/local/bin -xzf - golicense
308310
"""
311+
installAzureCLI()
312+
azureAuth()
313+
}
314+
315+
void azureAuth() {
316+
withCredentials([azureServicePrincipal('PERCONA-OPERATORS-SP')]) {
317+
sh '''
318+
az login --service-principal -u "$AZURE_CLIENT_ID" -p "$AZURE_CLIENT_SECRET" -t "$AZURE_TENANT_ID" --allow-no-subscriptions
319+
az account set -s "$AZURE_SUBSCRIPTION_ID"
320+
'''
321+
}
322+
}
323+
324+
void installAzureCLI() {
325+
sh """
326+
if ! command -v az &>/dev/null; then
327+
echo "Installing Azure CLI for Hetzner instances..."
328+
sudo rpm --import https://packages.microsoft.com/keys/microsoft.asc
329+
cat <<EOF | sudo tee /etc/yum.repos.d/azure-cli.repo
330+
[azure-cli]
331+
name=Azure CLI
332+
baseurl=https://packages.microsoft.com/yumrepos/azure-cli
333+
enabled=1
334+
gpgcheck=1
335+
gpgkey=https://packages.microsoft.com/keys/microsoft.asc
336+
EOF
337+
sudo dnf install azure-cli -y
338+
fi
339+
"""
309340
}
310341

311342
boolean isManualBuild() {

‎e2e-tests/data-at-rest-encryption/run‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -66,7 +66,8 @@ compare_mongos_cmd "find" "myApp:myPass@$cluster-mongos.$namespace"
6666
if [ -z "$SKIP_BACKUPS_TO_AWS_GCP_AZURE" ]; then
6767
desc 'check backup and restore -- gcp-cs'
6868
backup_dest_gcp=$(get_backup_dest "$backup_name_gcp")
69-
curl -s "https://storage.googleapis.com/${backup_dest_gcp}/rs0/myApp.test.gz" | gunzip >/dev/null
69+
setup_gcs_credentials
70+
check_backup_existence_gcs "$backup_dest_gcp" "/rs0/myApp.test.gz"
7071
run_mongos 'use myApp\n db.test.insert({ x: 100501 })' "myApp:myPass@$cluster-mongos.$namespace"
7172
compare_mongos_cmd "find" "myApp:myPass@$cluster-mongos.$namespace" "-2nd"
7273
run_restore "$backup_name_gcp"

‎e2e-tests/demand-backup-eks-credentials-irsa/run‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -116,7 +116,8 @@ sleep 5
116116

117117
desc 'check backup and restore -- aws-s3'
118118
backup_dest_aws=$(get_backup_dest "$backup_name_aws")
119-
curl -s "https://s3.amazonaws.com/${backup_dest_aws}/rs0/myApp.test.gz" | gunzip >/dev/null
119+
setup_aws_credentials
120+
check_backup_existence_aws "$backup_dest_aws" "/rs0/myApp.test.gz"
120121
run_mongo 'use myApp\n db.test.insert({ x: 100501 })' "myApp:myPass@$cluster-rs0.$namespace"
121122
compare_mongo_cmd "find" "myApp:myPass@$cluster-rs0-0.$cluster-rs0.$namespace" "-2nd"
122123
compare_mongo_cmd "find" "myApp:myPass@$cluster-rs0-1.$cluster-rs0.$namespace" "-2nd"

‎e2e-tests/demand-backup-eks-credentials/run‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,8 @@ sleep 5
5757

5858
desc 'check backup and restore -- aws-s3'
5959
backup_dest_aws=$(get_backup_dest "$backup_name_aws")
60-
curl -s "https://s3.amazonaws.com/${backup_dest_aws}/rs0/myApp.test.gz" | gunzip >/dev/null
60+
setup_aws_credentials
61+
check_backup_existence_aws "$backup_dest_aws" "/rs0/myApp.test.gz"
6162
run_mongo 'use myApp\n db.test.insert({ x: 100501 })' "myApp:myPass@$cluster.$namespace"
6263
compare_mongo_cmd "find" "myApp:myPass@$cluster-0.$cluster.$namespace" "-2nd"
6364
compare_mongo_cmd "find" "myApp:myPass@$cluster-1.$cluster.$namespace" "-2nd"
@@ -70,7 +71,7 @@ compare_mongo_cmd "find" "myApp:myPass@$cluster-2.$cluster.$namespace"
7071

7172
desc 'delete backup and check if it is removed from bucket -- aws-s3'
7273
kubectl_bin delete psmdb-backup --all
73-
check_backup_deletion "https://s3.amazonaws.com/${backup_dest_aws}" "aws-s3"
74+
check_backup_deletion_aws "$backup_dest_aws" "/rs0/myApp.test.gz"
7475

7576
destroy $namespace
7677

‎e2e-tests/demand-backup-sharded/run‎

Lines changed: 16 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -108,9 +108,10 @@ sleep 5
108108
if [ -z "$SKIP_BACKUPS_TO_AWS_GCP_AZURE" ]; then
109109
desc 'check backup and restore -- aws-s3'
110110
backup_dest_aws=$(get_backup_dest "$backup_name_aws")
111-
curl -s "https://s3.amazonaws.com/${backup_dest_aws}/rs0/myApp.test.gz" | gunzip >/dev/null
112-
curl -s "https://s3.amazonaws.com/${backup_dest_aws}/rs1/myApp1.test.gz" | gunzip >/dev/null
113-
curl -s "https://s3.amazonaws.com/${backup_dest_aws}/rs2/myApp2.test.gz" | gunzip >/dev/null
111+
setup_aws_credentials
112+
check_backup_existence_aws "$backup_dest_aws" "/rs0/myApp.test.gz"
113+
check_backup_existence_aws "$backup_dest_aws" "/rs1/myApp1.test.gz"
114+
check_backup_existence_aws "$backup_dest_aws" "/rs2/myApp2.test.gz"
114115
insert_data_mongos "100501" "myApp" "" "$custom_port"
115116
insert_data_mongos "100501" "myApp1" "" "$custom_port"
116117
insert_data_mongos "100501" "myApp2" "" "$custom_port"
@@ -121,9 +122,10 @@ if [ -z "$SKIP_BACKUPS_TO_AWS_GCP_AZURE" ]; then
121122

122123
desc 'check backup and restore -- gcp-cs'
123124
backup_dest_gcp=$(get_backup_dest "$backup_name_gcp")
124-
curl -s "https://storage.googleapis.com/${backup_dest_gcp}/rs0/myApp.test.gz" | gunzip >/dev/null
125-
curl -s "https://storage.googleapis.com/${backup_dest_gcp}/rs1/myApp1.test.gz" | gunzip >/dev/null
126-
curl -s "https://storage.googleapis.com/${backup_dest_gcp}/rs2/myApp2.test.gz" | gunzip >/dev/null
125+
setup_gcs_credentials
126+
check_backup_existence_gcs "$backup_dest_gcp" "/rs0/myApp.test.gz"
127+
check_backup_existence_gcs "$backup_dest_gcp" "/rs1/myApp1.test.gz"
128+
check_backup_existence_gcs "$backup_dest_gcp" "/rs2/myApp2.test.gz"
127129
insert_data_mongos "100501" "myApp" "" "$custom_port"
128130
insert_data_mongos "100501" "myApp1" "" "$custom_port"
129131
insert_data_mongos "100501" "myApp2" "" "$custom_port"
@@ -134,9 +136,10 @@ if [ -z "$SKIP_BACKUPS_TO_AWS_GCP_AZURE" ]; then
134136

135137
desc 'check backup and restore -- azure-blob'
136138
backup_dest_azure=$(get_backup_dest "$backup_name_azure")
137-
curl -s "https://engk8soperators.blob.core.windows.net/${backup_dest_azure}/rs0/myApp.test.gz" | gunzip >/dev/null
138-
curl -s "https://engk8soperators.blob.core.windows.net/${backup_dest_azure}/rs1/myApp1.test.gz" | gunzip >/dev/null
139-
curl -s "https://engk8soperators.blob.core.windows.net/${backup_dest_azure}/rs2/myApp2.test.gz" | gunzip >/dev/null
139+
setup_azure_credentials
140+
check_backup_existence_azure "$backup_dest_azure" "/rs0/myApp.test.gz"
141+
check_backup_existence_azure "$backup_dest_azure" "/rs1/myApp1.test.gz"
142+
check_backup_existence_azure "$backup_dest_azure" "/rs2/myApp2.test.gz"
140143
insert_data_mongos "100501" "myApp" "" "$custom_port"
141144
insert_data_mongos "100501" "myApp1" "" "$custom_port"
142145
insert_data_mongos "100501" "myApp2" "" "$custom_port"
@@ -169,9 +172,9 @@ if [[ $backup_exists -eq 1 ]]; then
169172
fi
170173

171174
if [ -z "$SKIP_BACKUPS_TO_AWS_GCP_AZURE" ]; then
172-
check_backup_deletion "https://s3.amazonaws.com/${backup_dest_aws}" "aws-s3"
173-
check_backup_deletion "https://storage.googleapis.com/${backup_dest_gcp}" "gcp-cs"
174-
check_backup_deletion "https://engk8soperators.blob.core.windows.net/${backup_dest_azure}" "azure-blob"
175+
check_backup_deletion_aws "$backup_dest_aws" "/rs0/myApp.test.gz"
176+
check_backup_deletion_gcs "$backup_dest_gcp" "/rs0/myApp.test.gz"
177+
check_backup_deletion_azure "$backup_dest_azure" "/rs0/myApp.test.gz"
175178
fi
176179

177180
# Temporarily skipping this check
@@ -180,3 +183,4 @@ fi
180183

181184
kubectl_bin delete -f "$conf_dir/container-rc.yaml"
182185
destroy "$namespace"
186+
desc 'test passed'

‎e2e-tests/demand-backup/run‎

Lines changed: 16 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -188,7 +188,7 @@ wait_backup_agent $cluster-1
188188
wait_backup_agent $cluster-2
189189

190190
if [ -z "$SKIP_BACKUPS_TO_AWS_GCP_AZURE" ]; then
191-
desc 'Check GCS pfofiles'
191+
desc 'Check GCS profiles'
192192
compare_pbm_profile_setup "some-name" "gcp-cs-s3"
193193
compare_pbm_profile_setup "some-name" "gcp-cs-sa"
194194
fi
@@ -223,22 +223,25 @@ sleep 5
223223
if [ -z "$SKIP_BACKUPS_TO_AWS_GCP_AZURE" ]; then
224224
desc 'check backup and restore -- aws-s3'
225225
backup_dest_aws=$(get_backup_dest "$backup_name_aws")
226-
check_backup_in_storage ${backup_name_aws} s3 rs0 'myApp.test.gz'
226+
setup_aws_credentials
227+
check_backup_existence_aws ${backup_dest_aws} '/rs0/myApp.test.gz'
227228
run_recovery_check "$backup_name_aws" "$cluster"
228229

229230
desc 'check backup and restore -- gcp-cs-s3'
230231
backup_dest_gcp_s3=$(get_backup_dest "$backup_name_gcp_s3")
231-
check_backup_in_storage ${backup_name_gcp_s3} gcs rs0 'myApp.test.gz'
232+
setup_gcs_credentials
233+
check_backup_existence_gcs ${backup_dest_gcp_s3} '/rs0/myApp.test.gz'
232234
run_recovery_check "$backup_name_gcp_s3" "$cluster"
233235

234236
desc 'check backup and restore -- gcp-cs-sa'
235237
backup_dest_gcp_sa=$(get_backup_dest "$backup_name_gcp_sa")
236-
check_backup_in_storage ${backup_name_gcp_sa} gcs rs0 'myApp.test.gz'
238+
check_backup_existence_gcs ${backup_dest_gcp_sa} '/rs0/myApp.test.gz'
237239
run_recovery_check "$backup_name_gcp_sa" "$cluster"
238240

239241
desc 'check backup and restore -- azure-blob'
240242
backup_dest_azure=$(get_backup_dest "$backup_name_azure")
241-
check_backup_in_storage ${backup_name_azure} azure rs0 'myApp.test.gz'
243+
setup_azure_credentials
244+
check_backup_existence_azure ${backup_dest_azure} '/rs0/myApp.test.gz'
242245
run_recovery_check "$backup_name_azure" "$cluster"
243246
fi
244247

@@ -285,10 +288,10 @@ if [[ $backup_exists -eq 1 ]]; then
285288
fi
286289

287290
if [ -z "$SKIP_BACKUPS_TO_AWS_GCP_AZURE" ]; then
288-
check_backup_deletion "https://s3.amazonaws.com/${backup_dest_aws}" "aws-s3"
289-
check_backup_deletion "https://storage.googleapis.com/${backup_dest_gcp_s3}" "gcp-cs-s3"
290-
check_backup_deletion "https://storage.googleapis.com/${backup_dest_gcp_sa}" "gcp-cs-sa"
291-
check_backup_deletion "${backup_dest_azure}" "azure-blob"
291+
check_backup_deletion_aws "$backup_dest_aws" "/rs0/myApp.test.gz"
292+
check_backup_deletion_gcs "$backup_dest_gcp_s3" "/rs0/myApp.test.gz"
293+
check_backup_deletion_gcs "$backup_dest_gcp_sa" "/rs0/myApp.test.gz"
294+
check_backup_deletion_azure "$backup_dest_azure" "/rs0/myApp.test.gz"
292295
fi
293296

294297
desc 'checking backup deletion without cr'
@@ -327,10 +330,10 @@ if [[ $backup_exists -eq 1 ]]; then
327330
fi
328331

329332
if [ -z "$SKIP_BACKUPS_TO_AWS_GCP_AZURE" ]; then
330-
check_backup_deletion "https://s3.amazonaws.com/${backup_dest_aws}" "aws-s3"
331-
check_backup_deletion "https://storage.googleapis.com/${backup_dest_gcp_s3}" "gcp-cs-s3"
332-
check_backup_deletion "https://storage.googleapis.com/${backup_dest_gcp_sa}" "gcp-cs-sa"
333-
check_backup_deletion "${backup_dest_azure}" "azure-blob"
333+
check_backup_deletion_aws "$backup_dest_aws" "/rs0/myApp.test.gz"
334+
check_backup_deletion_gcs "$backup_dest_gcp_s3" "/rs0/myApp.test.gz"
335+
check_backup_deletion_gcs "$backup_dest_gcp_sa" "/rs0/myApp.test.gz"
336+
check_backup_deletion_azure "$backup_dest_azure" "/rs0/myApp.test.gz"
334337
fi
335338

336339
# Temporarily skipping this check

0 commit comments

Comments
 (0)