From 591aa7d28bc46731cffb9e364a0dac3b251212fb Mon Sep 17 00:00:00 2001 From: Omkar Gaikwad Date: Fri, 24 Apr 2026 06:59:03 +0000 Subject: [PATCH 1/7] feat: add release version substitution to run configuration and update cloudbuild environment variables --- cloudbuild.yaml | 14 +++++++++++++- evals/run_config.yaml | 1 + evals/substitute_env.py | 21 +++++++++++---------- 3 files changed, 25 insertions(+), 11 deletions(-) diff --git a/cloudbuild.yaml b/cloudbuild.yaml index 402e43c..118cf1f 100644 --- a/cloudbuild.yaml +++ b/cloudbuild.yaml @@ -26,12 +26,24 @@ steps: - '-c' - | set -e + + # Determine Release Version + if [[ '$_PR_LABELS' =~ 'autorelease: triggered' ]]; then + if [[ '$_PR_TITLE' =~ release\ ([0-9]+\.[0-9]+\.[0-9]+) ]]; then + export RELEASE_VERSION="$${BASH_REMATCH[1]}" + else + export RELEASE_VERSION="unknown" + fi + else + export RELEASE_VERSION="unknown" + fi + # Workaround for evalbench bug: settings are only applied if path basename matches extension ID ln -s /workspace /workspace/cloud-sql-postgresql cd /evalbench export EVAL_GCP_PROJECT_ID=$PROJECT_ID - export EVAL_GCP_PROJECT_REGION=us-central1 + export EVAL_GCP_PROJECT_REGION=$_CLOUD_SQL_REGION export GOOGLE_CLOUD_PROJECT=$PROJECT_ID export CLOUD_SQL_POSTGRES_PROJECT=$PROJECT_ID export CLOUD_SQL_POSTGRES_INSTANCE=$_CLOUD_SQL_INSTANCE diff --git a/evals/run_config.yaml b/evals/run_config.yaml index b83b7e6..7c40df7 100644 --- a/evals/run_config.yaml +++ b/evals/run_config.yaml @@ -13,6 +13,7 @@ # limitations under the License. extension_id: cloud-sql-postgresql +release_version: ${RELEASE_VERSION} dataset_config: /workspace/evals/dataset.json dataset_format: gemini-cli-format diff --git a/evals/substitute_env.py b/evals/substitute_env.py index 3ef2295..f10c8e3 100644 --- a/evals/substitute_env.py +++ b/evals/substitute_env.py @@ -2,16 +2,17 @@ import re def main(): - yaml_path = '/workspace/evals/model_config.yaml' - if os.path.exists(yaml_path): - with open(yaml_path, 'r') as f: - content = f.read() - content = re.sub(r'\${(\w+)}', lambda m: os.environ.get(m.group(1), m.group(0)), content) - with open(yaml_path, 'w') as f: - f.write(content) - print(f"Successfully substituted environment variables in {yaml_path}") - else: - print(f"File not found: {yaml_path}") + yaml_paths = ['/workspace/evals/model_config.yaml', '/workspace/evals/run_config.yaml'] + for yaml_path in yaml_paths: + if os.path.exists(yaml_path): + with open(yaml_path, 'r') as f: + content = f.read() + content = re.sub(r'\${(\w+)}', lambda m: os.environ.get(m.group(1), m.group(0)), content) + with open(yaml_path, 'w') as f: + f.write(content) + print(f"Successfully substituted environment variables in {yaml_path}") + else: + print(f"File not found: {yaml_path}") if __name__ == '__main__': main() \ No newline at end of file From 34ed6197382dab4b0b4157f5c3b1ee9a4e171926 Mon Sep 17 00:00:00 2001 From: Omkar Gaikwad Date: Fri, 24 Apr 2026 07:25:15 +0000 Subject: [PATCH 2/7] refactor: remove comment from simulated user model configuration in run_config.yaml --- evals/run_config.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/evals/run_config.yaml b/evals/run_config.yaml index 7c40df7..eb745a9 100644 --- a/evals/run_config.yaml +++ b/evals/run_config.yaml @@ -20,7 +20,6 @@ dataset_format: gemini-cli-format orchestrator: geminicli model_config: /workspace/evals/model_config.yaml -# You can reference default simulated user models provided by the evalbench repo: simulated_user_model_config: /workspace/evals/gemini_2.5_pro_model.yaml scorers: From 0bb72aca91486e90f7675488bedb850a8a51ef08 Mon Sep 17 00:00:00 2001 From: Omkar Gaikwad Date: Fri, 24 Apr 2026 07:36:36 +0000 Subject: [PATCH 3/7] feat: add branch validation check to prevent execution on non-release branches in cloudbuild.yaml --- cloudbuild.yaml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/cloudbuild.yaml b/cloudbuild.yaml index 118cf1f..ce40a66 100644 --- a/cloudbuild.yaml +++ b/cloudbuild.yaml @@ -26,6 +26,13 @@ steps: - '-c' - | set -e + + # Only run on release branches + if [[ "$_HEAD_BRANCH" != release-please-* ]]; then + echo "Not a release-please branch. Exiting." + exit 0 + fi + echo "Release branch detected. Proceeding with Evalbench..." # Determine Release Version if [[ '$_PR_LABELS' =~ 'autorelease: triggered' ]]; then From ee863cb6d570cbec38dcd582b84e007c42fd1f82 Mon Sep 17 00:00:00 2001 From: Omkar Gaikwad Date: Fri, 24 Apr 2026 09:04:52 +0000 Subject: [PATCH 4/7] feat: update release logic to fetch PR labels and title via GitHub API using secret token --- cloudbuild.yaml | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/cloudbuild.yaml b/cloudbuild.yaml index ce40a66..b22421f 100644 --- a/cloudbuild.yaml +++ b/cloudbuild.yaml @@ -26,17 +26,18 @@ steps: - '-c' - | set -e - - # Only run on release branches - if [[ "$_HEAD_BRANCH" != release-please-* ]]; then - echo "Not a release-please branch. Exiting." - exit 0 - fi - echo "Release branch detected. Proceeding with Evalbench..." + + # Fetch PR data using curl approach + PR_DATA=$(curl -s -H "Authorization: token $$GITHUB_TOKEN" \ + "https://api.github.com/repos/$REPO_FULL_NAME/pulls/$_PR_NUMBER") + + # Extract labels and title from PR data + PR_LABELS=$(echo "$PR_DATA" | jq -r '.labels[].name' | paste -sd ',') + PR_TITLE=$(echo "$PR_DATA" | jq -r '.title') # Determine Release Version - if [[ '$_PR_LABELS' =~ 'autorelease: triggered' ]]; then - if [[ '$_PR_TITLE' =~ release\ ([0-9]+\.[0-9]+\.[0-9]+) ]]; then + if [[ '$PR_LABELS' =~ 'autorelease: triggered' ]]; then + if [[ '$PR_TITLE' =~ release\ ([0-9]+\.[0-9]+\.[0-9]+) ]]; then export RELEASE_VERSION="$${BASH_REMATCH[1]}" else export RELEASE_VERSION="unknown" @@ -77,3 +78,5 @@ availableSecrets: secretManager: - versionName: projects/$PROJECT_ID/secrets/daily-ci-evals-db-password/versions/latest env: 'DB_PASSWORD' + - versionName: projects/$PROJECT_ID/secrets/GITHUB_TOKEN/versions/latest + env: 'GITHUB_TOKEN' \ No newline at end of file From 1464f7398a6fbab7124f7abe072d8faca3d90dec Mon Sep 17 00:00:00 2001 From: Omkar Gaikwad Date: Fri, 24 Apr 2026 09:10:57 +0000 Subject: [PATCH 5/7] feat: add GITHUB_TOKEN to build environment and restrict execution to release branches --- cloudbuild.yaml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/cloudbuild.yaml b/cloudbuild.yaml index b22421f..349f84b 100644 --- a/cloudbuild.yaml +++ b/cloudbuild.yaml @@ -21,12 +21,19 @@ steps: - name: 'us-central1-docker.pkg.dev/cloud-db-nl2sql/evalbench/eval_server:latest' entrypoint: 'bash' # Decrypts the secret from Secret Manager into the DB_PASSWORD environment variable - secretEnv: ['DB_PASSWORD'] + secretEnv: ['DB_PASSWORD', 'GITHUB_TOKEN'] args: - '-c' - | set -e + # Only run on release branches + if [[ "$_HEAD_BRANCH" != release-please-* ]]; then + echo "Not a release-please branch. Exiting." + exit 0 + fi + echo "Release branch detected. Fetching PR data from GitHub API..." + # Fetch PR data using curl approach PR_DATA=$(curl -s -H "Authorization: token $$GITHUB_TOKEN" \ "https://api.github.com/repos/$REPO_FULL_NAME/pulls/$_PR_NUMBER") From c47e893cda29de9b4e62b21410d76d6091c23107 Mon Sep 17 00:00:00 2001 From: Omkar Gaikwad Date: Fri, 24 Apr 2026 09:14:55 +0000 Subject: [PATCH 6/7] fix: correct bash variable escaping and comparison syntax in cloudbuild.yaml --- cloudbuild.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/cloudbuild.yaml b/cloudbuild.yaml index 349f84b..c87e1ad 100644 --- a/cloudbuild.yaml +++ b/cloudbuild.yaml @@ -38,13 +38,13 @@ steps: PR_DATA=$(curl -s -H "Authorization: token $$GITHUB_TOKEN" \ "https://api.github.com/repos/$REPO_FULL_NAME/pulls/$_PR_NUMBER") - # Extract labels and title from PR data - PR_LABELS=$(echo "$PR_DATA" | jq -r '.labels[].name' | paste -sd ',') - PR_TITLE=$(echo "$PR_DATA" | jq -r '.title') + # Extract labels and title from PR data (Use $$ to escape bash variables) + PR_LABELS=$(echo "$$PR_DATA" | jq -r '.labels[].name' | paste -sd ',') + PR_TITLE=$(echo "$$PR_DATA" | jq -r '.title') - # Determine Release Version - if [[ '$PR_LABELS' =~ 'autorelease: triggered' ]]; then - if [[ '$PR_TITLE' =~ release\ ([0-9]+\.[0-9]+\.[0-9]+) ]]; then + # Determine Release Version (Use double quotes and $$ for bash variables) + if [[ "$$PR_LABELS" == *"autorelease: triggered"* ]]; then + if [[ "$$PR_TITLE" =~ release\ ([0-9]+\.[0-9]+\.[0-9]+) ]]; then export RELEASE_VERSION="$${BASH_REMATCH[1]}" else export RELEASE_VERSION="unknown" From 03525f0eacc5feb332f1228397b54af7aa3d4daf Mon Sep 17 00:00:00 2001 From: Omkar Gaikwad Date: Fri, 24 Apr 2026 17:02:56 +0000 Subject: [PATCH 7/7] fix: implement status code validation and improve label parsing in cloudbuild.yaml --- cloudbuild.yaml | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/cloudbuild.yaml b/cloudbuild.yaml index c87e1ad..2929cc7 100644 --- a/cloudbuild.yaml +++ b/cloudbuild.yaml @@ -34,12 +34,20 @@ steps: fi echo "Release branch detected. Fetching PR data from GitHub API..." - # Fetch PR data using curl approach - PR_DATA=$(curl -s -H "Authorization: token $$GITHUB_TOKEN" \ + # Fetch PR data and status code + HTTP_STATUS=$(curl -s -o pr_data.json -w "%{http_code}" -H "Authorization: token $$GITHUB_TOKEN" \ "https://api.github.com/repos/$REPO_FULL_NAME/pulls/$_PR_NUMBER") + if [ "$$HTTP_STATUS" -ne 200 ]; then + echo "Error fetching PR data: HTTP $$HTTP_STATUS" + cat pr_data.json + exit 1 + fi + + PR_DATA=$(cat pr_data.json) + # Extract labels and title from PR data (Use $$ to escape bash variables) - PR_LABELS=$(echo "$$PR_DATA" | jq -r '.labels[].name' | paste -sd ',') + PR_LABELS=$(echo "$$PR_DATA" | jq -r '[.labels[].name] | join(",")') PR_TITLE=$(echo "$$PR_DATA" | jq -r '.title') # Determine Release Version (Use double quotes and $$ for bash variables) @@ -86,4 +94,4 @@ availableSecrets: - versionName: projects/$PROJECT_ID/secrets/daily-ci-evals-db-password/versions/latest env: 'DB_PASSWORD' - versionName: projects/$PROJECT_ID/secrets/GITHUB_TOKEN/versions/latest - env: 'GITHUB_TOKEN' \ No newline at end of file + env: 'GITHUB_TOKEN'