Align eval labels with benchmarks build tiers (1, 50, 200) (#1254)

Co-authored-by: Claude <noreply@anthropic.com>
Co-authored-by: Xingyao Wang <xingyao@all-hands.dev>

Author: 3 people

.github/workflows/run-eval.yml

@@ -19,10 +19,9 @@ on:
                 type: choice
                 options:
                     - '1'
-                    - '2'
-                    - '10'
                     - '50'
-                    - '100'
+                    - '200'
+                    - '500'
             model_stubs:
                 description: Comma-separated model stubs to evaluate (must be allowlisted)
                 required: false
@@ -45,7 +44,7 @@ env:
     EVAL_AGENT_IMAGE: ghcr.io/openhands/eval-agent-server
     EVAL_AGENT_TARGET: source-minimal
     # Polling configuration for workflow status checks
-    MAX_POLL_ATTEMPTS: '80'  # 80 attempts × 60s = 80 minutes max wait
+    MAX_POLL_ATTEMPTS: '600'  # 600 attempts × 60s = 10 hours max wait
     POLL_INTERVAL_SECONDS: '60'
 
 jobs:
@@ -55,10 +54,10 @@ jobs:
             github.event_name == 'workflow_dispatch' ||
             (github.event_name == 'pull_request_target' &&
              (github.event.label.name == 'run-eval-1' ||
-              github.event.label.name == 'run-eval-2' ||
               github.event.label.name == 'run-eval-50' ||
-              github.event.label.name == 'run-eval-100'))
-        runs-on: blacksmith-32vcpu-ubuntu-2204
+              github.event.label.name == 'run-eval-200' ||
+              github.event.label.name == 'run-eval-500'))
+        runs-on: ubuntu-latest
         permissions:
             contents: read
             packages: write
@@ -71,7 +70,7 @@ jobs:
               uses: actions/checkout@v4
               with:
                   ref: ${{ github.event_name == 'pull_request_target' && github.event.pull_request.base.sha || (github.event_name == 
-                      'workflow_dispatch' && github.event.inputs.branch) || github.ref }}
+                      'workflow_dispatch' && github.event.inputs.sdk_ref) || github.ref }}
                   fetch-depth: 0
 
             - name: Load allowlists
@@ -117,9 +116,9 @@ jobs:
                     LABEL="${{ github.event.label.name }}"
                     case "$LABEL" in
                       run-eval-1) EVAL_LIMIT=1 ;;
-                      run-eval-2) EVAL_LIMIT=2 ;;
                       run-eval-50) EVAL_LIMIT=50 ;;
-                      run-eval-100) EVAL_LIMIT=100 ;;
+                      run-eval-200) EVAL_LIMIT=200 ;;
+                      run-eval-500) EVAL_LIMIT=500 ;;
                       *) echo "Unsupported label $LABEL" >&2; exit 1 ;;
                     esac
                     SDK_REF="${{ github.event.pull_request.head.ref }}"