FEAT: Improve adversarial benchmark dataset and scoring

.azuredevops/adversarial-benchmark.yml

@@ -1,5 +1,5 @@
 # Manually triggered prototype for comparing adversarial models and technique sets.
-# Completed results are retained with the Azure DevOps pipeline run as an artifact.
+# Completed results and reusable SQLite state are retained as pipeline artifacts.
 
 trigger: none
 pr: none
@@ -16,18 +16,38 @@ parameters:
   - name: techniques
     displayName: Space-separated technique names
     type: string
-    default: >-
-      role_play_movie_script role_play_video_game role_play_trivia_game
-      role_play_persuasion role_play_persuasion_written red_teaming
-      context_compliance tap crescendo_simulated
+    default: role_play_video_game crescendo_simulated tap
   - name: datasetName
     displayName: Dataset name
     type: string
-    default: harmbench-balanced-14-v1
+    default: adversarial_benchmark_v1
+  - name: benchmarkProfile
+    displayName: Benchmark profile
+    type: string
+    default: quick
+    values:
+      - quick
+      - full
   - name: maxDatasetSize
-    displayName: Maximum objectives
+    displayName: Maximum objectives override (0 uses profile)
+    type: number
+    default: 0
+  - name: tapTreeWidth
+    displayName: TAP tree width override (0 uses profile)
+    type: number
+    default: 0
+  - name: tapTreeDepth
+    displayName: TAP tree depth override (0 uses profile)
+    type: number
+    default: 0
+  - name: tapBranchingFactor
+    displayName: TAP branching factor override (0 uses profile)
+    type: number
+    default: 0
+  - name: tapBatchSize
+    displayName: TAP node batch size override (0 uses profile)
     type: number
-    default: 14
+    default: 0
   - name: maxConcurrency
     displayName: Maximum concurrency
     type: number
@@ -36,6 +56,10 @@ parameters:
     displayName: Maximum scenario retries
     type: number
     default: 0
+  - name: useCached
+    displayName: Reuse compatible results from prior runs
+    type: boolean
+    default: true
   - name: logLevel
     displayName: Log level
     type: string
@@ -57,6 +81,21 @@ jobs:
       - checkout: self
         fetchDepth: 1
 
+      - ${{ if eq(parameters.useCached, true) }}:
+          - task: DownloadPipelineArtifact@2
+            displayName: Restore prior benchmark database
+            continueOnError: true
+            inputs:
+              buildType: specific
+              project: $(System.TeamProject)
+              definition: $(System.DefinitionId)
+              buildVersionToDownload: latestFromBranch
+              branchName: $(Build.SourceBranch)
+              allowPartiallySucceededBuilds: true
+              allowFailedBuilds: true
+              artifactName: adversarial-benchmark-db
+              targetPath: $(Build.SourcesDirectory)/dbdata
+
       - task: UsePythonVersion@0
         displayName: Use Python 3.12
         inputs:
@@ -109,6 +148,61 @@ jobs:
           uv sync
         displayName: Install PyRIT
 
+      - bash: |
+          set -euo pipefail
+
+          case "$BENCHMARK_PROFILE_INPUT" in
+            quick)
+              profile_max_dataset_size=24
+              profile_tap_tree_width=2
+              profile_tap_tree_depth=3
+              profile_tap_branching_factor=2
+              profile_tap_batch_size=2
+              ;;
+            full)
+              profile_max_dataset_size=120
+              profile_tap_tree_width=3
+              profile_tap_tree_depth=5
+              profile_tap_branching_factor=2
+              profile_tap_batch_size=10
+              ;;
+            *)
+              echo "Unknown benchmark profile: $BENCHMARK_PROFILE_INPUT" >&2
+              exit 2
+              ;;
+          esac
+
+          resolve_override() {
+            local override="$1"
+            local profile_value="$2"
+            if [[ "$override" -gt 0 ]]; then
+              echo "$override"
+            else
+              echo "$profile_value"
+            fi
+          }
+
+          echo "##vso[task.setvariable variable=benchmarkMaxDatasetSize]$(resolve_override "$MAX_DATASET_SIZE_OVERRIDE" "$profile_max_dataset_size")"
+
+          # TAP search parameters reach the scenario through its technique-agnostic
+          # --technique-args flag, so the scenario itself stays free of per-technique knobs.
+          technique_args=(
+            "tap.tree_width=$(resolve_override "$TAP_TREE_WIDTH_OVERRIDE" "$profile_tap_tree_width")"
+            "tap.tree_depth=$(resolve_override "$TAP_TREE_DEPTH_OVERRIDE" "$profile_tap_tree_depth")"
+            "tap.branching_factor=$(resolve_override "$TAP_BRANCHING_FACTOR_OVERRIDE" "$profile_tap_branching_factor")"
+            "tap.batch_size=$(resolve_override "$TAP_BATCH_SIZE_OVERRIDE" "$profile_tap_batch_size")"
+          )
+          echo "##vso[task.setvariable variable=benchmarkTechniqueArgs]${technique_args[*]}"
+        displayName: Resolve benchmark profile
+        env:
+          BENCHMARK_PROFILE_INPUT: "${{ parameters.benchmarkProfile }}"
+          MAX_DATASET_SIZE_OVERRIDE: "${{ parameters.maxDatasetSize }}"
+          TAP_TREE_WIDTH_OVERRIDE: "${{ parameters.tapTreeWidth }}"
+          TAP_TREE_DEPTH_OVERRIDE: "${{ parameters.tapTreeDepth }}"
+          TAP_BRANCHING_FACTOR_OVERRIDE: "${{ parameters.tapBranchingFactor }}"
+          TAP_BATCH_SIZE_OVERRIDE: "${{ parameters.tapBatchSize }}"
+        condition: always()
+
       - task: AzureCLI@2
         displayName: Run benchmark and capture result snapshot
         inputs:
@@ -182,6 +276,8 @@ jobs:
             read -r -a techniques <<< "$TECHNIQUES_INPUT"
             memory_labels="$(printf '{"pipeline_build_id":"%s"}' "$BUILD_BUILDID")"
 
+            read -r -a technique_args <<< "$TECHNIQUE_ARGS_INPUT"
+
             echo "run-benchmark" > "$artifact_dir/phase.txt"
             set +e
             uv run pyrit_scan run benchmark.adversarial \
@@ -193,6 +289,8 @@ jobs:
               --max-dataset-size "$MAX_DATASET_SIZE_INPUT" \
               --max-concurrency "$MAX_CONCURRENCY_INPUT" \
               --max-retries "$MAX_RETRIES_INPUT" \
+              --use-cached "$USE_CACHED_INPUT" \
+              --technique-args "${technique_args[@]}" \
               --memory-labels "$memory_labels" \
               2>&1 | tee "$scenario_log"
             benchmark_exit_code=${PIPESTATUS[0]}
@@ -282,9 +380,11 @@ jobs:
           ADVERSARIAL_TARGETS_INPUT: "${{ parameters.adversarialTargets }}"
           TECHNIQUES_INPUT: "${{ parameters.techniques }}"
           DATASET_NAME_INPUT: "${{ parameters.datasetName }}"
-          MAX_DATASET_SIZE_INPUT: "${{ parameters.maxDatasetSize }}"
+          MAX_DATASET_SIZE_INPUT: "$(benchmarkMaxDatasetSize)"
           MAX_CONCURRENCY_INPUT: "${{ parameters.maxConcurrency }}"
           MAX_RETRIES_INPUT: "${{ parameters.maxRetries }}"
+          USE_CACHED_INPUT: "${{ parameters.useCached }}"
+          TECHNIQUE_ARGS_INPUT: "$(benchmarkTechniqueArgs)"
           LOG_LEVEL_INPUT: "${{ parameters.logLevel }}"
 
       - bash: |
@@ -336,9 +436,12 @@ jobs:
               "adversarial_targets": os.environ["ADVERSARIAL_TARGETS_INPUT"].split(),
               "techniques": os.environ["TECHNIQUES_INPUT"].split(),
               "dataset_name": os.environ["DATASET_NAME_INPUT"],
+              "benchmark_profile": os.environ["BENCHMARK_PROFILE_INPUT"],
               "max_dataset_size": int(os.environ["MAX_DATASET_SIZE_INPUT"]),
               "max_concurrency": int(os.environ["MAX_CONCURRENCY_INPUT"]),
               "max_retries": int(os.environ["MAX_RETRIES_INPUT"]),
+              "use_cached": os.environ["USE_CACHED_INPUT"].lower() == "true",
+              "technique_args": os.environ["TECHNIQUE_ARGS_INPUT"].split(),
               "log_level": os.environ["LOG_LEVEL_INPUT"],
           }
           (artifact_dir / "run-manifest.json").write_text(
@@ -353,11 +456,36 @@ jobs:
           ADVERSARIAL_TARGETS_INPUT: "${{ parameters.adversarialTargets }}"
           TECHNIQUES_INPUT: "${{ parameters.techniques }}"
           DATASET_NAME_INPUT: "${{ parameters.datasetName }}"
-          MAX_DATASET_SIZE_INPUT: "${{ parameters.maxDatasetSize }}"
+          BENCHMARK_PROFILE_INPUT: "${{ parameters.benchmarkProfile }}"
+          MAX_DATASET_SIZE_INPUT: "$(benchmarkMaxDatasetSize)"
           MAX_CONCURRENCY_INPUT: "${{ parameters.maxConcurrency }}"
           MAX_RETRIES_INPUT: "${{ parameters.maxRetries }}"
+          USE_CACHED_INPUT: "${{ parameters.useCached }}"
+          TECHNIQUE_ARGS_INPUT: "$(benchmarkTechniqueArgs)"
           LOG_LEVEL_INPUT: "${{ parameters.logLevel }}"
 
+      - bash: |
+          set -euo pipefail
+          db_artifact_dir="$(Build.ArtifactStagingDirectory)/adversarial-benchmark-db"
+          mkdir -p "$db_artifact_dir"
+          if [[ -f dbdata/pyrit.db ]]; then
+            cp dbdata/pyrit.db "$db_artifact_dir/pyrit.db"
+            echo "##vso[task.setvariable variable=hasBenchmarkDatabase]true"
+          else
+            echo "No benchmark database was created."
+            echo "##vso[task.setvariable variable=hasBenchmarkDatabase]false"
+          fi
+        displayName: Stage reusable benchmark database
+        condition: always()
+
+      - task: PublishPipelineArtifact@1
+        displayName: Publish reusable benchmark database
+        condition: and(always(), eq(variables['hasBenchmarkDatabase'], 'true'))
+        inputs:
+          targetPath: $(Build.ArtifactStagingDirectory)/adversarial-benchmark-db
+          artifactName: adversarial-benchmark-db
+          publishLocation: pipeline
+
       - task: PublishPipelineArtifact@1
         displayName: Publish benchmark snapshot
         condition: always()

pyrit/analytics/result_analysis.py

@@ -162,8 +162,11 @@ def _objective_target_eval_hash_for(attack_result: AttackResult) -> str | None:
     """
     Return the ObjectiveTargetEvaluationIdentifier eval hash for a result.
 
-    Walks ``atomic_attack_identifier.attack_technique.objective_target`` and
-    wraps the resulting identifier in ``ObjectiveTargetEvaluationIdentifier``.
+    Walks the current
+    ``atomic_attack_identifier.attack_technique.attack.objective_target``
+    shape and wraps the resulting identifier in
+    ``ObjectiveTargetEvaluationIdentifier``. The legacy direct
+    ``attack_technique.objective_target`` shape is also accepted.
 
     Args:
         attack_result (AttackResult): The attack result whose persisted
@@ -183,7 +186,8 @@ def _objective_target_eval_hash_for(attack_result: AttackResult) -> str | None:
     if technique is None:
         return None
 
-    target = technique.get_child("objective_target")
+    attack = technique.get_child("attack")
+    target = attack.get_child("objective_target") if attack else technique.get_child("objective_target")
     if target is None:
         return None