Arm backend: Enable Vulkan validation layer

.github/workflows/pull.yml

@@ -854,6 +854,46 @@ jobs:
         # Test test_arm_backend.sh with test
         backends/arm/test/test_arm_backend.sh "${ARM_TEST}"
 
+  test-arm-backend-vkml:
+    name: test-arm-backend-vkml
+    uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main
+    permissions:
+      id-token: write
+      contents: read
+    strategy:
+      matrix:
+        include:
+          - test_arm_backend: test_pytest_ops_vkml
+          - test_arm_backend: test_ootb_tests_vgf
+      fail-fast: false
+    with:
+      runner: linux.2xlarge.memory
+      docker-image: ci-image:executorch-ubuntu-24.04-arm-sdk
+      submodules: 'recursive'
+      ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
+      timeout: 120
+      script: |
+        CONDA_ENV=$(conda env list --json | jq -r ".envs | .[-1]")
+        conda activate "${CONDA_ENV}"
+
+        source .ci/scripts/utils.sh
+        install_executorch "--use-pt-pinned-commit"
+
+        .ci/scripts/setup-arm-baremetal-tools.sh \
+          --disable-ethos-u-deps \
+          --enable-mlsdk-deps \
+          --install-mlsdk-deps-with-pip
+
+        sudo sysctl fs.inotify.max_user_watches=1048576
+
+        ARM_TEST=${{ matrix.test_arm_backend }}
+
+        if [[ -n "${RUNNER_TEST_RESULTS_DIR:-}" ]]; then
+          export PYTEST_ADDOPTS="--junit-xml=${RUNNER_TEST_RESULTS_DIR}/${ARM_TEST}.xml ${PYTEST_ADDOPTS:-}"
+        fi
+
+        backends/arm/test/test_arm_backend.sh "${ARM_TEST}"
+
   test-arm-backend-public-api-backward-compatibility:
     needs: docker-image
     name: test-arm-backend-public-api-backward-compatibility

backends/arm/test/conftest.py

@@ -157,6 +157,39 @@ def pytest_sessionfinish(session, exitstatus):
 # ==== Pytest fixtures =====
 
 
+@pytest.fixture(autouse=True)
+def enable_vulkan_validation_for_vgf_tests(request, monkeypatch) -> None:
+    """Configure Vulkan validation for VGF tests only when explicitly requested.
+
+    Dedicated VGF/VKML test entry points set EXECUTORCH_VGF_VULKAN_VALIDATION
+    when validation is required. Generic pytest/Buck runs leave it unset, so
+    validation-specific integration tests can skip cleanly on hosts without the
+    Vulkan SDK or Khronos validation layer.
+
+    """
+    if "vgf" not in request.node.nodeid.lower():
+        return
+
+    from executorch.backends.arm.test import runner_utils
+
+    if not runner_utils._vulkan_validation_requested():
+        return
+
+    configured_env = runner_utils._enable_vulkan_validation(dict(os.environ))
+    for variable in (
+        "VK_LAYER_PATH",
+        "VK_ADD_LAYER_PATH",
+        "VK_INSTANCE_LAYERS",
+        "VK_KHRONOS_VALIDATION_REPORT_FLAGS",
+        "VK_KHRONOS_VALIDATION_LOG_FILENAME",
+        "VK_KHRONOS_VALIDATION_DEBUG_ACTION",
+        runner_utils.VULKAN_VALIDATION_MESSAGE_FILTER_ENV,
+    ):
+        value = configured_env.get(variable)
+        if value is not None:
+            monkeypatch.setenv(variable, value)
+
+
 @pytest.fixture(autouse=True)
 def set_random_seed(request):
     """Control random numbers in Arm test suite. Default behavior is to use a

backends/arm/test/misc/test_runner_utils.py

@@ -4,6 +4,7 @@
 # LICENSE file in the root directory of this source tree.
 
 import json
+import os
 from pathlib import Path
 from types import SimpleNamespace
 from typing import Any, cast
@@ -249,3 +250,74 @@ def fake_tensor(node):
     monkeypatch.setattr(runner_utils, "get_first_fake_tensor", fake_tensor)
 
     assert not runner_utils.user_inputs_need_shape_inference(cast(Any, program))
+
+
+def test_enable_vulkan_validation_puts_validation_first_and_deduplicates(
+    monkeypatch,
+    tmp_path: Path,
+) -> None:
+    monkeypatch.setenv(runner_utils.VULKAN_VALIDATION_ENV, "1")
+
+    vulkan_sdk = tmp_path / "vulkan-sdk"
+    validation_layer_dir = vulkan_sdk / "share/vulkan/explicit_layer.d"
+    validation_layer_dir.mkdir(parents=True)
+    (validation_layer_dir / "VkLayer_khronos_validation.json").write_text(
+        "{}", encoding="utf-8"
+    )
+
+    emulation_layer_dir = tmp_path / "emulation-layers"
+    emulation_layer_dir.mkdir()
+    emulation_layers = ["VK_LAYER_ARM_tensor", "VK_LAYER_ARM_graph"]
+    env = {
+        # Include a trailing path separator to cover setup_path.sh's CI form.
+        "VULKAN_SDK": f"{vulkan_sdk}{os.path.pathsep}",
+        "VK_LAYER_PATH": str(emulation_layer_dir),
+        "VK_ADD_LAYER_PATH": str(emulation_layer_dir),
+        "VK_INSTANCE_LAYERS": os.path.pathsep.join(
+            [
+                emulation_layers[0],
+                runner_utils.VULKAN_VALIDATION_LAYER,
+                emulation_layers[1],
+                runner_utils.VULKAN_VALIDATION_LAYER,
+            ]
+        ),
+    }
+
+    result = runner_utils._enable_vulkan_validation(env)
+    layers = result["VK_INSTANCE_LAYERS"].split(os.path.pathsep)
+
+    assert layers == [
+        runner_utils.VULKAN_VALIDATION_LAYER,
+        *emulation_layers,
+    ]
+    assert result["VK_LAYER_PATH"].split(os.path.pathsep) == [
+        str(validation_layer_dir),
+        str(emulation_layer_dir),
+    ]
+    assert result["VK_ADD_LAYER_PATH"].split(os.path.pathsep) == [
+        str(validation_layer_dir),
+        str(emulation_layer_dir),
+    ]
+    assert result["VK_KHRONOS_VALIDATION_REPORT_FLAGS"] == "error"
+
+
+def test_add_known_vkml_validation_filters_uses_platform_separator() -> None:
+    custom_vuid = "VUID-Test-existing-filter-00001"
+    first_known_vuid = runner_utils.KNOWN_VKML_VALIDATION_VUIDS[0]
+    env = {
+        runner_utils.VULKAN_VALIDATION_MESSAGE_FILTER_ENV: os.path.pathsep.join(
+            [custom_vuid, first_known_vuid]
+        )
+    }
+
+    runner_utils._add_known_vkml_validation_filters(env)
+
+    filters = env[runner_utils.VULKAN_VALIDATION_MESSAGE_FILTER_ENV].split(
+        os.path.pathsep
+    )
+
+    assert filters == [
+        custom_vuid,
+        *runner_utils.KNOWN_VKML_VALIDATION_VUIDS,
+    ]
+    assert filters.count(first_known_vuid) == 1