From 64f511f7823d9c361249fd78ca0c99d85c1ed41c Mon Sep 17 00:00:00 2001 From: Elena Zhelezina Date: Wed, 26 Aug 2026 22:18:30 +0100 Subject: [PATCH] Arm backend: Enable Vulkan validation layer Signed-off-by: Elena Zhelezina Change-Id: Ia693cf38b8a5ccf97fd6c4afed6e23611a0df556 --- .github/workflows/pull.yml | 40 ++ backends/arm/test/conftest.py | 33 ++ backends/arm/test/misc/test_runner_utils.py | 72 +++ .../test/misc/test_vulkan_validation_layer.py | 409 ++++++++++++++++++ .../arm/test/run_with_vulkan_validation.sh | 80 ++++ backends/arm/test/runner_utils.py | 159 ++++++- backends/arm/test/targets.bzl | 1 + backends/arm/test/test_arm_backend.sh | 117 ++++- 8 files changed, 901 insertions(+), 10 deletions(-) create mode 100644 backends/arm/test/misc/test_vulkan_validation_layer.py create mode 100755 backends/arm/test/run_with_vulkan_validation.sh diff --git a/.github/workflows/pull.yml b/.github/workflows/pull.yml index e51901411ab..8534f6a88ac 100644 --- a/.github/workflows/pull.yml +++ b/.github/workflows/pull.yml @@ -854,6 +854,46 @@ jobs: # Test test_arm_backend.sh with test backends/arm/test/test_arm_backend.sh "${ARM_TEST}" + test-arm-backend-vkml: + name: test-arm-backend-vkml + uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main + permissions: + id-token: write + contents: read + strategy: + matrix: + include: + - test_arm_backend: test_pytest_ops_vkml + - test_arm_backend: test_ootb_tests_vgf + fail-fast: false + with: + runner: linux.2xlarge.memory + docker-image: ci-image:executorch-ubuntu-24.04-arm-sdk + submodules: 'recursive' + ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} + timeout: 120 + script: | + CONDA_ENV=$(conda env list --json | jq -r ".envs | .[-1]") + conda activate "${CONDA_ENV}" + + source .ci/scripts/utils.sh + install_executorch "--use-pt-pinned-commit" + + .ci/scripts/setup-arm-baremetal-tools.sh \ + --disable-ethos-u-deps \ + --enable-mlsdk-deps \ + --install-mlsdk-deps-with-pip + + sudo sysctl fs.inotify.max_user_watches=1048576 + + ARM_TEST=${{ matrix.test_arm_backend }} + + if [[ -n "${RUNNER_TEST_RESULTS_DIR:-}" ]]; then + export PYTEST_ADDOPTS="--junit-xml=${RUNNER_TEST_RESULTS_DIR}/${ARM_TEST}.xml ${PYTEST_ADDOPTS:-}" + fi + + backends/arm/test/test_arm_backend.sh "${ARM_TEST}" + test-arm-backend-public-api-backward-compatibility: needs: docker-image name: test-arm-backend-public-api-backward-compatibility diff --git a/backends/arm/test/conftest.py b/backends/arm/test/conftest.py index 6f6c8296649..d32269ee900 100644 --- a/backends/arm/test/conftest.py +++ b/backends/arm/test/conftest.py @@ -157,6 +157,39 @@ def pytest_sessionfinish(session, exitstatus): # ==== Pytest fixtures ===== +@pytest.fixture(autouse=True) +def enable_vulkan_validation_for_vgf_tests(request, monkeypatch) -> None: + """Configure Vulkan validation for VGF tests only when explicitly requested. + + Dedicated VGF/VKML test entry points set EXECUTORCH_VGF_VULKAN_VALIDATION + when validation is required. Generic pytest/Buck runs leave it unset, so + validation-specific integration tests can skip cleanly on hosts without the + Vulkan SDK or Khronos validation layer. + + """ + if "vgf" not in request.node.nodeid.lower(): + return + + from executorch.backends.arm.test import runner_utils + + if not runner_utils._vulkan_validation_requested(): + return + + configured_env = runner_utils._enable_vulkan_validation(dict(os.environ)) + for variable in ( + "VK_LAYER_PATH", + "VK_ADD_LAYER_PATH", + "VK_INSTANCE_LAYERS", + "VK_KHRONOS_VALIDATION_REPORT_FLAGS", + "VK_KHRONOS_VALIDATION_LOG_FILENAME", + "VK_KHRONOS_VALIDATION_DEBUG_ACTION", + runner_utils.VULKAN_VALIDATION_MESSAGE_FILTER_ENV, + ): + value = configured_env.get(variable) + if value is not None: + monkeypatch.setenv(variable, value) + + @pytest.fixture(autouse=True) def set_random_seed(request): """Control random numbers in Arm test suite. Default behavior is to use a diff --git a/backends/arm/test/misc/test_runner_utils.py b/backends/arm/test/misc/test_runner_utils.py index 6acb04e054f..18543160acf 100644 --- a/backends/arm/test/misc/test_runner_utils.py +++ b/backends/arm/test/misc/test_runner_utils.py @@ -4,6 +4,7 @@ # LICENSE file in the root directory of this source tree. import json +import os from pathlib import Path from types import SimpleNamespace from typing import Any, cast @@ -249,3 +250,74 @@ def fake_tensor(node): monkeypatch.setattr(runner_utils, "get_first_fake_tensor", fake_tensor) assert not runner_utils.user_inputs_need_shape_inference(cast(Any, program)) + + +def test_enable_vulkan_validation_puts_validation_first_and_deduplicates( + monkeypatch, + tmp_path: Path, +) -> None: + monkeypatch.setenv(runner_utils.VULKAN_VALIDATION_ENV, "1") + + vulkan_sdk = tmp_path / "vulkan-sdk" + validation_layer_dir = vulkan_sdk / "share/vulkan/explicit_layer.d" + validation_layer_dir.mkdir(parents=True) + (validation_layer_dir / "VkLayer_khronos_validation.json").write_text( + "{}", encoding="utf-8" + ) + + emulation_layer_dir = tmp_path / "emulation-layers" + emulation_layer_dir.mkdir() + emulation_layers = ["VK_LAYER_ARM_tensor", "VK_LAYER_ARM_graph"] + env = { + # Include a trailing path separator to cover setup_path.sh's CI form. + "VULKAN_SDK": f"{vulkan_sdk}{os.path.pathsep}", + "VK_LAYER_PATH": str(emulation_layer_dir), + "VK_ADD_LAYER_PATH": str(emulation_layer_dir), + "VK_INSTANCE_LAYERS": os.path.pathsep.join( + [ + emulation_layers[0], + runner_utils.VULKAN_VALIDATION_LAYER, + emulation_layers[1], + runner_utils.VULKAN_VALIDATION_LAYER, + ] + ), + } + + result = runner_utils._enable_vulkan_validation(env) + layers = result["VK_INSTANCE_LAYERS"].split(os.path.pathsep) + + assert layers == [ + runner_utils.VULKAN_VALIDATION_LAYER, + *emulation_layers, + ] + assert result["VK_LAYER_PATH"].split(os.path.pathsep) == [ + str(validation_layer_dir), + str(emulation_layer_dir), + ] + assert result["VK_ADD_LAYER_PATH"].split(os.path.pathsep) == [ + str(validation_layer_dir), + str(emulation_layer_dir), + ] + assert result["VK_KHRONOS_VALIDATION_REPORT_FLAGS"] == "error" + + +def test_add_known_vkml_validation_filters_uses_platform_separator() -> None: + custom_vuid = "VUID-Test-existing-filter-00001" + first_known_vuid = runner_utils.KNOWN_VKML_VALIDATION_VUIDS[0] + env = { + runner_utils.VULKAN_VALIDATION_MESSAGE_FILTER_ENV: os.path.pathsep.join( + [custom_vuid, first_known_vuid] + ) + } + + runner_utils._add_known_vkml_validation_filters(env) + + filters = env[runner_utils.VULKAN_VALIDATION_MESSAGE_FILTER_ENV].split( + os.path.pathsep + ) + + assert filters == [ + custom_vuid, + *runner_utils.KNOWN_VKML_VALIDATION_VUIDS, + ] + assert filters.count(first_known_vuid) == 1 diff --git a/backends/arm/test/misc/test_vulkan_validation_layer.py b/backends/arm/test/misc/test_vulkan_validation_layer.py new file mode 100644 index 00000000000..e7202c876dc --- /dev/null +++ b/backends/arm/test/misc/test_vulkan_validation_layer.py @@ -0,0 +1,409 @@ +# Copyright 2026 Arm Limited and/or its affiliates. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. +"""Integration test for the Khronos Vulkan Validation Layer.""" + +from __future__ import annotations + +import os +import subprocess # nosec B404 +import sys +from pathlib import Path + +import pytest + + +_VALIDATION_LAYER = "VK_LAYER_KHRONOS_validation" +_VALIDATION_ENV = "EXECUTORCH_VGF_VULKAN_VALIDATION" +_EXPECTED_VUID = "VUID-VkApplicationInfo-sType-sType" + +# We check that validation layer is enabled. +# Run the actual Vulkan call in a child process so loading Vulkan/VVL does not +# modify the pytest process. +_VULKAN_VALIDATION_PROBE = r""" +import ctypes +import ctypes.util +import os +import sys +from pathlib import Path + + +VK_STRUCTURE_TYPE_APPLICATION_INFO = 0 +VK_STRUCTURE_TYPE_INSTANCE_CREATE_INFO = 1 +VK_SUCCESS = 0 + + +class VkApplicationInfo(ctypes.Structure): + _fields_ = [ + ("sType", ctypes.c_int32), + ("pNext", ctypes.c_void_p), + ("pApplicationName", ctypes.c_char_p), + ("applicationVersion", ctypes.c_uint32), + ("pEngineName", ctypes.c_char_p), + ("engineVersion", ctypes.c_uint32), + ("apiVersion", ctypes.c_uint32), + ] + + +class VkInstanceCreateInfo(ctypes.Structure): + _fields_ = [ + ("sType", ctypes.c_int32), + ("pNext", ctypes.c_void_p), + ("flags", ctypes.c_uint32), + ("pApplicationInfo", ctypes.POINTER(VkApplicationInfo)), + ("enabledLayerCount", ctypes.c_uint32), + ("ppEnabledLayerNames", ctypes.POINTER(ctypes.c_char_p)), + ("enabledExtensionCount", ctypes.c_uint32), + ("ppEnabledExtensionNames", ctypes.POINTER(ctypes.c_char_p)), + ] + + +def vulkan_library_candidates(): + candidates = [] + + sdk = os.environ.get("VULKAN_SDK", "") + if sdk: + sdk_root = Path(sdk.split(os.pathsep)[0]) + + if sys.platform == "darwin": + candidates.extend( + [ + sdk_root / "lib/libvulkan.dylib", + sdk_root / "lib/libvulkan.1.dylib", + ] + ) + elif sys.platform == "win32": + candidates.extend( + [ + sdk_root / "Bin/vulkan-1.dll", + sdk_root / "bin/vulkan-1.dll", + ] + ) + else: + candidates.extend( + [ + sdk_root / "lib/libvulkan.so.1", + sdk_root / "lib/libvulkan.so", + ] + ) + + discovered = ctypes.util.find_library("vulkan") + if discovered: + candidates.append(discovered) + + if sys.platform == "darwin": + candidates.extend(["libvulkan.dylib", "libvulkan.1.dylib"]) + elif sys.platform == "win32": + candidates.append("vulkan-1.dll") + else: + candidates.extend(["libvulkan.so.1", "libvulkan.so"]) + + return candidates + + +def load_vulkan(): + errors = [] + + for candidate in vulkan_library_candidates(): + try: + return ctypes.CDLL(str(candidate)) + except OSError as exc: + errors.append(f"{candidate}: {exc}") + + raise RuntimeError( + "Unable to load Vulkan loader.\n" + "\n".join(errors) + ) + + +vulkan = load_vulkan() + +vkCreateInstance = vulkan.vkCreateInstance +vkCreateInstance.argtypes = [ + ctypes.POINTER(VkInstanceCreateInfo), + ctypes.c_void_p, + ctypes.POINTER(ctypes.c_void_p), +] +vkCreateInstance.restype = ctypes.c_int32 + +vkDestroyInstance = vulkan.vkDestroyInstance +vkDestroyInstance.argtypes = [ + ctypes.c_void_p, + ctypes.c_void_p, +] +vkDestroyInstance.restype = None + + +# Deliberately invalid: +# +# VkApplicationInfo requires +# +# VK_STRUCTURE_TYPE_APPLICATION_INFO +# +# but we intentionally provide VK_STRUCTURE_TYPE_INSTANCE_CREATE_INFO. +# +# VK_LAYER_KHRONOS_validation must report: +# +# VUID-VkApplicationInfo-sType-sType +# +application_info = VkApplicationInfo( + sType=VK_STRUCTURE_TYPE_INSTANCE_CREATE_INFO, + pNext=None, + pApplicationName=b"executorch-vulkan-validation-probe", + applicationVersion=0, + pEngineName=b"executorch", + engineVersion=0, + apiVersion=0, +) + +instance_info = VkInstanceCreateInfo( + sType=VK_STRUCTURE_TYPE_INSTANCE_CREATE_INFO, + pNext=None, + flags=0, + pApplicationInfo=ctypes.pointer(application_info), + enabledLayerCount=0, + ppEnabledLayerNames=None, + enabledExtensionCount=0, + ppEnabledExtensionNames=None, +) + +instance = ctypes.c_void_p() + +result = vkCreateInstance( + ctypes.byref(instance_info), + None, + ctypes.byref(instance), +) + +print(f"vkCreateInstance result={result}") + +if result == VK_SUCCESS and instance.value: + vkDestroyInstance(instance, None) +""" + + +def _env_flag_enabled(name: str) -> bool: + value = os.environ.get(name) + + if value is None: + return False + + return value.strip().lower() not in { + "", + "0", + "false", + "off", + "no", + } + + +def _configured_instance_layers() -> list[str]: + value = os.environ.get("VK_INSTANCE_LAYERS", "") + + return [layer for layer in value.split(os.pathsep) if layer] + + +def _assert_validation_manifest_visible() -> None: + """Check VK_LAYER_PATH when it overrides normal loader discovery.""" + + layer_path = os.environ.get("VK_LAYER_PATH") + + if not layer_path: + # The loader can use its standard installation directories. + return + + manifest = "VkLayer_khronos_validation.json" + + candidates = [ + Path(entry) / manifest for entry in layer_path.split(os.pathsep) if entry + ] + + assert any(path.is_file() for path in candidates), ( + "VK_LAYER_PATH is set, but it does not contain the Khronos " + "validation-layer manifest.\n" + f"VK_LAYER_PATH={layer_path}\n" + "Checked:\n" + "\n".join(f" {path}" for path in candidates) + ) + + +def test_vgf_vulkan_validation_layer_reports_bad_stype(): + """Verify that the real Khronos layer detects invalid Vulkan usage.""" + + if not _env_flag_enabled(_VALIDATION_ENV): + pytest.skip(f"{_VALIDATION_ENV} is not enabled for this test run") + + layers = _configured_instance_layers() + + assert _VALIDATION_LAYER in layers, ( + f"{_VALIDATION_ENV}=1, but {_VALIDATION_LAYER} is not present in " + "VK_INSTANCE_LAYERS.\n" + f"VK_INSTANCE_LAYERS={os.environ.get('VK_INSTANCE_LAYERS', '')}" + ) + + _assert_validation_manifest_visible() + + env = os.environ.copy() + + # Test the Khronos layer itself in isolation. We already asserted above + # that the real VGF/VKML environment contains it alongside the emulation + # layers. + env["VK_INSTANCE_LAYERS"] = _VALIDATION_LAYER + + # Force deterministic diagnostic output from this negative probe. + env["VK_KHRONOS_VALIDATION_REPORT_FLAGS"] = "error" + env["VK_KHRONOS_VALIDATION_LOG_FILENAME"] = "stdout" + env["VK_KHRONOS_VALIDATION_DEBUG_ACTION"] = "VK_DBG_LAYER_ACTION_LOG_MSG" + + result = subprocess.run( # nosec B603 + [ + sys.executable, + "-c", + _VULKAN_VALIDATION_PROBE, + ], + env=env, + check=False, + capture_output=True, + text=True, + timeout=30, + ) + + output = result.stdout + "\n" + result.stderr + + assert _EXPECTED_VUID in output, ( + "VK_LAYER_KHRONOS_validation was enabled but did not report the " + "deliberately invalid VkApplicationInfo::sType.\n\n" + f"Expected VUID:\n {_EXPECTED_VUID}\n\n" + f"VK_INSTANCE_LAYERS={env.get('VK_INSTANCE_LAYERS', '')}\n" + f"VK_LAYER_PATH={env.get('VK_LAYER_PATH', '')}\n" + f"VK_ADD_LAYER_PATH={env.get('VK_ADD_LAYER_PATH', '')}\n" + f"VULKAN_SDK={env.get('VULKAN_SDK', '')}\n\n" + f"Probe return code: {result.returncode}\n\n" + f"stdout:\n{result.stdout}\n\n" + f"stderr:\n{result.stderr}" + ) + + +def test_vgf_validation_wrapper_fails_on_validation_error(): + """A real VVL error must make the direct-command wrapper fail.""" + + if not _env_flag_enabled(_VALIDATION_ENV): + pytest.skip(f"{_VALIDATION_ENV} is not enabled for this test run") + + validation_wrapper = ( + Path(__file__).resolve().parents[1] / "run_with_vulkan_validation.sh" + ) + + if not validation_wrapper.is_file(): + pytest.skip( + "Vulkan validation shell wrapper is not available in this " + "test environment" + ) + + env = os.environ.copy() + env[_VALIDATION_ENV] = "1" + env["VK_INSTANCE_LAYERS"] = _VALIDATION_LAYER + + # The wrapper itself must perform the error-to-failure conversion. + # Do not use VK_DBG_LAYER_ACTION_FAIL here. + env["VK_KHRONOS_VALIDATION_DEBUG_ACTION"] = "VK_DBG_LAYER_ACTION_LOG_MSG" + + result = subprocess.run( # nosec B603 + [ + str(validation_wrapper), + sys.executable, + "-c", + _VULKAN_VALIDATION_PROBE, + ], + env=env, + check=False, + capture_output=True, + text=True, + timeout=30, + ) + + output = result.stdout + "\n" + result.stderr + + assert result.returncode != 0, ( + "The Vulkan validation wrapper returned success even though the " + "probe deliberately generated invalid Vulkan usage.\n\n" + f"stdout:\n{result.stdout}\n\n" + f"stderr:\n{result.stderr}" + ) + + assert _EXPECTED_VUID in output, ( + "The wrapper failed, but the expected Vulkan validation diagnostic " + "was not observed.\n\n" + f"Expected VUID: {_EXPECTED_VUID}\n\n" + f"stdout:\n{result.stdout}\n\n" + f"stderr:\n{result.stderr}" + ) + + +def test_vgf_validation_wrapper_preserves_error_from_earlier_vulkan_child(): + """A later clean Vulkan child must not erase an earlier validation error.""" + + if not _env_flag_enabled(_VALIDATION_ENV): + pytest.skip(f"{_VALIDATION_ENV} is not enabled for this test run") + + validation_wrapper = ( + Path(__file__).resolve().parents[1] / "run_with_vulkan_validation.sh" + ) + if not validation_wrapper.is_file(): + pytest.skip( + "Vulkan validation shell wrapper is not available in this " + "test environment" + ) + + # Fix only VkApplicationInfo::sType to create a clean second probe. + clean_probe = _VULKAN_VALIDATION_PROBE.replace( + "sType=VK_STRUCTURE_TYPE_INSTANCE_CREATE_INFO,\n" + " pNext=None,\n" + ' pApplicationName=b"executorch-vulkan-validation-probe"', + "sType=VK_STRUCTURE_TYPE_APPLICATION_INFO,\n" + " pNext=None,\n" + ' pApplicationName=b"executorch-vulkan-validation-probe"', + 1, + ) + assert clean_probe != _VULKAN_VALIDATION_PROBE + + # Run the invalid child first and a clean Vulkan child second. With the old + # shared log file, the clean child could truncate the first child's VUID. + multi_child_probe = f""" +import subprocess +import sys + +invalid_probe = {_VULKAN_VALIDATION_PROBE!r} +clean_probe = {clean_probe!r} + +subprocess.run([sys.executable, "-c", invalid_probe], check=True) +subprocess.run([sys.executable, "-c", clean_probe], check=True) +""" + + env = os.environ.copy() + env[_VALIDATION_ENV] = "1" + env["VK_INSTANCE_LAYERS"] = _VALIDATION_LAYER + env["VK_KHRONOS_VALIDATION_DEBUG_ACTION"] = "VK_DBG_LAYER_ACTION_LOG_MSG" + + result = subprocess.run( # nosec B603 + [str(validation_wrapper), sys.executable, "-c", multi_child_probe], + env=env, + check=False, + capture_output=True, + text=True, + timeout=30, + ) + + output = result.stdout + "\n" + result.stderr + + assert result.returncode != 0, ( + "The validation wrapper returned success even though an earlier Vulkan " + "child generated a validation error and a clean child ran afterwards.\n\n" + f"stdout:\n{result.stdout}\n\n" + f"stderr:\n{result.stderr}" + ) + assert _EXPECTED_VUID in output, ( + "The validation error from the first Vulkan child was not preserved.\n\n" + f"Expected VUID: {_EXPECTED_VUID}\n\n" + f"stdout:\n{result.stdout}\n\n" + f"stderr:\n{result.stderr}" + ) diff --git a/backends/arm/test/run_with_vulkan_validation.sh b/backends/arm/test/run_with_vulkan_validation.sh new file mode 100755 index 00000000000..f888b3712c2 --- /dev/null +++ b/backends/arm/test/run_with_vulkan_validation.sh @@ -0,0 +1,80 @@ +#!/usr/bin/env bash +# Copyright 2026 Arm Limited and/or its affiliates. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +# Run a command with Khronos Vulkan validation and convert validation +# diagnostics into a failing command status. +# +# This intentionally does not depend on VK_DBG_LAYER_ACTION_FAIL because +# older Vulkan Validation Layer releases do not support that action. + +set -uo pipefail + + +vulkan_validation_enabled() { + case "${EXECUTORCH_VGF_VULKAN_VALIDATION:-0}" in + ""|0|false|False|FALSE|off|Off|OFF|no|No|NO) + return 1 + ;; + *) + return 0 + ;; + esac +} + + +if [[ $# -eq 0 ]]; then + echo "Usage: $0 [args ...]" >&2 + exit 2 +fi + + +# Preserve normal behavior when validation was explicitly disabled. +if ! vulkan_validation_enabled; then + exec "$@" +fi + + +tmp_dir="$(mktemp -d "${TMPDIR:-/tmp}/executorch-vulkan-validation.XXXXXX")" +command_log="${tmp_dir}/command.log" + +cleanup() { + rm -rf "${tmp_dir}" +} + +trap cleanup EXIT + + +# Do not rely on VK_DBG_LAYER_ACTION_FAIL. Older VVL versions understand +# LOG_MSG. Route diagnostics to stdout so every Vulkan subprocess contributes +# to the same wrapper-captured stream. A shared log file is unsafe because a +# later process can truncate diagnostics written by an earlier process. +export VK_KHRONOS_VALIDATION_REPORT_FLAGS="error" +export VK_KHRONOS_VALIDATION_DEBUG_ACTION="VK_DBG_LAYER_ACTION_LOG_MSG" +export VK_KHRONOS_VALIDATION_LOG_FILENAME="stdout" + + +# Keep normal command output visible in CI while retaining a copy that can be +# inspected as a fallback. PIPESTATUS[0] is the status of the command, not tee. +"$@" 2>&1 | tee "${command_log}" +command_status="${PIPESTATUS[0]}" + + +# Diagnostics from all Vulkan descendants are routed through stdout/stderr and +# captured by tee above, so this scan covers the entire wrapped command. +validation_error_pattern='Validation Error:|ERROR[[:space:]]*:[[:space:]]*VALIDATION|VUID-[A-Za-z0-9_]' + +if grep -Eq "${validation_error_pattern}" "${command_log}"; then + echo >&2 + echo "Vulkan validation error detected." >&2 + + # Use a deterministic non-zero status. The underlying command may have + # succeeded because VVL only logged the invalid Vulkan usage. + exit 1 +fi + + +# No validation error: preserve the underlying command's actual status. +exit "${command_status}" diff --git a/backends/arm/test/runner_utils.py b/backends/arm/test/runner_utils.py index 7c23ee9dbc2..184e37d95a3 100644 --- a/backends/arm/test/runner_utils.py +++ b/backends/arm/test/runner_utils.py @@ -111,6 +111,27 @@ def _get_qdq_memory_format_ops() -> tuple[object, ...]: } INFER_SHAPES_PATH = "infer_shapes" +VULKAN_VALIDATION_LAYER = "VK_LAYER_KHRONOS_validation" +VULKAN_VALIDATION_ENV = "EXECUTORCH_VGF_VULKAN_VALIDATION" +VULKAN_VALIDATION_MESSAGE_FILTER_ENV = "VK_LAYER_MESSAGE_ID_FILTER" + +# Temporary VKML workaround: BF16 shaders execute, but VKML currently +# does not advertise VK_KHR_shader_bfloat16/shaderBFloat16Type. +# Remove these filters when the VKML capability advertisement is fixed. +# See MLETORCH-2584. +# The same issue with 09748 error, MLETORCH-2582. +# The last 2 errors are the reported bug: +# Incomplete descriptor layout generated by +# Model Converter for duplicated custom-shader inputs. +# MLETORCH-2585 +KNOWN_VKML_VALIDATION_VUIDS = ( + "VUID-VkShaderModuleCreateInfo-pCode-08740", + "VUID-VkShaderModuleCreateInfo-pCode-08742", + "VUID-VkTensorViewCreateInfoARM-usage-09748", + "VUID-VkComputePipelineCreateInfo-layout-07988", + "VUID-vkCmdDispatch-None-08114", +) + class QuantizationParams: __slots__ = ["node_name", "zp", "scale", "qmin", "qmax", "dtype"] @@ -502,10 +523,14 @@ def run_vkml_emulation_layer( cmd_line += input_string cmd_line = cmd_line.split() - result = _run_cmd(cmd_line, env=_get_vkml_runtime_env()) + runtime_env = _get_vkml_runtime_env() + result = _run_cmd(cmd_line, env=runtime_env) + + result_stdout = result.stdout.decode(errors="replace") + result_stderr = result.stderr.decode(errors="replace") - # TODO: Add regex to check for error or fault messages in stdout from Emulation Layer - result_stdout = result.stdout.decode() # noqa: F841 + if _vulkan_validation_requested(): + _raise_on_vulkan_validation_errors(result_stdout, result_stderr) return get_output_from_file(exported_program, intermediate_path, output_base_name) @@ -744,6 +769,130 @@ def _prepend_env_path(existing: str | None, value: str) -> str: return os.path.pathsep.join([value, *parts]) +def _vulkan_validation_requested() -> bool: + value = os.environ.get(VULKAN_VALIDATION_ENV) + if value is None: + return False + + return value.strip().lower() not in { + "", + "0", + "false", + "off", + "no", + } + + +def _add_known_vkml_validation_filters(env: dict[str, str]) -> None: + """Mute only known VKML validation errors; preserve all other VVL errors.""" + existing = [ + value.strip() + for value in env.get(VULKAN_VALIDATION_MESSAGE_FILTER_ENV, "").split( + os.path.pathsep + ) + if value.strip() + ] + + for vuid in KNOWN_VKML_VALIDATION_VUIDS: + if vuid not in existing: + existing.append(vuid) + + env[VULKAN_VALIDATION_MESSAGE_FILTER_ENV] = os.path.pathsep.join(existing) + + +def _enable_vulkan_validation(env: dict[str, str]) -> dict[str, str]: + """Enable Khronos Vulkan validation for VKML test subprocesses. + + The layer is injected through the Vulkan loader rather than being added to + VGFBackend's VkInstanceCreateInfo, so production runtime behavior is + unchanged. + + """ + if not _vulkan_validation_requested(): + return env + + env = env.copy() + + _add_known_vkml_validation_filters(env) + + # Make the Khronos validation manifest visible when setup_path.sh has set + # VK_LAYER_PATH to only the ML emulation-layer manifests. VK_LAYER_PATH + # overrides the Vulkan loader's normal layer search path. + vulkan_sdk_roots = [ + entry for entry in env.get("VULKAN_SDK", "").split(os.path.pathsep) if entry + ] + if vulkan_sdk_roots: + validation_layer_dir = ( + Path(vulkan_sdk_roots[0]) / "share/vulkan/explicit_layer.d" + ) + validation_manifest = validation_layer_dir / "VkLayer_khronos_validation.json" + if validation_manifest.is_file(): + validation_layer_dir_text = str(validation_layer_dir) + for variable in ("VK_LAYER_PATH", "VK_ADD_LAYER_PATH"): + existing_paths = [ + entry + for entry in env.get(variable, "").split(os.path.pathsep) + if entry and entry != validation_layer_dir_text + ] + env[variable] = os.path.pathsep.join( + [validation_layer_dir_text, *existing_paths] + ) + + # Vulkan instance layers are ordered from closest to the application + # outward. Keep validation first and remove any pre-existing duplicate so + # the Python runner matches test_arm_backend.sh. + existing_layers = [ + layer + for layer in env.get("VK_INSTANCE_LAYERS", "").split(os.path.pathsep) + if layer and layer != VULKAN_VALIDATION_LAYER + ] + env["VK_INSTANCE_LAYERS"] = os.path.pathsep.join( + [VULKAN_VALIDATION_LAYER, *existing_layers] + ) + + # Make validation output deterministic and parseable by the test runner. + # These are Khronos Validation Layer settings. + env.setdefault("VK_KHRONOS_VALIDATION_REPORT_FLAGS", "error") + env.setdefault( + "VK_KHRONOS_VALIDATION_DEBUG_ACTION", + "VK_DBG_LAYER_ACTION_LOG_MSG", + ) + env.setdefault( + "VK_KHRONOS_VALIDATION_LOG_FILENAME", + "stdout", + ) + + logger.info( + "Vulkan validation enabled for VKML runtime: %s", + env["VK_INSTANCE_LAYERS"], + ) + + return env + + +def _raise_on_vulkan_validation_errors(stdout: str, stderr: str) -> None: + """Fail a VKML test when Khronos validation reports an API error.""" + combined = "\n".join(part for part in (stdout, stderr) if part) + + error_lines = [ + line + for line in combined.splitlines() + if "Validation Error:" in line or "ERROR : VALIDATION" in line + ] + + if not error_lines: + return + + summary = "\n".join(error_lines) + + raise RuntimeError( + "Vulkan validation failed during VGF execution.\n\n" + f"{summary}\n\n" + "Full executor output:\n" + f"{combined}" + ) + + def _find_local_vulkan_sdk_root() -> Path | None: repo_root = Path(__file__).resolve().parents[3] sdk_base_dir = repo_root / "examples/arm/arm-scratch/vulkan_sdk" @@ -778,7 +927,7 @@ def _get_vkml_runtime_env() -> dict[str, str]: env = os.environ.copy() sdk_root = _find_local_vulkan_sdk_root() if sdk_root is None: - return env + return _enable_vulkan_validation(env) env["VULKAN_SDK"] = str(sdk_root) env["PATH"] = _prepend_env_path(env.get("PATH"), str(sdk_root / "bin")) @@ -802,7 +951,7 @@ def _get_vkml_runtime_env() -> dict[str, str]: env.get("LD_LIBRARY_PATH"), str(sdk_root / "lib") ) - return env + return _enable_vulkan_validation(env) def _run_cmd( diff --git a/backends/arm/test/targets.bzl b/backends/arm/test/targets.bzl index 9f87daf5a44..9a9349db0bf 100644 --- a/backends/arm/test/targets.bzl +++ b/backends/arm/test/targets.bzl @@ -82,6 +82,7 @@ def define_arm_tests(): "misc/test_mxfp_linear_ao.py", "misc/test_post_quant_device_switch.py", "misc/test_vgf_check_env.py", + "misc/test_vulkan_validation_layer.py", "misc/test_vgf_backend.py", "misc/test_vgf_smoke.py", # "misc/test_dim_order.py", (TODO - T238390249) diff --git a/backends/arm/test/test_arm_backend.sh b/backends/arm/test/test_arm_backend.sh index 7e2e28d9d51..1e6694ba73b 100755 --- a/backends/arm/test/test_arm_backend.sh +++ b/backends/arm/test/test_arm_backend.sh @@ -7,6 +7,7 @@ set -e script_dir=$(cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd) +vulkan_validation_runner="${script_dir}/run_with_vulkan_validation.sh" # Executorch root et_root_dir=$(cd ${script_dir}/../../.. && pwd) @@ -20,14 +21,113 @@ setup_path_script=${scratch_dir}/setup_path.sh _setup_msg="please refer to ${et_root_dir}/examples/arm/setup.sh to properly install necessary tools." +# Configure Khronos Vulkan validation for VGF/VKML runtime tests. +# +# This must be called after setup_path.sh is sourced because setup_path.sh +# configures the ML Graph/Tensor emulation layers and their VK_LAYER_PATH. +enable_vgf_vulkan_validation() { + export EXECUTORCH_VGF_VULKAN_VALIDATION="${EXECUTORCH_VGF_VULKAN_VALIDATION:-1}" + + # Allow developers to explicitly disable validation when debugging. + case "${EXECUTORCH_VGF_VULKAN_VALIDATION}" in + ""|0|false|False|FALSE|off|Off|OFF|no|No|NO) + return 0 + ;; + esac + + # ML SDK setup normally sets VK_LAYER_PATH to the emulation-layer + # manifests. VK_LAYER_PATH overrides the loader's standard search paths, + # so make sure the Khronos validation manifest from the Vulkan SDK is + # included as well. + if [[ -n "${VULKAN_SDK:-}" ]]; then + # VULKAN_SDK should normally be a single path. Take the first entry + # defensively in case the environment contains duplicate path entries. + local vulkan_sdk_root="${VULKAN_SDK%%:*}" + local validation_layer_dir="${vulkan_sdk_root}/share/vulkan/explicit_layer.d" + + if [[ -f "${validation_layer_dir}/VkLayer_khronos_validation.json" ]]; then + if [[ -n "${VK_LAYER_PATH:-}" ]]; then + if [[ ":${VK_LAYER_PATH}:" != *":${validation_layer_dir}:"* ]]; then + export VK_LAYER_PATH="${validation_layer_dir}:${VK_LAYER_PATH}" + fi + else + # VK_ADD_LAYER_PATH is appropriate only when VK_LAYER_PATH is + # not already overriding the loader search path. + unset VK_LAYER_PATH + if [[ ":${VK_ADD_LAYER_PATH:-}:" != *":${validation_layer_dir}:"* ]]; then + export VK_ADD_LAYER_PATH="${validation_layer_dir}${VK_ADD_LAYER_PATH:+:${VK_ADD_LAYER_PATH}}" + fi + fi + fi + fi + + # Put validation closest to the application, ahead of the ML emulation + # layers already installed by setup_path.sh. + if [[ ":${VK_INSTANCE_LAYERS:-}:" != *":VK_LAYER_KHRONOS_validation:"* ]]; then + export VK_INSTANCE_LAYERS="VK_LAYER_KHRONOS_validation${VK_INSTANCE_LAYERS:+:${VK_INSTANCE_LAYERS}}" + fi + + # Log validation diagnostics, but also make invalid Vulkan commands fail. + # Restrict this to errors so warnings do not turn otherwise-valid tests + # into failures. + export VK_KHRONOS_VALIDATION_REPORT_FLAGS="error" + export VK_KHRONOS_VALIDATION_LOG_FILENAME="stdout" + export VK_KHRONOS_VALIDATION_DEBUG_ACTION="VK_DBG_LAYER_ACTION_LOG_MSG" + + # Temporary workarounds for known VKML / Model Converter validation + # defects. Keep validation enabled, but mute only these tracked VUIDs. + # + # BF16 capability advertisement: MLETORCH-2584 + # Tensor-view usage validation: MLETORCH-2582 + # Duplicated custom-shader descriptor layout: MLETORCH-2585 + # + # Keep this list in sync with KNOWN_VKML_VALIDATION_VUIDS in + # backends/arm/test/runner_utils.py. + # Layer-setting environment variables serialize string lists using the + # platform list separator: ':' on Unix-like systems and ';' on Windows. + local message_id_filter_separator=":" + case "${OSTYPE:-}" in + msys*|cygwin*|win32*) + message_id_filter_separator=";" + ;; + esac + + local known_vuid + for known_vuid in \ + "VUID-VkShaderModuleCreateInfo-pCode-08740" \ + "VUID-VkShaderModuleCreateInfo-pCode-08742" \ + "VUID-VkTensorViewCreateInfoARM-usage-09748" \ + "VUID-VkComputePipelineCreateInfo-layout-07988" \ + "VUID-vkCmdDispatch-None-08114"; do + if [[ "${message_id_filter_separator}${VK_LAYER_MESSAGE_ID_FILTER:-}${message_id_filter_separator}" != *"${message_id_filter_separator}${known_vuid}${message_id_filter_separator}"* ]]; then + if [[ -n "${VK_LAYER_MESSAGE_ID_FILTER:-}" ]]; then + export VK_LAYER_MESSAGE_ID_FILTER="${VK_LAYER_MESSAGE_ID_FILTER}${message_id_filter_separator}${known_vuid}" + else + export VK_LAYER_MESSAGE_ID_FILTER="${known_vuid}" + fi + fi + done + + echo "Vulkan validation enabled for ${TEST_SUITE}" + echo "VK_INSTANCE_LAYERS=${VK_INSTANCE_LAYERS}" +} + TEST_SUITE=$1 +# Enable Vulkan validation for VGF/VKML runtime tests. + # Source the tools # This should be prepared by the setup.sh [[ -f ${setup_path_script} ]] \ || { echo "Missing ${setup_path_script}. ${_setup_msg}"; exit 1; } source ${setup_path_script} +# Enable Vulkan validation for every VGF/VKML test path, including tests that +# launch scripts/executables directly instead of going through runner_utils.py. +if [[ "${TEST_SUITE}" == *vkml* || "${TEST_SUITE}" == *vgf* ]]; then + enable_vgf_vulkan_validation +fi + help() { echo "Usage:" echo " $0 " @@ -284,6 +384,13 @@ test_run_ethos_u85() { # ---------------------------------------------------------- # -------- Vulkan Graph Format (VGF) specific tests -------- # ---------------------------------------------------------- + +echo "EXECUTORCH_VGF_VULKAN_VALIDATION=${EXECUTORCH_VGF_VULKAN_VALIDATION:-UNSET}" +echo "VK_INSTANCE_LAYERS=${VK_INSTANCE_LAYERS:-UNSET}" +echo "VK_LAYER_PATH=${VK_LAYER_PATH:-UNSET}" +echo "VK_ADD_LAYER_PATH=${VK_ADD_LAYER_PATH:-UNSET}" +echo "VULKAN_SDK=${VULKAN_SDK:-UNSET}" + test_pytest_ops_vkml() { echo "${TEST_SUITE_NAME}: Run pytest operator tests with VKML runtime" @@ -329,11 +436,11 @@ test_run_vkml() { out_folder="arm_test/test_run" vkml_build_dir="${build_root_test_dir}" - examples/arm/run.sh --build-dir="${vkml_build_dir}" --et_build_root=${out_folder} --target=vgf --model_name=add --output=${out_folder}/runner - examples/arm/run.sh --build-dir="${vkml_build_dir}" --et_build_root=${out_folder} --target=vgf --model_name=mul --output=${out_folder}/runner + "${vulkan_validation_runner}" examples/arm/run.sh --build-dir="${vkml_build_dir}" --et_build_root=${out_folder} --target=vgf --model_name=add --output=${out_folder}/runner + "${vulkan_validation_runner}" examples/arm/run.sh --build-dir="${vkml_build_dir}" --et_build_root=${out_folder} --target=vgf --model_name=mul --output=${out_folder}/runner - examples/arm/run.sh --build-dir="${vkml_build_dir}" --et_build_root=${out_folder} --target=vgf --model_name=qadd --output=${out_folder}/runner - examples/arm/run.sh --build-dir="${vkml_build_dir}" --et_build_root=${out_folder} --target=vgf --model_name=qops --output=${out_folder}/runner + "${vulkan_validation_runner}" examples/arm/run.sh --build-dir="${vkml_build_dir}" --et_build_root=${out_folder} --target=vgf --model_name=qadd --output=${out_folder}/runner + "${vulkan_validation_runner}" examples/arm/run.sh --build-dir="${vkml_build_dir}" --et_build_root=${out_folder} --target=vgf --model_name=qops --output=${out_folder}/runner echo "${TEST_SUITE_NAME}: PASS" } @@ -364,7 +471,7 @@ test_ootb_tests_tosa() { test_ootb_tests_vgf() { echo "${TEST_SUITE_NAME}: Run out-of-the-box tests for VGF" - backends/arm/test/test_arm_ootb.sh run_ootb_tests_vgf + "${vulkan_validation_runner}" backends/arm/test/test_arm_ootb.sh run_ootb_tests_vgf echo "${TEST_SUITE_NAME}: PASS" }