NXP backend: handle tests of mlperf tiny visual wake words

backends/nxp/tests/generic_tests/test_aot_example.py

@@ -334,3 +334,77 @@ def test_aot_example__mlperf_tiny_kws__profiling():
     with _cleanup_generated_files(pte_file, etrecord_file):
         result = _run_compile(cmd)
         _assert_profiling(result, pte_file, etrecord_file)
+
+
+def test_aot_example__mlperf_tiny_vww():
+    """Test that the MLPerf Tiny Visual Wake Words model (MobileNetV1) can be lowered to Neutron backend via
+    `aot_neutron_compile.py` and all ops are delegated."""
+
+    # Number of random samples to generate, must be divisible by number of classes
+    num_random_samples = 60
+
+    # Run the compilation script as a module (like run_aot_example.sh does).
+    cmd = [
+        sys.executable,
+        "-m",
+        "examples.nxp.aot_neutron_compile",
+        "--model_name",
+        "mlperf_tiny_visual_wake_words",
+        "--delegate",
+        "--quantize",
+        "--target",
+        "imxrt700",
+        "--use_random_dataset",
+        "--num_random_samples",
+        str(num_random_samples),
+    ]
+
+    # Output file will be created in executorch_root
+    pte_file = Path(
+        os.path.join(EXECUTORCH_ROOT, "mlperf_tiny_visual_wake_words_nxp_delegate.pte")
+    )
+
+    with _cleanup_generated_files(pte_file):
+        result = _run_compile(cmd)
+        _assert_delegation(result, pte_file)
+
+
+def test_aot_example__mlperf_tiny_vww__profiling():
+    """Test that the MLPerf Tiny Visual Wake Words model (MobileNetV1) can be lowered to Neutron backend via
+    `aot_neutron_compile.py` and profiling works as intended."""
+
+    # Number of random samples to generate, must be divisible by number of classes
+    num_random_samples = 60
+
+    # Run the compilation script as a module (like run_aot_example.sh does)
+    cmd = [
+        sys.executable,
+        "-m",
+        "examples.nxp.aot_neutron_compile",
+        "--model_name",
+        "mlperf_tiny_visual_wake_words",
+        "--delegate",
+        "--quantize",
+        "--target",
+        "imxrt700",
+        "--remove-quant-io-ops",
+        "--use_profiling",  # Generate profilable model and create ETRecord
+        "--use_random_dataset",
+        "--num_random_samples",
+        str(num_random_samples),
+    ]
+
+    pte_file = Path(
+        os.path.join(
+            EXECUTORCH_ROOT, "mlperf_tiny_visual_wake_words_nxp_delegate_profile.pte"
+        )
+    )
+    etrecord_file = Path(
+        os.path.join(
+            EXECUTORCH_ROOT, "etrecord", "mlperf_tiny_visual_wake_words_etrecord.bin"
+        )
+    )
+
+    with _cleanup_generated_files(pte_file, etrecord_file):
+        result = _run_compile(cmd)
+        _assert_profiling(result, pte_file, etrecord_file)

backends/nxp/tests/models/test_mlperf_tiny_image_classification.py

@@ -97,8 +97,8 @@ def test_mlperf_tiny_classification_mse_cpu_vs_npu(
         request,
         dataset_creator=dataset_creator,
         output_comparator=comparator,
-        reference_model=ref_model,
         mocker=mocker,
+        reference_model=ref_model,
         use_qat=use_qat,
         train_fn=train_fn,
     )

backends/nxp/tests/models/test_mlperf_tiny_visual_wake_words.py

@@ -0,0 +1,119 @@
+# Copyright 2026 NXP
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+from functools import partial
+
+import numpy as np
+
+# noinspection PyUnusedImports
+import pytest
+import torch
+
+from executorch.backends.nxp.tests.dataset_creator import (
+    FromCalibrationDataDatasetCreator,
+)
+from executorch.backends.nxp.tests.executorch_pipeline import ModelInputSpec
+from executorch.backends.nxp.tests.graph_verifier import BaseGraphVerifier
+from executorch.backends.nxp.tests.model_output_comparator import (
+    ClassificationAccuracyOutputComparator,
+    NumericalStatsOutputComparator,
+)
+from executorch.backends.nxp.tests.nsys_testing import (
+    lower_run_compare,
+    lower_run_compare_ptq_qat,
+)
+from executorch.backends.nxp.tests.use_qat import *  # noqa F403
+from executorch.examples.nxp.models.mlperf_tiny.visual_wake_words.mlperf_tiny_visual_wake_words import (
+    MLPerfTinyVisualWakeWords,
+)
+
+BOUNDS_MSE = {
+    "PTQ": {"channels-last": 4.6e-8, "channels-first": 5.0e-8},
+    "QAT": {"channels-last": 3.0e-6, "channels-first": 3.7e-6},
+}
+
+
+@pytest.fixture(autouse=True)
+def reseed_model_per_test_run():
+    torch.manual_seed(23)
+    np.random.seed(23)
+
+
+@pytest.mark.parametrize("channels_last", [False, True])
+def test_mlperf_tiny_vww_mse_cpu_vs_npu(mocker, request, channels_last, use_qat):
+    # 20 samples per class
+    num_samples = 40
+
+    visual_wake_words = MLPerfTinyVisualWakeWords(
+        num_samples=num_samples, use_random_dataset=True
+    )
+    model = visual_wake_words.get_eager_model()
+    dataset = visual_wake_words.dataset
+    labels = visual_wake_words.labels
+
+    dataset_creator = FromCalibrationDataDatasetCreator(
+        dataset, num_examples=num_samples, idx_to_label=labels
+    )
+
+    input_spec = ModelInputSpec(visual_wake_words.input_shape)
+    if channels_last:
+        model.to(memory_format=torch.channels_last)
+        input_spec.dim_order = torch.channels_last
+
+    quant_type_key = "QAT" if use_qat else "PTQ"
+    format_key = "channels-last" if channels_last else "channels-first"
+    mse = BOUNDS_MSE[quant_type_key][format_key]
+    comparator = NumericalStatsOutputComparator(
+        max_mse_error=mse, use_softmax=True, is_classification_task=True
+    )
+    model_verifier = BaseGraphVerifier(1, [])
+    train_fn = (
+        partial(visual_wake_words.train_model_fn, channels_last=channels_last)
+        if use_qat
+        else None
+    )
+
+    lower_run_compare(
+        model,
+        [input_spec],
+        model_verifier,
+        request,
+        dataset_creator=dataset_creator,
+        output_comparator=comparator,
+        mocker=mocker,
+        use_qat=use_qat,
+        train_fn=train_fn,
+    )
+
+
+def test_mlperf_tiny_vww_ptq_qat_equivalence(request):
+    # 20 samples per class
+    num_samples = 40
+
+    visual_wake_words = MLPerfTinyVisualWakeWords(
+        num_samples=num_samples, use_random_dataset=True
+    )
+
+    model = visual_wake_words.get_eager_model()
+    dataset = visual_wake_words.dataset
+    labels = visual_wake_words.labels
+
+    dataset_creator = FromCalibrationDataDatasetCreator(
+        dataset, num_examples=num_samples, idx_to_label=labels
+    )
+    comparator = ClassificationAccuracyOutputComparator(class_dict=labels)
+
+    input_spec = ModelInputSpec(visual_wake_words.input_shape)
+    model_verifier = BaseGraphVerifier(1, [])
+
+    lower_run_compare_ptq_qat(
+        model,
+        [input_spec],
+        model_verifier,
+        request,
+        train_fn=visual_wake_words.train_model_fn,
+        dataset_creator=dataset_creator,
+        output_comparator=comparator,
+    )

examples/nxp/aot_neutron_compile.py

@@ -49,6 +49,9 @@
 from executorch.examples.nxp.models.mlperf_tiny.keyword_spotting.mlperf_tiny_keyword_spotting import (
     MLPerfTinyKeywordSpotting,
 )
+from executorch.examples.nxp.models.mlperf_tiny.visual_wake_words.mlperf_tiny_visual_wake_words import (
+    MLPerfTinyVisualWakeWords,
+)
 from executorch.examples.nxp.models.mobilenet_v2 import MobilenetV2
 from executorch.exir import (
     EdgeCompileConfig,
@@ -69,6 +72,7 @@
     "mobilenetv2": MobilenetV2,
     "mlperf_tiny_image_classification": MLPerfTinyImageClassification,
     "mlperf_tiny_keyword_spotting": MLPerfTinyKeywordSpotting,
+    "mlperf_tiny_visual_wake_words": MLPerfTinyVisualWakeWords,
 }
 
 FORMAT = "[%(levelname)s %(asctime)s %(filename)s:%(lineno)s] %(message)s"
@@ -126,7 +130,11 @@ def _get_model_info_from_name(
                 )
             model_cls_inst = model_cls()
 
-        elif model_cls in (MLPerfTinyImageClassification, MLPerfTinyKeywordSpotting):
+        elif model_cls in (
+            MLPerfTinyImageClassification,
+            MLPerfTinyKeywordSpotting,
+            MLPerfTinyVisualWakeWords,
+        ):
             model_cls_inst = model_cls(
                 dataset_path=dataset_path,
                 use_random_dataset=use_random_dataset,
@@ -337,7 +345,12 @@ def _get_arg_parser():
         if args.use_qat:
             if not isinstance(
                 model_cls_inst,
-                (CifarNet, MLPerfTinyImageClassification, MLPerfTinyKeywordSpotting),
+                (
+                    CifarNet,
+                    MLPerfTinyImageClassification,
+                    MLPerfTinyKeywordSpotting,
+                    MLPerfTinyVisualWakeWords,
+                ),
             ):
                 raise ValueError(
                     f"QAT training is not supported for model '{args.model_name}'"

examples/nxp/models/mlperf_tiny/visual_wake_words/__init__.py

@@ -0,0 +1,4 @@
+# Copyright 2026 NXP
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.

examples/nxp/models/mlperf_tiny/visual_wake_words/mlperf_tiny_visual_wake_words.py

@@ -0,0 +1,64 @@
+# Copyright 2026 NXP
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+import logging
+
+import torch
+
+from executorch.examples.models.mlperf_tiny import MobileNetV1025
+from executorch.examples.nxp.models.mlperf_tiny.mlperf_tiny_model import MLPerfTinyModel
+
+log = logging.getLogger(__name__)
+
+
+class MLPerfTinyVisualWakeWords(MLPerfTinyModel):
+    """MLPerf Tiny visual wake words model (MobileNetV1 width 0.25)."""
+
+    # MobileNetV1 specific QAT training hyperparameters.
+    TRAIN_HYPERPARAMETERS = {
+        "num_epochs": 15,
+        "batch_size": 20,
+        "lr": 2.5e-7,
+        "eps": 1e-7,
+        "weight_decay": 0.0,
+    }
+
+    INPUT_SHAPE = (1, 3, 96, 96)
+    IDX_TO_LABEL = {0: "person", 1: "non_person"}
+
+    # MobileNetV1 stacks 13 depthwise-separable blocks, each with a BatchNorm.
+    # Randomly initialized MobileNetV1 has the BatchNorm nodes with default parameters
+    # (running_mean=0, running_var=1), causing each depthwise-separable block to behave as identity.
+    # In other MLPerf Tiny models, this could be fixed by multiplying the random weights
+    # by constant, however in this case the model is too deep and that solution
+    # is no longer viable.
+    # Instead calibration of BatchNorm by running few forward passes in train mode fixes it.
+    BN_CALIBRATION_ITERS = 10
+    BN_CALIBRATION_BATCH_SIZE = 16
+
+    def _calibrate_batch_norm(self, model: torch.nn.Module) -> None:
+        model.train()
+        with torch.no_grad():
+            for _ in range(self.BN_CALIBRATION_ITERS):
+                inputs = torch.rand(
+                    (self.BN_CALIBRATION_BATCH_SIZE, *self.input_shape[1:]),
+                    dtype=torch.float32,
+                )
+                model(inputs)
+        model.eval()
+
+    @property
+    def input_shape(self):
+        return self.INPUT_SHAPE
+
+    @property
+    def labels(self):
+        return self.IDX_TO_LABEL
+
+    def _init_eager_model(self) -> torch.nn.Module:
+        model = MobileNetV1025()
+        self._calibrate_batch_norm(model)
+
+        return model.eval()