diff --git a/backends/nxp/tests/generic_tests/test_aot_example.py b/backends/nxp/tests/generic_tests/test_aot_example.py index b75d3605d34..99e5b23c644 100644 --- a/backends/nxp/tests/generic_tests/test_aot_example.py +++ b/backends/nxp/tests/generic_tests/test_aot_example.py @@ -334,3 +334,77 @@ def test_aot_example__mlperf_tiny_kws__profiling(): with _cleanup_generated_files(pte_file, etrecord_file): result = _run_compile(cmd) _assert_profiling(result, pte_file, etrecord_file) + + +def test_aot_example__mlperf_tiny_vww(): + """Test that the MLPerf Tiny Visual Wake Words model (MobileNetV1) can be lowered to Neutron backend via + `aot_neutron_compile.py` and all ops are delegated.""" + + # Number of random samples to generate, must be divisible by number of classes + num_random_samples = 60 + + # Run the compilation script as a module (like run_aot_example.sh does). + cmd = [ + sys.executable, + "-m", + "examples.nxp.aot_neutron_compile", + "--model_name", + "mlperf_tiny_visual_wake_words", + "--delegate", + "--quantize", + "--target", + "imxrt700", + "--use_random_dataset", + "--num_random_samples", + str(num_random_samples), + ] + + # Output file will be created in executorch_root + pte_file = Path( + os.path.join(EXECUTORCH_ROOT, "mlperf_tiny_visual_wake_words_nxp_delegate.pte") + ) + + with _cleanup_generated_files(pte_file): + result = _run_compile(cmd) + _assert_delegation(result, pte_file) + + +def test_aot_example__mlperf_tiny_vww__profiling(): + """Test that the MLPerf Tiny Visual Wake Words model (MobileNetV1) can be lowered to Neutron backend via + `aot_neutron_compile.py` and profiling works as intended.""" + + # Number of random samples to generate, must be divisible by number of classes + num_random_samples = 60 + + # Run the compilation script as a module (like run_aot_example.sh does) + cmd = [ + sys.executable, + "-m", + "examples.nxp.aot_neutron_compile", + "--model_name", + "mlperf_tiny_visual_wake_words", + "--delegate", + "--quantize", + "--target", + "imxrt700", + "--remove-quant-io-ops", + "--use_profiling", # Generate profilable model and create ETRecord + "--use_random_dataset", + "--num_random_samples", + str(num_random_samples), + ] + + pte_file = Path( + os.path.join( + EXECUTORCH_ROOT, "mlperf_tiny_visual_wake_words_nxp_delegate_profile.pte" + ) + ) + etrecord_file = Path( + os.path.join( + EXECUTORCH_ROOT, "etrecord", "mlperf_tiny_visual_wake_words_etrecord.bin" + ) + ) + + with _cleanup_generated_files(pte_file, etrecord_file): + result = _run_compile(cmd) + _assert_profiling(result, pte_file, etrecord_file) diff --git a/backends/nxp/tests/models/test_mlperf_tiny_image_classification.py b/backends/nxp/tests/models/test_mlperf_tiny_image_classification.py index fc0be12f500..a978249f154 100644 --- a/backends/nxp/tests/models/test_mlperf_tiny_image_classification.py +++ b/backends/nxp/tests/models/test_mlperf_tiny_image_classification.py @@ -97,8 +97,8 @@ def test_mlperf_tiny_classification_mse_cpu_vs_npu( request, dataset_creator=dataset_creator, output_comparator=comparator, - reference_model=ref_model, mocker=mocker, + reference_model=ref_model, use_qat=use_qat, train_fn=train_fn, ) diff --git a/backends/nxp/tests/models/test_mlperf_tiny_visual_wake_words.py b/backends/nxp/tests/models/test_mlperf_tiny_visual_wake_words.py new file mode 100644 index 00000000000..c90fd7135b2 --- /dev/null +++ b/backends/nxp/tests/models/test_mlperf_tiny_visual_wake_words.py @@ -0,0 +1,119 @@ +# Copyright 2026 NXP +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +from functools import partial + +import numpy as np + +# noinspection PyUnusedImports +import pytest +import torch + +from executorch.backends.nxp.tests.dataset_creator import ( + FromCalibrationDataDatasetCreator, +) +from executorch.backends.nxp.tests.executorch_pipeline import ModelInputSpec +from executorch.backends.nxp.tests.graph_verifier import BaseGraphVerifier +from executorch.backends.nxp.tests.model_output_comparator import ( + ClassificationAccuracyOutputComparator, + NumericalStatsOutputComparator, +) +from executorch.backends.nxp.tests.nsys_testing import ( + lower_run_compare, + lower_run_compare_ptq_qat, +) +from executorch.backends.nxp.tests.use_qat import * # noqa F403 +from executorch.examples.nxp.models.mlperf_tiny.visual_wake_words.mlperf_tiny_visual_wake_words import ( + MLPerfTinyVisualWakeWords, +) + +BOUNDS_MSE = { + "PTQ": {"channels-last": 4.6e-8, "channels-first": 5.0e-8}, + "QAT": {"channels-last": 3.0e-6, "channels-first": 3.7e-6}, +} + + +@pytest.fixture(autouse=True) +def reseed_model_per_test_run(): + torch.manual_seed(23) + np.random.seed(23) + + +@pytest.mark.parametrize("channels_last", [False, True]) +def test_mlperf_tiny_vww_mse_cpu_vs_npu(mocker, request, channels_last, use_qat): + # 20 samples per class + num_samples = 40 + + visual_wake_words = MLPerfTinyVisualWakeWords( + num_samples=num_samples, use_random_dataset=True + ) + model = visual_wake_words.get_eager_model() + dataset = visual_wake_words.dataset + labels = visual_wake_words.labels + + dataset_creator = FromCalibrationDataDatasetCreator( + dataset, num_examples=num_samples, idx_to_label=labels + ) + + input_spec = ModelInputSpec(visual_wake_words.input_shape) + if channels_last: + model.to(memory_format=torch.channels_last) + input_spec.dim_order = torch.channels_last + + quant_type_key = "QAT" if use_qat else "PTQ" + format_key = "channels-last" if channels_last else "channels-first" + mse = BOUNDS_MSE[quant_type_key][format_key] + comparator = NumericalStatsOutputComparator( + max_mse_error=mse, use_softmax=True, is_classification_task=True + ) + model_verifier = BaseGraphVerifier(1, []) + train_fn = ( + partial(visual_wake_words.train_model_fn, channels_last=channels_last) + if use_qat + else None + ) + + lower_run_compare( + model, + [input_spec], + model_verifier, + request, + dataset_creator=dataset_creator, + output_comparator=comparator, + mocker=mocker, + use_qat=use_qat, + train_fn=train_fn, + ) + + +def test_mlperf_tiny_vww_ptq_qat_equivalence(request): + # 20 samples per class + num_samples = 40 + + visual_wake_words = MLPerfTinyVisualWakeWords( + num_samples=num_samples, use_random_dataset=True + ) + + model = visual_wake_words.get_eager_model() + dataset = visual_wake_words.dataset + labels = visual_wake_words.labels + + dataset_creator = FromCalibrationDataDatasetCreator( + dataset, num_examples=num_samples, idx_to_label=labels + ) + comparator = ClassificationAccuracyOutputComparator(class_dict=labels) + + input_spec = ModelInputSpec(visual_wake_words.input_shape) + model_verifier = BaseGraphVerifier(1, []) + + lower_run_compare_ptq_qat( + model, + [input_spec], + model_verifier, + request, + train_fn=visual_wake_words.train_model_fn, + dataset_creator=dataset_creator, + output_comparator=comparator, + ) diff --git a/examples/nxp/aot_neutron_compile.py b/examples/nxp/aot_neutron_compile.py index b9e3298c26d..e151f729b22 100644 --- a/examples/nxp/aot_neutron_compile.py +++ b/examples/nxp/aot_neutron_compile.py @@ -49,6 +49,9 @@ from executorch.examples.nxp.models.mlperf_tiny.keyword_spotting.mlperf_tiny_keyword_spotting import ( MLPerfTinyKeywordSpotting, ) +from executorch.examples.nxp.models.mlperf_tiny.visual_wake_words.mlperf_tiny_visual_wake_words import ( + MLPerfTinyVisualWakeWords, +) from executorch.examples.nxp.models.mobilenet_v2 import MobilenetV2 from executorch.exir import ( EdgeCompileConfig, @@ -69,6 +72,7 @@ "mobilenetv2": MobilenetV2, "mlperf_tiny_image_classification": MLPerfTinyImageClassification, "mlperf_tiny_keyword_spotting": MLPerfTinyKeywordSpotting, + "mlperf_tiny_visual_wake_words": MLPerfTinyVisualWakeWords, } FORMAT = "[%(levelname)s %(asctime)s %(filename)s:%(lineno)s] %(message)s" @@ -126,7 +130,11 @@ def _get_model_info_from_name( ) model_cls_inst = model_cls() - elif model_cls in (MLPerfTinyImageClassification, MLPerfTinyKeywordSpotting): + elif model_cls in ( + MLPerfTinyImageClassification, + MLPerfTinyKeywordSpotting, + MLPerfTinyVisualWakeWords, + ): model_cls_inst = model_cls( dataset_path=dataset_path, use_random_dataset=use_random_dataset, @@ -337,7 +345,12 @@ def _get_arg_parser(): if args.use_qat: if not isinstance( model_cls_inst, - (CifarNet, MLPerfTinyImageClassification, MLPerfTinyKeywordSpotting), + ( + CifarNet, + MLPerfTinyImageClassification, + MLPerfTinyKeywordSpotting, + MLPerfTinyVisualWakeWords, + ), ): raise ValueError( f"QAT training is not supported for model '{args.model_name}'" diff --git a/examples/nxp/models/mlperf_tiny/visual_wake_words/__init__.py b/examples/nxp/models/mlperf_tiny/visual_wake_words/__init__.py new file mode 100644 index 00000000000..55dc5fccf45 --- /dev/null +++ b/examples/nxp/models/mlperf_tiny/visual_wake_words/__init__.py @@ -0,0 +1,4 @@ +# Copyright 2026 NXP +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. diff --git a/examples/nxp/models/mlperf_tiny/visual_wake_words/mlperf_tiny_visual_wake_words.py b/examples/nxp/models/mlperf_tiny/visual_wake_words/mlperf_tiny_visual_wake_words.py new file mode 100644 index 00000000000..45cdbe38206 --- /dev/null +++ b/examples/nxp/models/mlperf_tiny/visual_wake_words/mlperf_tiny_visual_wake_words.py @@ -0,0 +1,64 @@ +# Copyright 2026 NXP +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +import logging + +import torch + +from executorch.examples.models.mlperf_tiny import MobileNetV1025 +from executorch.examples.nxp.models.mlperf_tiny.mlperf_tiny_model import MLPerfTinyModel + +log = logging.getLogger(__name__) + + +class MLPerfTinyVisualWakeWords(MLPerfTinyModel): + """MLPerf Tiny visual wake words model (MobileNetV1 width 0.25).""" + + # MobileNetV1 specific QAT training hyperparameters. + TRAIN_HYPERPARAMETERS = { + "num_epochs": 15, + "batch_size": 20, + "lr": 2.5e-7, + "eps": 1e-7, + "weight_decay": 0.0, + } + + INPUT_SHAPE = (1, 3, 96, 96) + IDX_TO_LABEL = {0: "person", 1: "non_person"} + + # MobileNetV1 stacks 13 depthwise-separable blocks, each with a BatchNorm. + # Randomly initialized MobileNetV1 has the BatchNorm nodes with default parameters + # (running_mean=0, running_var=1), causing each depthwise-separable block to behave as identity. + # In other MLPerf Tiny models, this could be fixed by multiplying the random weights + # by constant, however in this case the model is too deep and that solution + # is no longer viable. + # Instead calibration of BatchNorm by running few forward passes in train mode fixes it. + BN_CALIBRATION_ITERS = 10 + BN_CALIBRATION_BATCH_SIZE = 16 + + def _calibrate_batch_norm(self, model: torch.nn.Module) -> None: + model.train() + with torch.no_grad(): + for _ in range(self.BN_CALIBRATION_ITERS): + inputs = torch.rand( + (self.BN_CALIBRATION_BATCH_SIZE, *self.input_shape[1:]), + dtype=torch.float32, + ) + model(inputs) + model.eval() + + @property + def input_shape(self): + return self.INPUT_SHAPE + + @property + def labels(self): + return self.IDX_TO_LABEL + + def _init_eager_model(self) -> torch.nn.Module: + model = MobileNetV1025() + self._calibrate_batch_norm(model) + + return model.eval()