diff --git a/backends/qualcomm/genai_pipeline/__init__.py b/backends/qualcomm/genai_pipeline/__init__.py index 2c72f93165d..62006847867 100644 --- a/backends/qualcomm/genai_pipeline/__init__.py +++ b/backends/qualcomm/genai_pipeline/__init__.py @@ -6,6 +6,21 @@ __version__ = "1.0.0" +from executorch.backends.qualcomm.genai_pipeline.artifact_keys import ( + ALL_ARTIFACT_KEYS, + ARTIFACT_ATTENTION_SINK_EVICTOR, + ARTIFACT_AUDIO_ENCODER, + ARTIFACT_TEXT_DECODER, + ARTIFACT_TEXT_ENCODER, + ARTIFACT_TOK_EMBEDDING, + ARTIFACT_VISION_ENCODER, + DECODE_QDQ_FILENAME, +) +from executorch.backends.qualcomm.genai_pipeline.compilation import ( + QnnCompileSpecBuilder, + resolve_backend_type, + resolve_soc_model, +) from executorch.backends.qualcomm.genai_pipeline.configs import ( CompilationInputConfig, CompilationOutputConfig, @@ -16,6 +31,7 @@ QuantizationInputConfig, QuantizationOutputConfig, ) +from executorch.backends.qualcomm.genai_pipeline.control_args import ControlArgs from executorch.backends.qualcomm.genai_pipeline.engine_proxy import EngineProxy from executorch.backends.qualcomm.genai_pipeline.exceptions import ( ConfigValidationError, @@ -24,6 +40,16 @@ StageError, ) from executorch.backends.qualcomm.genai_pipeline.genai_pipeline import GenAIPipeline +from executorch.backends.qualcomm.genai_pipeline.graph_bundle import GraphBundle +from executorch.backends.qualcomm.genai_pipeline.graph_names import ( + DECODER_GRAPH_NAMES, + GRAPH_FORWARD, + GRAPH_KV_FORWARD, + GRAPH_PREFILL_FORWARD, + GRAPH_TOK_EMBEDDING_KV_FORWARD, + GRAPH_TOK_EMBEDDING_PREFILL_FORWARD, + TOK_EMBEDDING_GRAPH_NAMES, +) from executorch.backends.qualcomm.genai_pipeline.pipeline_context import ( PipelineContext, PipelineContextBuilder, @@ -32,13 +58,29 @@ from executorch.backends.qualcomm.genai_pipeline.pipeline_types import EngineType __all__ = [ + "ALL_ARTIFACT_KEYS", + "ARTIFACT_ATTENTION_SINK_EVICTOR", + "ARTIFACT_AUDIO_ENCODER", + "ARTIFACT_TEXT_DECODER", + "ARTIFACT_TEXT_ENCODER", + "ARTIFACT_TOK_EMBEDDING", + "ARTIFACT_VISION_ENCODER", "CompilationInputConfig", "CompilationOutputConfig", "ConfigValidationError", + "ControlArgs", + "DECODER_GRAPH_NAMES", + "DECODE_QDQ_FILENAME", "EngineNotAvailableError", "EngineProxy", "EngineType", "GenAIPipeline", + "GRAPH_FORWARD", + "GRAPH_KV_FORWARD", + "GRAPH_PREFILL_FORWARD", + "GRAPH_TOK_EMBEDDING_KV_FORWARD", + "GRAPH_TOK_EMBEDDING_PREFILL_FORWARD", + "GraphBundle", "InferenceInputConfig", "InferenceOutputConfig", "ModelPreparationInputConfig", @@ -47,7 +89,11 @@ "PipelineContextBuilder", "PipelineError", "PipelineStage", + "QnnCompileSpecBuilder", "QuantizationInputConfig", "QuantizationOutputConfig", + "resolve_backend_type", + "resolve_soc_model", "StageError", + "TOK_EMBEDDING_GRAPH_NAMES", ] diff --git a/backends/qualcomm/genai_pipeline/artifact_keys.py b/backends/qualcomm/genai_pipeline/artifact_keys.py new file mode 100644 index 00000000000..73e450d9bf4 --- /dev/null +++ b/backends/qualcomm/genai_pipeline/artifact_keys.py @@ -0,0 +1,69 @@ +# Copyright (c) Qualcomm Innovation Center, Inc. +# All rights reserved +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +"""Keys identifying the compiled artifacts a GenAI model produces. + +``CompilationOutputConfig.artifact_paths`` is keyed by these names, one entry +per ``.pte`` file. The values match the on-device runner's ``pte_paths`` keys +exactly, so a compiled bundle can be handed to the runner without translation. + +.. note:: + These are **artifact** keys, not **graph** names. One artifact may hold + several methods: a hybrid text decoder lowers its AR-N prefill and AR-1 + decode graphs into a single multi-method ``.pte`` so they can share + weights, and that one file is reached through + ``ARTIFACT_TEXT_DECODER``. Graph names (``kv_forward`` / + ``prefill_forward``) live in a separate key space used *inside* a lowering + call. + + The same string constants exist in the legacy + ``examples/qualcomm/oss_scripts/llama/decoder_constants.py``. They are + duplicated here rather than imported so that this package does not depend + on the example scripts; ``tests/test_artifact_keys.py`` asserts the two + stay in agreement, and the legacy copy goes away with the legacy flow. +""" + +from __future__ import annotations + +# Text decoder: the language model itself. Always present. +ARTIFACT_TEXT_DECODER = "text_decoder" + +# Token embedding, lowered separately for multimodal models so that the +# embedding lookup can run while modality features are being inserted. +ARTIFACT_TOK_EMBEDDING = "tok_embedding" + +# Attention sink evictor, compiled only when the attention sink feature is on. +ARTIFACT_ATTENTION_SINK_EVICTOR = "attention_sink_evictor" + +# Modality encoders. Present only for the corresponding multimodal model. +ARTIFACT_AUDIO_ENCODER = "audio_encoder" +ARTIFACT_TEXT_ENCODER = "text_encoder" +ARTIFACT_VISION_ENCODER = "vision_encoder" + +# Every key this package may emit. +# +# Membership in ``artifact_paths`` is meaningful: the runner decides whether a +# model is multimodal by testing for the encoder keys, so an artifact that was +# not compiled must be **absent** rather than mapped to ``None``. +ALL_ARTIFACT_KEYS = frozenset( + { + ARTIFACT_ATTENTION_SINK_EVICTOR, + ARTIFACT_AUDIO_ENCODER, + ARTIFACT_TEXT_DECODER, + ARTIFACT_TEXT_ENCODER, + ARTIFACT_TOK_EMBEDDING, + ARTIFACT_VISION_ENCODER, + } +) + +# The decode graph's QDQ exported program, written by quantization before the +# calibration graph is released and read back by the SQNR evaluation. +# +# A **filename**, not an artifact key: it is a ``.pt2`` exported program rather +# than a lowered ``.pte``, the runner never sees it, and only the A-side stages +# touch it. Hence deliberately absent from ``ALL_ARTIFACT_KEYS``. It lives here +# because this module already owns the names the pipeline writes to disk. +DECODE_QDQ_FILENAME = "decode_qdq.pt2" diff --git a/backends/qualcomm/genai_pipeline/compilation/__init__.py b/backends/qualcomm/genai_pipeline/compilation/__init__.py new file mode 100644 index 00000000000..2e188da3d41 --- /dev/null +++ b/backends/qualcomm/genai_pipeline/compilation/__init__.py @@ -0,0 +1,17 @@ +# Copyright (c) Qualcomm Innovation Center, Inc. +# All rights reserved +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +from executorch.backends.qualcomm.genai_pipeline.compilation.compile_spec_builder import ( + QnnCompileSpecBuilder, + resolve_backend_type, + resolve_soc_model, +) + +__all__ = [ + "QnnCompileSpecBuilder", + "resolve_backend_type", + "resolve_soc_model", +] diff --git a/backends/qualcomm/genai_pipeline/compilation/compile_spec_builder.py b/backends/qualcomm/genai_pipeline/compilation/compile_spec_builder.py new file mode 100644 index 00000000000..460925946ce --- /dev/null +++ b/backends/qualcomm/genai_pipeline/compilation/compile_spec_builder.py @@ -0,0 +1,301 @@ +# Copyright (c) Qualcomm Innovation Center, Inc. +# All rights reserved +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +"""Construction of QNN compiler specifications. + +``generate_qnn_executorch_compiler_spec`` needs backend options from a +per-backend helper (``generate_htp_compiler_spec`` / +``generate_gpu_compiler_spec``) and a ``QcomChipset`` enum rather than a SoC +name. ``QnnCompileSpecBuilder`` pairs those two calls behind one method so +callers do not repeat the branch, and holds the settings that are properties of +the target rather than of an individual graph. +""" + +from __future__ import annotations + +import logging +from typing import Any, Dict, List, Optional, TYPE_CHECKING, Union + +if TYPE_CHECKING: + from executorch.backends.qualcomm.serialization.qc_schema import ( + QcomChipset, + QnnExecuTorchBackendType, + ) + from executorch.exir.backend.compile_spec_schema import CompileSpec + +logger = logging.getLogger(__name__) + +BACKEND_HTP = "htp" +BACKEND_GPU = "gpu" + +# Backends this builder can emit specs for. The QNN backend enum also has LPAI +# and DSP entries, but neither has a spec-generation path here yet. +SUPPORTED_BACKENDS = (BACKEND_HTP, BACKEND_GPU) + + +def resolve_soc_model(soc_model: Union[str, "QcomChipset"]) -> "QcomChipset": + """Convert a SoC name to its ``QcomChipset`` enum member. + + ``PipelineContext`` carries ``soc_model`` as a string so that nothing in the + pipeline has to import the QNN serialization schema; lowering needs the + enum, because ``generate_qnn_executorch_compiler_spec`` validates against + ``QcomChipset`` values and indexes a table with them. This is where that + conversion happens. + + Args: + soc_model: A SoC name as accepted on the command line (e.g. "SM8750"), + or an already-resolved ``QcomChipset``. + + Returns: + The matching ``QcomChipset`` member. + + Raises: + ValueError: If the name matches no supported SoC. The message lists the + valid names, since this is typically a user-supplied value. + """ + from executorch.backends.qualcomm.serialization.qc_schema import QcomChipset + from executorch.backends.qualcomm.utils.utils import get_soc_to_chipset_map + + if isinstance(soc_model, QcomChipset): + return soc_model + + chipset_map = get_soc_to_chipset_map() + if soc_model not in chipset_map: + raise ValueError( + f"Unknown SoC model '{soc_model}'. Supported: " f"{sorted(chipset_map)}" + ) + + return chipset_map[soc_model] + + +def _backend_type_map() -> Dict[str, "QnnExecuTorchBackendType"]: + """Map each backend name to its ``QnnExecuTorchBackendType`` member. + + ``QnnExecuTorchBackendType.__str__`` yields the backend name the command + line uses ("htp", "gpu", ...), so the enum itself is the source of the valid + names rather than a table here. ``kUndefinedBackend`` is excluded: it is the + unset value, not a target. + + Returns: + Backend name to enum member, for every selectable backend. + """ + from executorch.backends.qualcomm.serialization.qc_schema import ( + QnnExecuTorchBackendType, + ) + + return { + str(member): member + for member in QnnExecuTorchBackendType + if member is not QnnExecuTorchBackendType.kUndefinedBackend + } + + +def resolve_backend_type(backend: str) -> "QnnExecuTorchBackendType": + """Convert a backend name to its ``QnnExecuTorchBackendType`` member. + + Mirrors ``export_utils.get_backend_type``, but validates the name instead of + raising ``AttributeError`` from a failed ``getattr``. + + Args: + backend: A backend name, e.g. "htp", "gpu" or "lpai". + + Returns: + The matching ``QnnExecuTorchBackendType`` member. + + Raises: + ValueError: If the name matches no backend. + """ + backend_map = _backend_type_map() + if backend not in backend_map: + raise ValueError( + f"Unknown backend '{backend}'. Supported: {sorted(backend_map)}" + ) + + return backend_map[backend] + + +class QnnCompileSpecBuilder: + """Builds QNN compiler specifications for one compilation target. + + Holds the settings that describe the *target* -- which SoC, which backend, + and whether the target is the x86 emulator -- so that per-graph calls to + :meth:`build` only pass what varies between graphs. + + One instance describes one target. A model whose components go to different + backends -- an encoder on GPU with the decoder on HTP -- needs one builder + per backend rather than one builder reconfigured, since ``backend`` is + constructor state. + + ``enable_x86_64`` is a constructor argument because the emulator's + restrictions are properties of the target: it supports neither weight + sharing nor shared buffers, so both default to off when it is set. Passing + those explicitly to :meth:`build` still overrides the default. + + Example:: + + builder = QnnCompileSpecBuilder(soc_model="SM8750") + decoder_spec = builder.build(use_fp16=False, use_mha2sha=True) + encoder_spec = builder.build(use_fp16=True) + + Args: + soc_model: Target SoC, as a name or a ``QcomChipset``. + backend: Target backend; one of :data:`SUPPORTED_BACKENDS`. + enable_x86_64: Whether the target is the x86 emulator. + + Raises: + ValueError: If ``soc_model`` or ``backend`` is not supported. Both are + validated here rather than at ``build()`` time so that a bad target + fails before any graph work. + """ + + def __init__( + self, + soc_model: Union[str, "QcomChipset"], + backend: str = BACKEND_HTP, + enable_x86_64: bool = False, + ) -> None: + if backend not in SUPPORTED_BACKENDS: + # Distinguished from an unknown backend: the QNN enum has LPAI and + # DSP members that ``resolve_backend_type`` resolves happily, they + # just have no spec-generation path through this builder yet. + raise ValueError( + f"No compile spec generation path for backend '{backend}' yet. " + f"Supported here: {list(SUPPORTED_BACKENDS)}" + ) + + self._soc_model = resolve_soc_model(soc_model) + self._backend = backend + self._enable_x86_64 = enable_x86_64 + + @property + def soc_model(self) -> "QcomChipset": + """The resolved target SoC.""" + return self._soc_model + + @property + def backend(self) -> str: + """The target backend name.""" + return self._backend + + @property + def backend_type(self) -> "QnnExecuTorchBackendType": + """The target backend as a ``QnnExecuTorchBackendType``.""" + return resolve_backend_type(self._backend) + + def build( + self, + use_fp16: bool = False, + use_multi_contexts: bool = False, + use_weight_sharing: Optional[bool] = None, + shared_buffer: Optional[bool] = None, + online_prepare: bool = False, + use_mha2sha: bool = False, + ) -> List["CompileSpec"]: + """Build the compiler specs for a single graph. + + Args: + use_fp16: Compile for FP16 rather than quantized execution. HTP + only; ignored for GPU, whose precision comes from the tensor + data types. + use_multi_contexts: Emit multiple contexts within one ``.pte`` so a + single spill-fill allocation can be reused across them. Set + this when the graph is sharded. Sizing that allocation is a + post-lowering step and is not done here. + use_weight_sharing: Share identical weights across the graphs of a + multi-method ``.pte``. Defaults to on unless targeting the + emulator, which does not support it. + shared_buffer: Use a shared buffer for graph I/O. Defaults to on + unless targeting the emulator, which does not support it. + + Note that this default is *not* ``ControlArgs.shared_buffer``, + which defaults to ``False`` to match ``llama.py``'s parser. A + caller driving this from a ``ControlArgs`` must therefore pass + ``shared_buffer=control_args.shared_buffer`` explicitly; + omitting it silently turns the feature on for a device target. + online_prepare: Compose the QNN graph on device. Cannot be combined + with ``use_multi_contexts``, which the underlying spec + generation rejects. + use_mha2sha: Convert multi-head attention to single-head attention. + + Returns: + The compiler specs for one graph. + """ + from executorch.backends.qualcomm.utils.utils import ( + generate_gpu_compiler_spec, + generate_htp_compiler_spec, + generate_qnn_executorch_compiler_spec, + ) + + # The emulator supports neither feature; on device both are wanted. + if use_weight_sharing is None: + use_weight_sharing = not self._enable_x86_64 + if shared_buffer is None: + shared_buffer = not self._enable_x86_64 + + backend_options = self._build_backend_options( + generate_gpu_compiler_spec=generate_gpu_compiler_spec, + generate_htp_compiler_spec=generate_htp_compiler_spec, + use_fp16=use_fp16, + use_multi_contexts=use_multi_contexts, + use_weight_sharing=use_weight_sharing, + ) + + logger.debug( + "Building %s compile spec: soc=%s, fp16=%s, multi_contexts=%s, " + "weight_sharing=%s, shared_buffer=%s, online_prepare=%s, mha2sha=%s", + self._backend, + self._soc_model.name, + use_fp16, + use_multi_contexts, + use_weight_sharing, + shared_buffer, + online_prepare, + use_mha2sha, + ) + + return generate_qnn_executorch_compiler_spec( + soc_model=self._soc_model, + backend_options=backend_options, + shared_buffer=shared_buffer, + online_prepare=online_prepare, + use_mha2sha=use_mha2sha, + ) + + def _build_backend_options( + self, + generate_gpu_compiler_spec: Any, + generate_htp_compiler_spec: Any, + use_fp16: bool, + use_multi_contexts: bool, + use_weight_sharing: bool, + ) -> Any: + """Build the backend-specific options for the target backend. + + Args: + generate_gpu_compiler_spec: The GPU options helper. + generate_htp_compiler_spec: The HTP options helper. + use_fp16: Whether to compile for FP16. HTP only. + use_multi_contexts: Whether to emit multiple contexts. HTP only. + use_weight_sharing: Whether to share weights across graphs. + + Returns: + The backend options for the target backend. + """ + if self._backend == BACKEND_HTP: + return generate_htp_compiler_spec( + use_fp16=use_fp16, + use_multi_contexts=use_multi_contexts, + use_weight_sharing=use_weight_sharing, + ) + + # GPU takes its precision from the tensor data types, so use_fp16 has no + # equivalent; flag it rather than dropping a caller's request silently. + if use_fp16: + logger.warning( + "use_fp16 has no effect for the GPU backend; precision follows " + "the tensor data types" + ) + return generate_gpu_compiler_spec(use_weight_sharing=use_weight_sharing) diff --git a/backends/qualcomm/genai_pipeline/configs/compilation_input_config.py b/backends/qualcomm/genai_pipeline/configs/compilation_input_config.py index 43a9ece4584..5dfa004afb3 100644 --- a/backends/qualcomm/genai_pipeline/configs/compilation_input_config.py +++ b/backends/qualcomm/genai_pipeline/configs/compilation_input_config.py @@ -8,14 +8,15 @@ from dataclasses import dataclass, field from pathlib import Path -from typing import Any, List, Optional, Tuple, TYPE_CHECKING +from typing import Any, Dict, List, Optional, Tuple, TYPE_CHECKING if TYPE_CHECKING: + from executorch.backends.qualcomm.genai_pipeline.graph_bundle import GraphBundle from executorch.backends.qualcomm.serialization.qc_schema import ( QcomChipset, QnnExecuTorchBackendType, ) - from executorch.exir.backend.compile_spec import CompileSpec + from executorch.exir.backend.compile_spec_schema import CompileSpec from torch import nn @@ -28,19 +29,37 @@ class CompilationInputConfig: when those stages are skipped. **Both are required once the compilation stage executes**, and strategies should validate their presence. + There are two ways to name what to compile, and a strategy takes whichever + is populated. ``model`` plus ``example_inputs`` describes a single graph. + ``graphs`` describes several exported from the same weights -- a hybrid + decoder's AR-N prefill and AR-1 decode -- which are lowered together into + one multi-method ``.pte`` so they can share weights. The single-graph form + is the degenerate case of the second, kept because it is what a + non-decoder model needs. + Attributes: soc_model: The target SoC (e.g., QcomChipset.SM8750). Required. backend_type: QNN backend type (HTP, GPU, LPAI, etc.). Required. model: The nn.Module to compile (quantized or original for FP16 mode). Required when the stage runs. - example_inputs: Positional example inputs for ``torch.export``. Required - when the stage runs. Sourced from the **model** via + example_inputs: Positional example inputs for ``torch.export``. Belongs + to the single-graph form only, alongside ``model``: it is required + when the stage runs on that path and ``None`` when ``graphs`` is + populated, where each ``GraphBundle`` carries its own ``inputs``. + Sourced from the **model** via ``ModelLoaderAdapter.get_example_inputs``, never from calibration data: this tuple defines the exported graph's positional signature, supplies the zero-initialized KV caches a dataset sample does not carry, and fixes the AR length because HTP has no dynamic shapes. artifact_dir: Directory to store compiled artifacts. compile_specs: QNN compiler specifications for backend delegation. + graphs: The graphs to compile, keyed by graph name (see + ``decoder_constants.DECODER_GRAPH_NAMES``), each carrying its own + module, inputs and metadata. Produced by the quantization stage, + which reconciles their encodings and releases the calibration graph + first, so every bundle here is deployable. ``None`` on the + single-graph path, where ``model`` and ``example_inputs`` are used + instead. """ soc_model: "QcomChipset" @@ -49,3 +68,4 @@ class CompilationInputConfig: example_inputs: Optional[Tuple[Any, ...]] = None artifact_dir: Path = field(default_factory=lambda: Path(".")) compile_specs: Optional[List["CompileSpec"]] = None + graphs: Optional[Dict[str, "GraphBundle"]] = None diff --git a/backends/qualcomm/genai_pipeline/configs/compilation_output_config.py b/backends/qualcomm/genai_pipeline/configs/compilation_output_config.py index f68b0471c3f..4ad8d00b5b0 100644 --- a/backends/qualcomm/genai_pipeline/configs/compilation_output_config.py +++ b/backends/qualcomm/genai_pipeline/configs/compilation_output_config.py @@ -8,7 +8,7 @@ from dataclasses import dataclass from pathlib import Path -from typing import List, Optional, TYPE_CHECKING +from typing import Dict, Optional, TYPE_CHECKING if TYPE_CHECKING: from executorch.devtools.etrecord import ETRecord @@ -19,11 +19,14 @@ class CompilationOutputConfig: """Output produced by the compilation stage. Attributes: - artifact_paths: Paths to the compiled artifacts (.pte files). - List to support multi-split models where compilation produces - multiple .pte files (e.g., prefill + decode). + artifact_paths: Compiled artifacts (.pte files), keyed by artifact name + (see ``artifact_keys``). A model may produce several: a text decoder + always, plus a token embedding and modality encoders when + multimodal. Keyed rather than ordered because the inference stage + addresses them individually and the set present varies by model. + Absent artifacts are omitted rather than mapped to ``None``. etrecord: Optional ETRecord for debugging. ExecuTorch engine only. """ - artifact_paths: Optional[List[Path]] = None + artifact_paths: Optional[Dict[str, Path]] = None etrecord: Optional["ETRecord"] = None diff --git a/backends/qualcomm/genai_pipeline/configs/inference_input_config.py b/backends/qualcomm/genai_pipeline/configs/inference_input_config.py index 19612b15a2c..a66da66f7ce 100644 --- a/backends/qualcomm/genai_pipeline/configs/inference_input_config.py +++ b/backends/qualcomm/genai_pipeline/configs/inference_input_config.py @@ -20,8 +20,10 @@ class InferenceInputConfig: Attributes: soc_model: The target SoC (e.g., QcomChipset.SM8750). Required. - artifact_paths: Paths to compiled model artifacts (.pte files). - List to support multi-split models (e.g., prefill + decode). + artifact_paths: Compiled model artifacts (.pte files), keyed by artifact + name (see ``artifact_keys``), as produced by the compilation stage. + The on-device runner passes each to a different argument, so they + are addressed by name rather than by position. tokenizer: The tokenizer instance for encoding/decoding. runtime_tokenizer_path: Path to runtime tokenizer for on-device use. prompt: The user prompt(s) for text generation. @@ -29,7 +31,7 @@ class InferenceInputConfig: """ soc_model: "QcomChipset" - artifact_paths: Optional[List[Path]] = None + artifact_paths: Optional[Dict[str, Path]] = None tokenizer: Any = None runtime_tokenizer_path: Optional[Path] = None prompt: Optional[List[str]] = None diff --git a/backends/qualcomm/genai_pipeline/control_args.py b/backends/qualcomm/genai_pipeline/control_args.py new file mode 100644 index 00000000000..ce9a981365d --- /dev/null +++ b/backends/qualcomm/genai_pipeline/control_args.py @@ -0,0 +1,390 @@ +# Copyright (c) Qualcomm Innovation Center, Inc. +# All rights reserved +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +"""Typed configuration bridge between ``PipelineContext`` and the LLM flow. + +The LLM export/quantize/compile/eval code is driven by the +``argparse.Namespace`` produced by ``llama.py``'s parser: components read +attributes off it directly (``control_args.max_seq_len``, +``args.enable_x86_64``, ...) and ``QnnConfig.load_config`` reflects over it with +``vars()``. ``ControlArgs`` is a typed dataclass holding the same attributes, so +it can be built from a ``PipelineContext`` and handed to that code unchanged. + +It **subclasses** ``argparse.Namespace`` deliberately. ``QnnConfig.load_config`` +dispatches on ``isinstance(config, argparse.Namespace)`` and raises +``TypeError`` for anything else, so a bare dataclass would break the device +path. Subclassing keeps ``isinstance`` true and ``vars()`` working while adding +field names, defaults and type annotations. + +Defaults mirror ``llama.py``'s parser exactly; ``tests/test_control_args.py`` +asserts that against the real parser so the two cannot drift. + +:meth:`ControlArgs.build_parser` derives an ``argparse`` parser from these same +fields, so a command-line entry point gets one default per setting rather than +maintaining a second table of its own. Callers needing extra arguments pass the +result to their own parser via ``parents=[...]`` and add them there. + +.. note:: + This is a second configuration surface alongside ``PipelineContext``, and + that duplication is temporary: it exists so the pipeline can drive the + existing LLM components while they are migrated. As components move behind + pipeline interfaces, their fields leave this dataclass. New fields should be + added only when an existing component genuinely reads them. +""" + +from __future__ import annotations + +import argparse +import logging +from dataclasses import dataclass, field, fields, MISSING +from typing import Any, Dict, List, Optional, TYPE_CHECKING + +if TYPE_CHECKING: + from executorch.backends.qualcomm.genai_pipeline.pipeline_context import ( + PipelineContext, + ) + +logger = logging.getLogger(__name__) + +# Defaults, mirroring llama.py's argparse configuration. Named rather than +# inlined so callers can reference them instead of repeating literals. +DEFAULT_ARTIFACT = "./llama_qnn" +DEFAULT_BACKEND = "htp" +DEFAULT_BATCH_SIZE = 1 +DEFAULT_CALIB_HF_LIMIT = 1 +DEFAULT_CALIB_LIMIT = 1 +DEFAULT_DTYPE_OVERRIDE = "fp32" +DEFAULT_EVAL_LIMIT = 1 +DEFAULT_EVAL_METHOD = "prompt_eval" +DEFAULT_GCAP = 8 +DEFAULT_HTP_PERFORMANCE_MODE = 2 +DEFAULT_MAX_SEQ_LEN = 512 +DEFAULT_MODEL_MODE = "hybrid" +DEFAULT_NGRAM = 5 +DEFAULT_PORT = -1 +DEFAULT_PREFILL_AR_LEN = 32 +DEFAULT_PROFILE_LEVEL = 0 +DEFAULT_TARGET = "aarch64-android" +DEFAULT_TEMPERATURE = 0.8 +DEFAULT_TRAIN_HF_LIMIT = 1000 +DEFAULT_TRAIN_LIMIT = 1 +DEFAULT_TRAIN_VAL_RATIO = 1.0 +DEFAULT_WINDOW = 8 + +# Fields whose default is deliberately not ``llama.py``'s. Its defaults for +# these two are paths to YAML files that live under ``examples/``, which this +# package must not reference; ``None`` means "let the consumer fall back to its +# own default", which is what those paths are. +_PARSER_DEFAULT_EXEMPT_FIELDS = frozenset( + { + "lr_config", + "train_config", + } +) + +# Fields ``from_pipeline_context`` fills from the context itself. They are read +# back out of the resulting ``ControlArgs`` by the LLM components *and* out of +# the context by the stage configs, so the two must agree: ``extra_options`` +# cannot set them, and an explicit override is the only way to differ. +_CONTEXT_OWNED_FIELDS = frozenset( + { + "artifact", + "decoder_model", + "prompt", + "soc_model", + } +) + +# Fields ``build_parser`` gives ``nargs="+"``. Their annotation is a list, so a +# command line supplies them as repeated values rather than one string. +_LIST_VALUED_FIELDS = frozenset( + { + "audio_path", + "calib_samples", + "calib_tasks", + "eval_methods", + "eval_tasks", + "image_path", + "prompt", + "train_tasks", + } +) + +# Fields whose value is constrained. ``argparse`` rejects anything else, so a +# typo fails at parse time rather than deep inside the flow. +_FIELD_CHOICES = { + "backend": ("htp", "gpu"), + "dtype_override": ("fp32", "fp16"), + "model_mode": ("kv", "hybrid", "lookahead"), +} + + +@dataclass +class ControlArgs(argparse.Namespace): + """Configuration for the LLM export, quantization and evaluation flow. + + Every field is read by at least one existing component; the grouping below + follows which part of the flow consumes it. Field defaults match + ``llama.py``'s parser, so ``ControlArgs()`` is equivalent to running that + parser with only its required arguments supplied. + """ + + # --- Model identification and input paths --- + decoder_model: str = "" + artifact: str = DEFAULT_ARTIFACT + checkpoint: Optional[str] = None + params: Optional[str] = None + tokenizer_model: Optional[str] = None + tokenizer_bin: Optional[str] = None + + # --- Graph shapes and modes --- + model_mode: str = DEFAULT_MODEL_MODE + max_seq_len: int = DEFAULT_MAX_SEQ_LEN + max_context_len: Optional[int] = None + prefill_ar_len: int = DEFAULT_PREFILL_AR_LEN + dtype_override: str = DEFAULT_DTYPE_OVERRIDE + # Lookahead decoding shape parameters. + ngram: int = DEFAULT_NGRAM + window: int = DEFAULT_WINDOW + gcap: int = DEFAULT_GCAP + + # --- Quantization --- + use_fp16: bool = False + embedding_quantize: Optional[str] = None + quant_recipe_suggestion: bool = False + batch_size: int = DEFAULT_BATCH_SIZE + + # --- Calibration data selection --- + calib_tasks: Optional[List[str]] = None + calib_limit: int = DEFAULT_CALIB_LIMIT + calib_num_fewshot: Optional[int] = None + calib_samples: Optional[List[str]] = None + calib_hf_dataset: Optional[str] = None + calib_hf_limit: int = DEFAULT_CALIB_HF_LIMIT + + # --- Quantization-aware training --- + # + # ``train_config`` and ``lr_config`` name YAML files. ``llama.py`` defaults + # them to files under its own directory; here they default to ``None``, + # which the consumer reads as "use your own default", because this package + # does not reference paths in ``examples/``. See + # :data:`_PARSER_DEFAULT_EXEMPT_FIELDS`. + qat: bool = False + train_config: Optional[str] = None + lr_config: Optional[str] = None + train_tasks: Optional[List[str]] = None + train_limit: int = DEFAULT_TRAIN_LIMIT + train_hf_dataset: Optional[str] = None + train_hf_limit: int = DEFAULT_TRAIN_HF_LIMIT + train_val_ratio: float = DEFAULT_TRAIN_VAL_RATIO + freeze_all_params: bool = False + + # --- Features --- + use_attention_sink: Optional[str] = None + + # --- Runtime prompts and multimodal inputs --- + prompt: List[str] = field(default_factory=list) + system_prompt: str = "" + temperature: float = DEFAULT_TEMPERATURE + audio_path: List[str] = field(default_factory=list) + image_path: List[str] = field(default_factory=list) + + # --- Evaluation --- + eval_methods: List[str] = field(default_factory=lambda: [DEFAULT_EVAL_METHOD]) + eval_tasks: Optional[List[str]] = None + eval_limit: int = DEFAULT_EVAL_LIMIT + eval_num_fewshot: Optional[int] = None + + # --- Backend and SoC selection --- + # + # ``soc_model`` stays a string here, as it is in ``PipelineContext``; the + # conversion to ``QcomChipset`` happens in the compilation adapter layer. + soc_model: Optional[str] = None + backend: str = DEFAULT_BACKEND + online_prepare: bool = False + htp_performance_mode: int = DEFAULT_HTP_PERFORMANCE_MODE + + # --- Flow control --- + compile_only: bool = False + pre_gen_pte: Optional[str] = None + verbose: bool = False + + # --- Device and runner settings --- + # + # These are consumed by ``QnnConfig``, which reflects over this object with + # ``vars()`` and asserts that ``soc_model`` and ``build_folder`` are set. + build_folder: Optional[str] = None + direct_build_folder: Optional[str] = None + target: str = DEFAULT_TARGET + host: Optional[str] = None + device: Optional[str] = None + enable_x86_64: bool = False + shared_buffer: bool = False + skip_push: bool = False + config_file: Optional[str] = None + + # --- Partitioning overrides --- + skip_delegate_node_ids: Optional[str] = None + skip_delegate_node_ops: Optional[str] = None + + # --- Debug and CI --- + dump_intermediate_outputs: bool = False + profile_level: int = DEFAULT_PROFILE_LEVEL + ci: bool = False + seed: Optional[int] = None + # IPC endpoint the CI harness listens on for results. + ip: str = "" + port: int = DEFAULT_PORT + + @classmethod + def field_names(cls) -> frozenset: + """The set of field names this dataclass defines.""" + return frozenset(f.name for f in fields(cls)) + + @classmethod + def build_parser(cls, **parser_kwargs: Any) -> argparse.ArgumentParser: + """Derive an ``argparse`` parser from this dataclass's fields. + + Each field becomes ``--field-name`` (also accepting ``--field_name``) + with this dataclass's default, so a command-line entry point does not + maintain a second table of defaults that can drift from these. Booleans + become ``store_true`` flags; list-valued fields take ``nargs="+"``; + :data:`_FIELD_CHOICES` constrains the rest. + + The parser sets no help text: the authoritative description of each + setting is this dataclass's field grouping and comments. Entry points + wanting help strings, or arguments this dataclass does not carry, pass + this parser as a parent (``ArgumentParser(parents=[...])``). + + Args: + **parser_kwargs: Forwarded to ``argparse.ArgumentParser``. + + Returns: + A parser whose ``parse_args`` result :meth:`from_namespace` accepts. + """ + parser = argparse.ArgumentParser(**parser_kwargs) + + for spec in fields(cls): + if spec.default is not MISSING: + default = spec.default + elif spec.default_factory is not MISSING: + default = spec.default_factory() + else: + default = None + + flags = [f"--{spec.name.replace('_', '-')}"] + if "_" in spec.name: + flags.append(f"--{spec.name}") + + if isinstance(default, bool): + parser.add_argument( + *flags, dest=spec.name, action="store_true", default=default + ) + continue + + kwargs: Dict[str, Any] = {"dest": spec.name, "default": default} + if spec.name in _LIST_VALUED_FIELDS: + kwargs["nargs"] = "+" + kwargs["type"] = str + elif spec.name in _FIELD_CHOICES: + kwargs["choices"] = _FIELD_CHOICES[spec.name] + kwargs["type"] = str + else: + # int/float fields keep their type so the parser converts; the + # rest are strings. ``None`` defaults carry no type information, + # so they stay strings and the consumer coerces. + numeric = isinstance(default, (int, float)) + kwargs["type"] = type(default) if numeric else str + parser.add_argument(*flags, **kwargs) + + return parser + + @classmethod + def from_namespace(cls, namespace: argparse.Namespace) -> "ControlArgs": + """Build from an ``argparse.Namespace``, ignoring unknown attributes. + + Lets the legacy entry point and the pipeline share one configuration + type: ``llama.py``'s parser produces a superset of these fields, and + attributes without a matching field are dropped. + + Args: + namespace: A parsed namespace, e.g. from ``llama.py``'s parser. + + Returns: + A ``ControlArgs`` carrying every recognised attribute. + """ + known = cls.field_names() + supplied = vars(namespace) + + ignored = sorted(set(supplied) - known) + if ignored: + logger.debug("Ignoring unrecognised arguments: %s", ignored) + + return cls(**{k: v for k, v in supplied.items() if k in known}) + + @classmethod + def from_pipeline_context( + cls, + context: "PipelineContext", + **overrides: Any, + ) -> "ControlArgs": + """Build from a ``PipelineContext`` plus per-stage overrides. + + The context supplies the four settings it owns; anything else the LLM + components need is taken from ``context.extra_options`` when the key + names a field, and ``overrides`` wins over both. + + ``extra_options`` cannot reach the context-owned settings (see + :data:`_CONTEXT_OWNED_FIELDS`). Those have one authoritative source, and + letting a loosely-typed dict silently replace one would let the flow + compile for a SoC other than the one the context reports and the + quantization stage validated against -- a mismatch that only surfaces on + device. A caller that genuinely means to change them passes an explicit + override, which is deliberate and traceable at the call site. + + Args: + context: The pipeline context holding user inputs. + **overrides: Field values taking precedence over the context. + + Returns: + A populated ``ControlArgs``. + + Raises: + TypeError: If an override does not name a field, which would + otherwise be silently dropped. + """ + known = cls.field_names() + + unknown = sorted(set(overrides) - known) + if unknown: + raise TypeError( + f"ControlArgs has no field(s) {unknown}; valid fields: " + f"{sorted(known)}" + ) + + kwargs: Dict[str, Any] = { + "decoder_model": context.model_name, + "soc_model": context.soc_model, + "artifact": context.artifact_dir, + "prompt": list(context.prompt), + } + + extra = context.extra_options or {} + for key, value in extra.items(): + if key in _CONTEXT_OWNED_FIELDS: + logger.debug( + "extra_options['%s'] ignored: the context owns this field. " + "Pass it as an explicit override to change it.", + key, + ) + elif key in known: + kwargs[key] = value + else: + logger.debug("extra_options['%s'] is not a ControlArgs field", key) + + kwargs.update(overrides) + + return cls(**kwargs) diff --git a/backends/qualcomm/genai_pipeline/graph_bundle.py b/backends/qualcomm/genai_pipeline/graph_bundle.py new file mode 100644 index 00000000000..6c415ec8071 --- /dev/null +++ b/backends/qualcomm/genai_pipeline/graph_bundle.py @@ -0,0 +1,142 @@ +# Copyright (c) Qualcomm Innovation Center, Inc. +# All rights reserved +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +"""The per-graph unit handed from quantization to compilation. + +One decoder is exported several times from the same weights -- an AR-N prefill +graph and an AR-1 decode graph differ only in the ``ar_len`` baked into them -- +so the stages after model preparation operate on a *set* of graphs rather than a +single module. ``GraphBundle`` is that set's element: everything compilation +needs about one graph, in one object, so the pipeline threads a single +``{graph_name: GraphBundle}`` map instead of several parallel +``{graph_name: value}`` dicts that can fall out of step. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any, Dict, Optional, Tuple + +#: The exact key set ``GraphBundle.quant_io_dtypes`` carries when it is not +#: ``None``. ``compilation/pass_policy.py`` indexes both, so a mapping missing +#: either one is rejected at construction rather than at lowering. +_QUANT_IO_DTYPE_KEYS = frozenset({"kv_type", "io_type"}) + + +@dataclass(frozen=True) +class GraphBundle: + """One deployable graph and the inputs needed to lower it. + + Frozen so that a bundle cannot be edited in place as it moves between + stages: a stage that needs to change one field returns a new bundle via + ``dataclasses.replace``, which keeps the producer's output readable after + the consumer has run. + + .. note:: + Immutability stops at the field boundary. ``module`` is a + ``GraphModule`` and ``meta`` a dict, and both are mutated by the + quantization stage -- ``convert_pt2e`` writes the quantized logits and + KV-cache attributes into ``meta`` in place. ``frozen=True`` protects + which objects a bundle names, not their contents. + + .. note:: + Only **deployable** graphs are bundled. A hybrid decoder also builds a + full-auto-regressive calibration graph, but that exists solely to source + activation statistics: quantization propagates its scales onto the + deployed graphs and then releases it, so it never reaches compilation. + + Four things are deliberately *not* fields here: + + * ``compile_spec`` -- built by the compilation stage from ``ControlArgs`` + and the component the graph belongs to, since weight sharing and + multi-context are properties of how graphs are grouped into a ``.pte`` + rather than of a graph itself. + * ``dep_table`` / ``passes_job`` -- derived by the compilation stage from + ``meta``, ``ControlArgs`` and the shard count, for the same reason: graph + sharding and quantized-IO tagging are settings of the lowering call. The + shard count that inserts ``SplitGraph`` is the one that sets the + multi-context compile spec, so deriving both in one place keeps them from + disagreeing -- a model that is sharded but not split still compiles. + * ``dynamic_shapes`` -- HTP has no dynamic shapes, and the parameter was + removed from this package at review request. + * ``skip_node_id_set`` / ``skip_node_op_set`` -- per-run partitioning + overrides today, so they travel in ``extra_options``. They belong here + only once two graphs of one model need different values. + + Attributes: + module: The graph to lower, quantized and already reconciled against the + calibration graph. A ``torch.fx.GraphModule`` in practice. + inputs: Positional example inputs for ``torch.export``, derived from the + **model** and never from the calibration data: they carry this + graph's positional signature, its zero-initialized KV caches, and + the AR length baked in because HTP has no dynamic shapes. + meta: The graph's ``get_metadata()`` constants -- layer count, head dim, + context and AR lengths -- plus the quantized logits / KV-cache + attributes written during ``convert_pt2e``. Becomes the ``.pte``'s + constant methods, so it is complete only **after** quantization has + run. + quant_io_dtypes: The graph-boundary dtypes quantization chose, as + ``{"kv_type": torch.dtype, "io_type": torch.dtype}``. Compilation + builds the ``TagQuantIO`` pass settings from these; it cannot derive + them, since they come from the quantization recipe's KV and logits + bit widths. + + ``None`` means *this graph's boundary dtypes were not chosen by a + recipe* -- which happens for two different reasons, and compilation + must not treat them alike: + + * **Quantization was skipped** for the graph, so its IO stays + float32 and ``TagQuantIO`` is left inactive. + * **The component never derives dtypes from a recipe.** Encoder + graphs are the case: the legacy flow tags an encoder's boundary + ``torch.float32`` from a literal and never consults the encoder + recipe, which exposes no KV or logits bit width at all. Such a + graph still needs ``TagQuantIO`` active when it is sharded, to + tag the ``llama.fallback.default`` boundary -- so ``None`` here + must not be read as "nothing to tag". + + ``compilation/pass_policy.py`` distinguishes the two by component, + since only the component knows whether a recipe was meant to supply + these at all. + + **Both keys or neither.** The tagger indexes both unconditionally, + once per node, so a mapping carrying only one of them is not a + partially-quantized boundary -- it is a ``KeyError`` during + lowering. A recipe whose bit width maps to no fixed-point dtype + makes the whole mapping ``None``; it does not drop a single key. + ``__post_init__`` rejects anything else. + modality_inputs: Encoder inputs for a multimodal model, keyed by + modality. ``None`` for a text-only model. + executorch_config: Optional override for the ``to_executorch`` + configuration. A property of the *lowering call* rather than of one + graph, so a multi-method ``.pte`` uses one config for the whole + group; carried here so a caller can vary it without reaching into + ``extra_options``. + """ + + module: Any + inputs: Tuple[Any, ...] + meta: Dict[str, Any] = field(default_factory=dict) + quant_io_dtypes: Optional[Dict[str, Any]] = None + modality_inputs: Optional[Dict[str, Any]] = None + executorch_config: Optional[Any] = None + + def __post_init__(self) -> None: + """Reject a ``quant_io_dtypes`` mapping that is not both keys. + + Raises: + ValueError: If ``quant_io_dtypes`` is a mapping whose keys are not + exactly ``kv_type`` and ``io_type``. + """ + if self.quant_io_dtypes is None: + return + + keys = set(self.quant_io_dtypes) + if keys != _QUANT_IO_DTYPE_KEYS: + raise ValueError( + "quant_io_dtypes must carry exactly " + f"{sorted(_QUANT_IO_DTYPE_KEYS)} or be None; got {sorted(keys)}" + ) diff --git a/backends/qualcomm/genai_pipeline/graph_names.py b/backends/qualcomm/genai_pipeline/graph_names.py new file mode 100644 index 00000000000..bf61b505689 --- /dev/null +++ b/backends/qualcomm/genai_pipeline/graph_names.py @@ -0,0 +1,56 @@ +# Copyright (c) Qualcomm Innovation Center, Inc. +# All rights reserved +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +"""Names identifying the graphs (methods) inside one compiled artifact. + +A hybrid decoder is exported twice from the same weights -- an AR-1 decode graph +and an AR-N prefill graph -- and both become methods of a single multi-method +``.pte`` so they share weights. These are the method names. + +.. note:: + These are **graph** names, not **artifact** keys (see ``artifact_keys``). + Graph names are used *inside* one lowering call; artifact keys name the + ``.pte`` files that call produces. + + The deployed graph names also exist in the legacy + ``examples/qualcomm/oss_scripts/llama/decoder_constants.py``. They are + duplicated here rather than imported so that this package does not depend on + the example scripts; ``tests/test_graph_names.py`` asserts the two stay in + agreement, and the legacy copy goes away with the legacy flow. +""" + +from __future__ import annotations + +# Decode: AR-1, consumes and updates the KV cache one token at a time. +GRAPH_KV_FORWARD = "kv_forward" + +# Prefill: AR-N, processes the prompt in one pass. +GRAPH_PREFILL_FORWARD = "prefill_forward" + +# The decoder's graphs, decode first. +# +# Order is significant: in ``kv`` mode there is no prefill graph, and the legacy +# flow slices this list to the number of graphs a model actually built. Decode is +# also the authoritative graph for the artifact's constant methods, since the +# runtime uses its KV-cache scales for both graphs. +DECODER_GRAPH_NAMES = [GRAPH_KV_FORWARD, GRAPH_PREFILL_FORWARD] + +# The token-embedding graphs, in the same order for the same reason. +GRAPH_TOK_EMBEDDING_KV_FORWARD = "tok_embedding_kv_forward" +GRAPH_TOK_EMBEDDING_PREFILL_FORWARD = "tok_embedding_prefill_forward" + +TOK_EMBEDDING_GRAPH_NAMES = [ + GRAPH_TOK_EMBEDDING_KV_FORWARD, + GRAPH_TOK_EMBEDDING_PREFILL_FORWARD, +] + +# The name ExecuTorch gives a module exported as a single graph. +# +# Used for the components that build one graph rather than a decode/prefill pair +# -- an encoder -- and for the decoder's full-auto-regressive calibration graph, +# which sources encodings and is released before compilation. It is therefore +# absent from ``DECODER_GRAPH_NAMES``: those are the *deployed* graphs. +GRAPH_FORWARD = "forward" diff --git a/backends/qualcomm/genai_pipeline/strategies/compilation/compiler_adapter.py b/backends/qualcomm/genai_pipeline/strategies/compilation/compiler_adapter.py index 5318365e23d..e0f076eb604 100644 --- a/backends/qualcomm/genai_pipeline/strategies/compilation/compiler_adapter.py +++ b/backends/qualcomm/genai_pipeline/strategies/compilation/compiler_adapter.py @@ -8,7 +8,11 @@ from dataclasses import dataclass, field from pathlib import Path -from typing import Any, Dict, List, Optional, Protocol, runtime_checkable, Tuple +from typing import Any, Dict, Optional, Protocol, runtime_checkable, Tuple + +from executorch.backends.qualcomm.genai_pipeline.artifact_keys import ( + ARTIFACT_TEXT_DECODER, +) @dataclass @@ -16,11 +20,16 @@ class CompilationResult: """Result of a compilation operation. Attributes: - artifact_paths: Paths to the compiled .pte artifacts. + artifact_paths: Compiled ``.pte`` artifacts, keyed by artifact name (see + ``artifact_keys``). Keyed rather than ordered because the consumer + needs to address artifacts individually -- the on-device runner + takes a decoder path, a token-embedding path and an encoder path as + separate arguments -- and because which artifacts exist varies by + model, so position carries no reliable meaning. etrecord: Optional ETRecord for debugging. """ - artifact_paths: List[Path] = field(default_factory=list) + artifact_paths: Dict[str, Path] = field(default_factory=dict) etrecord: Optional[Any] = None @@ -58,6 +67,7 @@ def compile_model( constant_methods: Optional[Dict[str, Any]] = None, dep_table: Optional[Dict] = None, passes_job: Optional[Any] = None, + artifact_key: str = ARTIFACT_TEXT_DECODER, extra_options: Optional[Dict[str, Any]] = None, ) -> CompilationResult: """Compile the model to on-device .pte artifacts. @@ -75,13 +85,23 @@ def compile_model( compile_specs: QNN compiler specifications for backend delegation. artifact_dir: Directory to store compiled artifacts. file_name: Base name for the output .pte file. - soc_model: Target SoC chipset. - backend_type: QNN backend type (HTP, GPU, LPAI). + soc_model: Target SoC chipset. The target lowering actually uses is + the one carried in ``compile_specs``; implementations should + treat a disagreement as an error rather than silently preferring + one, since ops validated for one SoC and compiled for another + fail only on device. + backend_type: QNN backend type (HTP, GPU, LPAI). Carried in + ``compile_specs`` as well; see ``soc_model``. constant_methods: Methods returning constants in eager mode. For a decoder this carries the quantization attributes written during quantization, so it is only complete after that stage. dep_table: Per-graph pass dependency table. passes_job: Per-graph pass configuration. + artifact_key: Name to key the resulting artifact under (see + ``artifact_keys``). Supplied by the caller because a single-graph + adapter cannot tell a text decoder from a vision encoder -- both + arrive as ``model`` -- and the inference stage addresses + artifacts by name. extra_options: Optional tuning knobs (``skip_node_id_set``, ``skip_node_op_set``, ``skip_mutable_buffer``, ``convert_linear_to_conv2d``, ``generate_etrecord``, diff --git a/backends/qualcomm/genai_pipeline/strategies/compilation/default_compiler_adapter.py b/backends/qualcomm/genai_pipeline/strategies/compilation/default_compiler_adapter.py index e93de929b77..73dc7bafd8b 100644 --- a/backends/qualcomm/genai_pipeline/strategies/compilation/default_compiler_adapter.py +++ b/backends/qualcomm/genai_pipeline/strategies/compilation/default_compiler_adapter.py @@ -10,41 +10,157 @@ from pathlib import Path from typing import Any, Dict, Optional, Tuple +from executorch.backends.qualcomm.genai_pipeline.artifact_keys import ( + ARTIFACT_TEXT_DECODER, +) from executorch.backends.qualcomm.genai_pipeline.strategies.compilation.compiler_adapter import ( CompilationResult, ) logger = logging.getLogger(__name__) +# Options forwarded to lowering when present in ``extra_options``. Each maps to +# a parameter of ``to_edge_transform_and_lower_to_qnn`` of the same name. +_LOWERING_OPTION_KEYS = ( + "convert_linear_to_conv2d", + "generate_etrecord", + "skip_mutable_buffer", + "skip_node_id_set", + "skip_node_op_set", +) + +# ``extra_options`` key overriding the ``to_executorch`` configuration wholesale. +_KEY_EXECUTORCH_BACKEND_CONFIG = "executorch_backend_config" + + +def _decode_qnn_options(compile_specs: Any) -> Optional[Any]: + """Recover the ``QnnExecuTorchOptions`` carried by a list of compile specs. + + ``generate_qnn_executorch_compiler_spec`` serialises the target settings + into a single flatbuffer-valued ``CompileSpec``, so the target the graph + will actually be built for is readable back out of the specs themselves. + + Args: + compile_specs: The value passed as ``compile_specs``; any shape is + accepted, since callers may inject a stub. + + Returns: + The decoded options, or ``None`` if ``compile_specs`` does not carry a + QNN spec -- which is the case for a test double or a non-QNN backend. + Returning ``None`` rather than raising keeps this usable as a + best-effort consistency check. + """ + from executorch.backends.qualcomm.serialization.qc_schema_serialize import ( + flatbuffer_to_option, + ) + from executorch.backends.qualcomm.utils.constants import QCOM_QNN_COMPILE_SPEC + + try: + for spec in compile_specs: + if getattr(spec, "key", None) != QCOM_QNN_COMPILE_SPEC: + continue + return flatbuffer_to_option(spec.value) + except Exception: # noqa: BLE001 - a stub may not be iterable or decodable + return None + + return None + + +def _verify_target_matches_specs( + compile_specs: Any, + soc_model: Any, + backend_type: Any, +) -> None: + """Check that ``soc_model`` / ``backend_type`` agree with the compile specs. + + Lowering takes its target exclusively from ``compile_specs``; these two + arguments are informational. That makes a disagreement silent and expensive + -- a graph whose ops were validated for one SoC but compiled for another -- + so it is rejected here instead. + + Args: + compile_specs: QNN compiler specifications for this graph. + soc_model: The SoC the caller believes it is targeting. + backend_type: The backend the caller believes it is targeting. + + Raises: + ValueError: If either value contradicts the specs. + """ + options = _decode_qnn_options(compile_specs) + if options is None: + return + + spec_soc_model = options.soc_info.soc_model + if soc_model is not None and soc_model != spec_soc_model: + raise ValueError( + f"soc_model {soc_model!r} contradicts the compile specs, which " + f"target {spec_soc_model!r}. Lowering follows the specs, so the " + "mismatch would otherwise pass silently." + ) + + spec_backend_type = options.backend_options.backend_type + if backend_type is not None and backend_type != spec_backend_type: + raise ValueError( + f"backend_type {backend_type!r} contradicts the compile specs, " + f"which target {spec_backend_type!r}. Lowering follows the specs, " + "so the mismatch would otherwise pass silently." + ) + + +def _default_executorch_config() -> Any: + """Build the ``to_executorch`` configuration for QNN graph I/O. + + Graph inputs and outputs are deliberately left unallocated: with a shared + buffer the caller supplies the addresses, which are allocated from RPC + memory rather than by memory planning. ``BuildQuantIo`` then gives the + quantized I/O tensors their types. + + Returns: + An ``ExecutorchBackendConfig`` suitable for QNN lowering. + """ + from executorch.backends.qualcomm._passes.build_quant_io import BuildQuantIo + from executorch.exir.capture._config import ExecutorchBackendConfig + from executorch.exir.passes.memory_planning_pass import MemoryPlanningPass + + return ExecutorchBackendConfig( + memory_planning_pass=MemoryPlanningPass( + alloc_graph_input=False, + alloc_graph_output=False, + ), + passes=[BuildQuantIo()], + ) + class DefaultCompilerAdapter: """Default adapter delegating to ``to_edge_transform_and_lower_to_qnn``. + Lowers one graph to a single ``.pte``: the model is lowered to QNN, the + resulting edge program is converted to an ExecuTorch program, and that is + written to ``artifact_dir``. + .. note:: - The signature below is the final one -- it mirrors - ``to_edge_transform_and_lower_to_qnn`` argument-for-argument, so the - per-graph inputs (``compile_specs``, ``dep_table``, ``passes_job``, - ``constant_methods``) are explicit parameters rather than - ``extra_options`` keys. - - The **body** is not implemented in this PR. Lowering is implementation - work rather than interface work, and the version this package needs is - the multi-graph one: ``to_edge_transform_and_lower_to_qnn`` accepts - graph-name-keyed dicts for ``module`` / ``inputs`` / ``compiler_specs`` - / ``dep_table`` / ``passes_job``, and a hybrid decoder groups its graphs - into a single multi-method ``.pte`` for weight sharing. Writing a - single-graph body here and then replacing it would mean implementing the - lowering twice, so it lands with the strategy-level fan-out that calls - it. - - A recipe-based body (``ExportRecipe.get_recipe(QNNRecipeType.FP16)`` + - ``ExportSession``) was considered and rejected: ``QNNRecipeProvider`` - accepts only ``soc_model`` and the three ``skip_*`` keys and silently - ignores the rest, so ``dep_table``, ``passes_job``, ``constant_methods`` - and ``convert_linear_to_conv2d`` are unreachable through it, and it - hardcodes ``use_fp16=True``. - - Until the body lands, inject a custom ``CompilerAdapter``. + This adapter is a **1:1 wrapper over one graph**, mirroring the + single-graph form of ``to_edge_transform_and_lower_to_qnn``. Two things + therefore sit deliberately outside it: + + * **Multi-graph grouping.** That function also accepts graph-name-keyed + dicts, which is how several graphs (a hybrid decoder's prefill and + decode) become one multi-method ``.pte`` that shares weights. Fanning + out over graphs is the compilation *strategy*'s job; passing dicts + through here would make the adapter's contract depend on which of two + shapes its arguments take. + * **Spill-fill sizing.** A sharded model wants one spill-fill + allocation reused across its contexts, which ``update_spill_fill_size`` + computes from the lowered program. It is a property of the group, so + it belongs with the call that lowers the group; for a single graph + there is nothing to share. + + A recipe-based implementation (``ExportRecipe`` + ``ExportSession``) was + considered and rejected: ``QNNRecipeProvider`` accepts only ``soc_model`` + and the three ``skip_*`` keys and warns-and-ignores the rest, so + ``dep_table``, ``passes_job``, ``constant_methods`` and + ``convert_linear_to_conv2d`` cannot be expressed through it, and its + FP16 recipe hardcodes ``use_fp16=True``, leaving no quantized path. """ def compile_model( @@ -59,6 +175,7 @@ def compile_model( constant_methods: Optional[Dict[str, Any]] = None, dep_table: Optional[Dict] = None, passes_job: Optional[Any] = None, + artifact_key: str = ARTIFACT_TEXT_DECODER, extra_options: Optional[Dict[str, Any]] = None, ) -> CompilationResult: """Compile the model via ``to_edge_transform_and_lower_to_qnn``. @@ -68,16 +185,25 @@ def compile_model( example_inputs: Positional example inputs for ``torch.export``, sourced from the model itself. compile_specs: QNN compiler specifications for backend delegation. - artifact_dir: Directory to store compiled .pte artifacts. + artifact_dir: Directory to store the compiled .pte artifact. Created + if it does not exist. file_name: Base name for the output .pte file. - soc_model: Target SoC chipset enum value. - backend_type: QNN backend type (HTP, GPU, LPAI). + soc_model: Target SoC chipset. Lowering reads the target from + ``compile_specs``, so this is not applied independently; it is + logged and checked against the specs. + backend_type: QNN backend type. As with ``soc_model``, lowering + reads this from ``compile_specs``; it is logged and checked. constant_methods: Methods returning constants in eager mode. For a decoder this carries the quantization attributes written into ``meta`` during quantization, so it is only complete once that stage has run. - dep_table: Per-graph pass dependency table. - passes_job: Per-graph pass configuration. + dep_table: Pass dependency table for this graph. + passes_job: Pass configuration for this graph. + artifact_key: Name to key the written artifact under (see + ``artifact_keys``). A 1:1 adapter cannot tell which component it + was handed -- a vision encoder and a text decoder arrive as the + same argument -- so the caller names it. Defaults to the text + decoder, the only artifact a text-only model produces. extra_options: Optional tuning knobs forwarded to lowering: ``skip_node_id_set``, ``skip_node_op_set``, ``skip_mutable_buffer``, ``convert_linear_to_conv2d``, @@ -85,16 +211,68 @@ def compile_model( override the ``to_executorch`` configuration. Returns: - CompilationResult with artifact paths and optional etrecord. + CompilationResult holding the written artifact under + ``artifact_key``, and the ETRecord when one was requested. Raises: - NotImplementedError: Always raised in this PR; the body lands with - the multi-graph lowering that calls it. + ValueError: If ``example_inputs`` is missing, since ``torch.export`` + cannot trace without it, or if ``soc_model`` / ``backend_type`` + contradict ``compile_specs``. """ - raise NotImplementedError( - "DefaultCompilerAdapter has no body yet: lowering is implemented " - "together with the strategy-level multi-graph fan-out that calls " - "it, so that to_edge_transform_and_lower_to_qnn is wired up once " - "in its graph-keyed form. Inject a custom CompilerAdapter until " - "then." + from executorch.backends.qualcomm.utils.utils import ( + to_edge_transform_and_lower_to_qnn, + ) + + if example_inputs is None: + raise ValueError( + "example_inputs is required to compile a model; it defines the " + "exported graph's positional signature and is produced from " + "the model by ModelLoaderAdapter.get_example_inputs" + ) + + _verify_target_matches_specs(compile_specs, soc_model, backend_type) + + options = dict(extra_options or {}) + lowering_options = { + key: options[key] for key in _LOWERING_OPTION_KEYS if key in options + } + generate_etrecord = bool(lowering_options.get("generate_etrecord", False)) + + logger.info( + "Compiling '%s' for SoC=%s, backend=%s", + file_name, + getattr(soc_model, "name", soc_model), + backend_type, + ) + + edge_prog_mgr = to_edge_transform_and_lower_to_qnn( + module=model, + inputs=example_inputs, + compiler_specs=compile_specs, + constant_methods=constant_methods, + dep_table=dep_table, + passes_job=passes_job, + **lowering_options, + ) + + executorch_config = ( + options.get(_KEY_EXECUTORCH_BACKEND_CONFIG) or _default_executorch_config() + ) + exec_prog_mgr = edge_prog_mgr.to_executorch(executorch_config) + + artifact_dir = Path(artifact_dir) + artifact_dir.mkdir(parents=True, exist_ok=True) + pte_path = artifact_dir / f"{file_name}.pte" + with open(pte_path, "wb") as file: + exec_prog_mgr.write_to_file(file) + + logger.info("Wrote artifact to %s", pte_path) + + etrecord = None + if generate_etrecord: + etrecord = exec_prog_mgr.get_etrecord() + + return CompilationResult( + artifact_paths={artifact_key: pte_path}, + etrecord=etrecord, ) diff --git a/backends/qualcomm/genai_pipeline/strategies/compilation/executorch_compilation_strategy.py b/backends/qualcomm/genai_pipeline/strategies/compilation/executorch_compilation_strategy.py index 439c4b753f6..d2e1169f1e8 100644 --- a/backends/qualcomm/genai_pipeline/strategies/compilation/executorch_compilation_strategy.py +++ b/backends/qualcomm/genai_pipeline/strategies/compilation/executorch_compilation_strategy.py @@ -81,9 +81,7 @@ class ExecuTorchCompilationStrategy(CompilationStrategy): and compile specs, which is additive to ``CompilationInputConfig``, so deferring costs nothing structurally. Per the layering used throughout this package, the fan-out belongs in this *strategy* -- adapters stay - thin 1:1 wrappers over one graph. ``CompilationOutputConfig.artifact_paths`` - likewise stays a ``List`` here and becomes graph-name-keyed with the same - change. + thin 1:1 wrappers over one graph. Args: compiler_adapter: Injectable adapter for compilation operations. @@ -163,8 +161,9 @@ def invoke( ) logger.info( - "Compilation completed: %d artifact(s) produced", + "Compilation completed: %d artifact(s) produced (%s)", len(result.artifact_paths), + ", ".join(sorted(map(str, result.artifact_paths))), ) return CompilationOutputConfig( diff --git a/backends/qualcomm/genai_pipeline/strategies/inference/default_device_runner_adapter.py b/backends/qualcomm/genai_pipeline/strategies/inference/default_device_runner_adapter.py index ac41e684bc3..5d1c9e5b1a1 100644 --- a/backends/qualcomm/genai_pipeline/strategies/inference/default_device_runner_adapter.py +++ b/backends/qualcomm/genai_pipeline/strategies/inference/default_device_runner_adapter.py @@ -39,14 +39,17 @@ def __init__( def push_artifacts( self, - artifact_paths: List[Path], + artifact_paths: Dict[str, Path], input_data: Optional[List[Any]] = None, extra_files: Optional[List[str]] = None, ) -> None: """Push compiled artifacts and inputs to the device via ADB. Args: - artifact_paths: Paths to compiled .pte artifacts. + artifact_paths: Compiled .pte artifacts keyed by artifact name. + Every artifact is pushed; the keys select which on-device runner + argument each path is passed to, which is the runner's concern + rather than this adapter's. input_data: Optional input data to push to device. extra_files: Optional additional files to push. """ @@ -58,7 +61,7 @@ def push_artifacts( "replaced on device." ) - pte_paths = [str(p) for p in artifact_paths] + pte_paths = [str(p) for p in artifact_paths.values()] logger.info("Pushing artifacts to device: %s", pte_paths) self._adb = SimpleADB( diff --git a/backends/qualcomm/genai_pipeline/strategies/inference/device_runner_adapter.py b/backends/qualcomm/genai_pipeline/strategies/inference/device_runner_adapter.py index 0c428bdc326..4059c358d3f 100644 --- a/backends/qualcomm/genai_pipeline/strategies/inference/device_runner_adapter.py +++ b/backends/qualcomm/genai_pipeline/strategies/inference/device_runner_adapter.py @@ -41,14 +41,15 @@ class DeviceRunnerAdapter(Protocol): def push_artifacts( self, - artifact_paths: List[Path], + artifact_paths: Dict[str, Path], input_data: Optional[List[Any]] = None, extra_files: Optional[List[str]] = None, ) -> None: """Push compiled artifacts and inputs to the device. Args: - artifact_paths: Paths to compiled .pte artifacts. + artifact_paths: Compiled .pte artifacts keyed by artifact name (see + ``artifact_keys``), as produced by the compilation stage. input_data: Optional pre-encoded input data to push to device. For adapters that prepare inputs themselves -- turning a prompt into model inputs needs the tokenizer's chat template, BOS handling, diff --git a/backends/qualcomm/genai_pipeline/tests/compilation/__init__.py b/backends/qualcomm/genai_pipeline/tests/compilation/__init__.py new file mode 100644 index 00000000000..b5f86874fd4 --- /dev/null +++ b/backends/qualcomm/genai_pipeline/tests/compilation/__init__.py @@ -0,0 +1,5 @@ +# Copyright (c) Qualcomm Innovation Center, Inc. +# All rights reserved +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. diff --git a/backends/qualcomm/genai_pipeline/tests/compilation/test_compile_spec_builder.py b/backends/qualcomm/genai_pipeline/tests/compilation/test_compile_spec_builder.py new file mode 100644 index 00000000000..29369a90c48 --- /dev/null +++ b/backends/qualcomm/genai_pipeline/tests/compilation/test_compile_spec_builder.py @@ -0,0 +1,272 @@ +# Copyright (c) Qualcomm Innovation Center, Inc. +# All rights reserved +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +import unittest +from unittest.mock import MagicMock, patch + +from executorch.backends.qualcomm.genai_pipeline.compilation.compile_spec_builder import ( + QnnCompileSpecBuilder, + resolve_backend_type, + resolve_soc_model, +) +from executorch.backends.qualcomm.genai_pipeline.control_args import ControlArgs +from executorch.backends.qualcomm.serialization.qc_schema import ( + QcomChipset, + QnnExecuTorchBackendType, +) + +_TEST_SOC = "SM8750" + +# The builder imports these lazily from their defining module, so patching +# there intercepts the call. +_HTP_SPEC = "executorch.backends.qualcomm.utils.utils.generate_htp_compiler_spec" +_GPU_SPEC = "executorch.backends.qualcomm.utils.utils.generate_gpu_compiler_spec" +_QNN_SPEC = ( + "executorch.backends.qualcomm.utils.utils." "generate_qnn_executorch_compiler_spec" +) + + +def _build_with_mocks(builder, **build_kwargs): + """Invoke builder.build() with the spec generators mocked. + + Returns: + A (htp_mock, gpu_mock, qnn_mock) tuple of the patched generators. + """ + with patch(_HTP_SPEC) as htp, patch(_GPU_SPEC) as gpu, patch(_QNN_SPEC) as qnn: + builder.build(**build_kwargs) + return htp, gpu, qnn + + +class TestResolveSocModel(unittest.TestCase): + + def test_resolves_known_name(self): + """A CLI-style SoC name maps to its QcomChipset member.""" + self.assertEqual(resolve_soc_model("SM8750"), QcomChipset.SM8750) + + def test_passes_through_enum(self): + """An already-resolved QcomChipset is returned unchanged.""" + self.assertEqual(resolve_soc_model(QcomChipset.SM8650), QcomChipset.SM8650) + + def test_unknown_name_raises_with_supported_list(self): + """An unknown name raises ValueError naming the valid SoCs.""" + with self.assertRaises(ValueError) as cm: + resolve_soc_model("NOT_A_SOC") + + message = str(cm.exception) + self.assertIn("NOT_A_SOC", message) + self.assertIn("SM8750", message) + + +class TestResolveBackendType(unittest.TestCase): + + def test_resolves_supported_backends(self): + """Backend names map to their QnnExecuTorchBackendType members.""" + expected = { + "htp": QnnExecuTorchBackendType.kHtpBackend, + "gpu": QnnExecuTorchBackendType.kGpuBackend, + "lpai": QnnExecuTorchBackendType.kLpaiBackend, + "dsp": QnnExecuTorchBackendType.kDspBackend, + } + + for name, member in expected.items(): + with self.subTest(backend=name): + self.assertEqual(resolve_backend_type(name), member) + + def test_unknown_backend_raises(self): + """An unknown backend name raises ValueError rather than AttributeError.""" + with self.assertRaises(ValueError) as cm: + resolve_backend_type("not_a_backend") + + message = str(cm.exception) + self.assertIn("not_a_backend", message) + self.assertIn("htp", message) + + def test_the_undefined_backend_is_not_selectable(self): + """kUndefinedBackend is the unset value, so its name resolves to nothing.""" + with self.assertRaises(ValueError): + resolve_backend_type(str(QnnExecuTorchBackendType.kUndefinedBackend)) + + +class TestQnnCompileSpecBuilderConstruction(unittest.TestCase): + + def test_resolves_soc_model_eagerly(self): + """The SoC is resolved at construction, not at build time.""" + builder = QnnCompileSpecBuilder(soc_model=_TEST_SOC) + + self.assertEqual(builder.soc_model, QcomChipset.SM8750) + + def test_rejects_unknown_soc_model(self): + """A bad SoC fails at construction, before any graph work.""" + with self.assertRaises(ValueError): + QnnCompileSpecBuilder(soc_model="NOT_A_SOC") + + def test_rejects_backend_without_spec_support(self): + """A backend with no spec-generation path is rejected.""" + with self.assertRaises(ValueError) as cm: + QnnCompileSpecBuilder(soc_model=_TEST_SOC, backend="lpai") + + self.assertIn("lpai", str(cm.exception)) + + def test_backend_type_exposes_the_enum(self): + """backend_type converts the backend name to the QNN enum.""" + builder = QnnCompileSpecBuilder(soc_model=_TEST_SOC, backend="gpu") + + self.assertEqual(builder.backend_type, QnnExecuTorchBackendType.kGpuBackend) + + +class TestQnnCompileSpecBuilderDeviceDefaults(unittest.TestCase): + + def test_device_target_enables_weight_sharing_and_shared_buffer(self): + """On device, both weight sharing and shared buffer default to on.""" + builder = QnnCompileSpecBuilder(soc_model=_TEST_SOC) + + htp, _, qnn = _build_with_mocks(builder) + + self.assertTrue(htp.call_args.kwargs["use_weight_sharing"]) + self.assertTrue(qnn.call_args.kwargs["shared_buffer"]) + + def test_x86_target_disables_weight_sharing_and_shared_buffer(self): + """The emulator supports neither, so both default to off.""" + builder = QnnCompileSpecBuilder(soc_model=_TEST_SOC, enable_x86_64=True) + + htp, _, qnn = _build_with_mocks(builder) + + self.assertFalse(htp.call_args.kwargs["use_weight_sharing"]) + self.assertFalse(qnn.call_args.kwargs["shared_buffer"]) + + def test_explicit_values_override_x86_defaults(self): + """Passing the flags explicitly overrides the target-derived default.""" + builder = QnnCompileSpecBuilder(soc_model=_TEST_SOC, enable_x86_64=True) + + htp, _, qnn = _build_with_mocks( + builder, use_weight_sharing=True, shared_buffer=True + ) + + self.assertTrue(htp.call_args.kwargs["use_weight_sharing"]) + self.assertTrue(qnn.call_args.kwargs["shared_buffer"]) + + +class TestQnnCompileSpecBuilderForwarding(unittest.TestCase): + + def test_forwards_htp_precision_and_contexts(self): + """use_fp16 and use_multi_contexts reach the HTP options helper.""" + builder = QnnCompileSpecBuilder(soc_model=_TEST_SOC) + + htp, _, _ = _build_with_mocks(builder, use_fp16=True, use_multi_contexts=True) + + self.assertTrue(htp.call_args.kwargs["use_fp16"]) + self.assertTrue(htp.call_args.kwargs["use_multi_contexts"]) + + def test_forwards_soc_and_graph_options(self): + """The SoC enum and per-graph options reach the spec generator.""" + builder = QnnCompileSpecBuilder(soc_model=_TEST_SOC) + + _, _, qnn = _build_with_mocks(builder, online_prepare=True, use_mha2sha=True) + + self.assertEqual(qnn.call_args.kwargs["soc_model"], QcomChipset.SM8750) + self.assertTrue(qnn.call_args.kwargs["online_prepare"]) + self.assertTrue(qnn.call_args.kwargs["use_mha2sha"]) + + def test_passes_backend_options_through(self): + """The backend options object is handed to the spec generator as-is.""" + builder = QnnCompileSpecBuilder(soc_model=_TEST_SOC) + options = MagicMock(name="backend_options") + + with patch(_HTP_SPEC, return_value=options), patch(_QNN_SPEC) as qnn: + builder.build() + + self.assertIs(qnn.call_args.kwargs["backend_options"], options) + + +class TestQnnCompileSpecBuilderBackendSelection(unittest.TestCase): + + def test_htp_backend_uses_htp_options(self): + """The HTP target builds HTP options and not GPU options.""" + builder = QnnCompileSpecBuilder(soc_model=_TEST_SOC, backend="htp") + + htp, gpu, _ = _build_with_mocks(builder) + + htp.assert_called_once() + gpu.assert_not_called() + + def test_gpu_backend_uses_gpu_options(self): + """The GPU target builds GPU options and not HTP options.""" + builder = QnnCompileSpecBuilder(soc_model=_TEST_SOC, backend="gpu") + + htp, gpu, _ = _build_with_mocks(builder) + + gpu.assert_called_once() + htp.assert_not_called() + + def test_gpu_backend_warns_that_fp16_is_ignored(self): + """GPU has no fp16 switch, so requesting it warns rather than silently dropping.""" + builder = QnnCompileSpecBuilder(soc_model=_TEST_SOC, backend="gpu") + + with patch(_GPU_SPEC), patch(_QNN_SPEC): + with self.assertLogs( + "executorch.backends.qualcomm.genai_pipeline.compilation." + "compile_spec_builder", + level="WARNING", + ) as logs: + builder.build(use_fp16=True) + + self.assertIn("use_fp16", logs.output[0]) + + +class TestQnnCompileSpecBuilderAgainstRealApis(unittest.TestCase): + """Exercises the real QNN spec generation, which needs no device.""" + + def test_builds_real_compile_specs(self): + """A default HTP build produces usable CompileSpec objects.""" + from executorch.exir.backend.compile_spec_schema import CompileSpec + + specs = QnnCompileSpecBuilder(soc_model=_TEST_SOC).build() + + self.assertTrue(specs) + for spec in specs: + with self.subTest(spec=spec.key): + self.assertIsInstance(spec, CompileSpec) + + def test_multi_contexts_conflicts_with_online_prepare(self): + """The underlying generator rejects this combination; it is not masked.""" + builder = QnnCompileSpecBuilder(soc_model=_TEST_SOC) + + with self.assertRaises(ValueError): + builder.build(use_multi_contexts=True, online_prepare=True) + + +class TestQnnCompileSpecBuilderFromControlArgs(unittest.TestCase): + """Covers the seam where a ``ControlArgs`` drives the builder. + + The two disagree on ``shared_buffer``: ``ControlArgs`` mirrors + ``llama.py``'s parser, which defaults it off, while the builder defaults it + on for a device target. A caller must therefore thread the value rather + than rely on either default. + """ + + def test_control_args_shared_buffer_default_differs_from_the_builders(self): + """The two defaults differ, which is why callers must pass it explicitly.""" + control_args = ControlArgs() + + _, _, qnn = _build_with_mocks(QnnCompileSpecBuilder(soc_model=_TEST_SOC)) + + self.assertFalse(control_args.shared_buffer) + self.assertTrue(qnn.call_args.kwargs["shared_buffer"]) + + def test_threading_control_args_shared_buffer_is_honoured(self): + """Passing the ControlArgs value explicitly overrides the builder default.""" + control_args = ControlArgs() + + _, _, qnn = _build_with_mocks( + QnnCompileSpecBuilder(soc_model=_TEST_SOC), + shared_buffer=control_args.shared_buffer, + ) + + self.assertFalse(qnn.call_args.kwargs["shared_buffer"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/backends/qualcomm/genai_pipeline/tests/configs/test_compilation_input_config.py b/backends/qualcomm/genai_pipeline/tests/configs/test_compilation_input_config.py index e8407399183..718c418ecb6 100644 --- a/backends/qualcomm/genai_pipeline/tests/configs/test_compilation_input_config.py +++ b/backends/qualcomm/genai_pipeline/tests/configs/test_compilation_input_config.py @@ -11,6 +11,7 @@ from executorch.backends.qualcomm.genai_pipeline.configs.compilation_input_config import ( CompilationInputConfig, ) +from executorch.backends.qualcomm.genai_pipeline.graph_bundle import GraphBundle class TestCompilationInputConfig(unittest.TestCase): @@ -28,6 +29,7 @@ def test_optional_fields_default_to_none(self): self.assertIsNone(config.model) self.assertIsNone(config.example_inputs) self.assertIsNone(config.compile_specs) + self.assertIsNone(config.graphs) def test_example_inputs_carries_export_signature(self): example_inputs = (MagicMock(name="tokens"), MagicMock(name="attn_mask")) @@ -38,6 +40,18 @@ def test_example_inputs_carries_export_signature(self): ) self.assertIs(config.example_inputs, example_inputs) + def test_graphs_are_addressable_by_graph_name(self): + graphs = { + name: GraphBundle(module=MagicMock(name=name), inputs=(MagicMock(),)) + for name in ("kv_forward", "prefill_forward") + } + config = CompilationInputConfig( + soc_model=MagicMock(), + backend_type=MagicMock(), + graphs=graphs, + ) + self.assertIs(config.graphs["prefill_forward"], graphs["prefill_forward"]) + if __name__ == "__main__": unittest.main() diff --git a/backends/qualcomm/genai_pipeline/tests/stages/test_compilation_stage.py b/backends/qualcomm/genai_pipeline/tests/stages/test_compilation_stage.py index 5f382c11c24..05d012b2733 100644 --- a/backends/qualcomm/genai_pipeline/tests/stages/test_compilation_stage.py +++ b/backends/qualcomm/genai_pipeline/tests/stages/test_compilation_stage.py @@ -22,7 +22,7 @@ ) from executorch.backends.qualcomm.genai_pipeline.tests.test_utils import ( make_test_context, - TEST_PTE_PATH, + TEST_ARTIFACT_PATHS, ) @@ -36,7 +36,7 @@ def test_name(self): def test_invoke_delegates_to_strategy(self): mock_strategy = MagicMock(spec=CompilationStrategy) mock_strategy.invoke.return_value = CompilationOutputConfig( - artifact_paths=[TEST_PTE_PATH] + artifact_paths=TEST_ARTIFACT_PATHS ) stage = CompilationStage(mock_strategy) context = make_test_context() diff --git a/backends/qualcomm/genai_pipeline/tests/strategies/compilation/test_default_compiler_adapter.py b/backends/qualcomm/genai_pipeline/tests/strategies/compilation/test_default_compiler_adapter.py new file mode 100644 index 00000000000..26f04fbaf89 --- /dev/null +++ b/backends/qualcomm/genai_pipeline/tests/strategies/compilation/test_default_compiler_adapter.py @@ -0,0 +1,374 @@ +# Copyright (c) Qualcomm Innovation Center, Inc. +# All rights reserved +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +import tempfile +import unittest +from pathlib import Path +from unittest.mock import MagicMock, patch + +from executorch.backends.qualcomm.genai_pipeline.artifact_keys import ( + ARTIFACT_TEXT_DECODER, + ARTIFACT_VISION_ENCODER, +) +from executorch.backends.qualcomm.genai_pipeline.compilation import ( + QnnCompileSpecBuilder, +) +from executorch.backends.qualcomm.genai_pipeline.strategies.compilation.compiler_adapter import ( + CompilerAdapter, +) +from executorch.backends.qualcomm.genai_pipeline.strategies.compilation.default_compiler_adapter import ( + DefaultCompilerAdapter, +) +from executorch.backends.qualcomm.genai_pipeline.tests.test_utils import ( + TEST_BACKEND_TYPE, + TEST_SOC_CHIPSET, +) +from executorch.backends.qualcomm.serialization.qc_schema import ( + QcomChipset, + QnnExecuTorchBackendType, +) + +# The adapter imports lowering lazily from its defining module, so patching +# there intercepts the call. +_LOWER = "executorch.backends.qualcomm.utils.utils.to_edge_transform_and_lower_to_qnn" + +_FILE_NAME = "test_model" + + +def _make_edge_program_manager(): + """Create a mock EdgeProgramManager whose program writes bytes to file. + + ``write_to_file`` receives a real file object, so the mock writes to it to + keep the on-disk artifact non-empty. + """ + exec_prog_mgr = MagicMock(name="exec_prog_mgr") + exec_prog_mgr.write_to_file.side_effect = lambda f: f.write(b"pte-bytes") + + edge_prog_mgr = MagicMock(name="edge_prog_mgr") + edge_prog_mgr.to_executorch.return_value = exec_prog_mgr + return edge_prog_mgr + + +def _compile(adapter, artifact_dir, **overrides): + """Invoke compile_model with valid defaults and lowering mocked. + + Returns: + A (result, lower_mock, edge_prog_mgr) tuple. + """ + edge_prog_mgr = _make_edge_program_manager() + kwargs = { + "model": MagicMock(name="model"), + "example_inputs": (MagicMock(name="example_input"),), + "compile_specs": [MagicMock(name="compile_spec")], + "artifact_dir": artifact_dir, + "file_name": _FILE_NAME, + "soc_model": TEST_SOC_CHIPSET, + "backend_type": TEST_BACKEND_TYPE, + } + kwargs.update(overrides) + + with patch(_LOWER, return_value=edge_prog_mgr) as lower: + result = adapter.compile_model(**kwargs) + + return result, lower, edge_prog_mgr + + +class TestDefaultCompilerAdapterProtocol(unittest.TestCase): + + def test_satisfies_the_compiler_adapter_protocol(self): + """The adapter is usable where a CompilerAdapter is expected.""" + self.assertIsInstance(DefaultCompilerAdapter(), CompilerAdapter) + + +class TestDefaultCompilerAdapterArtifacts(unittest.TestCase): + + def test_writes_pte_named_after_file_name(self): + """The artifact is written as .pte in artifact_dir.""" + with tempfile.TemporaryDirectory() as tmp: + artifact_dir = Path(tmp) + + result, _, _ = _compile(DefaultCompilerAdapter(), artifact_dir) + + expected = artifact_dir / f"{_FILE_NAME}.pte" + self.assertEqual(result.artifact_paths, {ARTIFACT_TEXT_DECODER: expected}) + self.assertTrue(expected.exists()) + + def test_creates_missing_artifact_directories(self): + """A nested artifact_dir that does not exist is created.""" + with tempfile.TemporaryDirectory() as tmp: + artifact_dir = Path(tmp) / "nested" / "dir" + + _compile(DefaultCompilerAdapter(), artifact_dir) + + self.assertTrue(artifact_dir.is_dir()) + + def test_written_artifact_is_not_empty(self): + """The ExecuTorch program's bytes reach the file.""" + with tempfile.TemporaryDirectory() as tmp: + artifact_dir = Path(tmp) + + result, _, _ = _compile(DefaultCompilerAdapter(), artifact_dir) + + path = result.artifact_paths[ARTIFACT_TEXT_DECODER] + self.assertGreater(path.stat().st_size, 0) + + def test_keys_the_artifact_as_the_text_decoder_by_default(self): + """Omitting artifact_key yields the text decoder, the text-only case.""" + with tempfile.TemporaryDirectory() as tmp: + result, _, _ = _compile(DefaultCompilerAdapter(), Path(tmp)) + + self.assertEqual(list(result.artifact_paths), [ARTIFACT_TEXT_DECODER]) + + def test_keys_the_artifact_under_the_requested_name(self): + """A non-decoder component is keyed as itself, not as the decoder.""" + with tempfile.TemporaryDirectory() as tmp: + artifact_dir = Path(tmp) + + result, _, _ = _compile( + DefaultCompilerAdapter(), + artifact_dir, + artifact_key=ARTIFACT_VISION_ENCODER, + ) + + expected = artifact_dir / f"{_FILE_NAME}.pte" + self.assertEqual(result.artifact_paths, {ARTIFACT_VISION_ENCODER: expected}) + + +class TestDefaultCompilerAdapterLoweringArguments(unittest.TestCase): + + def test_forwards_required_lowering_inputs(self): + """model, inputs and compile specs reach lowering under its own names.""" + model = MagicMock(name="model") + example_inputs = (MagicMock(name="example_input"),) + compile_specs = [MagicMock(name="compile_spec")] + + with tempfile.TemporaryDirectory() as tmp: + _, lower, _ = _compile( + DefaultCompilerAdapter(), + Path(tmp), + model=model, + example_inputs=example_inputs, + compile_specs=compile_specs, + ) + + self.assertIs(lower.call_args.kwargs["module"], model) + self.assertIs(lower.call_args.kwargs["inputs"], example_inputs) + self.assertIs(lower.call_args.kwargs["compiler_specs"], compile_specs) + + def test_forwards_per_graph_lowering_inputs(self): + """constant_methods, dep_table and passes_job reach lowering.""" + constant_methods = {"get_max_seq_len": 512} + dep_table = {"pass": ["dependency"]} + passes_job = MagicMock(name="passes_job") + + with tempfile.TemporaryDirectory() as tmp: + _, lower, _ = _compile( + DefaultCompilerAdapter(), + Path(tmp), + constant_methods=constant_methods, + dep_table=dep_table, + passes_job=passes_job, + ) + + self.assertIs(lower.call_args.kwargs["constant_methods"], constant_methods) + self.assertIs(lower.call_args.kwargs["dep_table"], dep_table) + self.assertIs(lower.call_args.kwargs["passes_job"], passes_job) + + def test_forwards_known_extra_options(self): + """Recognised tuning knobs are forwarded to lowering.""" + skip_ops = {"llama.fallback.default"} + + with tempfile.TemporaryDirectory() as tmp: + _, lower, _ = _compile( + DefaultCompilerAdapter(), + Path(tmp), + extra_options={ + "skip_node_op_set": skip_ops, + "convert_linear_to_conv2d": True, + }, + ) + + self.assertIs(lower.call_args.kwargs["skip_node_op_set"], skip_ops) + self.assertTrue(lower.call_args.kwargs["convert_linear_to_conv2d"]) + + def test_ignores_unknown_extra_options(self): + """Options lowering does not accept are not forwarded to it.""" + with tempfile.TemporaryDirectory() as tmp: + _, lower, _ = _compile( + DefaultCompilerAdapter(), + Path(tmp), + extra_options={"not_a_lowering_option": True}, + ) + + self.assertNotIn("not_a_lowering_option", lower.call_args.kwargs) + + def test_does_not_forward_dynamic_shapes(self): + """HTP has no dynamic shapes, so none is passed even though the API has it.""" + with tempfile.TemporaryDirectory() as tmp: + _, lower, _ = _compile(DefaultCompilerAdapter(), Path(tmp)) + + self.assertNotIn("dynamic_shapes", lower.call_args.kwargs) + + +class TestDefaultCompilerAdapterExecutorchConfig(unittest.TestCase): + + def test_leaves_graph_io_unallocated_by_default(self): + """Graph I/O is not allocated, since a shared buffer supplies it.""" + with tempfile.TemporaryDirectory() as tmp: + _, _, edge_prog_mgr = _compile(DefaultCompilerAdapter(), Path(tmp)) + + config = edge_prog_mgr.to_executorch.call_args.args[0] + self.assertFalse(config.memory_planning_pass.alloc_graph_input) + self.assertFalse(config.memory_planning_pass.alloc_graph_output) + + def test_builds_quant_io_by_default(self): + """BuildQuantIo runs, so quantized I/O tensors get their types.""" + from executorch.backends.qualcomm._passes.build_quant_io import BuildQuantIo + + with tempfile.TemporaryDirectory() as tmp: + _, _, edge_prog_mgr = _compile(DefaultCompilerAdapter(), Path(tmp)) + + config = edge_prog_mgr.to_executorch.call_args.args[0] + self.assertTrue(any(isinstance(p, BuildQuantIo) for p in config.passes)) + + def test_extra_options_can_override_the_config(self): + """A caller-supplied backend config replaces the default.""" + override = MagicMock(name="executorch_backend_config") + + with tempfile.TemporaryDirectory() as tmp: + _, _, edge_prog_mgr = _compile( + DefaultCompilerAdapter(), + Path(tmp), + extra_options={"executorch_backend_config": override}, + ) + + self.assertIs(edge_prog_mgr.to_executorch.call_args.args[0], override) + + +class TestDefaultCompilerAdapterEtrecord(unittest.TestCase): + + def test_no_etrecord_unless_requested(self): + """ETRecord is not retrieved when it was not requested.""" + with tempfile.TemporaryDirectory() as tmp: + result, _, edge_prog_mgr = _compile(DefaultCompilerAdapter(), Path(tmp)) + + self.assertIsNone(result.etrecord) + edge_prog_mgr.to_executorch.return_value.get_etrecord.assert_not_called() + + def test_returns_etrecord_when_requested(self): + """Requesting an ETRecord both enables it in lowering and returns it.""" + with tempfile.TemporaryDirectory() as tmp: + result, lower, edge_prog_mgr = _compile( + DefaultCompilerAdapter(), + Path(tmp), + extra_options={"generate_etrecord": True}, + ) + + self.assertTrue(lower.call_args.kwargs["generate_etrecord"]) + self.assertIs( + result.etrecord, + edge_prog_mgr.to_executorch.return_value.get_etrecord.return_value, + ) + + +class TestDefaultCompilerAdapterValidation(unittest.TestCase): + + def test_missing_example_inputs_raises_before_lowering(self): + """Without example_inputs the adapter raises rather than calling lowering.""" + adapter = DefaultCompilerAdapter() + + with patch(_LOWER) as lower: + with self.assertRaises(ValueError) as cm: + adapter.compile_model( + model=MagicMock(name="model"), + example_inputs=None, + compile_specs=[MagicMock(name="compile_spec")], + artifact_dir=Path("/tmp/never_written"), + file_name=_FILE_NAME, + soc_model=TEST_SOC_CHIPSET, + backend_type=TEST_BACKEND_TYPE, + ) + + self.assertIn("example_inputs", str(cm.exception)) + lower.assert_not_called() + + +class TestDefaultCompilerAdapterTargetConsistency(unittest.TestCase): + """Uses real compile specs, whose target is readable back out of them. + + Lowering ignores the ``soc_model`` / ``backend_type`` arguments in favour of + the specs, so a disagreement between the two would otherwise only surface on + device. + """ + + def test_accepts_a_target_matching_the_specs(self): + """The SoC and backend the specs carry are accepted.""" + specs = QnnCompileSpecBuilder(soc_model=TEST_SOC_CHIPSET).build() + + with tempfile.TemporaryDirectory() as tmp: + result, lower, _ = _compile( + DefaultCompilerAdapter(), + Path(tmp), + compile_specs=specs, + ) + + lower.assert_called_once() + self.assertEqual(list(result.artifact_paths), [ARTIFACT_TEXT_DECODER]) + + def test_soc_model_contradicting_the_specs_raises_before_lowering(self): + """A SoC other than the specs' fails rather than being ignored.""" + specs = QnnCompileSpecBuilder(soc_model=QcomChipset.SM8650).build() + adapter = DefaultCompilerAdapter() + + with patch(_LOWER) as lower: + with self.assertRaises(ValueError) as cm: + adapter.compile_model( + model=MagicMock(name="model"), + example_inputs=(MagicMock(name="example_input"),), + compile_specs=specs, + artifact_dir=Path("/tmp/never_written"), + file_name=_FILE_NAME, + soc_model=QcomChipset.SM8750, + backend_type=TEST_BACKEND_TYPE, + ) + + self.assertIn("soc_model", str(cm.exception)) + lower.assert_not_called() + + def test_backend_type_contradicting_the_specs_raises_before_lowering(self): + """A backend other than the specs' fails rather than being ignored.""" + specs = QnnCompileSpecBuilder(soc_model=TEST_SOC_CHIPSET).build() + adapter = DefaultCompilerAdapter() + + with patch(_LOWER) as lower: + with self.assertRaises(ValueError) as cm: + adapter.compile_model( + model=MagicMock(name="model"), + example_inputs=(MagicMock(name="example_input"),), + compile_specs=specs, + artifact_dir=Path("/tmp/never_written"), + file_name=_FILE_NAME, + soc_model=TEST_SOC_CHIPSET, + backend_type=QnnExecuTorchBackendType.kGpuBackend, + ) + + self.assertIn("backend_type", str(cm.exception)) + lower.assert_not_called() + + def test_specs_without_a_qnn_entry_are_not_checked(self): + """A stub spec list disables the check rather than failing the call.""" + with tempfile.TemporaryDirectory() as tmp: + _, lower, _ = _compile( + DefaultCompilerAdapter(), + Path(tmp), + compile_specs=[MagicMock(name="not_a_qnn_spec")], + ) + + lower.assert_called_once() + + +if __name__ == "__main__": + unittest.main() diff --git a/backends/qualcomm/genai_pipeline/tests/strategies/compilation/test_executorch_compilation_strategy.py b/backends/qualcomm/genai_pipeline/tests/strategies/compilation/test_executorch_compilation_strategy.py index 20c7a1e9af4..8e20e088c18 100644 --- a/backends/qualcomm/genai_pipeline/tests/strategies/compilation/test_executorch_compilation_strategy.py +++ b/backends/qualcomm/genai_pipeline/tests/strategies/compilation/test_executorch_compilation_strategy.py @@ -8,6 +8,10 @@ from pathlib import Path from unittest.mock import MagicMock, patch +from executorch.backends.qualcomm.genai_pipeline.artifact_keys import ( + ARTIFACT_TEXT_DECODER, + ARTIFACT_TOK_EMBEDDING, +) from executorch.backends.qualcomm.genai_pipeline.configs.compilation_input_config import ( CompilationInputConfig, ) @@ -26,6 +30,8 @@ ) from executorch.backends.qualcomm.genai_pipeline.tests.test_utils import ( make_test_context, + TEST_BACKEND_TYPE, + TEST_SOC_CHIPSET, ) @@ -33,7 +39,7 @@ def _make_mock_adapter(): """Create a mock compiler adapter returning a valid CompilationResult.""" adapter = MagicMock() adapter.compile_model.return_value = CompilationResult( - artifact_paths=[Path("/tmp/test_model.pte")], + artifact_paths={ARTIFACT_TEXT_DECODER: Path("/tmp/test_model.pte")}, etrecord=None, ) return adapter @@ -42,8 +48,8 @@ def _make_mock_adapter(): def _make_valid_input_config(**overrides): """Create a valid CompilationInputConfig with defaults.""" defaults = { - "soc_model": MagicMock(name="SM8750"), - "backend_type": MagicMock(name="kHtpBackend"), + "soc_model": TEST_SOC_CHIPSET, + "backend_type": TEST_BACKEND_TYPE, "model": MagicMock(name="test_model"), "example_inputs": (MagicMock(name="example_input"),), "artifact_dir": Path("/tmp/artifacts"), @@ -87,7 +93,10 @@ def test_invoke_happy_path(self): result = strategy.invoke(context, input_config) self.assertIsInstance(result, CompilationOutputConfig) - self.assertEqual(result.artifact_paths, [Path("/tmp/test_model.pte")]) + self.assertEqual( + result.artifact_paths, + {ARTIFACT_TEXT_DECODER: Path("/tmp/test_model.pte")}, + ) def test_invoke_passes_correct_args_to_adapter(self): """compile_model receives model, specs, artifact_dir, etc.""" @@ -230,7 +239,7 @@ def test_invoke_returns_etrecord_when_present(self): adapter = _make_mock_adapter() mock_etrecord = MagicMock(name="etrecord") adapter.compile_model.return_value = CompilationResult( - artifact_paths=[Path("/tmp/test.pte")], + artifact_paths={ARTIFACT_TEXT_DECODER: Path("/tmp/test.pte")}, etrecord=mock_etrecord, ) strategy = ExecuTorchCompilationStrategy(compiler_adapter=adapter) @@ -243,16 +252,23 @@ def test_invoke_multiple_artifacts(self): """Multiple artifact paths from adapter are forwarded correctly.""" adapter = _make_mock_adapter() adapter.compile_model.return_value = CompilationResult( - artifact_paths=[Path("/tmp/prefill.pte"), Path("/tmp/decode.pte")], + artifact_paths={ + ARTIFACT_TEXT_DECODER: Path("/tmp/decode.pte"), + ARTIFACT_TOK_EMBEDDING: Path("/tmp/tok_embedding.pte"), + }, etrecord=None, ) strategy = ExecuTorchCompilationStrategy(compiler_adapter=adapter) result = strategy.invoke(make_test_context(), _make_valid_input_config()) - self.assertEqual(len(result.artifact_paths), 2) - self.assertEqual(result.artifact_paths[0], Path("/tmp/prefill.pte")) - self.assertEqual(result.artifact_paths[1], Path("/tmp/decode.pte")) + self.assertEqual( + result.artifact_paths, + { + ARTIFACT_TEXT_DECODER: Path("/tmp/decode.pte"), + ARTIFACT_TOK_EMBEDDING: Path("/tmp/tok_embedding.pte"), + }, + ) if __name__ == "__main__": diff --git a/backends/qualcomm/genai_pipeline/tests/strategies/inference/test_executorch_inference_strategy.py b/backends/qualcomm/genai_pipeline/tests/strategies/inference/test_executorch_inference_strategy.py index 8c9392b0806..ce814812dc7 100644 --- a/backends/qualcomm/genai_pipeline/tests/strategies/inference/test_executorch_inference_strategy.py +++ b/backends/qualcomm/genai_pipeline/tests/strategies/inference/test_executorch_inference_strategy.py @@ -8,6 +8,10 @@ from pathlib import Path from unittest.mock import MagicMock +from executorch.backends.qualcomm.genai_pipeline.artifact_keys import ( + ARTIFACT_TEXT_DECODER, + ARTIFACT_TOK_EMBEDDING, +) from executorch.backends.qualcomm.genai_pipeline.configs.inference_input_config import ( InferenceInputConfig, ) @@ -46,7 +50,7 @@ def _make_valid_input_config(**overrides): """Create a valid InferenceInputConfig with defaults.""" defaults = { "soc_model": MagicMock(name="SM8750"), - "artifact_paths": [Path("/tmp/test.pte")], + "artifact_paths": {ARTIFACT_TEXT_DECODER: Path("/tmp/test.pte")}, } defaults.update(overrides) return InferenceInputConfig(**defaults) @@ -99,7 +103,10 @@ def test_invoke_passes_artifact_paths_to_push(self): """push_artifacts receives the artifact paths from input config.""" adapter = _make_mock_adapter() strategy = ExecuTorchInferenceStrategy(device_runner_adapter=adapter) - artifact_paths = [Path("/tmp/a.pte"), Path("/tmp/b.pte")] + artifact_paths = { + ARTIFACT_TEXT_DECODER: Path("/tmp/decode.pte"), + ARTIFACT_TOK_EMBEDDING: Path("/tmp/tok_embedding.pte"), + } input_config = _make_valid_input_config(artifact_paths=artifact_paths) strategy.invoke(make_test_context(), input_config) @@ -171,10 +178,10 @@ def test_invoke_no_adapter_raises_stage_error(self): self.assertEqual(cm.exception.stage_name, "inference") def test_invoke_missing_artifacts_raises_stage_error(self): - """StageError raised when artifact_paths is empty.""" + """StageError raised when artifact_paths holds no artifacts.""" adapter = _make_mock_adapter() strategy = ExecuTorchInferenceStrategy(device_runner_adapter=adapter) - input_config = _make_valid_input_config(artifact_paths=[]) + input_config = _make_valid_input_config(artifact_paths={}) with self.assertRaises(StageError) as cm: strategy.invoke(make_test_context(), input_config) diff --git a/backends/qualcomm/genai_pipeline/tests/strategies/quantization/test_executorch_quantization_strategy.py b/backends/qualcomm/genai_pipeline/tests/strategies/quantization/test_executorch_quantization_strategy.py index f6fb4281242..a1e1bbc49ae 100644 --- a/backends/qualcomm/genai_pipeline/tests/strategies/quantization/test_executorch_quantization_strategy.py +++ b/backends/qualcomm/genai_pipeline/tests/strategies/quantization/test_executorch_quantization_strategy.py @@ -25,6 +25,8 @@ ) from executorch.backends.qualcomm.genai_pipeline.tests.test_utils import ( make_test_context, + TEST_BACKEND_TYPE, + TEST_SOC_CHIPSET, ) @@ -56,8 +58,8 @@ def _make_mock_adapter(): def _make_valid_input_config(**overrides): """Create a valid QuantizationInputConfig with defaults.""" defaults = { - "soc_model": MagicMock(name="SM8750"), - "backend_type": MagicMock(name="kHtpBackend"), + "soc_model": TEST_SOC_CHIPSET, + "backend_type": TEST_BACKEND_TYPE, "model_module": MagicMock(name="test_model"), "example_inputs": (MagicMock(name="example_input"),), "calibration_data": [(MagicMock(),)], diff --git a/backends/qualcomm/genai_pipeline/tests/test_artifact_keys.py b/backends/qualcomm/genai_pipeline/tests/test_artifact_keys.py new file mode 100644 index 00000000000..f4719e2dbb3 --- /dev/null +++ b/backends/qualcomm/genai_pipeline/tests/test_artifact_keys.py @@ -0,0 +1,69 @@ +# Copyright (c) Qualcomm Innovation Center, Inc. +# All rights reserved +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +import unittest + +from executorch.backends.qualcomm.genai_pipeline.artifact_keys import ( + ALL_ARTIFACT_KEYS, + ARTIFACT_ATTENTION_SINK_EVICTOR, + ARTIFACT_AUDIO_ENCODER, + ARTIFACT_TEXT_DECODER, + ARTIFACT_TEXT_ENCODER, + ARTIFACT_TOK_EMBEDDING, + ARTIFACT_VISION_ENCODER, + DECODE_QDQ_FILENAME, +) + + +class TestArtifactKeys(unittest.TestCase): + + def test_keys_match_legacy_decoder_constants(self): + """Every artifact key equals the legacy runner's pte_paths key.""" + from executorch.examples.qualcomm.oss_scripts.llama import decoder_constants + + pairs = { + ARTIFACT_ATTENTION_SINK_EVICTOR: decoder_constants.ATTENTION_SINK_EVICTOR, + ARTIFACT_AUDIO_ENCODER: decoder_constants.AUDIO_ENCODER, + ARTIFACT_TEXT_DECODER: decoder_constants.TEXT_DECODER, + ARTIFACT_TEXT_ENCODER: decoder_constants.TEXT_ENCODER, + ARTIFACT_TOK_EMBEDDING: decoder_constants.TOK_EMBEDDING, + ARTIFACT_VISION_ENCODER: decoder_constants.VISION_ENCODER, + } + + for ours, legacy in pairs.items(): + with self.subTest(key=ours): + self.assertEqual(ours, legacy) + + def test_all_artifact_keys_is_complete(self): + """ALL_ARTIFACT_KEYS holds every individually exported key.""" + individual = { + ARTIFACT_ATTENTION_SINK_EVICTOR, + ARTIFACT_AUDIO_ENCODER, + ARTIFACT_TEXT_DECODER, + ARTIFACT_TEXT_ENCODER, + ARTIFACT_TOK_EMBEDDING, + ARTIFACT_VISION_ENCODER, + } + + self.assertEqual(ALL_ARTIFACT_KEYS, individual) + + +class TestDecodeQdqFilename(unittest.TestCase): + """The QDQ export is a file the A-side stages exchange, not an artifact.""" + + def test_matches_legacy_filename(self): + """A2's SQNR path reads what llama.py writes, so the name must agree.""" + from executorch.examples.qualcomm.oss_scripts.llama import decoder_constants + + self.assertEqual(DECODE_QDQ_FILENAME, decoder_constants.DECODE_QDQ_FILENAME) + + def test_is_not_an_artifact_key(self): + """It is not a ``.pte``, so it must never reach ``artifact_paths``.""" + self.assertNotIn(DECODE_QDQ_FILENAME, ALL_ARTIFACT_KEYS) + + +if __name__ == "__main__": + unittest.main() diff --git a/backends/qualcomm/genai_pipeline/tests/test_control_args.py b/backends/qualcomm/genai_pipeline/tests/test_control_args.py new file mode 100644 index 00000000000..01ac55e31a7 --- /dev/null +++ b/backends/qualcomm/genai_pipeline/tests/test_control_args.py @@ -0,0 +1,278 @@ +# Copyright (c) Qualcomm Innovation Center, Inc. +# All rights reserved +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +import argparse +import unittest +from dataclasses import fields + +from executorch.backends.qualcomm.genai_pipeline.control_args import ( + _PARSER_DEFAULT_EXEMPT_FIELDS, + ControlArgs, +) +from executorch.backends.qualcomm.genai_pipeline.pipeline_context import PipelineContext + +# Arguments llama.py's parser marks as required, so it can be run with no +# user-facing options and still produce a namespace full of defaults. +_REQUIRED_PARSER_ARGS = ["--decoder_model", "stories260k", "--prompt", "hi"] + + +def _make_context(**overrides): + """Create a PipelineContext with valid defaults.""" + defaults = { + "model_name": "stories260k", + "soc_model": "SM8750", + "prompt": ["hello"], + "artifact_dir": "/tmp/artifacts", + "extra_options": {}, + } + defaults.update(overrides) + return PipelineContext(**defaults) + + +class TestControlArgsDefaults(unittest.TestCase): + + def test_defaults_match_llama_parser(self): + """Every shared field defaults to the same value as llama.py's parser.""" + from executorch.examples.qualcomm.oss_scripts.llama.llama import _build_parser + + parsed = vars(_build_parser().parse_args(_REQUIRED_PARSER_ARGS)) + ours = ControlArgs() + + shared = ( + ControlArgs.field_names() & set(parsed) + ) - _PARSER_DEFAULT_EXEMPT_FIELDS + # The parser supplies these two, so they are not defaults to compare. + shared -= {"decoder_model", "prompt"} + + for name in sorted(shared): + with self.subTest(field=name): + self.assertEqual(getattr(ours, name), parsed[name]) + + def test_every_field_is_known_to_the_parser(self): + """No field exists that llama.py's parser cannot supply.""" + from executorch.examples.qualcomm.oss_scripts.llama.llama import _build_parser + + parsed = vars(_build_parser().parse_args(_REQUIRED_PARSER_ARGS)) + + self.assertEqual(ControlArgs.field_names() - set(parsed), set()) + + def test_exempt_fields_default_to_none_where_the_parser_has_a_path(self): + """The exempted fields differ from the parser deliberately, not by drift. + + llama.py defaults them to YAML paths under examples/, which this package + does not reference; None means "use the consumer's own default". + """ + from executorch.examples.qualcomm.oss_scripts.llama.llama import _build_parser + + parsed = vars(_build_parser().parse_args(_REQUIRED_PARSER_ARGS)) + ours = ControlArgs() + + for name in sorted(_PARSER_DEFAULT_EXEMPT_FIELDS): + with self.subTest(field=name): + self.assertIsNone(getattr(ours, name)) + self.assertIsNotNone(parsed[name]) + + def test_is_argparse_namespace(self): + """ControlArgs is a Namespace, as QnnConfig.load_config requires.""" + self.assertIsInstance(ControlArgs(), argparse.Namespace) + + def test_vars_exposes_all_fields(self): + """vars() yields every field, so reflection-based consumers see them.""" + self.assertEqual( + set(vars(ControlArgs())), + {f.name for f in fields(ControlArgs)}, + ) + + def test_mutable_defaults_are_not_shared(self): + """Each instance gets its own list defaults.""" + first, second = ControlArgs(), ControlArgs() + + first.prompt.append("mutated") + + self.assertEqual(second.prompt, []) + + +class TestControlArgsFromNamespace(unittest.TestCase): + + def test_copies_recognised_attributes(self): + """Attributes matching a field are carried over.""" + namespace = argparse.Namespace(decoder_model="qwen3-0_6b", max_seq_len=2048) + + result = ControlArgs.from_namespace(namespace) + + self.assertEqual(result.decoder_model, "qwen3-0_6b") + self.assertEqual(result.max_seq_len, 2048) + + def test_ignores_unrecognised_attributes(self): + """Attributes without a field are dropped rather than raising.""" + namespace = argparse.Namespace(decoder_model="x", not_a_field=object()) + + result = ControlArgs.from_namespace(namespace) + + self.assertFalse(hasattr(result, "not_a_field")) + + def test_unsupplied_attributes_keep_defaults(self): + """Fields absent from the namespace fall back to their defaults.""" + result = ControlArgs.from_namespace(argparse.Namespace(decoder_model="x")) + + self.assertEqual(result.model_mode, "hybrid") + + def test_accepts_the_real_llama_parser_namespace(self): + """A namespace from llama.py's parser converts without error.""" + from executorch.examples.qualcomm.oss_scripts.llama.llama import _build_parser + + namespace = _build_parser().parse_args(_REQUIRED_PARSER_ARGS) + + result = ControlArgs.from_namespace(namespace) + + self.assertEqual(result.decoder_model, "stories260k") + self.assertEqual(result.prompt, ["hi"]) + + +class TestControlArgsFromPipelineContext(unittest.TestCase): + + def test_maps_context_owned_fields(self): + """model_name, soc_model, artifact_dir and prompt come from the context.""" + context = _make_context( + model_name="llama3_2-1b_instruct", + soc_model="SM8650", + artifact_dir="/tmp/out", + prompt=["a", "b"], + ) + + result = ControlArgs.from_pipeline_context(context) + + self.assertEqual(result.decoder_model, "llama3_2-1b_instruct") + self.assertEqual(result.soc_model, "SM8650") + self.assertEqual(result.artifact, "/tmp/out") + self.assertEqual(result.prompt, ["a", "b"]) + + def test_applies_matching_extra_options(self): + """extra_options keys naming a field are applied.""" + context = _make_context(extra_options={"max_seq_len": 1024, "use_fp16": True}) + + result = ControlArgs.from_pipeline_context(context) + + self.assertEqual(result.max_seq_len, 1024) + self.assertTrue(result.use_fp16) + + def test_ignores_unrelated_extra_options(self): + """extra_options keys that are not fields are skipped silently.""" + context = _make_context(extra_options={"generate_etrecord": True}) + + result = ControlArgs.from_pipeline_context(context) + + self.assertFalse(hasattr(result, "generate_etrecord")) + + def test_overrides_take_precedence_over_extra_options(self): + """Explicit overrides beat extra_options and context values.""" + context = _make_context(extra_options={"max_seq_len": 1024}) + + result = ControlArgs.from_pipeline_context(context, max_seq_len=256) + + self.assertEqual(result.max_seq_len, 256) + + def test_extra_options_cannot_replace_context_owned_fields(self): + """The context wins over extra_options for the settings it owns. + + A SoC arriving via extra_options would otherwise diverge from the one the + context reports and the stage configs use, compiling for one target while + validating ops for another. + """ + context = _make_context( + extra_options={ + "soc_model": "SM8650", + "decoder_model": "other_model", + "artifact": "/tmp/elsewhere", + "prompt": ["injected"], + }, + ) + + result = ControlArgs.from_pipeline_context(context) + + self.assertEqual(result.soc_model, context.soc_model) + self.assertEqual(result.decoder_model, context.model_name) + self.assertEqual(result.artifact, context.artifact_dir) + self.assertEqual(result.prompt, context.prompt) + + def test_explicit_override_still_replaces_a_context_owned_field(self): + """Overrides remain the deliberate way to change a context-owned field.""" + context = _make_context(extra_options={"soc_model": "SM8650"}) + + result = ControlArgs.from_pipeline_context(context, soc_model="SM8450") + + self.assertEqual(result.soc_model, "SM8450") + + def test_rejects_unknown_override(self): + """An override that is not a field raises rather than being dropped.""" + context = _make_context() + + with self.assertRaises(TypeError) as cm: + ControlArgs.from_pipeline_context(context, not_a_field=1) + + self.assertIn("not_a_field", str(cm.exception)) + + def test_prompt_is_copied_from_the_context(self): + """Mutating the result's prompt does not affect the context.""" + context = _make_context(prompt=["original"]) + + result = ControlArgs.from_pipeline_context(context) + result.prompt.append("added") + + self.assertEqual(context.prompt, ["original"]) + + +class TestControlArgsBuildParser(unittest.TestCase): + + def test_parsing_no_arguments_reproduces_the_dataclass_defaults(self): + """The parser is the same defaults, so an empty command line matches.""" + parsed = ControlArgs.from_namespace(ControlArgs.build_parser().parse_args([])) + + self.assertEqual(parsed, ControlArgs()) + + def test_values_are_converted_to_their_field_type(self): + """Numeric fields arrive as numbers, not the strings argparse read.""" + args = ControlArgs.build_parser().parse_args( + ["--max-seq-len", "1024", "--temperature", "0.5"] + ) + + self.assertEqual(args.max_seq_len, 1024) + self.assertEqual(args.temperature, 0.5) + + def test_underscore_and_hyphen_spellings_both_work(self): + """``llama.py`` uses both conventions, so accept either.""" + parser = ControlArgs.build_parser() + + self.assertEqual(parser.parse_args(["--max-seq-len", "8"]).max_seq_len, 8) + self.assertEqual(parser.parse_args(["--max_seq_len", "8"]).max_seq_len, 8) + + def test_list_valued_fields_accept_several_values(self): + """Prompts and task lists are repeated arguments, not one string.""" + args = ControlArgs.build_parser().parse_args(["--prompt", "one", "two"]) + + self.assertEqual(args.prompt, ["one", "two"]) + + def test_constrained_fields_reject_an_unlisted_value(self): + """A misspelled mode fails at parse time, not deep in the flow.""" + parser = ControlArgs.build_parser() + + with self.assertRaises(SystemExit): + parser.parse_args(["--model-mode", "not_a_mode"]) + + def test_parser_can_be_extended_as_a_parent(self): + """Entry points add their own arguments rather than redeclaring these.""" + parent = ControlArgs.build_parser(add_help=False) + parser = argparse.ArgumentParser(parents=[parent]) + parser.add_argument("--list-models", action="store_true") + + args = parser.parse_args(["--list-models", "--backend", "gpu"]) + + self.assertTrue(args.list_models) + self.assertEqual(args.backend, "gpu") + + +if __name__ == "__main__": + unittest.main() diff --git a/backends/qualcomm/genai_pipeline/tests/test_engine_proxy.py b/backends/qualcomm/genai_pipeline/tests/test_engine_proxy.py index 2d6b4f5908e..e7f8c74e6e0 100644 --- a/backends/qualcomm/genai_pipeline/tests/test_engine_proxy.py +++ b/backends/qualcomm/genai_pipeline/tests/test_engine_proxy.py @@ -5,7 +5,6 @@ # LICENSE file in the root directory of this source tree. import unittest -from unittest.mock import MagicMock from executorch.backends.qualcomm.genai_pipeline.engine_proxy import EngineProxy from executorch.backends.qualcomm.genai_pipeline.pipeline_types import ( @@ -15,6 +14,9 @@ STAGE_MODEL_PREPARATION, STAGE_QUANTIZATION, ) +from executorch.backends.qualcomm.genai_pipeline.tests.test_utils import ( + TEST_BACKEND_TYPE, +) class TestEngineProxy(unittest.TestCase): @@ -27,7 +29,7 @@ def test_full_executorch_workflow(self): STAGE_COMPILATION: EngineType.EXECUTORCH, STAGE_INFERENCE: EngineType.EXECUTORCH, }, - backend_type=MagicMock(name="kHtpBackend"), + backend_type=TEST_BACKEND_TYPE, ) self.assertEqual( proxy.get_engine(STAGE_MODEL_PREPARATION), EngineType.EXECUTORCH @@ -37,7 +39,7 @@ def test_full_executorch_workflow(self): self.assertEqual(proxy.get_engine(STAGE_INFERENCE), EngineType.EXECUTORCH) def test_default_engine_is_executorch(self): - proxy = EngineProxy({}, backend_type=MagicMock(name="kHtpBackend")) + proxy = EngineProxy({}, backend_type=TEST_BACKEND_TYPE) self.assertEqual( proxy.get_engine(STAGE_MODEL_PREPARATION), EngineType.EXECUTORCH ) @@ -46,7 +48,7 @@ def test_default_engine_is_executorch(self): self.assertEqual(proxy.get_engine(STAGE_INFERENCE), EngineType.EXECUTORCH) def test_backend_type_is_stored(self): - backend = MagicMock(name="kHtpBackend") + backend = TEST_BACKEND_TYPE proxy = EngineProxy( {STAGE_INFERENCE: EngineType.EXECUTORCH}, backend_type=backend ) @@ -55,7 +57,7 @@ def test_backend_type_is_stored(self): def test_stage_engines_returns_copy(self): proxy = EngineProxy( {STAGE_INFERENCE: EngineType.EXECUTORCH}, - backend_type=MagicMock(name="kHtpBackend"), + backend_type=TEST_BACKEND_TYPE, ) engines = proxy.stage_engines engines[STAGE_INFERENCE] = None @@ -65,13 +67,13 @@ def test_invalid_stage_name_raises(self): with self.assertRaises(ValueError) as cm: EngineProxy( {"invalid_stage": EngineType.EXECUTORCH}, - backend_type=MagicMock(name="kHtpBackend"), + backend_type=TEST_BACKEND_TYPE, ) self.assertIn("Unknown stage", str(cm.exception)) self.assertIn("invalid_stage", str(cm.exception)) def test_empty_stage_engines(self): - proxy = EngineProxy({}, backend_type=MagicMock(name="kHtpBackend")) + proxy = EngineProxy({}, backend_type=TEST_BACKEND_TYPE) self.assertEqual(proxy.get_engine(STAGE_QUANTIZATION), EngineType.EXECUTORCH) self.assertEqual(proxy.get_engine(STAGE_INFERENCE), EngineType.EXECUTORCH) diff --git a/backends/qualcomm/genai_pipeline/tests/test_genai_pipeline.py b/backends/qualcomm/genai_pipeline/tests/test_genai_pipeline.py index 1083950035b..34045047734 100644 --- a/backends/qualcomm/genai_pipeline/tests/test_genai_pipeline.py +++ b/backends/qualcomm/genai_pipeline/tests/test_genai_pipeline.py @@ -45,12 +45,12 @@ ) from executorch.backends.qualcomm.genai_pipeline.tests.test_utils import ( make_test_context, - TEST_PTE_PATH, + TEST_ARTIFACT_PATHS, + TEST_BACKEND_TYPE, ) TEST_MOCK_GENERATED_TEXT = "Mock generated text" TEST_MOCK_TOKENS_PER_SEC = 42.0 -TEST_MOCK_BACKEND_TYPE = MagicMock(name="kHtpBackend") class _MockQuantizationStrategy(QuantizationStrategy): @@ -60,7 +60,7 @@ def invoke(self, context, input_config): class _MockCompilationStrategy(CompilationStrategy): def invoke(self, context, input_config): - return CompilationOutputConfig(artifact_paths=[TEST_PTE_PATH]) + return CompilationOutputConfig(artifact_paths=TEST_ARTIFACT_PATHS) class _MockInferenceStrategy(InferenceStrategy): @@ -81,7 +81,7 @@ def test_full_executorch_creates_all_stages(self): STAGE_COMPILATION: EngineType.EXECUTORCH, STAGE_INFERENCE: EngineType.EXECUTORCH, }, - backend_type=TEST_MOCK_BACKEND_TYPE, + backend_type=TEST_BACKEND_TYPE, ) pipeline = GenAIPipeline.from_proxy(proxy) self.assertIsNotNone(pipeline._model_preparation_stage) @@ -92,7 +92,7 @@ def test_full_executorch_creates_all_stages(self): def test_skip_stages(self): proxy = EngineProxy( {STAGE_INFERENCE: EngineType.EXECUTORCH}, - backend_type=TEST_MOCK_BACKEND_TYPE, + backend_type=TEST_BACKEND_TYPE, ) pipeline = GenAIPipeline.from_proxy( proxy, @@ -109,7 +109,7 @@ def test_skip_stages(self): self.assertIsNone(pipeline._inference_stage) def test_default_engines_all_executorch(self): - proxy = EngineProxy({}, backend_type=TEST_MOCK_BACKEND_TYPE) + proxy = EngineProxy({}, backend_type=TEST_BACKEND_TYPE) pipeline = GenAIPipeline.from_proxy(proxy) self.assertIsNotNone(pipeline._model_preparation_stage) self.assertIsNotNone(pipeline._quantization_stage) @@ -164,7 +164,7 @@ def test_invoke_with_mock_strategies(self): STAGE_COMPILATION: EngineType.EXECUTORCH, STAGE_INFERENCE: EngineType.EXECUTORCH, }, - backend_type=TEST_MOCK_BACKEND_TYPE, + backend_type=TEST_BACKEND_TYPE, ) pipeline = GenAIPipeline( model_preparation_stage=None, @@ -187,7 +187,7 @@ def test_invoke_compile_only(self): STAGE_QUANTIZATION: EngineType.EXECUTORCH, STAGE_COMPILATION: EngineType.EXECUTORCH, }, - backend_type=TEST_MOCK_BACKEND_TYPE, + backend_type=TEST_BACKEND_TYPE, ) pipeline = GenAIPipeline( model_preparation_stage=None, @@ -201,7 +201,7 @@ def test_invoke_compile_only(self): self.assertIsInstance(result, InferenceOutputConfig) def test_invoke_no_stages(self): - proxy = EngineProxy({}, backend_type=TEST_MOCK_BACKEND_TYPE) + proxy = EngineProxy({}, backend_type=TEST_BACKEND_TYPE) pipeline = GenAIPipeline( model_preparation_stage=None, quantization_stage=None, @@ -222,7 +222,7 @@ def test_quantization_receives_soc_model(self): test_soc = "SM8650" proxy = EngineProxy( {STAGE_QUANTIZATION: EngineType.EXECUTORCH}, - backend_type=TEST_MOCK_BACKEND_TYPE, + backend_type=TEST_BACKEND_TYPE, ) pipeline = GenAIPipeline( model_preparation_stage=None, @@ -240,7 +240,7 @@ def test_quantization_receives_soc_model(self): def test_compilation_receives_backend_type(self): mock_compile = MagicMock(spec=CompilationStrategy) mock_compile.invoke.return_value = CompilationOutputConfig( - artifact_paths=[TEST_PTE_PATH] + artifact_paths=TEST_ARTIFACT_PATHS ) mock_backend_type = MagicMock() @@ -270,7 +270,7 @@ def test_inference_receives_prompt(self): test_prompt = ["What is AI?"] proxy = EngineProxy( {STAGE_INFERENCE: EngineType.EXECUTORCH}, - backend_type=TEST_MOCK_BACKEND_TYPE, + backend_type=TEST_BACKEND_TYPE, ) pipeline = GenAIPipeline( model_preparation_stage=None, diff --git a/backends/qualcomm/genai_pipeline/tests/test_graph_bundle.py b/backends/qualcomm/genai_pipeline/tests/test_graph_bundle.py new file mode 100644 index 00000000000..342840a3df5 --- /dev/null +++ b/backends/qualcomm/genai_pipeline/tests/test_graph_bundle.py @@ -0,0 +1,118 @@ +# Copyright (c) Qualcomm Innovation Center, Inc. +# All rights reserved +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +import dataclasses +import unittest +from unittest.mock import MagicMock + +import torch +from executorch.backends.qualcomm.genai_pipeline.graph_bundle import GraphBundle + + +def _make_bundle(**overrides) -> GraphBundle: + """Create a GraphBundle with valid defaults.""" + kwargs = { + "module": MagicMock(name="graph_module"), + "inputs": (MagicMock(name="tokens"), MagicMock(name="attn_mask")), + } + kwargs.update(overrides) + return GraphBundle(**kwargs) + + +class TestGraphBundleFields(unittest.TestCase): + + def test_module_and_inputs_are_required(self): + """A bundle without a graph or its export signature is not constructible.""" + with self.assertRaises(TypeError): + GraphBundle() + + def test_lowering_options_default_to_absent(self): + """The optional lowering inputs default to None, and meta to empty.""" + bundle = _make_bundle() + + self.assertEqual(bundle.meta, {}) + self.assertIsNone(bundle.quant_io_dtypes) + self.assertIsNone(bundle.modality_inputs) + self.assertIsNone(bundle.executorch_config) + + def test_meta_is_not_shared_between_bundles(self): + """Each bundle gets its own meta, since quantization mutates it in place.""" + first, second = _make_bundle(), _make_bundle() + + first.meta["get_logits_scale"] = 0.5 + + self.assertEqual(second.meta, {}) + + +class TestQuantIoDtypes(unittest.TestCase): + """The tagger indexes both keys per node, so a partial mapping is invalid.""" + + def test_both_dtypes_are_accepted(self): + """The mapping quantization publishes carries both boundary dtypes.""" + quant_io_dtypes = {"kv_type": torch.uint8, "io_type": torch.uint16} + + bundle = _make_bundle(quant_io_dtypes=quant_io_dtypes) + + self.assertIs(bundle.quant_io_dtypes, quant_io_dtypes) + + def test_skipped_quantization_is_none_not_a_partial_mapping(self): + """None is how a graph says its IO stays float32.""" + bundle = _make_bundle(quant_io_dtypes=None) + + self.assertIsNone(bundle.quant_io_dtypes) + + def test_partial_mapping_is_rejected(self): + """One key alone would be a KeyError at lowering, so reject it here.""" + for partial in ({"kv_type": torch.uint8}, {"io_type": torch.uint16}, {}): + with self.subTest(quant_io_dtypes=partial): + with self.assertRaises(ValueError): + _make_bundle(quant_io_dtypes=partial) + + def test_unknown_key_is_rejected(self): + """An unrecognised key means the producer and the tagger disagree.""" + with self.assertRaises(ValueError): + _make_bundle( + quant_io_dtypes={ + "kv_type": torch.uint8, + "io_type": torch.uint16, + "logits_type": torch.uint16, + } + ) + + def test_replace_is_validated_too(self): + """``dataclasses.replace`` re-runs __post_init__, so it cannot bypass this.""" + bundle = _make_bundle( + quant_io_dtypes={"kv_type": torch.uint8, "io_type": torch.uint16} + ) + + with self.assertRaises(ValueError): + dataclasses.replace(bundle, quant_io_dtypes={"kv_type": torch.uint8}) + + +class TestGraphBundleImmutability(unittest.TestCase): + + def test_fields_cannot_be_reassigned(self): + """Frozen, so a consumer cannot repoint a bundle's fields in place.""" + bundle = _make_bundle() + + with self.assertRaises(dataclasses.FrozenInstanceError): + bundle.module = MagicMock(name="other_module") + + def test_replace_returns_a_new_bundle_carrying_the_rest(self): + """``dataclasses.replace`` is how a stage adds to a bundle.""" + bundle = _make_bundle() + quant_io_dtypes = {"kv_type": torch.uint8, "io_type": torch.uint16} + + updated = dataclasses.replace(bundle, quant_io_dtypes=quant_io_dtypes) + + self.assertIs(updated.quant_io_dtypes, quant_io_dtypes) + self.assertIs(updated.module, bundle.module) + self.assertIs(updated.inputs, bundle.inputs) + self.assertIsNone(bundle.quant_io_dtypes) + + +if __name__ == "__main__": + unittest.main() diff --git a/backends/qualcomm/genai_pipeline/tests/test_graph_names.py b/backends/qualcomm/genai_pipeline/tests/test_graph_names.py new file mode 100644 index 00000000000..d9504890743 --- /dev/null +++ b/backends/qualcomm/genai_pipeline/tests/test_graph_names.py @@ -0,0 +1,40 @@ +# Copyright (c) Qualcomm Innovation Center, Inc. +# All rights reserved +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +import unittest + +from executorch.backends.qualcomm.genai_pipeline.graph_names import ( + DECODER_GRAPH_NAMES, + GRAPH_FORWARD, + GRAPH_KV_FORWARD, + TOK_EMBEDDING_GRAPH_NAMES, +) + + +class TestGraphNames(unittest.TestCase): + + def test_names_match_legacy_decoder_constants(self): + """Both graph-name lists equal the legacy flow's, order included.""" + from executorch.examples.qualcomm.oss_scripts.llama import decoder_constants + + self.assertEqual(DECODER_GRAPH_NAMES, decoder_constants.DECODER_GRAPH_NAMES) + self.assertEqual( + TOK_EMBEDDING_GRAPH_NAMES, decoder_constants.TOK_EMBEDDING_GRAPH_NAMES + ) + + def test_decode_graph_is_first(self): + """Decode leads both lists: it is sliced first in kv mode and owns meta.""" + self.assertEqual(DECODER_GRAPH_NAMES[0], GRAPH_KV_FORWARD) + self.assertIn(GRAPH_KV_FORWARD, TOK_EMBEDDING_GRAPH_NAMES[0]) + + def test_single_graph_name_is_not_a_deployed_decoder_graph(self): + """``GRAPH_FORWARD`` names the calibration / encoder graph, never a method.""" + self.assertNotIn(GRAPH_FORWARD, DECODER_GRAPH_NAMES) + self.assertNotIn(GRAPH_FORWARD, TOK_EMBEDDING_GRAPH_NAMES) + + +if __name__ == "__main__": + unittest.main() diff --git a/backends/qualcomm/genai_pipeline/tests/test_utils.py b/backends/qualcomm/genai_pipeline/tests/test_utils.py index 211dd13cb68..71010cac79a 100644 --- a/backends/qualcomm/genai_pipeline/tests/test_utils.py +++ b/backends/qualcomm/genai_pipeline/tests/test_utils.py @@ -6,7 +6,14 @@ from pathlib import Path +from executorch.backends.qualcomm.genai_pipeline.artifact_keys import ( + ARTIFACT_TEXT_DECODER, +) from executorch.backends.qualcomm.genai_pipeline.pipeline_context import PipelineContext +from executorch.backends.qualcomm.serialization.qc_schema import ( + QcomChipset, + QnnExecuTorchBackendType, +) # Shared test constants TEST_MODEL_NAME = "test_model" @@ -15,6 +22,16 @@ TEST_ARTIFACT_DIR = "/tmp/test_artifacts" TEST_PTE_PATH = Path("/tmp/test.pte") +# Real enum values rather than mocks: both import without a device or the QNN +# SDK, and a mock named after an enum member does not actually carry that +# member's identity, so it cannot catch a wrong value being passed through. +TEST_BACKEND_TYPE = QnnExecuTorchBackendType.kHtpBackend +TEST_SOC_CHIPSET = QcomChipset.SM8750 + +# The compiled-artifact map a single-graph text-only model produces, in the +# graph-keyed shape the compilation stage emits and inference consumes. +TEST_ARTIFACT_PATHS = {ARTIFACT_TEXT_DECODER: TEST_PTE_PATH} + def make_test_context(**kwargs) -> PipelineContext: defaults = {