diff --git a/.github/scripts/ci/run_colossalai_batch.sh b/.github/scripts/ci/run_colossalai_batch.sh index 0eb7072ad34e..ac384732f4a7 100644 --- a/.github/scripts/ci/run_colossalai_batch.sh +++ b/.github/scripts/ci/run_colossalai_batch.sh @@ -26,6 +26,7 @@ Batches: 4c Single-GPU Triton inference kernels 4d Single-GPU SmoothQuant tests 4e Single-GPU flash-decoding-attention parameter matrix + 4f Single-GPU strict FA3 training attention (requires the optional FA3 package) 5 Two-GPU DDP/FSDP/LowLevelZeRO/LoRA 6 Four-GPU DeviceMesh/DTensor/communication foundations 7 Four-GPU FP8 collectives and DDP/FSDP communication hooks @@ -85,6 +86,7 @@ case "${BATCH}" in tests/test_pipeline/test_pipeline_utils/test_t5_pipeline_utils.py tests/test_pipeline/test_pipeline_utils/test_whisper_pipeline_utils.py tests/test_pipeline/test_schedule/test_pipeline_schedule_utils.py + tests/test_shardformer/test_layer/test_attention_backends.py tests/test_tensor/test_dtensor/test_dtensor_sharding_spec.py tests/test_tensor/test_shape_consistency.py tests/test_tensor/test_sharding_spec.py @@ -168,6 +170,18 @@ case "${BATCH}" in NEEDS_CUDA_TOOLKIT=1 TESTS=(tests/test_infer/test_kernels/cuda/test_flash_decoding_attention.py) ;; + 4f|04f) + LABEL="04f-fa3-training-attention" + DESCRIPTION="Strict FA3 dense/varlen forward, backward and numerical alignment" + EXPECTED_GPUS=1 + DEFAULT_TIMEOUT_MIN=30 + TESTS=(tests/test_shardformer/test_layer/test_attention_backends.py) + PYTEST_EXTRA=(-k gpu -o junit_family=xunit1) + # This is acceptance for a requested backend, not an optional discovery + # run. Missing FA3/CUDA must fail rather than produce a green skip. + export COLOSSALAI_TEST_ATTENTION_BACKEND=fa3 + export COLOSSALAI_ATTENTION_TRACE_DIR="${RESULTS_ROOT}/fa3-traces" + ;; 5|05) LABEL="05-booster-lora-2gpu" DESCRIPTION="DDP, FSDP, LowLevelZeRO, LoRA and the numerical alignment tool on its plugins" diff --git a/.github/scripts/ci/run_cpu_unit_tests.sh b/.github/scripts/ci/run_cpu_unit_tests.sh index 4f8944c839fa..46da75b93ca0 100644 --- a/.github/scripts/ci/run_cpu_unit_tests.sh +++ b/.github/scripts/ci/run_cpu_unit_tests.sh @@ -23,6 +23,7 @@ TESTS=( tests/test_pipeline/test_pipeline_utils/test_t5_pipeline_utils.py tests/test_pipeline/test_pipeline_utils/test_whisper_pipeline_utils.py tests/test_pipeline/test_schedule/test_pipeline_schedule_utils.py + tests/test_shardformer/test_layer/test_attention_backends.py tests/test_tensor/test_dtensor/test_dtensor_sharding_spec.py tests/test_tensor/test_shape_consistency.py tests/test_tensor/test_sharding_spec.py diff --git a/.github/scripts/ci/run_gpu_batch.py b/.github/scripts/ci/run_gpu_batch.py index b46cb1e624ed..d5f296e548c8 100644 --- a/.github/scripts/ci/run_gpu_batch.py +++ b/.github/scripts/ci/run_gpu_batch.py @@ -26,6 +26,7 @@ "4c": 1, "4d": 1, "4e": 1, + "4f": 1, "5": 2, "6": 4, "7": 4, diff --git a/colossalai/kernel/kernel_loader.py b/colossalai/kernel/kernel_loader.py index 8f4196b7dfa1..fc968881ab3b 100644 --- a/colossalai/kernel/kernel_loader.py +++ b/colossalai/kernel/kernel_loader.py @@ -1,10 +1,18 @@ import warnings -from typing import List +from dataclasses import dataclass, field +from functools import lru_cache, partial +from importlib.metadata import version +from importlib.util import find_spec +from typing import Callable, List, Optional, Tuple, Type + +import torch +from packaging.version import Version from .extensions import ( CpuAdamArmExtension, CpuAdamX86Extension, FlashAttentionDaoCudaExtension, + FlashAttentionDaoV3CudaExtension, FlashAttentionNpuExtension, FlashAttentionSdpaCudaExtension, FusedOptimizerCudaExtension, @@ -124,6 +132,105 @@ class FlashAttentionDaoLoader(KernelLoader): REGISTRY = [FlashAttentionDaoCudaExtension] +def _flash_attention_support(q, v, attention_mask, dropout_p, is_causal, causal_alignment, non_square, *, hopper=False): + # ROCm reuses torch.cuda, but its architecture numbers are not NVIDIA SMs. + # These adapters are validated for NVIDIA; ROCm needs its own capability policy. + if torch.version.hip is not None: + return "requires NVIDIA CUDA in this adapter; ROCm is not supported" + if q.dtype not in (torch.float16, torch.bfloat16): + return "requires float16 or bfloat16 inputs" + if attention_mask is not None: + return "does not support an arbitrary attention_mask" + if q.shape[-1] != v.shape[-1] or q.shape[-1] > 256 or q.shape[-1] % 8 != 0: + return "requires equal Q/K/V head dimensions divisible by 8 and at most 256" + if not hopper and q.shape[-1] > 192: + return "supports head dimensions at most 192 in the explicit training adapter" + if is_causal and non_square and causal_alignment == "upper_left": + return "does not support non-square upper-left causal attention" + if hopper and dropout_p != 0: + return "does not support nonzero dropout" + major, minor = torch.cuda.get_device_capability(q.device) + if hopper and (major, minor) != (9, 0): + return "requires a Hopper (SM90) GPU in this adapter" + if not hopper and major < 8: + return "requires an Ampere or newer GPU" + return None + + +@dataclass(frozen=True) +class _AttentionBackend: + extension: Type[_Extension] + device_types: Optional[Tuple[str, ...]] = None + capability: Optional[Callable] = None + package: Optional[str] = None + minimum_version: Optional[Tuple[str, str]] = None + load_kwargs: dict = field(default_factory=dict) + host_lengths: bool = False + + +class FlashAttentionBackendLoader: + """Load an explicitly selected attention backend without an execution-time fallback. + + Unlike the legacy priority-based loader, a broken installed extension must surface its + import/ABI error. Only a missing optional package makes a backend unavailable to auto. + REGISTRY insertion order defines auto priority within a device type. Generic + backends (device_types=None) come last. Adapters accept causal_alignment and + obey True=allowed masks; native conventions are converted inside the adapter. + Register before first use; clear load's cache if replacing an existing entry. + """ + + REGISTRY = { + "fa3": _AttentionBackend( + FlashAttentionDaoV3CudaExtension, ("cuda",), partial(_flash_attention_support, hopper=True), "flash_attn_3" + ), + "fa2": _AttentionBackend( + FlashAttentionDaoCudaExtension, ("cuda",), _flash_attention_support, "flash_attn", ("flash-attn", "2.1") + ), + "sdpa": _AttentionBackend(FlashAttentionSdpaCudaExtension, load_kwargs={"strict": True}, host_lengths=True), + } + + @classmethod + def get_backend(cls, backend): + if not isinstance(backend, str) or backend not in cls.REGISTRY: + raise ValueError(f"Unknown attention backend {backend!r}; expected one of {tuple(cls.REGISTRY)}") + return cls.REGISTRY[backend] + + @classmethod + def candidates(cls, device_type): + specific = [ + name for name, spec in cls.REGISTRY.items() if spec.device_types and device_type in spec.device_types + ] + generic = [name for name, spec in cls.REGISTRY.items() if spec.device_types is None] + return specific + generic + + @staticmethod + def is_available(backend: str) -> bool: + spec = FlashAttentionBackendLoader.get_backend(backend) + if spec.package is not None and find_spec(spec.package) is None: + return False + # FA2 changed non-square causal alignment in 2.1. Earlier releases do + # not implement the shared lower-right contract. Do not infer a version + # for an unidentifiable/broken installation. + if spec.minimum_version is not None: + distribution, minimum = spec.minimum_version + return Version(version(distribution)) >= Version(minimum) + return True + + @staticmethod + @lru_cache(maxsize=None) + def load(backend: str, packed: bool = False): + spec = FlashAttentionBackendLoader.get_backend(backend) + if not FlashAttentionBackendLoader.is_available(backend): + if spec.minimum_version is not None and find_spec(spec.package) is not None: + distribution, minimum = spec.minimum_version + raise NotImplementedError(f"Attention backend {backend!r} requires {distribution} >= {minimum}") + raise ModuleNotFoundError(f"Attention backend {backend!r} requires its optional package to be installed") + extension = spec.extension() + if packed: + return extension.load_varlen() + return extension.load(**spec.load_kwargs) + + class FlashAttentionWithCustomMaskLoader(KernelLoader): REGISTRY = [FlashAttentionNpuExtension, FlashAttentionSdpaCudaExtension] diff --git a/colossalai/shardformer/layer/attn.py b/colossalai/shardformer/layer/attn.py index 71e96c5b0d3e..d89c684adc7b 100644 --- a/colossalai/shardformer/layer/attn.py +++ b/colossalai/shardformer/layer/attn.py @@ -1,5 +1,8 @@ +import math +from collections.abc import Sequence from enum import Enum -from typing import Callable, Dict, Optional, Tuple +from numbers import Integral +from typing import Callable, Dict, Optional, Tuple, Union import torch import torch.distributed @@ -9,6 +12,7 @@ from packaging import version from colossalai.kernel.kernel_loader import ( + FlashAttentionBackendLoader, FlashAttentionDaoLoader, FlashAttentionForFloatAndCustomMaskLoader, FlashAttentionLoader, @@ -52,7 +56,9 @@ def invert_mask(mask: torch.Tensor) -> torch.Tensor: # adapted from https://github.com/Dao-AILab/flash-attention/blob/main/flash_attn/bert_padding.py def get_pad_info( - padding_mask: torch.Tensor, invert: Optional[bool] = False, return_indices: Optional[bool] = True + padding_mask: torch.Tensor, + invert: Optional[bool] = False, + return_indices: Optional[bool] = True, ) -> Tuple[int, torch.Tensor, torch.Tensor]: """Get padding information from padding mask. @@ -83,6 +89,217 @@ class ColoAttention: _kernel_dispatch_map: Optional[Dict[torch.dtype, Dict[Optional[AttnMaskType], Callable]]] = None _flash_kernel_dispatch: Optional[Dict[torch.dtype, Dict[Optional[AttnMaskType], Callable]]] = None + @staticmethod + def _validate_inputs(q, k, v, packed, dropout_p, scale): + ndim = 3 if packed else 4 + if any(x.ndim != ndim for x in (q, k, v)): + raise ValueError(f"Attention expects {'THD' if packed else 'BHSD'} tensors") + if not (q.device == k.device == v.device and q.dtype == k.dtype == v.dtype): + raise ValueError("Q, K and V must have the same dtype and device") + if q.dtype not in (torch.float16, torch.bfloat16, torch.float32, torch.float64): + raise TypeError("Attention expects float16, bfloat16, float32 or float64 inputs") + if any(size <= 0 for x in (q, k, v) for size in x.shape): + raise ValueError("Attention does not support empty sequences, batches or heads") + head_axis = 1 + seq_axis = 0 if packed else 2 + if not packed and not (q.shape[0] == k.shape[0] == v.shape[0]): + raise ValueError("Q, K and V batch sizes must match") + if k.shape[seq_axis] != v.shape[seq_axis] or k.shape[head_axis] != v.shape[head_axis]: + raise ValueError("K and V must have the same sequence length and number of heads") + if q.shape[-1] != k.shape[-1]: + raise ValueError("Q and K head dimensions must match") + if q.shape[head_axis] % k.shape[head_axis] != 0: + raise ValueError("The number of query heads must be divisible by the number of key/value heads") + if not isinstance(dropout_p, (int, float)) or not 0 <= dropout_p < 1: + raise ValueError("dropout_p must be a number in [0, 1)") + if scale is not None and (not isinstance(scale, (int, float)) or not math.isfinite(scale)): + raise ValueError("scale must be a finite number or None") + + @staticmethod + def _validate_causal(is_causal, causal_alignment, non_square): + if not isinstance(is_causal, bool): + raise TypeError("is_causal must be a bool") + if causal_alignment not in (None, "upper_left", "lower_right"): + raise ValueError("causal_alignment must be 'upper_left', 'lower_right' or None") + if not is_causal and causal_alignment is not None: + raise ValueError("causal_alignment requires is_causal=True") + if is_causal and non_square and causal_alignment is None: + raise ValueError("Non-square causal attention requires an explicit causal_alignment") + + @staticmethod + def _select_backend( + q, + v, + backend, + attention_mask, + dropout_p, + is_causal, + causal_alignment, + non_square, + ): + if backend != "auto": + FlashAttentionBackendLoader.get_backend(backend) + candidates = FlashAttentionBackendLoader.candidates(q.device.type) if backend == "auto" else (backend,) + for candidate in candidates: + spec = FlashAttentionBackendLoader.get_backend(candidate) + reason = None + if spec.device_types is not None and q.device.type not in spec.device_types: + reason = f"requires tensors on {spec.device_types}" + elif spec.capability is not None: + reason = spec.capability(q, v, attention_mask, dropout_p, is_causal, causal_alignment, non_square) + if reason is not None: + if backend != "auto": + raise NotImplementedError(f"Attention backend {candidate!r} {reason}") + continue + if backend != "auto" or FlashAttentionBackendLoader.is_available(candidate): + return candidate + raise RuntimeError("No attention backend supports this request") + + @staticmethod + def _attention_explicit(q, k, v, attention_mask, dropout_p, scale, backend, is_causal, causal_alignment): + ColoAttention._validate_inputs(q, k, v, False, dropout_p, scale) + sq, sk = q.shape[2], k.shape[2] + ColoAttention._validate_causal(is_causal, causal_alignment, sq != sk) + if attention_mask is not None: + if attention_mask.device != q.device: + raise ValueError("attention_mask must be on the same device as Q") + if attention_mask.dtype not in (torch.bool, torch.float32, q.dtype): + raise TypeError("attention_mask must be bool, float32 or the input dtype") + shape = (q.shape[0], q.shape[1], sq, sk) + if torch.broadcast_shapes(attention_mask.shape, shape) != shape: + raise ValueError("attention_mask must broadcast to [B, Hq, Sq, Skv]") + selected = ColoAttention._select_backend( + q, + v, + backend, + attention_mask, + dropout_p, + is_causal, + causal_alignment, + sq != sk, + ) + # Never retry another backend after loading or executing the selected one. + attn_func = FlashAttentionBackendLoader.load(selected) + return attn_func( + q, + k, + v, + dropout_p=dropout_p, + scale=scale, + attention_mask=attention_mask, + is_causal=is_causal, + causal_alignment=causal_alignment, + ) + + @staticmethod + def attention_varlen( + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + cu_seqlens_q: Union[torch.Tensor, Sequence[int]], + cu_seqlens_kv: Union[torch.Tensor, Sequence[int]], + max_seqlen_q: int, + max_seqlen_kv: int, + *, + backend: str = "auto", + is_causal: bool = False, + causal_alignment: Optional[str] = None, + dropout_p: float = 0.0, + scale: Optional[float] = None, + validate: bool = True, + ) -> torch.Tensor: + """Attention over packed samples, without padding or cross-sample attention. + + Q is [Tq, Hq, D], K/V are [Tkv, Hkv, D]. Hq must be divisible by Hkv. + The output is [Tq, Hq, Dv]. Cumulative lengths are contiguous int32 + tensors on the input device or host integer sequences. They start at zero, + end at Tq/Tkv and contain one positive-length interval per sample. Max + lengths must cover the longest interval. Zero-length samples are rejected. + + validate=True checks tensor values on the host. For previously validated + device tensors, validate=False avoids these reads in FA2/FA3; callers must + guarantee valid boundaries and maxima. Shape/dtype/device checks remain, + and host sequences are always validated. Causal calls with unchecked + tensor values require an explicit alignment and are conservatively treated + as non-square (FA2/FA3 require lower_right). SDPA still reads boundaries + for slicing. Reuse device tensors across layers to avoid repeated host-list + conversion. Dynamic packed fullgraph compilation is not guaranteed. + + Explicit backends are strict: unsupported requests, missing packages and + execution errors do not fall back. Auto checks registered device backends + before SDPA (FA3, FA2, SDPA on NVIDIA CUDA). SDPA loops over samples as a + correctness fallback, not a fused varlen kernel. FA3 supports dropout=0 only. Non-square causal samples + require 'upper_left' or 'lower_right'; FA2/FA3 support only lower-right. + Query rows with no visible keys produce zero output and no gradient contribution. + Dropout is applied according to dropout_p even during evaluation: pass 0 + explicitly for inference. This method always returns a Tensor. + """ + ColoAttention._validate_inputs(q, k, v, True, dropout_p, scale) + if not isinstance(validate, bool): + raise TypeError("validate must be a bool") + prefixes = [] + host_bounds = [] + counts = [] + for name, prefix, total, maximum in ( + ("q", cu_seqlens_q, q.shape[0], max_seqlen_q), + ("kv", cu_seqlens_kv, k.shape[0], max_seqlen_kv), + ): + if not isinstance(maximum, int) or isinstance(maximum, bool) or maximum <= 0: + raise ValueError(f"max_seqlen_{name} must be a positive integer") + if total > torch.iinfo(torch.int32).max: + raise ValueError(f"cu_seqlens_{name} exceeds the int32 token limit") + if isinstance(prefix, torch.Tensor): + if prefix.ndim != 1 or prefix.numel() < 2 or prefix.dtype != torch.int32: + raise ValueError(f"cu_seqlens_{name} must be a 1D int32 tensor with at least two elements") + if prefix.device != q.device or not prefix.is_contiguous(): + raise ValueError(f"cu_seqlens_{name} must be contiguous and on the input device") + counts.append(prefix.numel()) + bounds = prefix.tolist() if validate else None + elif isinstance(prefix, Sequence) and not isinstance(prefix, (str, bytes)): + if len(prefix) < 2 or any(isinstance(x, bool) or not isinstance(x, Integral) for x in prefix): + raise ValueError(f"cu_seqlens_{name} must be a sequence of at least two integers") + bounds = [int(x) for x in prefix] + counts.append(len(bounds)) + else: + raise TypeError(f"cu_seqlens_{name} must be a tensor or an integer sequence") + if bounds is not None: + sizes = [end - start for start, end in zip(bounds[:-1], bounds[1:])] + if bounds[0] != 0 or bounds[-1] != total or min(sizes) <= 0: + raise ValueError(f"cu_seqlens_{name} must partition all tokens into nonempty samples") + if max(sizes) > maximum: + raise ValueError(f"max_seqlen_{name} is smaller than the longest sample") + prefixes.append(prefix) + host_bounds.append(bounds) + if counts[0] != counts[1]: + raise ValueError("Q and KV cumulative lengths must describe the same number of samples") + # Unknown lengths must not be assumed square: equal totals/maxima do not + # imply equal per-sample lengths. Requiring alignment preserves semantics. + non_square = any(bounds is None for bounds in host_bounds) or host_bounds[0] != host_bounds[1] + ColoAttention._validate_causal(is_causal, causal_alignment, non_square) + selected = ColoAttention._select_backend( + q, v, backend, None, dropout_p, is_causal, causal_alignment, non_square + ) + attn_func = FlashAttentionBackendLoader.load(selected, packed=True) + spec = FlashAttentionBackendLoader.get_backend(selected) + for i, (prefix, bounds) in enumerate(zip(prefixes, host_bounds)): + if spec.host_lengths: + prefixes[i] = bounds if bounds is not None else prefix.tolist() + elif not isinstance(prefix, torch.Tensor): + prefixes[i] = torch.tensor(bounds, dtype=torch.int32, device=q.device) + return attn_func( + q, + k, + v, + prefixes[0], + prefixes[1], + max_seqlen_q, + max_seqlen_kv, + dropout_p=dropout_p, + scale=scale, + is_causal=is_causal, + causal_alignment=causal_alignment, + ) + @staticmethod def _init_kernels_dispatch(): if ColoAttention._kernel_dispatch_map is None: @@ -240,6 +457,10 @@ def attention( kv_indices: Optional[torch.Tensor] = None, dropout_p: float = 0.0, scale: Optional[float] = None, + *, + backend: Optional[str] = None, + is_causal: bool = False, + causal_alignment: Optional[str] = None, **kwargs, ) -> torch.Tensor: """Flash Attention function. It supports 4 mask type. @@ -266,10 +487,54 @@ def attention( Shape should be [NUM_TOKENS]. Defaults to None. dropout_p (float, optional): Dropout probability. Defaults to 0.0. scale (Optional[float], optional): Scaling factor applied prior to softmax. Defaults to None. + backend (Optional[str]): None preserves the legacy dispatch and mask contract. + Explicit 'fa2', 'fa3' and 'sdpa' never fall back; 'auto' selects a + supported backend before execution. New calls use is_causal and a + CUSTOM attention_mask (bool True means allowed; float is additive). + All explicit backends must preserve this convention; adapters convert + to native mask conventions. Fully masked rows produce zero output + and no gradient contribution. + Legacy mask enums/padding metadata cannot be mixed with an explicit + backend; use attention_varlen for packed samples instead. FA3 requires + dropout_p=0. SDPA supports CPU and CUDA; FA2/FA3 require CUDA half types. + The explicit FA2 training path requires version >=2.1 and D<=192. + is_causal (bool): Causal attention on the explicit backend path only. + causal_alignment (Optional[str]): 'upper_left' or 'lower_right'. Required + for non-square causal requests; FA2/FA3 only support lower-right. Returns: torch.Tensor: Output tensor. Shape should be [B, nHeads, Sq, D] """ + if backend is not None: + if kwargs: + raise TypeError(f"Unexpected attention arguments: {', '.join(sorted(kwargs))}") + if attention_mask_type != AttnMaskType.CUSTOM or any( + x is not None + for x in ( + cu_seqlens_q, + cu_seqlens_kv, + max_seqlen_q, + max_seqlen_kv, + q_indices, + kv_indices, + ) + ): + raise ValueError("Explicit backends use is_causal/CUSTOM masks; use attention_varlen for packed inputs") + return ColoAttention._attention_explicit( + q, + k, + v, + attention_mask, + dropout_p, + scale, + backend, + is_causal, + causal_alignment, + ) + if is_causal or causal_alignment is not None: + raise ValueError( + "is_causal/causal_alignment require an explicit backend; legacy calls use attention_mask_type" + ) # known issue: sdpa does not support attention mask which contains whole row of masked tokens, which leads to nan # this case is usaul when padding mask is used and self attention is performed # thus, we don't use sdpa when padding mask is used diff --git a/docs/source/en/features/shardformer.md b/docs/source/en/features/shardformer.md index 40b8954b55b5..327824846ab8 100644 --- a/docs/source/en/features/shardformer.md +++ b/docs/source/en/features/shardformer.md @@ -310,6 +310,65 @@ if dist.get_world_size() > 1: 2. When you use Shardformer to process classification models such as `GPT2ForSequenceClassification`, `ViTForImageClassification`, please ensure that the total number of labels should be integer multiple of tensor parallel size, otherwise Shardformer can't process the classifier layer correctly. A simple fix could be appending dummy labels in transformers config. This bug will be fixed in future version of Shardformer. +## Explicit Attention Backends + +`ColoAttention.attention(q, k, v, backend="sdpa")` accepts dense `[B, H, S, D]` +tensors and returns a tensor in the same layout. Explicit `fa2`, `fa3` and `sdpa` +requests never fall back; `auto` selects a compatible implementation before +execution. Omitting `backend` preserves the legacy mask and dispatch contract. +The explicit API uses `is_causal` and an optional custom `attention_mask` (boolean +True means allowed, floating masks are additive), rather than legacy mask enums. +Every explicit backend must preserve this convention; adapters convert to their +native mask convention. Fully masked query rows have zero output and contribute +no gradient. This includes lower-right causal attention with `Sq > Skv`. +Non-square causal requests require `causal_alignment="upper_left"` or +`"lower_right"`; FA2/FA3 support only the latter for non-square inputs. + +`ColoAttention.attention_varlen` accepts packed Q `[Tq, Hq, D]` and K/V +`[Tkv, Hkv, D]`, with separate `cu_seqlens_q`, `cu_seqlens_kv`, `max_seqlen_q` +and `max_seqlen_kv`. Prefixes are host integer sequences or contiguous int32 +tensors on the input device. They start at zero and partition every token into +nonempty samples; zero-length samples are rejected. For lengths +`[3, 5]`, the prefix is `[0, 3, 8]`. Samples never attend across boundaries. +By default, tensor value validation reads prefixes on the host. For previously +validated device tensors, `validate=False` skips value reads in FA2/FA3 while +retaining shape, dtype, device and sample-count checks. The caller then guarantees +valid boundaries and maximum lengths. Such causal calls must explicitly specify +alignment; FA2/FA3 require `lower_right` because unchecked lengths cannot be +assumed equal. Host sequences are always validated and converted to device int32 +only for adapters that need tensors. Reuse prepared tensors across layers to avoid +repeated conversions. SDPA reuses host boundaries, or reads them once for slicing; +it remains a per-sample correctness path, not a fused varlen kernel. Dynamic packed +fullgraph compilation is not guaranteed by this option. + +The explicit loader uses one internal backend registry: device-specific entries +are tried in registration order, followed by generic entries such as SDPA. The +built-in CUDA order remains FA3, FA2, SDPA. Adding a device backend does not require +editing the shared selector; its adapter must implement the common contracts. + +FA3 is optional and uses the `flash_attn_3` training API from +[`54b044c8`](https://github.com/Dao-AILab/flash-attention/tree/54b044c8cf897dfcc2577a665559d1bfda0bfc80/hopper). +Install that revision's `hopper` package in an isolated, compatible CUDA/PyTorch +environment. The adapter requires SM90 and FP16/BF16, with zero dropout and no +arbitrary mask. An installed but broken extension propagates its import or kernel +error. GPU/platform compatibility must be verified on the actual build. +Importing Diffusers 0.35.2's attention dispatcher overwrites this FA3 operator's +schema; the combined video path has been validated with Diffusers 0.36.0 instead. +The explicit FA2 path requires `flash-attn>=2.1` and limits head dimensions to +192 or less, avoiding version/device-dependent backward support above that size. +Both explicit FA adapters currently require NVIDIA CUDA. Upstream FA2 also has +ROCm implementations, but these require a separate capability policy and validation +before being enabled here. FA3 accepts Tensor and `(output, softmax_lse)` returns; +this does not guarantee compatibility with every historical FA3 build. + +Run CPU contract tests with `pytest tests/test_shardformer/test_layer/test_attention_backends.py -k 'not gpu'`. +For strict single-GPU FA3 acceptance, use the existing batch runner: +`bash .github/scripts/ci/run_colossalai_batch.sh 4f `. +It fails if CUDA or FA3 is unavailable and saves numerical results and profiler +traces under `COLOSSAL_RESULTS`. The existing full-matrix workflow can select +batch `4f` manually after its image includes compatible FA3. It is opt-in only, +not part of the default matrix or standard Layer 2 batches (`2`, `4a`, `5`). + ## How Shardformer Works ### Main Idea diff --git a/docs/source/zh-Hans/features/shardformer.md b/docs/source/zh-Hans/features/shardformer.md index 02290f3d6eae..88df1ca57678 100644 --- a/docs/source/zh-Hans/features/shardformer.md +++ b/docs/source/zh-Hans/features/shardformer.md @@ -304,6 +304,56 @@ if dist.get_world_size() > 1: 2. 当使用Shardformer处理`GPT2ForSequenceClassification`、`ViTForImageClassification`等分类模型时,请确保labels的总数为张量并行度的整数倍,否则Shardformer无法正确地处理classifier层。一个简单的修复方法就是在transformers的config中添加虚拟的标签。这一bug将在 Shardformer的未来版本中修复。 +## 显式选择 Attention 后端 + +`ColoAttention.attention(q, k, v, backend="sdpa")` 接收 `[B, H, S, D]` +布局的 dense 张量,并返回相同布局的 Tensor。显式指定 `fa2`、`fa3` 或 +`sdpa` 时不会 fallback;`auto` 在执行前选择支持该请求的实现。不传 +`backend` 保留旧的 mask 与分发合同。新入口通过 `is_causal` 和可选的自定义 +`attention_mask` 表达关注规则,bool True 表示允许读取,浮点 mask 是加性偏置, +不能混用旧 mask 枚举。所有显式后端必须遵守上述 mask 语义,由适配器转换到原生 +接口约定。整行被遮挡的 query 输出为零,且不贡献梯度;这包括 `Sq > Skv` 的右下 causal 对齐。不等长 causal 请求必须指定 `causal_alignment="upper_left"` +或 `"lower_right"`;此时 FA2/FA3 只支持后者。 + +`ColoAttention.attention_varlen` 接收 packed Q `[Tq, Hq, D]` 和 K/V +`[Tkv, Hkv, D]`,分别提供 `cu_seqlens_q`、`cu_seqlens_kv`、`max_seqlen_q` +与 `max_seqlen_kv`。累计长度可以是主机整数序列,或输入设备上的连续 int32 张量, +从 0 开始,覆盖所有 token;拒绝零长度样本。例如长度 `[3, 5]` 的前缀是 `[0, 3, 8]`, +两条样本不会互相读取。 + +默认会将 tensor 边界值读回主机校验。对已校验的设备 tensor,可以设置 +`validate=False`,省去 FA2/FA3 路径上的边界值读取,仍保留形状、dtype、设备和 +样本数检查;此时调用方负责保证边界和最大长度合法。未校验的 causal 调用必须 +显式指定对齐方式,FA2/FA3 要求 `lower_right`,因为不能假设每条样本的 Q/K 等长。 +主机整数序列始终校验,只有需要 tensor 的适配器才将其转换为设备 int32。 +应跨层复用准备好的 tensor,避免重复转换。SDPA 复用已读取的主机边界,或者读取 +一次用于切片;它仍是逐样本的正确性路径,不是融合 varlen 内核。此选项不保证 +动态 packed fullgraph 编译支持。 + +显式 loader 使用单一内部注册表,先按注册顺序选择对应设备的后端,再尝试 SDPA +等通用后端。内置 CUDA 顺序保持 FA3、FA2、SDPA。新增设备后端不必修改共享 +选择器,但其适配器必须遵守统一接口合同。 + +FA3 是可选依赖,使用官方 +[`54b044c8`](https://github.com/Dao-AILab/flash-attention/tree/54b044c8cf897dfcc2577a665559d1bfda0bfc80/hopper) +的 `flash_attn_3` 训练接口。在隔离且兼容的 CUDA/PyTorch 环境中安装该提交的 +`hopper` 包。适配器要求 SM90、FP16/BF16、零 dropout,且不支持任意 mask。 +已安装但损坏的扩展会直接暴露导入或 kernel 错误;实际平台兼容性需要上机验证。 +Diffusers 0.35.2 的 attention dispatcher 会覆盖该 FA3 算子的 schema; +组合视频路径已在 Diffusers 0.36.0 验证。 +显式 FA2 路径要求 `flash-attn>=2.1`,head dimension 限制为不超过 192, +避免更大维度的反向能力依赖具体版本和显卡。 +这两个显式 FA 适配器当前限定 NVIDIA CUDA。上游 FA2 也有 ROCm 实现,但在此 +启用前需要独立的能力判断和验证。FA3 兼容 Tensor 和 `(output, softmax_lse)` +两种返回形式;这不代表所有历史 FA3 构建都已经兼容。 + +CPU 合同测试命令为 `pytest tests/test_shardformer/test_layer/test_attention_backends.py -k 'not gpu'`。 +单卡 FA3 严格验收复用现有 runner: +`bash .github/scripts/ci/run_colossalai_batch.sh 4f `。 +缺少 CUDA 或 FA3 会失败,数值结果和 profiler 记录保存在 `COLOSSAL_RESULTS` 下。 +现有 full-matrix workflow 在镜像具备兼容 FA3 后可手动选择 batch `4f`;它仅供主动选择, +不属于默认矩阵,也不属于标准 Layer 2 的 `2`、`4a`、`5` 批次。 + ## Shardformer的工作原理 ### 设计思想 diff --git a/extensions/__init__.py b/extensions/__init__.py index c392a16b5a61..82d01326648d 100644 --- a/extensions/__init__.py +++ b/extensions/__init__.py @@ -1,6 +1,7 @@ from .pybind.cpu_adam import CpuAdamArmExtension, CpuAdamX86Extension from .pybind.flash_attention import ( FlashAttentionDaoCudaExtension, + FlashAttentionDaoV3CudaExtension, FlashAttentionNpuExtension, FlashAttentionSdpaCudaExtension, ) @@ -20,6 +21,7 @@ ScaledMaskedSoftmaxCudaExtension, ScaledUpperTriangleMaskedSoftmaxCudaExtension, FlashAttentionDaoCudaExtension, + FlashAttentionDaoV3CudaExtension, FlashAttentionSdpaCudaExtension, FlashAttentionNpuExtension, ] @@ -34,6 +36,7 @@ "ScaledMaskedSoftmaxCudaExtension", "ScaledUpperTriangleMaskedSoftmaxCudaExtension", "FlashAttentionDaoCudaExtension", + "FlashAttentionDaoV3CudaExtension", "FlashAttentionSdpaCudaExtension", "FlashAttentionNpuExtension", ] diff --git a/extensions/pybind/flash_attention/__init__.py b/extensions/pybind/flash_attention/__init__.py index ea5b442aa58d..09a83c59752b 100644 --- a/extensions/pybind/flash_attention/__init__.py +++ b/extensions/pybind/flash_attention/__init__.py @@ -1,4 +1,5 @@ from .flash_attention_dao_cuda import FlashAttentionDaoCudaExtension +from .flash_attention_dao_v3_cuda import FlashAttentionDaoV3CudaExtension from .flash_attention_npu import FlashAttentionNpuExtension from .flash_attention_sdpa_cuda import FlashAttentionSdpaCudaExtension @@ -11,4 +12,9 @@ HAS_FLASH_ATTN = False -__all__ = ["FlashAttentionDaoCudaExtension", "FlashAttentionSdpaCudaExtension", "FlashAttentionNpuExtension"] +__all__ = [ + "FlashAttentionDaoCudaExtension", + "FlashAttentionDaoV3CudaExtension", + "FlashAttentionSdpaCudaExtension", + "FlashAttentionNpuExtension", +] diff --git a/extensions/pybind/flash_attention/flash_attention_dao_cuda.py b/extensions/pybind/flash_attention/flash_attention_dao_cuda.py index 560d952f6926..b3745a8dac84 100644 --- a/extensions/pybind/flash_attention/flash_attention_dao_cuda.py +++ b/extensions/pybind/flash_attention/flash_attention_dao_cuda.py @@ -56,6 +56,7 @@ def flash_attention( max_seqlen_kv: Optional[int] = None, q_indices: Optional[torch.Tensor] = None, kv_indices: Optional[torch.Tensor] = None, + causal_alignment: Optional[str] = None, ): # [B, H, S, D] -> [B, S, H, D] q = q.transpose(1, 2) @@ -94,3 +95,33 @@ def flash_attention( return attn_output.transpose(1, 2) return flash_attention + + def load_varlen(self): + from flash_attn import flash_attn_varlen_func + + def flash_attention_varlen( + q, + k, + v, + cu_seqlens_q, + cu_seqlens_kv, + max_seqlen_q, + max_seqlen_kv, + dropout_p=0.0, + scale=None, + is_causal=False, + causal_alignment=None, + ): + inputs = [x if x.stride(-1) == 1 else x.contiguous() for x in (q, k, v)] + return flash_attn_varlen_func( + *inputs, + cu_seqlens_q, + cu_seqlens_kv, + max_seqlen_q, + max_seqlen_kv, + dropout_p=dropout_p, + softmax_scale=scale, + causal=is_causal, + ) + + return flash_attention_varlen diff --git a/extensions/pybind/flash_attention/flash_attention_dao_v3_cuda.py b/extensions/pybind/flash_attention/flash_attention_dao_v3_cuda.py new file mode 100644 index 000000000000..7a2369244d9b --- /dev/null +++ b/extensions/pybind/flash_attention/flash_attention_dao_v3_cuda.py @@ -0,0 +1,89 @@ +from importlib.util import find_spec + +from ...base_extension import _Extension + + +def _output_tensor(result): + import torch + + # Older Hopper APIs return (out, softmax_lse). Keep out's autograd graph. + if isinstance(result, tuple) and len(result) == 2: + result = result[0] + if not isinstance(result, torch.Tensor): + raise TypeError("FA3 must return a Tensor or (Tensor, softmax_lse)") + return result + + +class FlashAttentionDaoV3CudaExtension(_Extension): + """FA3 training API at Dao-AILab/flash-attention commit 54b044c8cf897dfcc2577a665559d1bfda0bfc80. + + Install the ``hopper`` subdirectory of that revision as ``flash_attn_3``. + This API has no dropout argument. It is deliberately not registered in the + legacy automatic dispatcher, so existing callers keep their backend choice. + """ + + def __init__(self): + super().__init__(name="flash_attention_dao_v3_cuda", support_aot=False, support_jit=False) + + def is_available(self) -> bool: + import torch + + return torch.version.hip is None and torch.cuda.is_available() and find_spec("flash_attn_3") is not None + + def assert_compatible(self) -> None: + pass # Request-specific device/dtype checks belong to ColoAttention. + + def build_aot(self) -> None: + raise NotImplementedError("Install the FA3 hopper package from Dao-AILab/flash-attention separately.") + + def build_jit(self) -> None: + self.build_aot() + + def load(self): + from flash_attn_3 import flash_attn_interface + + def flash_attention( + q, k, v, dropout_p=0.0, scale=None, attention_mask=None, is_causal=False, causal_alignment=None + ): + if dropout_p != 0 or attention_mask is not None: + raise ValueError("FA3 does not support dropout or an arbitrary attention mask") + # A transpose is a view. Copy only when the kernel's last-dimension + # stride requirement is not met; keep the autograd connection. + inputs = [x.transpose(1, 2) for x in (q, k, v)] + inputs = [x if x.stride(-1) == 1 else x.contiguous() for x in inputs] + result = flash_attn_interface.flash_attn_func(*inputs, softmax_scale=scale, causal=is_causal) + return _output_tensor(result).transpose(1, 2) + + return flash_attention + + def load_varlen(self): + from flash_attn_3 import flash_attn_interface + + def flash_attention_varlen( + q, + k, + v, + cu_seqlens_q, + cu_seqlens_kv, + max_seqlen_q, + max_seqlen_kv, + dropout_p=0.0, + scale=None, + is_causal=False, + causal_alignment=None, + ): + if dropout_p != 0: + raise ValueError("FA3 does not support dropout") + inputs = [x if x.stride(-1) == 1 else x.contiguous() for x in (q, k, v)] + result = flash_attn_interface.flash_attn_varlen_func( + *inputs, + cu_seqlens_q, + cu_seqlens_kv, + max_seqlen_q, + max_seqlen_kv, + softmax_scale=scale, + causal=is_causal, + ) + return _output_tensor(result) + + return flash_attention_varlen diff --git a/extensions/pybind/flash_attention/flash_attention_sdpa_cuda.py b/extensions/pybind/flash_attention/flash_attention_sdpa_cuda.py index 2f920db61006..dd32acf0a6bf 100644 --- a/extensions/pybind/flash_attention/flash_attention_sdpa_cuda.py +++ b/extensions/pybind/flash_attention/flash_attention_sdpa_cuda.py @@ -24,10 +24,12 @@ def build_aot(self) -> None: def build_jit(self) -> None: raise NotImplementedError("Flash attention SDPA does not require just-in-time compilation.") - def load(self): + def load(self, strict=False): + from contextlib import nullcontext from typing import Optional import torch + from torch.nn.attention import SDPBackend, sdpa_kernel def flash_attention( q: torch.Tensor, @@ -43,14 +45,96 @@ def flash_attention( max_seqlen_kv: Optional[int] = None, q_indices: Optional[torch.Tensor] = None, kv_indices: Optional[torch.Tensor] = None, + causal_alignment: Optional[str] = None, ): - return torch.nn.functional.scaled_dot_product_attention( - q, - k, - v, - attn_mask=attention_mask, - dropout_p=dropout_p, - scale=scale, - ) + kwargs = {} + if strict: + sq, sk = q.shape[2], k.shape[2] + if is_causal and (attention_mask is not None or (sq != sk and causal_alignment == "lower_right")): + offset = sk - sq if causal_alignment == "lower_right" else 0 + keep = ( + torch.arange(sk, device=q.device)[None, :] + <= torch.arange(sq, device=q.device)[:, None] + offset + ) + if attention_mask is None: + attention_mask = keep + elif attention_mask.dtype == torch.bool: + attention_mask = attention_mask & keep + else: + attention_mask = attention_mask.masked_fill(~keep, float("-inf")) + is_causal = False + # Legacy calls already encode causal attention in attention_mask. + # The new explicit API can use native causal attention and GQA. + kwargs["is_causal"] = is_causal + if q.shape[1] != k.shape[1]: + kwargs["enable_gqa"] = True + context = nullcontext() + if ( + strict + and q.is_cuda + and q.dtype in (torch.float16, torch.bfloat16) + and attention_mask is not None + and attention_mask.dtype == torch.float32 + ): + # cuDNN SDPA in Torch 2.13 can produce NaNs or incorrect results + # for fp32 additive masks with half inputs. Preserve the mask's + # precision and the caller's other enabled SDPA kernels. + candidates = ( + (SDPBackend.FLASH_ATTENTION, torch.backends.cuda.flash_sdp_enabled()), + (SDPBackend.EFFICIENT_ATTENTION, torch.backends.cuda.mem_efficient_sdp_enabled()), + (SDPBackend.MATH, torch.backends.cuda.math_sdp_enabled()), + ) + context = sdpa_kernel([kernel for kernel, enabled in candidates if enabled]) + with context: + return torch.nn.functional.scaled_dot_product_attention( + q, + k, + v, + attn_mask=attention_mask, + dropout_p=dropout_p, + scale=scale, + **kwargs, + ) return flash_attention + + def load_varlen(self): + import torch + + dense_attention = self.load(strict=True) + + def flash_attention_varlen( + q, + k, + v, + cu_seqlens_q, + cu_seqlens_kv, + max_seqlen_q, + max_seqlen_kv, + dropout_p=0.0, + scale=None, + is_causal=False, + causal_alignment=None, + ): + # Correctness fallback, not a fused varlen implementation. Slice each + # sample independently so no token can attend across sample boundaries. + q_bounds = cu_seqlens_q.tolist() if isinstance(cu_seqlens_q, torch.Tensor) else cu_seqlens_q + kv_bounds = cu_seqlens_kv.tolist() if isinstance(cu_seqlens_kv, torch.Tensor) else cu_seqlens_kv + outputs = [] + for q_start, q_end, kv_start, kv_end in zip(q_bounds[:-1], q_bounds[1:], kv_bounds[:-1], kv_bounds[1:]): + qi = q[q_start:q_end].transpose(0, 1).unsqueeze(0) + ki = k[kv_start:kv_end].transpose(0, 1).unsqueeze(0) + vi = v[kv_start:kv_end].transpose(0, 1).unsqueeze(0) + out = dense_attention( + qi, + ki, + vi, + dropout_p=dropout_p, + scale=scale, + is_causal=is_causal, + causal_alignment=causal_alignment, + ) + outputs.append(out.squeeze(0).transpose(0, 1)) + return torch.cat(outputs, dim=0) + + return flash_attention_varlen diff --git a/tests/test_shardformer/test_layer/test_attention_backends.py b/tests/test_shardformer/test_layer/test_attention_backends.py new file mode 100644 index 000000000000..c18c2bfc089f --- /dev/null +++ b/tests/test_shardformer/test_layer/test_attention_backends.py @@ -0,0 +1,972 @@ +"""Explicit attention contracts and strict GPU acceptance. + +CPU: pytest -k 'not gpu' +FA3 acceptance (missing CUDA/package is a failure): + COLOSSALAI_TEST_ATTENTION_BACKEND=fa3 pytest -k gpu +Set COLOSSALAI_ATTENTION_TRACE_DIR to retain CUDA profiler traces as CI artifacts. +GPU skips in an ordinary CPU run do not establish GPU compatibility. +""" + +import hashlib +import json +import os +import sys +from contextlib import nullcontext +from importlib.metadata import distribution +from importlib.util import find_spec +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import patch + +import pytest +import torch + +from colossalai.kernel import kernel_loader +from colossalai.kernel.kernel_loader import FlashAttentionBackendLoader +from colossalai.shardformer.layer.attn import AttnMaskType, ColoAttention +from tests.kit.numerics import assert_aligned, run_fwd_bwd +from tests.kit.numerics.scenarios import independent_impl + + +class _LowPrecisionAttentionReference(torch.autograd.Function): + """Plain attention with the fused backward's intermediate precision recipe. + + FA2/FA3 accumulate dP in fp32, compute delta = sum(dO * O) from the + rounded output, then round dS before the dQ/dK matmuls. Autograd through + ``softmax(scores).to(v.dtype) @ v`` instead rounds dP and keeps dS in + fp32, so it does not measure the same low-precision baseline. + """ + + @staticmethod + def forward(ctx, scores, v, keep): + probabilities = scores.softmax(dim=-1).masked_fill(~keep, 0) + output = probabilities.to(v.dtype) @ v + ctx.save_for_backward(probabilities, v, output) + return output + + @staticmethod + def backward(ctx, grad_output): + probabilities, v, output = ctx.saved_tensors + grad = grad_output.to(probabilities.dtype) + grad_probabilities = grad @ v.to(probabilities.dtype).transpose(-1, -2) + delta = (grad * output.to(probabilities.dtype)).sum(dim=-1, keepdim=True) + grad_scores = (probabilities * (grad_probabilities - delta)).to(v.dtype) + grad_v = probabilities.to(v.dtype).transpose(-1, -2) @ grad_output + return grad_scores.to(probabilities.dtype), grad_v, None + + +def _reference(q, k, v, *, causal=False, alignment=None, mask=None, scale=None): + """Independent formula, including GQA and all-masked rows; no production helpers.""" + groups = q.shape[1] // k.shape[1] + k, v = (x.repeat_interleave(groups, dim=1) for x in (k, v)) + # CUDA attention accumulates QK and computes softmax at higher precision. + # Do not inflate the allowed error with an already rounded low-precision QK. + compute_dtype = torch.float64 if q.dtype == torch.float64 else torch.float32 + scores = q.to(compute_dtype) @ k.to(compute_dtype).transpose(-2, -1) + scores = scores * (q.shape[-1] ** -0.5 if scale is None else scale) + keep = torch.ones_like(scores, dtype=torch.bool) + if causal: + sq, sk = q.shape[-2], k.shape[-2] + offset = sk - sq if alignment == "lower_right" else 0 + # Deliberately construct the reference by rows, not the production helper. + causal_mask = torch.tensor([[j <= i + offset for j in range(sk)] for i in range(sq)], device=q.device) + keep = keep & causal_mask + if mask is not None: + if mask.dtype == torch.bool: + keep = keep & mask + else: + scores = scores + mask + keep = keep & ~torch.isneginf(mask) + scores = scores.masked_fill(~keep, float("-inf")) + # Do not backpropagate through softmax([-inf, ...]); the all-masked result is zero. + scores = torch.where(keep.any(dim=-1, keepdim=True), scores, torch.zeros_like(scores)) + if q.dtype in (torch.float16, torch.bfloat16): + return _LowPrecisionAttentionReference.apply(scores, v, keep) + softmax_dtype = torch.float64 if q.dtype == torch.float64 else torch.float32 + weights = scores.softmax(dim=-1, dtype=softmax_dtype).to(q.dtype).masked_fill(~keep, 0) + return weights @ v + + +def test_attention_reference_backward_formula(): + # Validate the handwritten derivative independently of low-precision rounding, + # including a row with no visible keys. Production kernels are not involved. + generator = torch.Generator().manual_seed(41) + scores = torch.randn(1, 2, 3, 4, generator=generator, dtype=torch.float64, requires_grad=True) + v = torch.randn(1, 2, 4, 5, generator=generator, dtype=torch.float64, requires_grad=True) + keep = torch.ones_like(scores, dtype=torch.bool) + keep[:, :, 0] = False + assert torch.autograd.gradcheck(lambda s, value: _LowPrecisionAttentionReference.apply(s, value, keep), (scores, v)) + + +def _packed_reference(q, k, v, q_lengths, kv_lengths, **kwargs): + outputs = [] + qs = ks = 0 + for q_len, kv_len in zip(q_lengths, kv_lengths): + out = _reference( + q[qs : qs + q_len].transpose(0, 1).unsqueeze(0), + k[ks : ks + kv_len].transpose(0, 1).unsqueeze(0), + v[ks : ks + kv_len].transpose(0, 1).unsqueeze(0), + **kwargs, + ) + outputs.append(out.squeeze(0).transpose(0, 1)) + qs += q_len + ks += kv_len + return torch.cat(outputs) + + +def _inputs(sq=5, sk=7, heads=2, kv_heads=2, dim=8, device="cpu", seed=41): + generator = torch.Generator(device=device).manual_seed(seed) + return { + name: torch.randn(2, nh, seq, dim, generator=generator, device=device, dtype=torch.float64) + for name, nh, seq in ( + ("q", heads, sq), + ("k", kv_heads, sk), + ("v", kv_heads, sk), + ) + } + + +def _assert_traces(test, ref, truth): + required = {"out", "grad/q", "grad/k", "grad/v"} + for trace in (truth, ref, test): + assert set(trace) == required + for tensor in trace.values(): + assert tensor.numel() > 0 + assert torch.isfinite(tensor).all() + assert_aligned(test, ref, truth) + + +@pytest.mark.parametrize("sq,sk", [(5, 5), (2, 5), (5, 2)]) +@pytest.mark.parametrize("alignment", [None, "upper_left", "lower_right"]) +def test_attention_dense_cpu(sq, sk, alignment): + inputs = _inputs(sq, sk) + options = { + "is_causal": alignment is not None, + "causal_alignment": alignment, + "scale": 0.2, + } + reference = lambda **kw: _reference(**kw, causal=alignment is not None, alignment=alignment, scale=0.2) + actual = lambda **kw: ColoAttention.attention(**kw, backend="sdpa", **options) + truth = run_fwd_bwd(reference, inputs, torch.float64) + test = run_fwd_bwd(actual, inputs, torch.float64) + _assert_traces(test, truth, truth) + if alignment == "lower_right" and sq > sk: + assert torch.count_nonzero(test["out"][:, :, : sq - sk]) == 0 + assert torch.count_nonzero(test["grad/q"][:, :, : sq - sk]) == 0 + + +@pytest.mark.parametrize("alignment,expected", [("upper_left", [1.0, 1.5]), ("lower_right", [2.5, 3.0])]) +def test_attention_causal_hand_example(alignment, expected): + q, k = torch.zeros(1, 1, 2, 4), torch.zeros(1, 1, 5, 4) + v = torch.arange(1, 6, dtype=torch.float32).reshape(1, 1, 5, 1) + out = ColoAttention.attention(q, k, v, backend="sdpa", is_causal=True, causal_alignment=alignment) + torch.testing.assert_close(out.flatten(), torch.tensor(expected)) + + +@pytest.mark.parametrize("floating", [False, True]) +def test_attention_custom_mask_gqa_and_gradients(floating): + inputs = _inputs(heads=4, kv_heads=2) + mask = torch.ones(5, 7, dtype=torch.bool) + mask[0] = False + mask[1, 1:] = False + mask[:, -1] = False + if floating: + mask = torch.zeros(5, 7, dtype=torch.float64).masked_fill(~mask, float("-inf")) + options = {"causal": True, "alignment": "lower_right", "mask": mask} + truth = run_fwd_bwd(lambda **kw: _reference(**kw, **options), inputs, torch.float64) + test = run_fwd_bwd( + lambda **kw: ColoAttention.attention( + **kw, + backend="sdpa", + attention_mask=mask, + is_causal=True, + causal_alignment="lower_right", + ), + inputs, + torch.float64, + ) + _assert_traces(test, truth, truth) + assert torch.count_nonzero(test["out"][:, :, 0]) == 0 + + +@pytest.mark.parametrize("alignment", [None, "upper_left", "lower_right"]) +@pytest.mark.parametrize("metadata", ["tensor", "host", "mixed", "unchecked"]) +def test_attention_varlen_cpu_and_isolation(alignment, metadata): + torch.manual_seed(7) + inputs = {"q": torch.randn(8, 4, 8, dtype=torch.float64)} + inputs.update({name: torch.randn(6, 2, 8, dtype=torch.float64) for name in ("k", "v")}) + cuq, cuk = torch.tensor([0, 3, 8], dtype=torch.int32), torch.tensor([0, 2, 6], dtype=torch.int32) + if metadata in ("host", "mixed"): + cuq = cuq.tolist() + if metadata == "host": + cuk = tuple(cuk.tolist()) + + def actual(**kw): + return ColoAttention.attention_varlen( + **kw, + cu_seqlens_q=cuq, + cu_seqlens_kv=cuk, + max_seqlen_q=5, + max_seqlen_kv=4, + backend="sdpa", + is_causal=alignment is not None, + causal_alignment=alignment, + validate=metadata != "unchecked", + ) + + truth = run_fwd_bwd( + lambda **kw: _packed_reference( + **kw, + q_lengths=[3, 5], + kv_lengths=[2, 4], + causal=alignment is not None, + alignment=alignment, + ), + inputs, + torch.float64, + ) + _assert_traces(run_fwd_bwd(actual, inputs, torch.float64), truth, truth) + changed = {name: x.clone() for name, x in inputs.items()} + for name in changed: + changed[name][: 3 if name == "q" else 2] += 10 + torch.testing.assert_close(actual(**changed)[3:], actual(**inputs)[3:], rtol=0, atol=0) + + +@pytest.mark.parametrize( + "prefix,maximum,message", + [ + ([1, 3, 8], 5, "partition"), + ([0, 3, 7], 5, "partition"), + ([0, 0, 8], 8, "nonempty"), + ([0, 9, 8], 9, "nonempty"), + ([0, 3, 8], 4, "smaller"), + ], +) +def test_attention_invalid_prefix(prefix, maximum, message): + q = torch.randn(8, 2, 8) + with pytest.raises(ValueError, match=message): + ColoAttention.attention_varlen( + q, + q, + q, + torch.tensor(prefix, dtype=torch.int32), + torch.tensor([0, 3, 8], dtype=torch.int32), + maximum, + 5, + backend="sdpa", + ) + + +@pytest.mark.parametrize( + "options,error", + [ + ({"backend": "unknown"}, ValueError), + ({"backend": "fa3"}, NotImplementedError), + ({"backend": "fa2"}, NotImplementedError), + ({"backend": "sdpa", "is_causal": True}, ValueError), + ({"backend": "sdpa", "causal_alignment": "lower_right"}, ValueError), + ({"backend": "sdpa", "dropout_p": -0.1}, ValueError), + ({"backend": "sdpa", "scale": float("nan")}, ValueError), + ({"backend": "sdpa", "attention_mask_type": AttnMaskType.CAUSAL}, ValueError), + ({"backend": "sdpa", "q_indices": torch.tensor([0])}, ValueError), + ({"backend": "sdpa", "typo": True}, TypeError), + ], +) +def test_attention_invalid_requests(options, error): + with pytest.raises(error): + ColoAttention.attention(**_inputs(), **options) + + +def test_attention_auto_cpu_and_legacy_dispatch(monkeypatch): + inputs = _inputs() + torch.testing.assert_close( + ColoAttention.attention(**inputs, backend="auto"), + ColoAttention.attention(**inputs, backend="sdpa"), + ) + calls = [] + + def legacy(q, k, v, **kwargs): + calls.append(kwargs) + return q + + monkeypatch.setattr(ColoAttention, "_dispatch_kernel", lambda *args: legacy) + out = ColoAttention.attention( + **inputs, + attention_mask=torch.zeros(5, 7), + attention_mask_type=AttnMaskType.CAUSAL, + ) + assert out is inputs["q"] + assert calls[0]["is_causal"] is True + + +def test_attention_strict_capability_and_missing_package(monkeypatch): + # CPU-only capability tests do not pretend to execute a CUDA kernel. + q = SimpleNamespace(device=torch.device("cuda"), dtype=torch.bfloat16, shape=(1, 2, 5, 128)) + monkeypatch.setattr(torch.cuda, "get_device_capability", lambda device: (9, 0)) + for kwargs, message in [ + ({"dropout_p": 0.1}, "dropout"), + ({"attention_mask": torch.ones(5, 5)}, "mask"), + ( + {"is_causal": True, "causal_alignment": "upper_left", "non_square": True}, + "upper-left", + ), + ]: + options = dict( + attention_mask=None, + dropout_p=0.0, + is_causal=False, + causal_alignment=None, + non_square=False, + ) + options.update(kwargs) + with pytest.raises(NotImplementedError, match=message): + ColoAttention._select_backend(q, q, "fa3", **options) + monkeypatch.setattr(FlashAttentionBackendLoader, "is_available", lambda backend: backend == "sdpa") + assert ColoAttention._select_backend(q, q, "auto", None, 0.0, False, None, False) == "sdpa" + FlashAttentionBackendLoader.load.cache_clear() + with pytest.raises(ModuleNotFoundError, match="fa3"): + FlashAttentionBackendLoader.load("fa3") + wide = SimpleNamespace(device=torch.device("cuda"), dtype=torch.bfloat16, shape=(1, 2, 5, 256)) + with pytest.raises(NotImplementedError, match="at most 192"): + ColoAttention._select_backend(wide, wide, "fa2", None, 0, False, None, False) + + +def test_attention_old_fa2_cannot_change_causal_semantics(monkeypatch): + monkeypatch.setattr( + kernel_loader, + "find_spec", + lambda name: object() if name == "flash_attn" else None, + ) + monkeypatch.setattr(kernel_loader, "version", lambda name: "2.0.9") + monkeypatch.setattr(torch.cuda, "get_device_capability", lambda device: (9, 0)) + q = SimpleNamespace(device=torch.device("cuda"), dtype=torch.bfloat16, shape=(1, 2, 5, 128)) + assert not FlashAttentionBackendLoader.is_available("fa2") + assert ColoAttention._select_backend(q, q, "auto", None, 0, True, "lower_right", True) == "sdpa" + FlashAttentionBackendLoader.load.cache_clear() + with pytest.raises(NotImplementedError, match=r"flash-attn >= 2.1"): + FlashAttentionBackendLoader.load("fa2") + monkeypatch.setattr(kernel_loader, "version", lambda name: "2.1.0") + assert FlashAttentionBackendLoader.is_available("fa2") + + +def test_attention_broken_fa3_import_is_not_unavailable(monkeypatch): + monkeypatch.setattr(kernel_loader, "find_spec", lambda name: object()) + + def broken(self): + raise ImportError("undefined symbol ABI sentinel") + + monkeypatch.setattr(kernel_loader.FlashAttentionDaoV3CudaExtension, "load", broken) + FlashAttentionBackendLoader.load.cache_clear() + assert FlashAttentionBackendLoader.is_available("fa3") + with pytest.raises(ImportError, match="ABI sentinel"): + FlashAttentionBackendLoader.load("fa3") + + +@pytest.mark.parametrize("case", ["dtype", "heads", "kv_length", "qk_dim", "empty"]) +def test_attention_invalid_tensor_shapes(case): + inputs = _inputs() + if case == "dtype": + inputs["v"] = inputs["v"].float() + elif case == "heads": + inputs["q"] = torch.randn(2, 3, 5, 8, dtype=torch.float64) + elif case == "kv_length": + inputs["v"] = inputs["v"][:, :, :-1] + elif case == "qk_dim": + inputs["q"] = inputs["q"][..., :-1] + else: + inputs["q"] = inputs["q"][:, :, :0] + with pytest.raises(ValueError): + ColoAttention.attention(**inputs, backend="sdpa") + + +def test_attention_noncontiguous_last_dimension(): + inputs = {name: x[..., ::2].requires_grad_() for name, x in _inputs(dim=16).items()} + assert all(x.stride(-1) == 2 for x in inputs.values()) + out = ColoAttention.attention(**inputs, backend="sdpa") + ref = _reference(**inputs) + torch.testing.assert_close(out, ref) + grads = torch.autograd.grad(out.sum(), tuple(inputs.values()), retain_graph=True) + ref_grads = torch.autograd.grad(ref.sum(), tuple(inputs.values())) + for actual, expected in zip(grads, ref_grads): + torch.testing.assert_close(actual, expected) + + +@pytest.mark.parametrize("case", ["dtype", "noncontiguous", "sample_count"]) +def test_attention_invalid_prefix_metadata(case): + q = torch.randn(8, 2, 8) + cq = torch.tensor([0, 3, 8], dtype=torch.int32) + ck = cq + if case == "dtype": + cq = cq.long() + elif case == "noncontiguous": + cq = torch.tensor([0, 0, 3, 0, 8, 0], dtype=torch.int32)[::2] + else: + ck = torch.tensor([0, 8], dtype=torch.int32) + with pytest.raises(ValueError): + ColoAttention.attention_varlen(q, q, q, cq, ck, 8, 8, backend="sdpa") + + +@pytest.mark.parametrize("backend", ["fa3", "auto"]) +def test_attention_execution_error_is_not_fallback(monkeypatch, backend): + calls = [] + + def broken(*args, **kwargs): + raise RuntimeError("native backend failure sentinel") + + def load(backend, packed=False): + calls.append(backend) + return broken + + monkeypatch.setattr(ColoAttention, "_select_backend", lambda *args: "fa3") + monkeypatch.setattr(FlashAttentionBackendLoader, "load", load) + with pytest.raises(RuntimeError, match="sentinel"): + ColoAttention.attention(**_inputs(), backend=backend) + assert calls == ["fa3"] + + +@pytest.mark.parametrize("operation", ["is_available", "load"]) +def test_attention_loader_unknown_backend(operation): + with pytest.raises(ValueError, match="Unknown attention backend"): + getattr(FlashAttentionBackendLoader, operation)("unknown") + + +def test_attention_registered_backend(monkeypatch): + calls = [] + + class Extension: + def load(self): + def dense(q, k, v, **kwargs): + calls.append(kwargs) + return q + + return dense + + def load_varlen(self): + def packed(q, k, v, cq, ck, *args, **kwargs): + assert cq.dtype == ck.dtype == torch.int32 + calls.append(kwargs) + return q + + return packed + + spec = kernel_loader._AttentionBackend(Extension, ("cpu", "npu")) + monkeypatch.setitem(FlashAttentionBackendLoader.REGISTRY, "test_device", spec) + FlashAttentionBackendLoader.load.cache_clear() + try: + assert FlashAttentionBackendLoader.candidates("cuda") == ["fa3", "fa2", "sdpa"] + assert FlashAttentionBackendLoader.candidates("npu") == ["test_device", "sdpa"] + q = torch.randn(1, 2, 4, 8) + assert ColoAttention.attention(q, q, q, backend="auto") is q + packed = q.squeeze(0).transpose(0, 1) + assert ColoAttention.attention_varlen(packed, packed, packed, [0, 4], [0, 4], 4, 4) is packed + assert len(calls) == 2 + assert all("causal_alignment" in call for call in calls) + # A rejected request must skip the registered backend only in auto mode. + monkeypatch.setitem( + FlashAttentionBackendLoader.REGISTRY, + "test_device", + kernel_loader._AttentionBackend(Extension, ("cpu",), lambda *args: "unsupported request"), + ) + with pytest.raises(NotImplementedError, match="unsupported request"): + ColoAttention.attention(q, q, q, backend="test_device") + assert ColoAttention._select_backend(q, q, "auto", None, 0, False, None, False) == "sdpa" + finally: + FlashAttentionBackendLoader.load.cache_clear() + + +@pytest.mark.parametrize("backend", ["fa2", "fa3"]) +def test_attention_rocm_is_not_nvidia(backend, monkeypatch): + q = SimpleNamespace(device=torch.device("cuda"), dtype=torch.bfloat16, shape=(1, 2, 5, 128)) + monkeypatch.setattr(torch.version, "hip", "6.2") + # A HIP request must be rejected before interpreting its architecture as SM. + monkeypatch.setattr(torch.cuda, "get_device_capability", lambda device: pytest.fail("queried HIP as NVIDIA")) + monkeypatch.setattr(FlashAttentionBackendLoader, "is_available", lambda backend: True) + with pytest.raises(NotImplementedError, match="NVIDIA"): + ColoAttention._select_backend(q, q, backend, None, 0, False, None, False) + assert ColoAttention._select_backend(q, q, "auto", None, 0, False, None, False) == "sdpa" + + +@pytest.mark.parametrize("packed", [False, True]) +@pytest.mark.parametrize("result_type", ["tensor", "tuple", "invalid"]) +def test_attention_fa3_return_contract(packed, result_type, monkeypatch): + def native(q, k, v, *args, **kwargs): + out = 2 * q + k + v + return (out, None) if result_type == "tuple" else (None if result_type == "invalid" else out) + + module = SimpleNamespace(flash_attn_func=native, flash_attn_varlen_func=native) + monkeypatch.setitem(sys.modules, "flash_attn_3", SimpleNamespace(flash_attn_interface=module)) + extension = kernel_loader.FlashAttentionDaoV3CudaExtension() + shape = (5, 2, 8) if packed else (1, 2, 5, 8) + inputs = [torch.randn(shape, requires_grad=True) for _ in range(3)] + cu = torch.tensor([0, 5], dtype=torch.int32) + call = (lambda: extension.load_varlen()(*inputs, cu, cu, 5, 5)) if packed else (lambda: extension.load()(*inputs)) + if result_type == "invalid": + with pytest.raises(TypeError, match="FA3 must return"): + call() + return + out = call() + torch.testing.assert_close(out, 2 * inputs[0] + inputs[1] + inputs[2]) + for gradient, expected in zip(torch.autograd.grad(out.sum(), inputs), (2, 1, 1)): + torch.testing.assert_close(gradient, torch.full_like(gradient, expected)) + + +@pytest.mark.parametrize("validate", [False, True]) +def test_attention_varlen_sdpa_reads_prefix_once(validate, monkeypatch): + q = torch.randn(8, 2, 8) + prefix = torch.tensor([0, 3, 8], dtype=torch.int32) + reads = [] + original = torch.Tensor.tolist + + def tolist(tensor): + reads.append(tensor) + return original(tensor) + + monkeypatch.setattr(torch.Tensor, "tolist", tolist) + ColoAttention.attention_varlen(q, q, q, prefix, prefix, 5, 5, backend="sdpa", validate=validate) + assert len(reads) == 2 + assert all(tensor is prefix for tensor in reads) + + +def test_attention_varlen_unchecked_tensor_contract(monkeypatch): + q = torch.randn(8, 2, 8) + prefix = torch.tensor([0, 3, 8], dtype=torch.int32) + # Exercise routing without a GPU; this stub does not claim kernel validation. + monkeypatch.setattr(ColoAttention, "_select_backend", lambda *args: "fa3") + monkeypatch.setattr(FlashAttentionBackendLoader, "load", lambda *args, **kwargs: lambda q, *a, **kw: q) + monkeypatch.setattr(torch.Tensor, "tolist", lambda self: pytest.fail("unexpected host read")) + assert ColoAttention.attention_varlen(q, q, q, prefix, prefix, 5, 5, validate=False) is q + with pytest.raises(ValueError, match="explicit causal_alignment"): + ColoAttention.attention_varlen(q, q, q, prefix, prefix, 5, 5, validate=False, is_causal=True) + with pytest.raises(ValueError, match="int32"): + ColoAttention.attention_varlen(q, q, q, prefix.long(), prefix, 5, 5, validate=False) + with pytest.raises(ValueError, match="same number"): + ColoAttention.attention_varlen(q, q, q, prefix, prefix[:2], 5, 5, validate=False) + with pytest.raises(TypeError, match="validate"): + ColoAttention.attention_varlen(q, q, q, prefix, prefix, 5, 5, validate=0) + + +@pytest.mark.parametrize("prefix", [[0, 0, 8], [0, 3, 7], [0, True, 8], [0, 3.5, 8], [0], "038"]) +def test_attention_varlen_host_validation(prefix): + q = torch.randn(8, 2, 8) + # Host values remain cheap to validate, even with tensor validation disabled. + with pytest.raises((ValueError, TypeError)): + ColoAttention.attention_varlen(q, q, q, prefix, [0, 3, 8], 5, 5, backend="sdpa", validate=False) + + +def test_attention_compile_dense_forward_backward(): + fn = lambda q, k, v: ColoAttention.attention( + q, k, v, backend="sdpa", is_causal=True, causal_alignment="lower_right" + ) + inputs = _inputs() + fn(**inputs) # Resolve the optional backend before entering the compiled graph. + compiled = torch.compile(fn, backend="aot_eager", fullgraph=True) + reference = run_fwd_bwd(fn, inputs, torch.float64) + _assert_traces(run_fwd_bwd(compiled, inputs, torch.float64), reference, reference) + + +def test_attention_numerics_detect_wrong_scale(): + inputs = _inputs() + truth = run_fwd_bwd(_reference, inputs, torch.float64) + wrong = run_fwd_bwd( + lambda **kw: ColoAttention.attention(**kw, backend="sdpa", scale=0), + inputs, + torch.float64, + ) + with pytest.raises(AssertionError): + _assert_traces(wrong, truth, truth) + + +def _forbid_sdpa_fallback(*args, **kwargs): + # Dynamo inspects torch function metadata; keep this a real function, not a Mock. + raise AssertionError("SDPA fallback") + + +def _require_gpu(backend): + required = os.environ.get("COLOSSALAI_TEST_ATTENTION_BACKEND") + if required and required not in ("fa2", "fa3", "sdpa", "all"): + pytest.fail("COLOSSALAI_TEST_ATTENTION_BACKEND must be fa2, fa3, sdpa or all") + if required not in (None, "all", backend): + pytest.skip(f"This acceptance run selects {required}") + if not torch.cuda.is_available(): + if required: + pytest.fail("Strict GPU attention acceptance requires a CUDA GPU") + pytest.skip("CUDA unavailable; this is not GPU validation") + # Ordinary regression runs may lack optional packages. Explicit acceptance + # must fail instead; installed-but-broken packages always surface their error. + if backend == "sdpa": + return + package = "flash_attn_3" if backend == "fa3" else "flash_attn" + if required is None and find_spec(package) is None: + pytest.skip(f"Optional {package} absent; this is not {backend} validation") + + +def _watch_real_backend(backend, packed, monkeypatch): + if backend == "sdpa": + module = torch.nn.functional + others = (kernel_loader.FlashAttentionDaoCudaExtension, kernel_loader.FlashAttentionDaoV3CudaExtension) + function_name = "scaled_dot_product_attention" + elif backend == "fa3": + from flash_attn_3 import flash_attn_interface as module + + others = (kernel_loader.FlashAttentionDaoCudaExtension,) + function_name = "flash_attn_varlen_func" if packed else "flash_attn_func" + else: + import flash_attn as module + + others = (kernel_loader.FlashAttentionDaoV3CudaExtension,) + function_name = "flash_attn_varlen_func" if packed else "flash_attn_func" + original = getattr(module, function_name) + calls = [] + + def entered(*args, **kwargs): + calls.append(function_name) + return original(*args, **kwargs) + + def forbidden(*args, **kwargs): + raise AssertionError("Fallback to the other FlashAttention backend") + + for other in others: + monkeypatch.setattr(other, "load", forbidden) + monkeypatch.setattr(other, "load_varlen", forbidden) + monkeypatch.setattr(module, function_name, entered) + FlashAttentionBackendLoader.load.cache_clear() + return module, function_name, calls + + +def test_attention_required_device_does_not_skip(monkeypatch): + monkeypatch.setenv("COLOSSALAI_TEST_ATTENTION_BACKEND", "fa3") + monkeypatch.setattr(torch.cuda, "is_available", lambda: False) + with pytest.raises(pytest.fail.Exception, match="requires a CUDA GPU"): + _require_gpu("fa3") + + +@pytest.mark.parametrize("backend", ["fa2", "fa3", "sdpa"]) +@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16]) +@pytest.mark.parametrize("dim", [32, 64, 128]) +@pytest.mark.parametrize("heads", [2, 4]) +@pytest.mark.parametrize("seed", [0, 41]) +@pytest.mark.parametrize( + "packed,causal,sq,sk", + [ + (False, False, 127, 129), + (False, True, 129, 129), + (False, True, 2, 5), + (False, True, 5, 2), + (True, False, 127, 129), + (True, True, 5, 2), + (True, False, 1, 1), + ], +) +def test_attention_gpu_numerics_no_fallback(backend, dtype, dim, heads, seed, packed, causal, sq, sk, monkeypatch): + _require_gpu(backend) + alignment = "lower_right" if causal else None + if packed: + torch.manual_seed(seed) + q_lengths, kv_lengths = [3, sq], [4, sk] + inputs = { + "q": torch.randn(sum(q_lengths), heads, dim, device="cuda", dtype=torch.float64), + "k": torch.randn(sum(kv_lengths), 2, dim, device="cuda", dtype=torch.float64), + "v": torch.randn(sum(kv_lengths), 2, dim, device="cuda", dtype=torch.float64), + } + cuq = torch.tensor([0, 3, sum(q_lengths)], dtype=torch.int32, device="cuda") + cuk = torch.tensor([0, 4, sum(kv_lengths)], dtype=torch.int32, device="cuda") + reference = lambda **kw: _packed_reference( + **kw, + q_lengths=q_lengths, + kv_lengths=kv_lengths, + causal=causal, + alignment=alignment, + ) + actual = lambda **kw: ColoAttention.attention_varlen( + **kw, + cu_seqlens_q=cuq, + cu_seqlens_kv=cuk, + max_seqlen_q=max(q_lengths), + max_seqlen_kv=max(kv_lengths), + backend=backend, + is_causal=causal, + causal_alignment=alignment, + ) + else: + inputs = _inputs(sq, sk, heads=heads, kv_heads=2, dim=dim, device="cuda", seed=seed) + reference = lambda **kw: _reference(**kw, causal=causal, alignment=alignment) + actual = lambda **kw: ColoAttention.attention( + **kw, backend=backend, is_causal=causal, causal_alignment=alignment + ) + trace = _check_backend_numerics(actual, reference, inputs, dtype, backend, packed, monkeypatch) + if causal and sq > sk: + for name in ("out", "grad/q"): + rows = trace[name][3 : 3 + sq - sk] if packed else trace[name][:, :, : sq - sk] + assert torch.count_nonzero(rows) == 0 + + +def _check_backend_numerics(actual, reference, inputs, dtype, backend, packed, monkeypatch): + module, function_name, calls = _watch_real_backend(backend, packed, monkeypatch) + original_load = FlashAttentionBackendLoader.load + capture = independent_impl.run_fwd_bwd + traces = [] + + def record_trace(*args, **kwargs): + trace = capture(*args, **kwargs) + traces.append((kwargs["role"], trace)) + return trace + + def strict_load(selected, packed=False): + assert selected == backend, f"Unexpected fallback to {selected}" + return original_load(selected, packed) + + monkeypatch.setattr(FlashAttentionBackendLoader, "load", strict_load) + with ( + patch.object(independent_impl, "run_fwd_bwd", new=record_trace), + ( + nullcontext() + if backend == "sdpa" + else patch("torch.nn.functional.scaled_dot_product_attention", new=_forbid_sdpa_fallback) + ), + ): + report = independent_impl.check_fwd_bwd( + actual, reference, inputs, dtype, expect_calls=[(module, function_name)] + ) + assert calls and set(calls) == {function_name} + assert [role for role, _ in traces] == ["truth", "ref", "test"] + call_checks = [ + {"name": entry.name, "passed": entry.passed, "reason": entry.reason} + for entry in report.entries + if entry.name.startswith("/") + ] + for _, trace in traces: + trace.meta["test_call_checks"] = call_checks + _assert_traces(traces[2][1], traces[1][1], traces[0][1]) + report.assert_pass() + original_load.cache_clear() + return traces[2][1] + + +@pytest.mark.parametrize("backend", ["fa2", "fa3", "sdpa"]) +@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16]) +@pytest.mark.parametrize("length", [512, 2048]) +def test_attention_gpu_long_sequence(backend, dtype, length, monkeypatch): + _require_gpu(backend) + inputs = _inputs(length, length, heads=4, kv_heads=2, dim=128, device="cuda") + actual = lambda **kw: ColoAttention.attention(**kw, backend=backend) + _check_backend_numerics(actual, _reference, inputs, dtype, backend, False, monkeypatch) + + +@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16]) +@pytest.mark.parametrize("floating", [False, True]) +@pytest.mark.parametrize("alignment", ["upper_left", "lower_right"]) +@pytest.mark.parametrize("seed", [0, 41]) +def test_attention_gpu_sdpa_mask(dtype, floating, alignment, seed, monkeypatch): + _require_gpu("sdpa") + inputs = _inputs(5, 7, heads=4, kv_heads=2, dim=64, device="cuda", seed=seed) + mask = torch.ones(5, 7, device="cuda", dtype=torch.bool) + mask[0] = False + mask[1, 2:] = False + mask[:, -1] = False + if floating: + mask = torch.full((5, 7), 0.25, device="cuda", dtype=torch.float32).masked_fill(~mask, float("-inf")) + reference = lambda **kw: _reference(**kw, mask=mask, causal=True, alignment=alignment) + actual = lambda **kw: ColoAttention.attention( + **kw, attention_mask=mask, backend="sdpa", is_causal=True, causal_alignment=alignment + ) + _check_backend_numerics(actual, reference, inputs, dtype, "sdpa", False, monkeypatch) + + +@pytest.mark.parametrize("backend", ["fa2", "fa3", "sdpa"]) +@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16]) +@pytest.mark.parametrize("seed", [0, 41]) +def test_attention_gpu_varlen_isolation(backend, dtype, seed, monkeypatch): + _require_gpu(backend) + generator = torch.Generator(device="cuda").manual_seed(seed) + inputs = { + name: torch.randn(tokens, heads, 64, device="cuda", dtype=torch.float64, generator=generator) + for name, tokens, heads in (("q", 8, 4), ("k", 6, 2), ("v", 6, 2)) + } + cuq = torch.tensor([0, 3, 8], device="cuda", dtype=torch.int32) + cuk = torch.tensor([0, 2, 6], device="cuda", dtype=torch.int32) + + def actual(q, k, v): + return ColoAttention.attention_varlen(q, k, v, cuq, cuk, 5, 4, backend=backend)[3:] + + def reference(q, k, v): + # Only the second sample contributes to the loss. The first sample's + # input gradients must be exactly zero, not just numerically close. + return _packed_reference(q, k, v, [3, 5], [2, 4])[3:] + + changed = {name: value.clone() for name, value in inputs.items()} + for name in changed: + changed[name][: 3 if name == "q" else 2] += 3 + original = run_fwd_bwd(actual, inputs, dtype) + perturbed = run_fwd_bwd(actual, changed, dtype) + torch.testing.assert_close(original["out"], perturbed["out"], rtol=0, atol=0) + for trace in (original, perturbed): + for name in inputs: + assert torch.count_nonzero(trace[f"grad/{name}"][: 3 if name == "q" else 2]) == 0 + # Compare both executions to the independent reference rather than allowing + # an arbitrary epsilon between two potentially nondeterministic backwards. + for data in (inputs, changed): + with monkeypatch.context() as scoped: + _check_backend_numerics(actual, reference, data, dtype, backend, True, scoped) + + +def test_attention_numerics_detect_wrong_causal_alignment(): + inputs = _inputs(2, 5) + reference = lambda **kw: _reference(**kw, causal=True, alignment="lower_right") + wrong = lambda **kw: _reference(**kw, causal=True, alignment="upper_left") + with pytest.raises(AssertionError, match="numerical alignment"): + independent_impl.check_fwd_bwd(wrong, reference, inputs, torch.float64).assert_pass() + + +def test_attention_numerics_require_backend_call(): + backend = SimpleNamespace(attention=_reference) + report = independent_impl.check_fwd_bwd( + _reference, _reference, _inputs(), torch.float64, expect_calls=[(backend, "attention")] + ) + assert len(report.failures) == 1 + assert report.failures[0].reason == "never called" + with pytest.raises(AssertionError, match="never called"): + report.assert_pass() + + +@pytest.mark.parametrize("backend", ["fa2", "fa3"]) +@pytest.mark.parametrize("packed", [False, True]) +def test_attention_gpu_actual_kernel_and_error(backend, packed, monkeypatch, tmp_path, record_property): + _require_gpu(backend) + module, function_name, calls = _watch_real_backend(backend, packed, monkeypatch) + + torch.manual_seed(41) + inputs = {name: torch.randn(8, 2, 128, device="cuda", dtype=torch.float64) for name in ("q", "k", "v")} + + def actual(q, k, v): + # Exercise last-dimension copies while retaining gradients to Q/K/V. + inputs = [torch.stack((x, x), dim=-1).flatten(-2)[..., ::2] for x in (q, k, v)] + assert all(x.stride(-1) == 2 for x in inputs) + if packed: + cu = torch.tensor([0, 3, 8], device="cuda", dtype=torch.int32) + return ColoAttention.attention_varlen(*inputs, cu, cu, 5, 5, backend=backend) + return ColoAttention.attention(*(x.transpose(0, 1).unsqueeze(0) for x in inputs), backend=backend) + + def reference(q, k, v): + if packed: + return _packed_reference(q, k, v, q_lengths=[3, 5], kv_lengths=[3, 5]) + return _reference(*(x.transpose(0, 1).unsqueeze(0) for x in (q, k, v))) + + truth = run_fwd_bwd(reference, inputs, torch.float64) + ref = run_fwd_bwd(reference, inputs, torch.bfloat16) + with patch( + "torch.nn.functional.scaled_dot_product_attention", + new=_forbid_sdpa_fallback, + ): + with torch.profiler.profile( + activities=[ + torch.profiler.ProfilerActivity.CPU, + torch.profiler.ProfilerActivity.CUDA, + ] + ) as prof: + test = run_fwd_bwd(actual, inputs, torch.bfloat16) + torch.cuda.synchronize() + assert calls == [function_name] + _assert_traces(test, ref, truth) + names = [event.name for event in prof.events()] + if backend == "fa3": + assert any("flash_attn_3::_flash_attn_forward" in name for name in names), names + assert any("flash_attn_3::_flash_attn_backward" in name for name in names), names + else: + assert any("FlashAttn" in name and "Backward" in name for name in names), names + assert any(event.device_type == torch.autograd.DeviceType.CUDA for event in prof.events()) + trace_dir = Path(os.environ.get("COLOSSALAI_ATTENTION_TRACE_DIR", tmp_path)) + trace_dir.mkdir(parents=True, exist_ok=True) + trace_path = trace_dir / f"{backend}-{'packed' if packed else 'dense'}.json" + prof.export_chrome_trace(str(trace_path)) + for role, trace in (("truth", truth), ("ref", ref), ("test", test)): + trace.save(trace_dir / f"{trace_path.stem}-{role}.safetensors") + record_property("backend_module", module.__file__) + installed = distribution("flash_attn_3" if backend == "fa3" else "flash-attn") + record_property("backend_version", installed.version) + direct_url = installed.read_text("direct_url.json") + record_property("backend_direct_url", direct_url or "unavailable") + provenance = json.loads(direct_url) if direct_url else {} + record_property("installed_vcs_commit", provenance.get("vcs_info", {}).get("commit_id", "unavailable")) + # The expected source revision in the adapter docstring is not proof of the + # installed build's origin. Retain a wheel/hash when the build job provides it. + if backend == "fa3" and os.environ.get("COLOSSALAI_FA3_WHEEL_PATH"): + wheel_path = Path(os.environ["COLOSSALAI_FA3_WHEEL_PATH"]) + wheel_hash = hashlib.sha256() + with wheel_path.open("rb") as wheel_file: + for chunk in iter(lambda: wheel_file.read(1024 * 1024), b""): + wheel_hash.update(chunk) + record_property("wheel_path", str(wheel_path)) + record_property("wheel_sha256", wheel_hash.hexdigest()) + record_property("torch_cuda", f"{torch.__version__}/{torch.version.cuda}") + record_property("gpu", torch.cuda.get_device_name()) + record_property("profiler", str(trace_path)) + + def broken(*args, **kwargs): + raise RuntimeError("GPU backend failure sentinel") + + monkeypatch.setattr(module, function_name, broken) + FlashAttentionBackendLoader.load.cache_clear() + with pytest.raises(RuntimeError, match="GPU backend failure sentinel"): + actual(**{name: x.bfloat16() for name, x in inputs.items()}) + FlashAttentionBackendLoader.load.cache_clear() + + +@pytest.mark.parametrize("backend", ["fa2", "fa3"]) +def test_attention_gpu_compile_forward_backward(backend): + _require_gpu(backend) + inputs = _inputs(17, 17, dim=128, device="cuda") + fn = lambda q, k, v: ColoAttention.attention(q, k, v, backend=backend) + # Resolve imports before compilation. Keep real CUDA execution and backward + # in both runs; a fullgraph compiler failure must remain an acceptance failure. + reference = run_fwd_bwd(fn, inputs, torch.bfloat16) + compiled = torch.compile(fn, fullgraph=True) + with patch("torch.nn.functional.scaled_dot_product_attention", new=_forbid_sdpa_fallback): + test = run_fwd_bwd(compiled, inputs, torch.bfloat16) + truth = run_fwd_bwd(_reference, inputs, torch.float64) + ref = run_fwd_bwd(_reference, inputs, torch.bfloat16) + _assert_traces(reference, ref, truth) + _assert_traces(test, ref, truth) + + +@pytest.mark.parametrize("backend", ["fa2", "fa3", "sdpa"]) +@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16]) +@pytest.mark.parametrize("metadata", ["host", "mixed", "unchecked"]) +def test_attention_gpu_varlen_metadata(backend, dtype, metadata, monkeypatch): + _require_gpu(backend) + generator = torch.Generator(device="cuda").manual_seed(41) + inputs = { + name: torch.randn(tokens, heads, 64, device="cuda", dtype=torch.float64, generator=generator) + for name, tokens, heads in (("q", 8, 4), ("k", 6, 2), ("v", 6, 2)) + } + cuq, cuk = [0, 3, 8], [0, 4, 6] + if metadata in ("mixed", "unchecked"): + cuk = torch.tensor(cuk, dtype=torch.int32, device="cuda") + if metadata == "unchecked": + cuq = torch.tensor(cuq, dtype=torch.int32, device="cuda") + reference = lambda **kw: _packed_reference( + **kw, q_lengths=[3, 5], kv_lengths=[4, 2], causal=True, alignment="lower_right" + ) + actual = lambda **kw: ColoAttention.attention_varlen( + **kw, + cu_seqlens_q=cuq, + cu_seqlens_kv=cuk, + max_seqlen_q=5, + max_seqlen_kv=4, + backend=backend, + is_causal=True, + causal_alignment="lower_right", + validate=metadata != "unchecked", + ) + # For the trusted fused path, fail if production tries a device-to-host read. + if metadata == "unchecked" and backend != "sdpa": + original = torch.Tensor.tolist + + def tolist(tensor): + if tensor is cuq or tensor is cuk: + pytest.fail("unexpected host read") + return original(tensor) + + monkeypatch.setattr(torch.Tensor, "tolist", tolist) + trace = _check_backend_numerics(actual, reference, inputs, dtype, backend, True, monkeypatch) + for name in ("out", "grad/q"): + assert torch.count_nonzero(trace[name][3:6]) == 0