Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 14 additions & 0 deletions .github/scripts/ci/run_colossalai_batch.sh
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ Batches:
4c Single-GPU Triton inference kernels
4d Single-GPU SmoothQuant tests
4e Single-GPU flash-decoding-attention parameter matrix
4f Single-GPU strict FA3 training attention (requires the optional FA3 package)
5 Two-GPU DDP/FSDP/LowLevelZeRO/LoRA
6 Four-GPU DeviceMesh/DTensor/communication foundations
7 Four-GPU FP8 collectives and DDP/FSDP communication hooks
Expand Down Expand Up @@ -85,6 +86,7 @@ case "${BATCH}" in
tests/test_pipeline/test_pipeline_utils/test_t5_pipeline_utils.py
tests/test_pipeline/test_pipeline_utils/test_whisper_pipeline_utils.py
tests/test_pipeline/test_schedule/test_pipeline_schedule_utils.py
tests/test_shardformer/test_layer/test_attention_backends.py
tests/test_tensor/test_dtensor/test_dtensor_sharding_spec.py
tests/test_tensor/test_shape_consistency.py
tests/test_tensor/test_sharding_spec.py
Expand Down Expand Up @@ -168,6 +170,18 @@ case "${BATCH}" in
NEEDS_CUDA_TOOLKIT=1
TESTS=(tests/test_infer/test_kernels/cuda/test_flash_decoding_attention.py)
;;
4f|04f)
LABEL="04f-fa3-training-attention"
DESCRIPTION="Strict FA3 dense/varlen forward, backward and numerical alignment"
EXPECTED_GPUS=1
DEFAULT_TIMEOUT_MIN=30
TESTS=(tests/test_shardformer/test_layer/test_attention_backends.py)
PYTEST_EXTRA=(-k gpu -o junit_family=xunit1)
# This is acceptance for a requested backend, not an optional discovery
# run. Missing FA3/CUDA must fail rather than produce a green skip.
export COLOSSALAI_TEST_ATTENTION_BACKEND=fa3
export COLOSSALAI_ATTENTION_TRACE_DIR="${RESULTS_ROOT}/fa3-traces"
;;
5|05)
LABEL="05-booster-lora-2gpu"
DESCRIPTION="DDP, FSDP, LowLevelZeRO, LoRA and the numerical alignment tool on its plugins"
Expand Down
1 change: 1 addition & 0 deletions .github/scripts/ci/run_cpu_unit_tests.sh
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ TESTS=(
tests/test_pipeline/test_pipeline_utils/test_t5_pipeline_utils.py
tests/test_pipeline/test_pipeline_utils/test_whisper_pipeline_utils.py
tests/test_pipeline/test_schedule/test_pipeline_schedule_utils.py
tests/test_shardformer/test_layer/test_attention_backends.py
tests/test_tensor/test_dtensor/test_dtensor_sharding_spec.py
tests/test_tensor/test_shape_consistency.py
tests/test_tensor/test_sharding_spec.py
Expand Down
1 change: 1 addition & 0 deletions .github/scripts/ci/run_gpu_batch.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
"4c": 1,
"4d": 1,
"4e": 1,
"4f": 1,
"5": 2,
"6": 4,
"7": 4,
Expand Down
109 changes: 108 additions & 1 deletion colossalai/kernel/kernel_loader.py
Original file line number Diff line number Diff line change
@@ -1,10 +1,18 @@
import warnings
from typing import List
from dataclasses import dataclass, field
from functools import lru_cache, partial
from importlib.metadata import version
from importlib.util import find_spec
from typing import Callable, List, Optional, Tuple, Type

import torch
from packaging.version import Version

from .extensions import (
CpuAdamArmExtension,
CpuAdamX86Extension,
FlashAttentionDaoCudaExtension,
FlashAttentionDaoV3CudaExtension,
FlashAttentionNpuExtension,
FlashAttentionSdpaCudaExtension,
FusedOptimizerCudaExtension,
Expand Down Expand Up @@ -124,6 +132,105 @@ class FlashAttentionDaoLoader(KernelLoader):
REGISTRY = [FlashAttentionDaoCudaExtension]


def _flash_attention_support(q, v, attention_mask, dropout_p, is_causal, causal_alignment, non_square, *, hopper=False):
# ROCm reuses torch.cuda, but its architecture numbers are not NVIDIA SMs.
# These adapters are validated for NVIDIA; ROCm needs its own capability policy.
if torch.version.hip is not None:
return "requires NVIDIA CUDA in this adapter; ROCm is not supported"
if q.dtype not in (torch.float16, torch.bfloat16):
return "requires float16 or bfloat16 inputs"
if attention_mask is not None:
return "does not support an arbitrary attention_mask"
if q.shape[-1] != v.shape[-1] or q.shape[-1] > 256 or q.shape[-1] % 8 != 0:
return "requires equal Q/K/V head dimensions divisible by 8 and at most 256"
if not hopper and q.shape[-1] > 192:
return "supports head dimensions at most 192 in the explicit training adapter"
if is_causal and non_square and causal_alignment == "upper_left":
return "does not support non-square upper-left causal attention"
if hopper and dropout_p != 0:
return "does not support nonzero dropout"
major, minor = torch.cuda.get_device_capability(q.device)
if hopper and (major, minor) != (9, 0):
return "requires a Hopper (SM90) GPU in this adapter"
if not hopper and major < 8:
return "requires an Ampere or newer GPU"
return None


@dataclass(frozen=True)
class _AttentionBackend:
extension: Type[_Extension]
device_types: Optional[Tuple[str, ...]] = None
capability: Optional[Callable] = None
package: Optional[str] = None
minimum_version: Optional[Tuple[str, str]] = None
load_kwargs: dict = field(default_factory=dict)
host_lengths: bool = False


class FlashAttentionBackendLoader:
"""Load an explicitly selected attention backend without an execution-time fallback.

Unlike the legacy priority-based loader, a broken installed extension must surface its
import/ABI error. Only a missing optional package makes a backend unavailable to auto.
REGISTRY insertion order defines auto priority within a device type. Generic
backends (device_types=None) come last. Adapters accept causal_alignment and
obey True=allowed masks; native conventions are converted inside the adapter.
Register before first use; clear load's cache if replacing an existing entry.
"""

REGISTRY = {
"fa3": _AttentionBackend(
FlashAttentionDaoV3CudaExtension, ("cuda",), partial(_flash_attention_support, hopper=True), "flash_attn_3"
),
"fa2": _AttentionBackend(
FlashAttentionDaoCudaExtension, ("cuda",), _flash_attention_support, "flash_attn", ("flash-attn", "2.1")
),
"sdpa": _AttentionBackend(FlashAttentionSdpaCudaExtension, load_kwargs={"strict": True}, host_lengths=True),
}

@classmethod
def get_backend(cls, backend):
if not isinstance(backend, str) or backend not in cls.REGISTRY:
raise ValueError(f"Unknown attention backend {backend!r}; expected one of {tuple(cls.REGISTRY)}")
return cls.REGISTRY[backend]

@classmethod
def candidates(cls, device_type):
specific = [
name for name, spec in cls.REGISTRY.items() if spec.device_types and device_type in spec.device_types
]
generic = [name for name, spec in cls.REGISTRY.items() if spec.device_types is None]
return specific + generic

@staticmethod
def is_available(backend: str) -> bool:
spec = FlashAttentionBackendLoader.get_backend(backend)
if spec.package is not None and find_spec(spec.package) is None:
return False
# FA2 changed non-square causal alignment in 2.1. Earlier releases do
# not implement the shared lower-right contract. Do not infer a version
# for an unidentifiable/broken installation.
if spec.minimum_version is not None:
distribution, minimum = spec.minimum_version
return Version(version(distribution)) >= Version(minimum)
return True

@staticmethod
@lru_cache(maxsize=None)
def load(backend: str, packed: bool = False):
spec = FlashAttentionBackendLoader.get_backend(backend)
if not FlashAttentionBackendLoader.is_available(backend):
if spec.minimum_version is not None and find_spec(spec.package) is not None:
distribution, minimum = spec.minimum_version
raise NotImplementedError(f"Attention backend {backend!r} requires {distribution} >= {minimum}")
raise ModuleNotFoundError(f"Attention backend {backend!r} requires its optional package to be installed")
extension = spec.extension()
if packed:
return extension.load_varlen()
return extension.load(**spec.load_kwargs)


class FlashAttentionWithCustomMaskLoader(KernelLoader):
REGISTRY = [FlashAttentionNpuExtension, FlashAttentionSdpaCudaExtension]

Expand Down
Loading
Loading