Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 14 additions & 0 deletions .github/scripts/ci/run_colossalai_batch.sh
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ Batches:
4c Single-GPU Triton inference kernels
4d Single-GPU SmoothQuant tests
4e Single-GPU flash-decoding-attention parameter matrix
4f Single-GPU strict FA3 training attention (requires the optional FA3 package)
5 Two-GPU DDP/FSDP/LowLevelZeRO/LoRA
6 Four-GPU DeviceMesh/DTensor/communication foundations
7 Four-GPU FP8 collectives and DDP/FSDP communication hooks
Expand Down Expand Up @@ -85,6 +86,7 @@ case "${BATCH}" in
tests/test_pipeline/test_pipeline_utils/test_t5_pipeline_utils.py
tests/test_pipeline/test_pipeline_utils/test_whisper_pipeline_utils.py
tests/test_pipeline/test_schedule/test_pipeline_schedule_utils.py
tests/test_shardformer/test_layer/test_attention_backends.py
tests/test_tensor/test_dtensor/test_dtensor_sharding_spec.py
tests/test_tensor/test_shape_consistency.py
tests/test_tensor/test_sharding_spec.py
Expand Down Expand Up @@ -168,6 +170,18 @@ case "${BATCH}" in
NEEDS_CUDA_TOOLKIT=1
TESTS=(tests/test_infer/test_kernels/cuda/test_flash_decoding_attention.py)
;;
4f|04f)
LABEL="04f-fa3-training-attention"
DESCRIPTION="Strict FA3 dense/varlen forward, backward and numerical alignment"
EXPECTED_GPUS=1
DEFAULT_TIMEOUT_MIN=30
TESTS=(tests/test_shardformer/test_layer/test_attention_backends.py)
PYTEST_EXTRA=(-k gpu -o junit_family=xunit1)
# This is acceptance for a requested backend, not an optional discovery
# run. Missing FA3/CUDA must fail rather than produce a green skip.
export COLOSSALAI_TEST_ATTENTION_BACKEND=fa3
export COLOSSALAI_ATTENTION_TRACE_DIR="${RESULTS_ROOT}/fa3-traces"
;;
5|05)
LABEL="05-booster-lora-2gpu"
DESCRIPTION="DDP, FSDP, LowLevelZeRO, LoRA and the numerical alignment tool on its plugins"
Expand Down
1 change: 1 addition & 0 deletions .github/scripts/ci/run_cpu_unit_tests.sh
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ TESTS=(
tests/test_pipeline/test_pipeline_utils/test_t5_pipeline_utils.py
tests/test_pipeline/test_pipeline_utils/test_whisper_pipeline_utils.py
tests/test_pipeline/test_schedule/test_pipeline_schedule_utils.py
tests/test_shardformer/test_layer/test_attention_backends.py
tests/test_tensor/test_dtensor/test_dtensor_sharding_spec.py
tests/test_tensor/test_shape_consistency.py
tests/test_tensor/test_sharding_spec.py
Expand Down
1 change: 1 addition & 0 deletions .github/scripts/ci/run_gpu_batch.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
"4c": 1,
"4d": 1,
"4e": 1,
"4f": 1,
"5": 2,
"6": 4,
"7": 4,
Expand Down
45 changes: 45 additions & 0 deletions colossalai/kernel/kernel_loader.py
Original file line number Diff line number Diff line change
@@ -1,10 +1,16 @@
import warnings
from functools import lru_cache
from importlib.metadata import version
from importlib.util import find_spec
from typing import List

from packaging.version import Version

from .extensions import (
CpuAdamArmExtension,
CpuAdamX86Extension,
FlashAttentionDaoCudaExtension,
FlashAttentionDaoV3CudaExtension,
FlashAttentionNpuExtension,
FlashAttentionSdpaCudaExtension,
FusedOptimizerCudaExtension,
Expand Down Expand Up @@ -124,6 +130,45 @@ class FlashAttentionDaoLoader(KernelLoader):
REGISTRY = [FlashAttentionDaoCudaExtension]


class FlashAttentionBackendLoader:
"""Load an explicitly selected attention backend without an execution-time fallback.

Unlike the legacy priority-based loader, a broken installed extension must surface its
import/ABI error. Only a missing optional package makes a backend unavailable to auto.
Callers check request-specific capabilities before using this loader.
"""

@staticmethod
def is_available(backend: str) -> bool:
if backend == "sdpa":
return True
package = {"fa2": "flash_attn", "fa3": "flash_attn_3"}[backend]
if find_spec(package) is None:
return False
# FA2 changed non-square causal alignment in 2.1. Earlier releases do
# not implement the shared lower-right contract. Do not infer a version
# for an unidentifiable/broken installation.
return backend != "fa2" or Version(version("flash-attn")) >= Version("2.1")

@staticmethod
@lru_cache(maxsize=6)
def load(backend: str, packed: bool = False):
extension = {
"fa2": FlashAttentionDaoCudaExtension,
"fa3": FlashAttentionDaoV3CudaExtension,
"sdpa": FlashAttentionSdpaCudaExtension,
}[backend]()
if not FlashAttentionBackendLoader.is_available(backend):
if backend == "fa2" and find_spec("flash_attn") is not None:
raise NotImplementedError("The explicit FA2 backend requires flash-attn >= 2.1")
raise ModuleNotFoundError(f"Attention backend {backend!r} requires its optional package to be installed")
if packed:
return extension.load_varlen()
if backend == "sdpa":
return extension.load(strict=True)
return extension.load()


class FlashAttentionWithCustomMaskLoader(KernelLoader):
REGISTRY = [FlashAttentionNpuExtension, FlashAttentionSdpaCudaExtension]

Expand Down
Loading
Loading