From a126452dc2d31065301d244130ab3bc804fd9c04 Mon Sep 17 00:00:00 2001 From: sanyalington Date: Sun, 10 Nov 2024 11:22:43 +0000 Subject: [PATCH 01/10] opts v1 --- csrc/rocm/attention.cu | 402 +++++++++++++++++++++++++++++++++-------- 1 file changed, 331 insertions(+), 71 deletions(-) diff --git a/csrc/rocm/attention.cu b/csrc/rocm/attention.cu index efda714f53c6..a62f4b30a703 100644 --- a/csrc/rocm/attention.cu +++ b/csrc/rocm/attention.cu @@ -139,6 +139,14 @@ __device__ __forceinline__ _B16x4 from_floatx4(const floatx4& inp) { __hip_bfloat16 b; } t16; _B16x4 ret; +#if 0 + #pragma unroll + for (int i = 0; i < 4; i++) { + t16.f = (_Float16)inp[i]; + ret[i] = t16.u; + } + return ret; +#else if constexpr (std::is_same::value) { #pragma unroll for (int i = 0; i < 4; i++) { @@ -149,13 +157,20 @@ __device__ __forceinline__ _B16x4 from_floatx4(const floatx4& inp) { } else if constexpr (std::is_same::value) { #pragma unroll for (int i = 0; i < 4; i++) { - t16.b = __float2bfloat16(inp[i]); - ret[i] = t16.u; + union fcvt { + uint32_t i32; + float f32; + } u; + u.f32 = inp[i]; + ret[i] = uint16_t(u.i32 >> 16); + //t16.b = __float2bfloat16(inp[i]); + //ret[i] = t16.u; } return ret; } else { static_assert(false, "unsupported 16b dtype"); } +#endif } template @@ -167,7 +182,7 @@ __device__ __forceinline__ _B16x4 addx4(const _B16x4& inp1, __hip_bfloat16 b; } t1, t2, res; _B16x4 ret; - if constexpr (std::is_same::value) { +#if 0 #pragma unroll for (int i = 0; i < 4; i++) { t1.u = inp1[i]; @@ -176,18 +191,37 @@ __device__ __forceinline__ _B16x4 addx4(const _B16x4& inp1, ret[i] = res.u; } return ret; - } else if constexpr (std::is_same::value) { +#else + if constexpr (std::is_same::value) { #pragma unroll for (int i = 0; i < 4; i++) { t1.u = inp1[i]; t2.u = inp2[i]; - res.b = t1.b + t2.b; + res.f = t1.f + t2.f; ret[i] = res.u; } return ret; + } else if constexpr (std::is_same::value) { + #pragma unroll + for (int i = 0; i < 4; i++) { + union fcvt { + float f32; + uint32_t i32; + } u1,u2,s; + u1.i32 = uint32_t(inp1[i])<<16; + u2.i32 = uint32_t(inp2[i])<<16; + s.f32 = u1.f32 + u2.f32; + ret[i] = uint16_t(s.i32>>16); + //t1.u = inp1[i]; + //t2.u = inp2[i]; + //res.b = t1.b + t2.b; + //ret[i] = res.u; + } + return ret; } else { static_assert(false, "unsupported 16b dtype"); } +#endif } template @@ -210,6 +244,36 @@ __device__ __forceinline__ _B16x8 scaled_convert_b8x8(const _B8x8 input, } } +template +__device__ __forceinline__ _B16x8 scaled_convert_b8x8_custom(const _B8x8 input, + const float scale) { + union { + floatx4 f32x4[2]; + vllm::Float8_ f32x8; + } tmpf8; + tmpf8.f32x8 = vllm::fp8::vec_conversion(*reinterpret_cast(&input)); + + tmpf8.f32x4[0] *= scale; + tmpf8.f32x4[1] *= scale; + + _B16x8 ret; + ret.xy[0] = from_floatx4(tmpf8.f32x4[0]); + ret.xy[1] = from_floatx4(tmpf8.f32x4[1]); + return ret; +} + +template +__device__ __forceinline__ _B16x8 convert_b8x8_custom(const _B8x8 input) { + union { + floatx4 f32x4[2]; + vllm::Float8_ f32x8; + } tmpf8; + tmpf8.f32x8 = vllm::fp8::vec_conversion(*reinterpret_cast(&input)); + _B16x8 ret; + ret.xy[0] = from_floatx4(tmpf8.f32x4[0]); + ret.xy[1] = from_floatx4(tmpf8.f32x4[1]); + return ret; +} /////////////////////////////////////// // grid (num_seqs, num_partitions,num_heads/gqa_ratio) @@ -270,6 +334,8 @@ __global__ __launch_bounds__(NUM_THREADS) void paged_attention_ll4mi_QKV_kernel( WARP_SIZE; // v head_size dimension is distributed across lanes constexpr int VTLOOP = 8; // 16 separate 4xtokens across warp -> 16/2 // 8xtokens + constexpr int VBLOCKS = 8 * VTLOOP / BLOCK_SIZE; + int vphysical_blocks[VBLOCKS]; _B16x8 Vlocal[VHELOOP][VTLOOP]; _B8x8 Vlocalb8[VHELOOP][VTLOOP]; floatx4 dout[QHLOOP]; @@ -312,8 +378,6 @@ __global__ __launch_bounds__(NUM_THREADS) void paged_attention_ll4mi_QKV_kernel( static_cast(block_table[block_idx]); // fetch vphysical block numbers up front - constexpr int VBLOCKS = 8 * VTLOOP / BLOCK_SIZE; - int vphysical_blocks[VBLOCKS]; const int warp_start_block_idx = warp_start_token_idx / BLOCK_SIZE; if constexpr (GQA_RATIO < 12) { @@ -392,6 +456,7 @@ __global__ __launch_bounds__(NUM_THREADS) void paged_attention_ll4mi_QKV_kernel( } } +#if 1 //fetch vcache in normal case const cache_t* v_ptr = v_cache + wg_start_kv_head_idx * kv_head_stride; if constexpr (KV_DTYPE == vllm::Fp8KVCacheDataType::kAuto) { const _B16x8* v_ptrh8 = reinterpret_cast(v_ptr); @@ -416,7 +481,10 @@ __global__ __launch_bounds__(NUM_THREADS) void paged_attention_ll4mi_QKV_kernel( } } } - } else { + } //if constexpr (KV_DTYPE == vllm::Fp8KVCacheDataType::kAuto) +#endif +#if 1 //fetch vcache in fp8 case + else { // if constexpr (KV_DTYPE != vllm::Fp8KVCacheDataType::kAuto) const _B8x8* v_ptrh8 = reinterpret_cast(v_ptr); // iterate over each v block #pragma unroll @@ -435,23 +503,73 @@ __global__ __launch_bounds__(NUM_THREADS) void paged_attention_ll4mi_QKV_kernel( // iterate over all velems within block #pragma unroll for (int d = 0; d < BLOCK_SIZE / 8; d++) { - // Vlocalb8[h][b * BLOCK_SIZE / 8 + d] = v_ptrh8be[d]; - const _B8x8 Vlocalb8 = v_ptrh8be[d]; - Vlocal[h][b * BLOCK_SIZE / 8 + d] = - scaled_convert_b8x8(Vlocalb8, v_scale); + Vlocalb8[h][b * BLOCK_SIZE / 8 + d] = v_ptrh8be[d]; + //const _B8x8 Vlocalb8 = v_ptrh8be[d]; + //Vlocal[h][b * BLOCK_SIZE / 8 + d] = + // scaled_convert_b8x8(Vlocalb8, v_scale); } } } } - +#endif +#if 0 //cvt kf8 to kf/bf16 up front if constexpr (KV_DTYPE != vllm::Fp8KVCacheDataType::kAuto) { #pragma unroll for (int d = 0; d < KHELOOP; d++) { Klocal[d] = - scaled_convert_b8x8(Klocalb8[d], k_scale); + //scaled_convert_b8x8(Klocalb8[d], k_scale); + convert_b8x8_custom(Klocalb8[d]); } } +#endif + + /*Klocal[x] = scaled_convert_b8x8(Klocalb8[x], k_scale); \*/ + /*Klocal[x] = scaled_convert_b8x8_custom(Klocalb8[x], k_scale); \*/ +#define QK_mfma(x) \ + if constexpr (KV_DTYPE != vllm::Fp8KVCacheDataType::kAuto) { \ + Klocal[x] = convert_b8x8_custom(Klocalb8[x]); \ + } \ + for (int h = 0; h < QHLOOP; h++) { \ + dout[h] = gcn_mfma_instr(Qlocal[h].xy[0], \ + Klocal[x].xy[0], dout[h]);\ + dout[h] = gcn_mfma_instr(Qlocal[h].xy[1], \ + Klocal[x].xy[1], dout[h]);\ + } + //#pragma unroll + //for (int h = 0; h < QHLOOP; h++) { + QK_mfma(0); + QK_mfma(1); + QK_mfma(2); + QK_mfma(3); + QK_mfma(4); + QK_mfma(5); + QK_mfma(6); + QK_mfma(7); + if constexpr (KHELOOP > 8) { + QK_mfma(8); + QK_mfma(9); + QK_mfma(10); + QK_mfma(11); + QK_mfma(12); + QK_mfma(13); + QK_mfma(14); + QK_mfma(15); + } + //} +#undef QK_mfma + float scale2 = scale; + if constexpr (KV_DTYPE != vllm::Fp8KVCacheDataType::kAuto) { + scale2 *= k_scale; + } + #pragma unroll + for (int h = 0; h < QHLOOP; h++) { + dout[h] *= scale2; + //if constexpr (KV_DTYPE != vllm::Fp8KVCacheDataType::kAuto) { + // dout[h] *= k_scale; + //} + } +#if 0 #pragma unroll for (int h = 0; h < QHLOOP; h++) { dout[h] = gcn_mfma_instr(Qlocal[h].xy[0], @@ -522,6 +640,7 @@ __global__ __launch_bounds__(NUM_THREADS) void paged_attention_ll4mi_QKV_kernel( } // KHELOOP>8 dout[h] *= scale; } +#endif // transpose dout so that 4 token ids are in each lane, and 4 heads are across // 4 lanes #pragma unroll @@ -641,6 +760,139 @@ __global__ __launch_bounds__(NUM_THREADS) void paged_attention_ll4mi_QKV_kernel( } } } else { // warp in context +#if 0 //fetch v cache + const cache_t* v_ptr = v_cache + wg_start_kv_head_idx * kv_head_stride; + if constexpr (KV_DTYPE == vllm::Fp8KVCacheDataType::kAuto) { + const _B16x8* v_ptrh8 = reinterpret_cast(v_ptr); + // iterate over each v block + #pragma unroll + for (int b = 0; b < VBLOCKS; b++) { + // int32 physical_block_number leads to overflow when multiplied with + // kv_block_stride + const int64_t vphysical_block_number = + static_cast(vphysical_blocks[b]); + const _B16x8* v_ptrh8b = + v_ptrh8 + (vphysical_block_number * kv_block_stride) / 8; + // iterate over each head elem (within head_size) + #pragma unroll + for (int h = 0; h < VHELOOP; h++) { + const int head_size_elem = h * WARP_SIZE + laneid; + const _B16x8* v_ptrh8be = v_ptrh8b + head_size_elem * BLOCK_SIZE / 8; + // iterate over all velems within block + #pragma unroll + for (int d = 0; d < BLOCK_SIZE / 8; d++) { + Vlocal[h][b * BLOCK_SIZE / 8 + d] = v_ptrh8be[d]; + } + } + } + } //if constexpr (KV_DTYPE == vllm::Fp8KVCacheDataType::kAuto) + + if constexpr (KV_DTYPE != vllm::Fp8KVCacheDataType::kAuto) { + const _B8x8* v_ptrh8 = reinterpret_cast(v_ptr); + // iterate over each v block + #pragma unroll + for (int b = 0; b < VBLOCKS; b++) { + // int32 physical_block_number leads to overflow when multiplied with + // kv_block_stride + const int64_t vphysical_block_number = + static_cast(vphysical_blocks[b]); + const _B8x8* v_ptrh8b = + v_ptrh8 + (vphysical_block_number * kv_block_stride) / 8; + // iterate over each head elem (within head_size) + #pragma unroll + for (int h = 0; h < VHELOOP; h++) { + const int head_size_elem = h * WARP_SIZE + laneid; + const _B8x8* v_ptrh8be = v_ptrh8b + head_size_elem * BLOCK_SIZE / 8; + // iterate over all velems within block + #pragma unroll + for (int d = 0; d < BLOCK_SIZE / 8; d++) { + Vlocalb8[h][b * BLOCK_SIZE / 8 + d] = v_ptrh8be[d]; + //const _B8x8 Vlocalb8 = v_ptrh8be[d]; + //Vlocal[h][b * BLOCK_SIZE / 8 + d] = + // scaled_convert_b8x8(Vlocalb8, v_scale); + } + } + } + } +#endif +#if 0 //cvt vf8 ->f16/bf16 up front + if constexpr (KV_DTYPE != vllm::Fp8KVCacheDataType::kAuto) { + for (int vh = 0; vh < VHELOOP; vh++) { + for (int b=0; b < VTLOOP; b++) { + //Vlocal[vh][b] = scaled_convert_b8x8(Vlocalb8[vh][b], v_scale); + Vlocal[vh][b] = convert_b8x8_custom(Vlocalb8[vh][b]); + } + } + } +#endif + + /*Vlocal[vh][x] = scaled_convert_b8x8(Vlocalb8[vh][x], v_scale);\*/ + /*Vlocal[vh][x] = scaled_convert_b8x8_custom(Vlocalb8[vh][x], v_scale);\*/ + #define SV_mfma(x) \ + if constexpr (KV_DTYPE != vllm::Fp8KVCacheDataType::kAuto) {\ + Vlocal[vh][x] = convert_b8x8_custom(Vlocalb8[vh][x]);\ + }\ + for (int qh = 0; qh < QHLOOP; qh++) { \ + acc[qh] = gcn_mfma_instr(logits[qh], Vlocal[vh][x].xy[0], \ + acc[qh]); \ + acc[qh] = gcn_mfma_instr(logits[qh], Vlocal[vh][x].xy[1], \ + acc[qh]); \ + } +#if 0 + floatx4 acc[QHLOOP][VHELOOP]; + for (int qh = 0; qh < QHLOOP; qh++) { + for (int vh = 0; vh < VHELOOP; vh++) { + acc[qh][vh] = {0}; + } + } +#endif + //#pragma unroll + // for (int qh = 0; qh < QHLOOP; qh++) { + // iterate over each v head elem (within head_size) + //#pragma unroll + for (int vh = 0; vh < VHELOOP; vh++) { + floatx4 acc[QHLOOP]; + for (int qh = 0; qh < QHLOOP; qh++) { + acc[qh] = {0}; + } + // iterate over tokens + SV_mfma(0); + SV_mfma(1); + SV_mfma(2); + SV_mfma(3); + SV_mfma(4); + SV_mfma(5); + SV_mfma(6); + SV_mfma(7); +#if 0 + SV_mfma(8); + SV_mfma(9); + SV_mfma(10); + SV_mfma(11); + SV_mfma(12); + SV_mfma(13); + SV_mfma(14); + SV_mfma(15); +#endif + for (int qh = 0; qh < QHLOOP; qh++) { + if constexpr (KV_DTYPE != vllm::Fp8KVCacheDataType::kAuto) { + acc[qh] *= v_scale; + } + vout_shared[qh][vh][laneid][warpid] = from_floatx4(acc[qh]); + } + } + //} + +#if 0 + for (int qh = 0; qh < QHLOOP; qh++) { + for (int vh = 0; vh < VHELOOP; vh++) { + vout_shared[qh][vh][laneid][warpid] = from_floatx4(acc[qh][vh]); + } + } +#endif + +#undef SV_mfma +#if 0 // iterate across heads #pragma unroll for (int qh = 0; qh < QHLOOP; qh++) { @@ -684,6 +936,7 @@ __global__ __launch_bounds__(NUM_THREADS) void paged_attention_ll4mi_QKV_kernel( vout_shared[qh][vh][laneid][warpid] = from_floatx4(acc); } } +#endif } // warp in context __syncthreads(); @@ -1088,54 +1341,54 @@ void paged_attention_custom_launcher( const at::cuda::OptionalCUDAGuard device_guard(device_of(query)); const cudaStream_t stream = at::cuda::getCurrentCUDAStream(); switch (gqa_ratio) { - case 1: - LAUNCH_CUSTOM_ATTENTION(1); - break; - case 2: - LAUNCH_CUSTOM_ATTENTION(2); - break; - case 3: - LAUNCH_CUSTOM_ATTENTION(3); - break; - case 4: - LAUNCH_CUSTOM_ATTENTION(4); - break; - case 5: - LAUNCH_CUSTOM_ATTENTION(5); - break; - case 6: - LAUNCH_CUSTOM_ATTENTION(6); - break; - case 7: - LAUNCH_CUSTOM_ATTENTION(7); - break; + //case 1: + // LAUNCH_CUSTOM_ATTENTION(1); + // break; + //case 2: + // LAUNCH_CUSTOM_ATTENTION(2); + // break; + //case 3: + // LAUNCH_CUSTOM_ATTENTION(3); + // break; + //case 4: + // LAUNCH_CUSTOM_ATTENTION(4); + // break; + //case 5: + // LAUNCH_CUSTOM_ATTENTION(5); + // break; + //case 6: + // LAUNCH_CUSTOM_ATTENTION(6); + // break; + //case 7: + // LAUNCH_CUSTOM_ATTENTION(7); + // break; case 8: LAUNCH_CUSTOM_ATTENTION(8); break; - case 9: - LAUNCH_CUSTOM_ATTENTION(9); - break; - case 10: - LAUNCH_CUSTOM_ATTENTION(10); - break; - case 11: - LAUNCH_CUSTOM_ATTENTION(11); - break; - case 12: - LAUNCH_CUSTOM_ATTENTION(12); - break; - case 13: - LAUNCH_CUSTOM_ATTENTION(13); - break; - case 14: - LAUNCH_CUSTOM_ATTENTION(14); - break; - case 15: - LAUNCH_CUSTOM_ATTENTION(15); - break; - case 16: - LAUNCH_CUSTOM_ATTENTION(16); - break; + //case 9: + // LAUNCH_CUSTOM_ATTENTION(9); + // break; + //case 10: + // LAUNCH_CUSTOM_ATTENTION(10); + // break; + //case 11: + // LAUNCH_CUSTOM_ATTENTION(11); + // break; + //case 12: + // LAUNCH_CUSTOM_ATTENTION(12); + // break; + //case 13: + // LAUNCH_CUSTOM_ATTENTION(13); + // break; + //case 14: + // LAUNCH_CUSTOM_ATTENTION(14); + // break; + //case 15: + // LAUNCH_CUSTOM_ATTENTION(15); + // break; + //case 16: + // LAUNCH_CUSTOM_ATTENTION(16); + // break; default: TORCH_CHECK(false, "Unsupported gqa ratio: ", gqa_ratio); break; @@ -1197,14 +1450,15 @@ void paged_attention_custom_launcher( case 256: \ CALL_CUSTOM_LAUNCHER(T, KVT, KV_DTYPE, BLK_SIZE, HEAD_SIZE, OUTT, 256); \ break; \ - case 512: \ - CALL_CUSTOM_LAUNCHER(T, KVT, KV_DTYPE, BLK_SIZE, HEAD_SIZE, OUTT, 512); \ - break; \ default: \ TORCH_CHECK(false, "Unsupported partition size: ", partition_size); \ break; \ } - +/* + case 512: \ + CALL_CUSTOM_LAUNCHER(T, KVT, KV_DTYPE, BLK_SIZE, HEAD_SIZE, OUTT, 512); \ + break; \ +*/ #if defined(__HIPCC__) && defined(__gfx90a__) #define CALL_CUSTOM_LAUNCHER_OUT(T, KVT, KV_DTYPE, BLK_SIZE, HEAD_SIZE) \ if (fp8_out_scale) { \ @@ -1213,6 +1467,9 @@ void paged_attention_custom_launcher( CALL_CUSTOM_LAUNCHER_PSIZE(T, KVT, KV_DTYPE, BLK_SIZE, HEAD_SIZE, T); \ } #else + #define CALL_CUSTOM_LAUNCHER_OUT(T, KVT, KV_DTYPE, BLK_SIZE, HEAD_SIZE) \ + CALL_CUSTOM_LAUNCHER_PSIZE(T, KVT, KV_DTYPE, BLK_SIZE, HEAD_SIZE, T); +/* #define CALL_CUSTOM_LAUNCHER_OUT(T, KVT, KV_DTYPE, BLK_SIZE, HEAD_SIZE) \ if (fp8_out_scale) { \ CALL_CUSTOM_LAUNCHER_PSIZE(T, KVT, KV_DTYPE, BLK_SIZE, HEAD_SIZE, \ @@ -1220,25 +1477,24 @@ void paged_attention_custom_launcher( } else { \ CALL_CUSTOM_LAUNCHER_PSIZE(T, KVT, KV_DTYPE, BLK_SIZE, HEAD_SIZE, T); \ } + */ #endif #define CALL_CUSTOM_LAUNCHER_BLK(T, KVT, KV_DTYPE, HEAD_SIZE) \ switch (block_size) { \ case 16: \ CALL_CUSTOM_LAUNCHER_OUT(T, KVT, KV_DTYPE, 16, HEAD_SIZE); \ break; \ - case 32: \ - CALL_CUSTOM_LAUNCHER_OUT(T, KVT, KV_DTYPE, 32, HEAD_SIZE); \ - break; \ default: \ TORCH_CHECK(false, "Unsupported block size: ", block_size); \ break; \ } - +/* + case 32: \ + CALL_CUSTOM_LAUNCHER_OUT(T, KVT, KV_DTYPE, 32, HEAD_SIZE); \ + break; \ +*/ #define CALL_CUSTOM_LAUNCHER_BLK_HEAD(T, KVT, KV_DTYPE) \ switch (head_size) { \ - case 64: \ - CALL_CUSTOM_LAUNCHER_BLK(T, KVT, KV_DTYPE, 64); \ - break; \ case 128: \ CALL_CUSTOM_LAUNCHER_BLK(T, KVT, KV_DTYPE, 128); \ break; \ @@ -1246,7 +1502,11 @@ void paged_attention_custom_launcher( TORCH_CHECK(false, "Unsupported head size: ", head_size); \ break; \ } - +/* + case 64: \ + CALL_CUSTOM_LAUNCHER_BLK(T, KVT, KV_DTYPE, 64); \ + break; \ +*/ void paged_attention( torch::Tensor& out, // [num_seqs, num_heads, head_size] torch::Tensor& exp_sums, // [num_seqs, num_heads, max_num_partitions] From 98270da18902393f11a86f8bfe3e1920d5cc8d22 Mon Sep 17 00:00:00 2001 From: sanyalington Date: Mon, 18 Nov 2024 16:10:32 +0000 Subject: [PATCH 02/10] opt logits and expsum calc --- .../kernels/benchmark_paged_attention.py | 9 +- csrc/rocm/attention.cu | 137 +++++++++++++++++- csrc/rocm/torch_bindings.cpp | 4 +- 3 files changed, 138 insertions(+), 12 deletions(-) diff --git a/benchmarks/kernels/benchmark_paged_attention.py b/benchmarks/kernels/benchmark_paged_attention.py index 483584dd804e..1090754d1566 100644 --- a/benchmarks/kernels/benchmark_paged_attention.py +++ b/benchmarks/kernels/benchmark_paged_attention.py @@ -10,8 +10,7 @@ create_kv_caches_with_random) NUM_BLOCKS = 1024 * 1024 -PARTITION_SIZE = 512 - +PARTITION_SIZE = 256 @torch.inference_mode() def main( @@ -101,7 +100,7 @@ def run_cuda_benchmark(num_iters: int, profile: bool = False) -> float: start_time = time.perf_counter() # Using default kv_scale - k_scale = v_scale = 1.0 + k_scale = v_scale = 0.1 for _ in range(num_iters): if version == "v1": @@ -161,6 +160,8 @@ def run_cuda_benchmark(num_iters: int, profile: bool = False) -> float: kv_cache_dtype, k_scale, v_scale, + None, + PARTITION_SIZE ) else: raise ValueError(f"Invalid version: {version}") @@ -180,7 +181,7 @@ def run_cuda_benchmark(num_iters: int, profile: bool = False) -> float: if do_profile: latency = run_benchmark(num_iters=1, profile=True) else: - latency = run_benchmark(num_iters=1000, profile=False) + latency = run_benchmark(num_iters=5000, profile=False) print(f"Kernel running time: {latency * 1000000:.3f} us") diff --git a/csrc/rocm/attention.cu b/csrc/rocm/attention.cu index a62f4b30a703..4f9a626ccc97 100644 --- a/csrc/rocm/attention.cu +++ b/csrc/rocm/attention.cu @@ -148,12 +148,22 @@ __device__ __forceinline__ _B16x4 from_floatx4(const floatx4& inp) { return ret; #else if constexpr (std::is_same::value) { +#if 0 #pragma unroll for (int i = 0; i < 4; i++) { t16.f = (_Float16)inp[i]; ret[i] = t16.u; } return ret; +#else + union h2cvt { + __half2 h2[2]; + _B16x4 b16x4; + } u; + u.h2[0] = __float22half2_rn(make_float2(inp[0],inp[1])); + u.h2[1] = __float22half2_rn(make_float2(inp[2],inp[3])); + return u.b16x4; +#endif } else if constexpr (std::is_same::value) { #pragma unroll for (int i = 0; i < 4; i++) { @@ -193,6 +203,7 @@ __device__ __forceinline__ _B16x4 addx4(const _B16x4& inp1, return ret; #else if constexpr (std::is_same::value) { +#if 0 #pragma unroll for (int i = 0; i < 4; i++) { t1.u = inp1[i]; @@ -201,6 +212,17 @@ __device__ __forceinline__ _B16x4 addx4(const _B16x4& inp1, ret[i] = res.u; } return ret; +#else + union h2cvt { + _B16x4 b16x4; + __half2 h2[2]; + } u1,u2,s; + u1.b16x4 = inp1; + u2.b16x4 = inp2; + s.h2[0] = u1.h2[0] + u2.h2[0]; + s.h2[1] = u1.h2[1] + u2.h2[1]; + return s.b16x4; +#endif } else if constexpr (std::is_same::value) { #pragma unroll for (int i = 0; i < 4; i++) { @@ -340,6 +362,7 @@ __global__ __launch_bounds__(NUM_THREADS) void paged_attention_ll4mi_QKV_kernel( _B8x8 Vlocalb8[VHELOOP][VTLOOP]; floatx4 dout[QHLOOP]; float qk_max[QHLOOP]; + __shared__ _B16x4 vout_shared[QHLOOP][VHELOOP][WARP_SIZE][NWARPS + 1]; #pragma unroll for (int h = 0; h < QHLOOP; h++) { dout[h] = {0}; @@ -434,6 +457,7 @@ __global__ __launch_bounds__(NUM_THREADS) void paged_attention_ll4mi_QKV_kernel( } } +#if 1 float alibi_slope[QHLOOP]; if (alibi_slopes != nullptr) { #pragma unroll @@ -444,6 +468,26 @@ __global__ __launch_bounds__(NUM_THREADS) void paged_attention_ll4mi_QKV_kernel( : 0.f; } } +#endif +#if 0 + float alibi_slope; + const int lane16id = laneid % 16; + if (alibi_slopes != nullptr) { + alibi_slope = (lane16id < GQA_RATIO) + ? alibi_slopes[wg_start_head_idx + lane16id] + : 0.f; + //#pragma unroll + // for (int h = 0; h < QHLOOP; h++) { + // for (int i=0; i<4; i++) { + // const int qhead_idx = h * 4 + i; + // alibi_slope[qhead_idx] = (qhead_idx < GQA_RATIO) + // ? alibi_slopes[wg_start_head_idx + qhead_idx] + // : 0.f; + // } + //} + //} + } +#endif // fetch vphysical block numbers up front if constexpr (GQA_RATIO >= 12) { @@ -641,10 +685,38 @@ __global__ __launch_bounds__(NUM_THREADS) void paged_attention_ll4mi_QKV_kernel( dout[h] *= scale; } #endif + +#if 0 + if (alibi_slopes != nullptr) { + float alibi_slope_local[GQA_RATIO]; +#define DPP_BCAST_ASM(id) asm("s_nop 0\n\tv_mov_b32_dpp %0, %1 row_newbcast:id " : "=v"(alibi_slope_local[id]) : "v"(alibi_slope)); + //for (int head=0; head < 16; head++) { + //DPP_BCAST_ASM(0); + if constexpr(GQA_RATIO>0) { asm("s_nop 0\n\tv_mov_b32_dpp %0, %1 row_newbcast:0 " : "=v"(alibi_slope_local[0]) : "v"(alibi_slope));} + if constexpr(GQA_RATIO>1) { asm("s_nop 0\n\tv_mov_b32_dpp %0, %1 row_newbcast:1 " : "=v"(alibi_slope_local[1]) : "v"(alibi_slope));} + if constexpr(GQA_RATIO>2) { asm("s_nop 0\n\tv_mov_b32_dpp %0, %1 row_newbcast:2 " : "=v"(alibi_slope_local[2]) : "v"(alibi_slope));} + if constexpr(GQA_RATIO>3) { asm("s_nop 0\n\tv_mov_b32_dpp %0, %1 row_newbcast:3 " : "=v"(alibi_slope_local[3]) : "v"(alibi_slope));} + if constexpr(GQA_RATIO>4) { asm("s_nop 0\n\tv_mov_b32_dpp %0, %1 row_newbcast:4 " : "=v"(alibi_slope_local[4]) : "v"(alibi_slope));} + if constexpr(GQA_RATIO>5) { asm("s_nop 0\n\tv_mov_b32_dpp %0, %1 row_newbcast:5 " : "=v"(alibi_slope_local[5]) : "v"(alibi_slope));} + if constexpr(GQA_RATIO>6) { asm("s_nop 0\n\tv_mov_b32_dpp %0, %1 row_newbcast:6 " : "=v"(alibi_slope_local[6]) : "v"(alibi_slope));} + if constexpr(GQA_RATIO>7) { asm("s_nop 0\n\tv_mov_b32_dpp %0, %1 row_newbcast:7 " : "=v"(alibi_slope_local[7]) : "v"(alibi_slope));} + //} + + const int alibi_offset = global_token_idx - context_len + 1; + #pragma unroll + for (int h = 0; h < QHLOOP; h++) { + #pragma unroll + for (int i = 0; i < 4; i++) { + dout[h][i] += alibi_slope_local[4*h+i] * alibi_offset; + } + } + } +#endif // transpose dout so that 4 token ids are in each lane, and 4 heads are across // 4 lanes #pragma unroll for (int h = 0; h < QHLOOP; h++) { +#if 1 floatx4 tmp = {0}; #pragma unroll for (int i = 0; i < 4; i++) { @@ -654,9 +726,38 @@ __global__ __launch_bounds__(NUM_THREADS) void paged_attention_ll4mi_QKV_kernel( // tmp = __builtin_amdgcn_mfma_f32_4x4x1f32(A, B, tmp, 0, 0, 0); } dout[h] = tmp; +#endif +#if 0 + asm("s_nop 0\n\t v_mov_b32_dpp %0, %1 quad_perm:[1,0,3,2] " : "=v"(dout[h][1]) : "v"(dout[h][1]) ); + asm("s_nop 0\n\t v_mov_b32_dpp %0, %1 quad_perm:[2,3,0,1] " : "=v"(dout[h][2]) : "v"(dout[h][2]) ); + asm("s_nop 0\n\t v_mov_b32_dpp %0, %1 quad_perm:[3,2,1,0] " : "=v"(dout[h][3]) : "v"(dout[h][3]) ); + + bool mask = (lane4id % 2) == 1; + float tmp = dout[h][1]; + dout[h][1] = mask ? dout[h][0] : dout[h][1]; + dout[h][0] = mask ? tmp : dout[h][0]; + tmp = dout[h][3]; + dout[h][3] = mask ? dout[h][2] : dout[h][3]; + dout[h][2] = mask ? tmp : dout[h][2]; + + mask = (lane4id>>1) == 1; + tmp = dout[h][2]; + dout[h][2] = mask ? dout[h][0] : dout[h][2]; + dout[h][0] = mask ? tmp : dout[h][0]; + tmp = dout[h][3]; + dout[h][3] = mask ? dout[h][1] : dout[h][3]; + dout[h][1] = mask ? tmp : dout[h][1]; + + + asm("s_nop 0\n\t v_mov_b32_dpp %0, %1 quad_perm:[1,0,3,2] " : "=v"(dout[h][1]) : "v"(dout[h][1]) ); + asm("s_nop 0\n\t v_mov_b32_dpp %0, %1 quad_perm:[2,3,0,1] " : "=v"(dout[h][2]) : "v"(dout[h][2]) ); + asm("s_nop 0\n\t v_mov_b32_dpp %0, %1 quad_perm:[3,2,1,0] " : "=v"(dout[h][3]) : "v"(dout[h][3]) ); + +#endif } const int lane4_token_idx = 4 * (global_token_idx >> 2); +#if 1 //alibi after transpose const int alibi_offset = lane4_token_idx - context_len + 1; if (alibi_slopes != nullptr) { #pragma unroll @@ -667,6 +768,9 @@ __global__ __launch_bounds__(NUM_THREADS) void paged_attention_ll4mi_QKV_kernel( } } } +#endif + + const int bpermute_mask = 4*(16*((laneid>>2)%4) + lane4id); #pragma unroll for (int h = 0; h < QHLOOP; h++) { @@ -678,11 +782,22 @@ __global__ __launch_bounds__(NUM_THREADS) void paged_attention_ll4mi_QKV_kernel( : qk_max[h]; } #pragma unroll - for (int mask = WARP_SIZE / 2; mask >= 4; mask /= 2) { + for (int mask = WARP_SIZE / 2; mask >= 64; mask /= 2) { qk_max[h] = fmaxf(qk_max[h], __shfl_xor(qk_max[h], mask)); } + asm("v_nop\n v_nop\n v_max_f32_dpp %0, %1, %2 row_ror:4" : "=v"(qk_max[h]) : "v"(qk_max[h]), "v"(qk_max[h]) ); + asm("v_nop\n v_nop\n v_max_f32_dpp %0, %1, %2 row_ror:8" : "=v"(qk_max[h]) : "v"(qk_max[h]), "v"(qk_max[h]) ); + + //asm("v_nop\n v_nop\n ds_bpermute_b32 %0, %1, %2 \n s_waitcnt lgkmcnt(0)" : "=v"(qk_max[h]) : "v"(bpermute_mask), "v"(qk_max[h]) ); + + //qk_max[h] = __builtin_amdgcn_ds_bpermute(bpermute_mask, qk_max[h]); + auto tmp = __builtin_amdgcn_ds_bpermute(bpermute_mask, *reinterpret_cast(&qk_max[h])); + qk_max[h] = *reinterpret_cast(&tmp); + asm("v_nop\n v_nop\n v_max_f32_dpp %0, %1, %2 row_ror:4" : "=v"(qk_max[h]) : "v"(qk_max[h]), "v"(qk_max[h]) ); + asm("v_nop\n v_nop\n v_max_f32_dpp %0, %1, %2 row_ror:8" : "=v"(qk_max[h]) : "v"(qk_max[h]), "v"(qk_max[h]) ); } + float exp_sum[QHLOOP]; #pragma unroll for (int h = 0; h < QHLOOP; h++) { @@ -695,17 +810,28 @@ __global__ __launch_bounds__(NUM_THREADS) void paged_attention_ll4mi_QKV_kernel( exp_sum[h] += dout[h][i]; } #pragma unroll - for (int mask = WARP_SIZE / 2; mask >= 4; mask /= 2) { + for (int mask = WARP_SIZE / 2; mask >= 64; mask /= 2) { exp_sum[h] += __shfl_xor(exp_sum[h], mask); } + asm("v_nop\n v_nop\n v_add_f32_dpp %0, %1, %2 row_ror:4" : "=v"(exp_sum[h]) : "v"(exp_sum[h]), "v"(exp_sum[h]) ); + asm("v_nop\n v_nop\n v_add_f32_dpp %0, %1, %2 row_ror:8" : "=v"(exp_sum[h]) : "v"(exp_sum[h]), "v"(exp_sum[h]) ); + + //asm("v_nop\n v_nop\n ds_bpermute_b32 %0, %1, %2 \n s_waitcnt lgkmcnt(0)" : "=v"(exp_sum[h]) : "v"(bpermute_mask), "v"(exp_sum[h]) ); + //exp_sum[h] = __builtin_amdgcn_ds_bpermute(bpermute_mask, exp_sum[h]); + auto tmp = __builtin_amdgcn_ds_bpermute(bpermute_mask, *reinterpret_cast(&exp_sum[h])); + exp_sum[h] = *reinterpret_cast(&tmp); + asm("v_nop\n v_nop\n v_add_f32_dpp %0, %1, %2 row_ror:4" : "=v"(exp_sum[h]) : "v"(exp_sum[h]), "v"(exp_sum[h]) ); + asm("v_nop\n v_nop\n v_add_f32_dpp %0, %1, %2 row_ror:8" : "=v"(exp_sum[h]) : "v"(exp_sum[h]), "v"(exp_sum[h]) ); } + if (laneid<4) { #pragma unroll for (int h = 0; h < QHLOOP; h++) { const int head_idx = 4 * h + lane4id; shared_qk_max[warpid][head_idx] = qk_max[h]; shared_exp_sum[warpid][head_idx] = exp_sum[h]; } + } } // warp within context __syncthreads(); @@ -749,7 +875,6 @@ __global__ __launch_bounds__(NUM_THREADS) void paged_attention_ll4mi_QKV_kernel( logits[h] = from_floatx4(dout[h]); } - __shared__ _B16x4 vout_shared[QHLOOP][VHELOOP][WARP_SIZE][NWARPS + 1]; if (warp_start_token_idx >= context_len) { // warp out of context #pragma unroll @@ -1450,14 +1575,14 @@ void paged_attention_custom_launcher( case 256: \ CALL_CUSTOM_LAUNCHER(T, KVT, KV_DTYPE, BLK_SIZE, HEAD_SIZE, OUTT, 256); \ break; \ + case 512: \ + CALL_CUSTOM_LAUNCHER(T, KVT, KV_DTYPE, BLK_SIZE, HEAD_SIZE, OUTT, 512); \ + break; \ default: \ TORCH_CHECK(false, "Unsupported partition size: ", partition_size); \ break; \ } /* - case 512: \ - CALL_CUSTOM_LAUNCHER(T, KVT, KV_DTYPE, BLK_SIZE, HEAD_SIZE, OUTT, 512); \ - break; \ */ #if defined(__HIPCC__) && defined(__gfx90a__) #define CALL_CUSTOM_LAUNCHER_OUT(T, KVT, KV_DTYPE, BLK_SIZE, HEAD_SIZE) \ diff --git a/csrc/rocm/torch_bindings.cpp b/csrc/rocm/torch_bindings.cpp index 6402a3b2b2b6..2df0d90e542f 100644 --- a/csrc/rocm/torch_bindings.cpp +++ b/csrc/rocm/torch_bindings.cpp @@ -26,8 +26,8 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, rocm_ops) { // Compute the attention between an input query and the cached // keys/values using PagedAttention. rocm_ops.def( - "paged_attention(Tensor! out, Tensor exp_sums," - " Tensor max_logits, Tensor tmp_out," + "paged_attention(Tensor! out, Tensor! exp_sums," + " Tensor! max_logits, Tensor! tmp_out," " Tensor query, Tensor key_cache," " Tensor value_cache, int num_kv_heads," " float scale, Tensor block_tables," From 73a1edb6f13cba7d257641c068ce54f9037d0e03 Mon Sep 17 00:00:00 2001 From: sanyalington Date: Wed, 20 Nov 2024 16:27:38 +0000 Subject: [PATCH 03/10] mfma16x16 load patterns only --- csrc/rocm/attention.cu | 268 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 266 insertions(+), 2 deletions(-) diff --git a/csrc/rocm/attention.cu b/csrc/rocm/attention.cu index 4f9a626ccc97..6920ed8321e5 100644 --- a/csrc/rocm/attention.cu +++ b/csrc/rocm/attention.cu @@ -297,7 +297,233 @@ __device__ __forceinline__ _B16x8 convert_b8x8_custom(const _B8x8 input) { return ret; } /////////////////////////////////////// +// grid (num_seqs, num_partitions,num_heads/gqa_ratio) +// block (partition size) +template +__global__ __launch_bounds__(NUM_THREADS) void paged_attention_ll4mi_QKV_mfma16_kernel( + const scalar_t* __restrict__ q, // [num_seqs, num_heads, head_size] + const cache_t* __restrict__ k_cache, // [num_blocks, num_kv_heads, + // head_size/x, block_size, x] + const cache_t* __restrict__ v_cache, // [num_blocks, num_kv_heads, + // head_size, block_size] + const int num_kv_heads, const float scale, + const int* __restrict__ block_tables, // [num_seqs, max_num_blocks_per_seq] + const int* __restrict__ context_lens, // [num_seqs] + const int max_num_blocks_per_seq, + const float* __restrict__ alibi_slopes, // [num_heads] + const int q_stride, const int kv_block_stride, const int kv_head_stride, + float* __restrict__ exp_sums, // [num_seqs, num_heads, max_num_partitions] + float* __restrict__ max_logits, // [num_seqs, num_heads, + // max_num_partitions] + scalar_t* __restrict__ out, // [num_seqs, num_heads, max_num_partitions, + // head_size] + OUTT* __restrict__ final_out, // [num_seqs, num_heads, head_size] + int max_ctx_blocks, float k_scale, float v_scale, + const float* __restrict__ fp8_out_scale_ptr) { + constexpr int NWARPS = NUM_THREADS / WARP_SIZE; + const int warpid = threadIdx.x / WARP_SIZE; + const int laneid = threadIdx.x % WARP_SIZE; + const int lane4id = laneid % 4; + const int lane16id = laneid % 16; + const int rowid = laneid / 16; + + const int seq_idx = blockIdx.x; + const int partition_idx = blockIdx.y; + + const int partition_size = 256; //blockDim.x; //TODO this could be head_size or partition_size + + const int max_num_partitions = gridDim.y; + + const int context_len = context_lens[seq_idx]; + + const int partition_start_token_idx = partition_idx * partition_size; + // exit if partition is out of context for seq + if (partition_start_token_idx >= context_len) { + return; + } + + constexpr int GQA_RATIO4 = DIVIDE_ROUND_UP(GQA_RATIO,4); + + __shared__ float shared_qk_max[NWARPS][GQA_RATIO4 + 1]; + __shared__ float shared_exp_sum[NWARPS][GQA_RATIO4 + 1]; + + //for QK mfma16x16, layout is QHead/Tokenx16 across every 16 lanes, 16 Bytes HeadElements in each lane, 4x16B HeadElements across warp + constexpr int ROWS_PER_WARP = WARP_SIZE / 16; //rows refers to 16 lanes; refer dpp terminology + constexpr int CONTIGUOUS_KV_ELEMS_16B_LOAD = 16 / sizeof(cache_t); + constexpr int QKHE_PER_WARP = CONTIGUOUS_KV_ELEMS_16B_LOAD * ROWS_PER_WARP; //TODO 8B form? + constexpr int QKHELOOP = HEAD_SIZE / QKHE_PER_WARP; //4xQKHE_16B across warp + + _B16x8 Qlocal[QKHELOOP]; //this could be B8x16 too + + constexpr int CONTIGUOUS_SCALAR_ELEMS_16B = 16 / sizeof(scalar_t); + constexpr int x = CONTIGUOUS_SCALAR_ELEMS_16B; //x is defined by vLLM as 16Bytes + + constexpr int TLOOP1 = CONTIGUOUS_KV_ELEMS_16B_LOAD / 4; //mfma16x16x16 outputs 4 elements per lane: will be moved to match layout for V dwordx4 loads + constexpr int TOKENS_PER_WARP1 = 16 * TLOOP1; //16 tokens across lanes * TLOOP factor + constexpr int T_PAR_SIZE = 256; + constexpr int T_PAR_LOOP = T_PAR_SIZE / TOKENS_PER_WARP1 / NWARPS; + constexpr int TLOOP = TLOOP1 * T_PAR_LOOP; + constexpr int TOKENS_PER_WARP = TOKENS_PER_WARP1 * T_PAR_LOOP; + + _B16x8 Klocal[TLOOP][QKHELOOP]; //this could be B8x16 too + const int wg_start_head_idx = blockIdx.z * GQA_RATIO; + const int wg_start_kv_head_idx = blockIdx.z; + + //TODO implement warp out of context logic + + //for QK mfma, tokens in multiples of TOKENS_PER_WARP are spread across warps + //each mfma takes QH16xT16x16HE across warp + //repeat mfmas across QKHELOOP dimension + //output layout from QKmfma : QH16xT4x4 16 qheads across 16 lanes, 16 tokens across 4 rowsx4 tokens per lane + + const int num_context_blocks = DIVIDE_ROUND_UP(context_len, BLOCK_SIZE); + const int last_ctx_block = num_context_blocks - 1; + + const int* block_table_seq = block_tables + seq_idx * max_num_blocks_per_seq; + + int kphysical_block_number[TLOOP]; + + //fetch k physical block numbers + for (int token_depth = 0; token_depth < TLOOP; token_depth++) { + const int klocal_token_idx = TOKENS_PER_WARP * warpid + token_depth * 16 + lane16id; + const int kglobal_token_idx = partition_start_token_idx + klocal_token_idx; + const int kblock_idx = (kglobal_token_idx < context_len) + ? kglobal_token_idx / BLOCK_SIZE + : last_ctx_block; + kphysical_block_number[token_depth] = block_table_seq[kblock_idx]; + //static_cast(block_table_seq[kblock_idx]); + } + + const int local_qhead_idx = lane16id; + const int global_qhead_idx = wg_start_head_idx + local_qhead_idx; + const scalar_t* q_ptr = q + seq_idx * q_stride + global_qhead_idx * HEAD_SIZE + rowid * CONTIGUOUS_KV_ELEMS_16B_LOAD; + + for (int qkhe_depth = 0; qkhe_depth < QKHELOOP; qkhe_depth++) { + const scalar_t* q_fetch_ptr = q_ptr + qkhe_depth * QKHE_PER_WARP; + const _B16x8* q_fetch_ptr_16B = reinterpret_cast(q_fetch_ptr); + Qlocal[qkhe_depth] = *q_fetch_ptr_16B; + } + + constexpr int KX = 16 / sizeof(cache_t); + const cache_t* k_ptr = k_cache + wg_start_kv_head_idx * kv_head_stride; + + const int row_head_elem = rowid * CONTIGUOUS_KV_ELEMS_16B_LOAD; + + for (int token_depth = 0; token_depth < TLOOP; token_depth++) { + const int klocal_token_idx = TOKENS_PER_WARP * warpid + token_depth * 16 + lane16id; + const int kglobal_token_idx = partition_start_token_idx + klocal_token_idx; + const int kphysical_block_offset = klocal_token_idx % BLOCK_SIZE; + const int64_t kblock_number = static_cast(kphysical_block_number[token_depth]); + const cache_t* k_ptr2 = k_ptr + kblock_number * kv_block_stride + kphysical_block_offset * KX; + + for (int qkhe_depth = 0; qkhe_depth < QKHELOOP; qkhe_depth++) { + const int head_elem = row_head_elem + qkhe_depth * QKHE_PER_WARP; + const int offset1 = head_elem / KX; + const int offset2 = head_elem % KX; + const cache_t* k_fetch_ptr = k_ptr2 + offset1 * BLOCK_SIZE * KX + offset2; + const _B16x8* k_fetch_ptr_16B = reinterpret_cast(k_fetch_ptr); + Klocal[token_depth][qkhe_depth] = *k_fetch_ptr_16B; + } + } + + constexpr int VTOKENS_PER_LANE = 16; + constexpr int VTLOOP = NWARPS * TOKENS_PER_WARP / ROWS_PER_WARP / VTOKENS_PER_LANE; + constexpr int VTLANELOOP = VTOKENS_PER_LANE / CONTIGUOUS_KV_ELEMS_16B_LOAD; //optimized for 16B fetches; assumes minimum block size is 16 + constexpr int VHELOOP = HEAD_SIZE / 16 / NWARPS; + int vphysical_block_number[VTLOOP]; + + //fetch v physical block numbers + for (int vtoken_depth = 0; vtoken_depth < VTLOOP; vtoken_depth++) { + const int vlocal_token_idx = vtoken_depth * VTOKENS_PER_LANE * ROWS_PER_WARP + rowid * VTOKENS_PER_LANE; + const int vglobal_token_idx = partition_start_token_idx + vlocal_token_idx; + const int vblock_idx = (vglobal_token_idx < context_len) + ? vglobal_token_idx / BLOCK_SIZE + : last_ctx_block; + vphysical_block_number[vtoken_depth] = + block_table_seq[vblock_idx]; + //static_cast(block_table_seq[vblock_idx]); + } + + _B16x8 Vlocal[VTLOOP][VHELOOP][VTLANELOOP]; //this could be B8x16 too + + const cache_t* v_ptr = v_cache + wg_start_kv_head_idx * kv_head_stride; + + //16he across lanes x 16 tokens per lane + for (int vhe_depth = 0; vhe_depth < VHELOOP; vhe_depth++) { + const int vhead_elem = vhe_depth * NWARPS * 16 + warpid * 16 + lane16id; + const cache_t* v_ptr2 = v_ptr + vhead_elem * BLOCK_SIZE; + + for (int vtoken_depth = 0; vtoken_depth < VTLOOP; vtoken_depth++) { + const int64_t vblock_number = static_cast(vphysical_block_number[vtoken_depth]); + const cache_t* v_ptr3 = v_ptr2 + vblock_number * kv_block_stride; + + for (int vfetch_depth = 0; vfetch_depth < VTLANELOOP; vfetch_depth++) { + const cache_t* v_fetch_ptr = v_ptr3 + vfetch_depth * CONTIGUOUS_KV_ELEMS_16B_LOAD; + const _B16x8* v_fetch_ptr_16B = reinterpret_cast(v_fetch_ptr); + Vlocal[vtoken_depth][vhe_depth][vfetch_depth] = *v_fetch_ptr_16B; + } + } + } + + floatx4 dout[TLOOP]; + __shared__ _B16x4 shared_tokens[NWARPS][TLOOP][16][VTOKENS_PER_LANE/4 + 1]; + for (int token_depth = 0; token_depth < TLOOP; token_depth++) { + dout[token_depth] = {0}; + for (int qkhe_depth = 0; qkhe_depth < QKHELOOP; qkhe_depth++) { + for (int i=0; i<2; i++) { + dout[token_depth] = __builtin_amdgcn_mfma_f32_4x4x4f16(Klocal[token_depth][qkhe_depth].xy[i], Qlocal[qkhe_depth].xy[i], dout[token_depth], 0, 0, 0); + } + } + //shared_tokens[warpid][token_depth][lane16id][rowid] = from_floatx4(dout[token_depth]); + } +#if 1 + for (int token_depth = 0; token_depth < TLOOP; token_depth++) { + shared_tokens[warpid][token_depth][lane16id][rowid] = from_floatx4(dout[token_depth]); + } +#endif + __syncthreads(); + + floatx4 vout[VHELOOP]; //this could be B8x16 too + //16he across lanes x 16 tokens per lane + + for (int vhe_depth = 0; vhe_depth < VHELOOP; vhe_depth++) { + vout[vhe_depth] = {0}; + const int vhead_elem = vhe_depth * NWARPS * 16 + warpid * 16 + lane16id; + + for (int vtoken_depth = 0; vtoken_depth < VTLOOP; vtoken_depth++) { + + for (int vfetch_depth = 0; vfetch_depth < VTLANELOOP; vfetch_depth++) { + for (int i=0; i<2; i++) { + vout[vhe_depth] = __builtin_amdgcn_mfma_f32_4x4x4f16(Vlocal[vtoken_depth][vhe_depth][vfetch_depth].xy[i], + shared_tokens[vtoken_depth][rowid][lane16id][2*vfetch_depth+i], vout[vhe_depth], 0, 0, 0); + } + } + } + } + + + + if (laneid < GQA_RATIO) { + auto* exp_sums_ptr = exp_sums + seq_idx * 8 * max_num_partitions + partition_idx; + floatx4 tmp = {0}; + //for (int t=0; t(from_floatx4(tmp), shared_tokens[warpid][lane4id][lane16id][rowid]); + + float2 tmpf = *reinterpret_cast(&tmp16); + *exp_sums_ptr = laneid%2 == 0 ? tmpf.x : tmpf.y; + } + +} +///////////////////////////////////////////////////////////// // grid (num_seqs, num_partitions,num_heads/gqa_ratio) // block (partition size) template +__global__ __launch_bounds__(NUM_THREADS) void paged_attention_ll4mi_QKV_mfma16_kernel( + const scalar_t* __restrict__ q, // [num_seqs, num_heads, head_size] + const cache_t* __restrict__ k_cache, // [num_blocks, num_kv_heads, + // head_size/x, block_size, x] + const cache_t* __restrict__ v_cache, // [num_blocks, num_kv_heads, + // head_size, block_size] + const int num_kv_heads, const float scale, + const int* __restrict__ block_tables, // [num_seqs, max_num_blocks_per_seq] + const int* __restrict__ context_lens, // [num_seqs] + const int max_num_blocks_per_seq, + const float* __restrict__ alibi_slopes, // [num_heads] + const int q_stride, const int kv_block_stride, const int kv_head_stride, + float* __restrict__ exp_sums, // [num_seqs, num_heads, max_num_partitions] + float* __restrict__ max_logits, // [num_seqs, num_heads, + // max_num_partitions] + scalar_t* __restrict__ out, // [num_seqs, num_heads, max_num_partitions, + // head_size] + OUTT* __restrict__ final_out, // [num_seqs, num_heads, head_size] + int max_ctx_blocks, float k_scale, float v_scale, + const float* __restrict__ fp8_out_scale_ptr) { + UNREACHABLE_CODE +} + template \ + <<>>( \ + query_ptr, key_cache_ptr, value_cache_ptr, num_kv_heads, scale, \ + block_tables_ptr, context_lens_ptr, max_num_blocks_per_seq, \ + alibi_slopes_ptr, q_stride, kv_block_stride, kv_head_stride, \ + exp_sums_ptr, max_logits_ptr, tmp_out_ptr, out_ptr, max_ctx_blocks, \ + k_scale, v_scale, fp8_out_scale_ptr); + #define LAUNCH_CUSTOM_ATTENTION(GQA_RATIO) \ paged_attention_ll4mi_QKV_kernel \ @@ -1455,7 +1718,7 @@ void paged_attention_custom_launcher( const int max_ctx_blocks = DIVIDE_ROUND_UP(max_context_len, BLOCK_SIZE); const int max_num_partitions = - DIVIDE_ROUND_UP(max_context_len, PARTITION_SIZE); + DIVIDE_ROUND_UP(max_context_len, 256); //PARTITION_SIZE); const int gqa_ratio = num_heads / num_kv_heads; assert(num_heads % num_kv_heads == 0); assert(head_size == HEAD_SIZE); @@ -1488,7 +1751,8 @@ void paged_attention_custom_launcher( // LAUNCH_CUSTOM_ATTENTION(7); // break; case 8: - LAUNCH_CUSTOM_ATTENTION(8); + //LAUNCH_CUSTOM_ATTENTION(8); + LAUNCH_CUSTOM_ATTENTION_MFMA16(8); break; //case 9: // LAUNCH_CUSTOM_ATTENTION(9); From 50208aa82d7b8debb47e240c3b1e786c3eeb7938 Mon Sep 17 00:00:00 2001 From: sanyalington Date: Wed, 20 Nov 2024 19:04:46 +0000 Subject: [PATCH 04/10] checkpoint --- csrc/rocm/attention.cu | 40 +++++++++++++++++++++++++++++++++++----- 1 file changed, 35 insertions(+), 5 deletions(-) diff --git a/csrc/rocm/attention.cu b/csrc/rocm/attention.cu index 6920ed8321e5..ae3c331a85b0 100644 --- a/csrc/rocm/attention.cu +++ b/csrc/rocm/attention.cu @@ -347,8 +347,8 @@ __global__ __launch_bounds__(NUM_THREADS) void paged_attention_ll4mi_QKV_mfma16_ constexpr int GQA_RATIO4 = DIVIDE_ROUND_UP(GQA_RATIO,4); - __shared__ float shared_qk_max[NWARPS][GQA_RATIO4 + 1]; - __shared__ float shared_exp_sum[NWARPS][GQA_RATIO4 + 1]; + __shared__ float shared_qk_max[NWARPS][16 + 1]; + __shared__ float shared_exp_sum[NWARPS][16 + 1]; //for QK mfma16x16, layout is QHead/Tokenx16 across every 16 lanes, 16 Bytes HeadElements in each lane, 4x16B HeadElements across warp constexpr int ROWS_PER_WARP = WARP_SIZE / 16; //rows refers to 16 lanes; refer dpp terminology @@ -478,8 +478,38 @@ __global__ __launch_bounds__(NUM_THREADS) void paged_attention_ll4mi_QKV_mfma16_ dout[token_depth] = __builtin_amdgcn_mfma_f32_4x4x4f16(Klocal[token_depth][qkhe_depth].xy[i], Qlocal[qkhe_depth].xy[i], dout[token_depth], 0, 0, 0); } } - //shared_tokens[warpid][token_depth][lane16id][rowid] = from_floatx4(dout[token_depth]); } + + float qk_max = -FLT_MAX; + + for (int token_depth = 0; token_depth < TLOOP; token_depth++) { + for (int i=0; i<4; i++) { + qk_max = fmaxf(qk_max, dout[token_depth][i]); + } + } + + for (int mask = WARP_SIZE/2; mask >= 16; mask/=2) { + qk_max = fmaxf(qk_max, __shfl_xor(qk_max,mask)); + } + + float exp_sum = 0.0f; + + for (int token_depth = 0; token_depth < TLOOP; token_depth++) { + for (int i=0; i<4; i++) { + dout[token_depth][i] = __expf(dout[token_depth][i] - qk_max); + exp_sum += dout[token_depth][i]; + } + } + + for (int mask = WARP_SIZE/2; mask >= 16; mask/=2) { + exp_sum += __shfl_xor(exp_sum,mask); + } + + if (laneid < 16) { + shared_qk_max[warpid][lane16id] = qk_max; + shared_exp_sum[warpid][lane16id] = exp_sum; + } + #if 1 for (int token_depth = 0; token_depth < TLOOP; token_depth++) { shared_tokens[warpid][token_depth][lane16id][rowid] = from_floatx4(dout[token_depth]); @@ -505,8 +535,6 @@ __global__ __launch_bounds__(NUM_THREADS) void paged_attention_ll4mi_QKV_mfma16_ } } - - if (laneid < GQA_RATIO) { auto* exp_sums_ptr = exp_sums + seq_idx * 8 * max_num_partitions + partition_idx; floatx4 tmp = {0}; @@ -516,6 +544,8 @@ __global__ __launch_bounds__(NUM_THREADS) void paged_attention_ll4mi_QKV_mfma16_ for (int h=0; h(from_floatx4(tmp), shared_tokens[warpid][lane4id][lane16id][rowid]); float2 tmpf = *reinterpret_cast(&tmp16); From 0977a1c4894571f94936265337502751cc66f2ab Mon Sep 17 00:00:00 2001 From: sanyalington Date: Fri, 29 Nov 2024 10:48:00 +0000 Subject: [PATCH 05/10] checkpoint working mfma16 kernel --- csrc/rocm/attention.cu | 171 +++++++++++++++++++++++++++++++++++------ 1 file changed, 149 insertions(+), 22 deletions(-) diff --git a/csrc/rocm/attention.cu b/csrc/rocm/attention.cu index ae3c331a85b0..c2cd7cd4fa62 100644 --- a/csrc/rocm/attention.cu +++ b/csrc/rocm/attention.cu @@ -92,6 +92,21 @@ __device__ __forceinline__ floatx4 gcn_mfma_instr(const _B16x4& inpA, } } +template +__device__ __forceinline__ floatx4 gcn_mfma16x16x16_instr(const _B16x4& inpA, + const _B16x4& inpB, + const floatx4& inpC) { + if constexpr (std::is_same::value) { + return __builtin_amdgcn_mfma_f32_16x16x16f16(inpA, inpB, inpC, absz, cbid, + blgp); + } else if constexpr (std::is_same::value) { + return __builtin_amdgcn_mfma_f32_16x16x16bf16_1k(inpA, inpB, inpC, absz, cbid, + blgp); + } else { + static_assert(false, "unsupported 16b dtype"); + } +} + template __device__ __forceinline__ float to_float(const T& inp) { if constexpr (std::is_same::value) { @@ -353,8 +368,8 @@ __global__ __launch_bounds__(NUM_THREADS) void paged_attention_ll4mi_QKV_mfma16_ //for QK mfma16x16, layout is QHead/Tokenx16 across every 16 lanes, 16 Bytes HeadElements in each lane, 4x16B HeadElements across warp constexpr int ROWS_PER_WARP = WARP_SIZE / 16; //rows refers to 16 lanes; refer dpp terminology constexpr int CONTIGUOUS_KV_ELEMS_16B_LOAD = 16 / sizeof(cache_t); - constexpr int QKHE_PER_WARP = CONTIGUOUS_KV_ELEMS_16B_LOAD * ROWS_PER_WARP; //TODO 8B form? - constexpr int QKHELOOP = HEAD_SIZE / QKHE_PER_WARP; //4xQKHE_16B across warp + constexpr int QKHE_PER_FETCH = CONTIGUOUS_KV_ELEMS_16B_LOAD * ROWS_PER_WARP; //TODO 8B form? + constexpr int QKHELOOP = HEAD_SIZE / QKHE_PER_FETCH; //4xQKHE_16B across warp _B16x8 Qlocal[QKHELOOP]; //this could be B8x16 too @@ -366,12 +381,14 @@ __global__ __launch_bounds__(NUM_THREADS) void paged_attention_ll4mi_QKV_mfma16_ constexpr int T_PAR_SIZE = 256; constexpr int T_PAR_LOOP = T_PAR_SIZE / TOKENS_PER_WARP1 / NWARPS; constexpr int TLOOP = TLOOP1 * T_PAR_LOOP; - constexpr int TOKENS_PER_WARP = TOKENS_PER_WARP1 * T_PAR_LOOP; + constexpr int TOKENS_PER_WARP = T_PAR_SIZE / NWARPS; //TOKENS_PER_WARP1 * T_PAR_LOOP; _B16x8 Klocal[TLOOP][QKHELOOP]; //this could be B8x16 too const int wg_start_head_idx = blockIdx.z * GQA_RATIO; const int wg_start_kv_head_idx = blockIdx.z; + const int total_num_heads = gridDim.z * GQA_RATIO; + const bool warp_in_context = (partition_start_token_idx + warpid * TOKENS_PER_WARP) < context_len; //TODO implement warp out of context logic @@ -398,12 +415,13 @@ __global__ __launch_bounds__(NUM_THREADS) void paged_attention_ll4mi_QKV_mfma16_ //static_cast(block_table_seq[kblock_idx]); } - const int local_qhead_idx = lane16id; + const int local_qhead_idx = lane16id % GQA_RATIO; const int global_qhead_idx = wg_start_head_idx + local_qhead_idx; - const scalar_t* q_ptr = q + seq_idx * q_stride + global_qhead_idx * HEAD_SIZE + rowid * CONTIGUOUS_KV_ELEMS_16B_LOAD; + const int64_t seq_idx64 = static_cast(seq_idx); + const scalar_t* q_ptr = q + seq_idx64 * q_stride + global_qhead_idx * HEAD_SIZE + rowid * CONTIGUOUS_KV_ELEMS_16B_LOAD; for (int qkhe_depth = 0; qkhe_depth < QKHELOOP; qkhe_depth++) { - const scalar_t* q_fetch_ptr = q_ptr + qkhe_depth * QKHE_PER_WARP; + const scalar_t* q_fetch_ptr = q_ptr + qkhe_depth * QKHE_PER_FETCH; const _B16x8* q_fetch_ptr_16B = reinterpret_cast(q_fetch_ptr); Qlocal[qkhe_depth] = *q_fetch_ptr_16B; } @@ -414,24 +432,25 @@ __global__ __launch_bounds__(NUM_THREADS) void paged_attention_ll4mi_QKV_mfma16_ const int row_head_elem = rowid * CONTIGUOUS_KV_ELEMS_16B_LOAD; for (int token_depth = 0; token_depth < TLOOP; token_depth++) { + const int64_t kblock_number = static_cast(kphysical_block_number[token_depth]); + const cache_t* k_ptr2 = k_ptr + kblock_number * kv_block_stride; const int klocal_token_idx = TOKENS_PER_WARP * warpid + token_depth * 16 + lane16id; const int kglobal_token_idx = partition_start_token_idx + klocal_token_idx; const int kphysical_block_offset = klocal_token_idx % BLOCK_SIZE; - const int64_t kblock_number = static_cast(kphysical_block_number[token_depth]); - const cache_t* k_ptr2 = k_ptr + kblock_number * kv_block_stride + kphysical_block_offset * KX; + const cache_t* k_ptr3 = k_ptr2 + kphysical_block_offset * KX; for (int qkhe_depth = 0; qkhe_depth < QKHELOOP; qkhe_depth++) { - const int head_elem = row_head_elem + qkhe_depth * QKHE_PER_WARP; + const int head_elem = row_head_elem + qkhe_depth * QKHE_PER_FETCH; const int offset1 = head_elem / KX; const int offset2 = head_elem % KX; - const cache_t* k_fetch_ptr = k_ptr2 + offset1 * BLOCK_SIZE * KX + offset2; + const cache_t* k_fetch_ptr = k_ptr3 + offset1 * BLOCK_SIZE * KX + offset2; const _B16x8* k_fetch_ptr_16B = reinterpret_cast(k_fetch_ptr); Klocal[token_depth][qkhe_depth] = *k_fetch_ptr_16B; } } constexpr int VTOKENS_PER_LANE = 16; - constexpr int VTLOOP = NWARPS * TOKENS_PER_WARP / ROWS_PER_WARP / VTOKENS_PER_LANE; + constexpr int VTLOOP = NWARPS; //was * TOKENS_PER_WARP / ROWS_PER_WARP / VTOKENS_PER_LANE; constexpr int VTLANELOOP = VTOKENS_PER_LANE / CONTIGUOUS_KV_ELEMS_16B_LOAD; //optimized for 16B fetches; assumes minimum block size is 16 constexpr int VHELOOP = HEAD_SIZE / 16 / NWARPS; int vphysical_block_number[VTLOOP]; @@ -459,7 +478,7 @@ __global__ __launch_bounds__(NUM_THREADS) void paged_attention_ll4mi_QKV_mfma16_ for (int vtoken_depth = 0; vtoken_depth < VTLOOP; vtoken_depth++) { const int64_t vblock_number = static_cast(vphysical_block_number[vtoken_depth]); - const cache_t* v_ptr3 = v_ptr2 + vblock_number * kv_block_stride; + const cache_t* v_ptr3 = v_ptr2 + (vblock_number * kv_block_stride); for (int vfetch_depth = 0; vfetch_depth < VTLANELOOP; vfetch_depth++) { const cache_t* v_fetch_ptr = v_ptr3 + vfetch_depth * CONTIGUOUS_KV_ELEMS_16B_LOAD; @@ -475,10 +494,20 @@ __global__ __launch_bounds__(NUM_THREADS) void paged_attention_ll4mi_QKV_mfma16_ dout[token_depth] = {0}; for (int qkhe_depth = 0; qkhe_depth < QKHELOOP; qkhe_depth++) { for (int i=0; i<2; i++) { - dout[token_depth] = __builtin_amdgcn_mfma_f32_4x4x4f16(Klocal[token_depth][qkhe_depth].xy[i], Qlocal[qkhe_depth].xy[i], dout[token_depth], 0, 0, 0); + //dout[token_depth] = __builtin_amdgcn_mfma_f32_16x16x16f16(Klocal[token_depth][qkhe_depth].xy[i], Qlocal[qkhe_depth].xy[i], dout[token_depth], 0, 0, 0); + dout[token_depth] = gcn_mfma16x16x16_instr(Klocal[token_depth][qkhe_depth].xy[i], Qlocal[qkhe_depth].xy[i], dout[token_depth]); } } + dout[token_depth] *= scale; + } +#if 0 //qk * scale + for (int token_depth = 0; token_depth < TLOOP; token_depth++) { + auto qkout_ptr2 = qkout_ptr + warpid * TLOOP * 16 + token_depth * 16 + rowid * 4; + auto qkout_write_ptr = reinterpret_cast<_B16x4 *>(qkout_ptr2); + auto tmp = from_floatx4(dout[token_depth]); + *qkout_write_ptr = tmp; } +#endif float qk_max = -FLT_MAX; @@ -510,31 +539,127 @@ __global__ __launch_bounds__(NUM_THREADS) void paged_attention_ll4mi_QKV_mfma16_ shared_exp_sum[warpid][lane16id] = exp_sum; } -#if 1 +#if 0 + //scalar_t* qkout_ptr = out + + // seq_idx * total_num_heads * T_PAR_SIZE + lane16id * T_PAR_SIZE; for (int token_depth = 0; token_depth < TLOOP; token_depth++) { - shared_tokens[warpid][token_depth][lane16id][rowid] = from_floatx4(dout[token_depth]); + //auto qkout_ptr2 = qkout_ptr + warpid * TLOOP * 16 + token_depth * 16 + rowid * 4; + //auto qkout_write_ptr = reinterpret_cast<_B16x4 *>(qkout_ptr2); + auto tmp = from_floatx4(dout[token_depth]); + shared_tokens[warpid][token_depth][lane16id][rowid] = tmp; + //*qkout_write_ptr = tmp; } #endif __syncthreads(); - floatx4 vout[VHELOOP]; //this could be B8x16 too + float partition_qk_max = -FLT_MAX; + float warp_qk_max_exp[NWARPS]; + float partition_exp_sum = 0.0f; + + for (int w=0; w(dout[token_depth]); + } + + if (threadIdx.x < GQA_RATIO) { + const int qhead_idx = lane16id; + const int offset = seq_idx * total_num_heads * max_num_partitions + (wg_start_head_idx + qhead_idx) * max_num_partitions + partition_idx; + max_logits[offset] = partition_qk_max; + exp_sums[offset] = partition_exp_sum; + } + + __syncthreads(); + +#if 0 + scalar_t* qkout_ptr = out + + seq_idx * total_num_heads * T_PAR_SIZE + lane16id * T_PAR_SIZE; + for (int token_depth = 0; token_depth < TLOOP; token_depth++) { + auto qkout_ptr2 = qkout_ptr + warpid * TLOOP * 16 + token_depth * 16 + rowid * 4; + auto qkout_write_ptr = reinterpret_cast<_B16x4 *>(qkout_ptr2); + //dout[token_depth] *= inv_sum_scale[warpid]; + //auto tmp = from_floatx4(dout[token_depth]); + auto tmp = shared_tokens[warpid][token_depth][lane16id][rowid]; + *qkout_write_ptr = tmp; + } +#endif + + floatx4 partition_out[VHELOOP]; + _B16x4 outelems[VHELOOP]; + //floatx4 vout[VHELOOP][VTLOOP]; //this could be B8x16 too //16he across lanes x 16 tokens per lane + //const int total_num_heads = gridDim.z * GQA_RATIO; for (int vhe_depth = 0; vhe_depth < VHELOOP; vhe_depth++) { - vout[vhe_depth] = {0}; - const int vhead_elem = vhe_depth * NWARPS * 16 + warpid * 16 + lane16id; + partition_out[vhe_depth] = {0}; for (int vtoken_depth = 0; vtoken_depth < VTLOOP; vtoken_depth++) { for (int vfetch_depth = 0; vfetch_depth < VTLANELOOP; vfetch_depth++) { for (int i=0; i<2; i++) { - vout[vhe_depth] = __builtin_amdgcn_mfma_f32_4x4x4f16(Vlocal[vtoken_depth][vhe_depth][vfetch_depth].xy[i], - shared_tokens[vtoken_depth][rowid][lane16id][2*vfetch_depth+i], vout[vhe_depth], 0, 0, 0); + const int offset = 4*rowid + 2*vfetch_depth + i; //8=num elems fetched per load + const int offset1 = offset % 4; + const int offset2 = offset / 4; + //partition_out[vhe_depth] = __builtin_amdgcn_mfma_f32_16x16x16f16(shared_tokens[vtoken_depth][offset2][lane16id][offset1], + // Vlocal[vtoken_depth][vhe_depth][vfetch_depth].xy[i], + // partition_out[vhe_depth], 0, 0, 0); + partition_out[vhe_depth] = gcn_mfma16x16x16_instr(shared_tokens[vtoken_depth][offset2][lane16id][offset1], + Vlocal[vtoken_depth][vhe_depth][vfetch_depth].xy[i], partition_out[vhe_depth]); } } + //vout *= inv_sum_scale[vtoken_depth]; + //partition_out[vhe_depth] += vout; + }//sub partition loop + outelems[vhe_depth] = from_floatx4(partition_out[vhe_depth]); + //output format is 16 he across 16 lanes, 16 qheads spread across 4 rows + } + + //scalar_t* out_ptr = final_out + + // seq_idx * total_num_heads * HEAD_SIZE; + + const int hsz_maxp_mult = HEAD_SIZE * max_num_partitions; + scalar_t* out_ptr = out + + seq_idx * total_num_heads * hsz_maxp_mult + partition_idx * HEAD_SIZE; + + const int vhe_offset = warpid * 16 + lane16id; + + #pragma unroll + for (int i=0; i<4; i++) { + const int local_head_idx = 4*rowid + i; + if (local_head_idx < GQA_RATIO) { + const int out_head_idx = wg_start_head_idx + local_head_idx; + scalar_t* out_ptr2 = out_ptr + out_head_idx * hsz_maxp_mult; + for (int vhe_depth = 0; vhe_depth < VHELOOP; vhe_depth++) { + const int vhead_elem = vhe_depth * NWARPS * 16 + vhe_offset; + scalar_t* out_ptr3 = out_ptr2 + vhead_elem; + bit16_t* out_ptr_b16 = reinterpret_cast(out_ptr3); + *out_ptr_b16 = outelems[vhe_depth][i]; + } } } + +#if 0 + floatx4 partition_out[VHELOOP]; + for (int vhe_depth = 0; vhe_depth < VHELOOP; vhe_depth++) { + partition_out[vhe_depth] = {0}; + for (int vtoken_depth = 0; vtoken_depth < VTLOOP; vtoken_depth++) { + partition_out[vhe_depth] += inv_sum_scale[vtoken_depth] * vout[vhe_depth][vtoken_depth]; + } + } +#endif +#if 0 if (laneid < GQA_RATIO) { auto* exp_sums_ptr = exp_sums + seq_idx * 8 * max_num_partitions + partition_idx; floatx4 tmp = {0}; @@ -542,7 +667,7 @@ __global__ __launch_bounds__(NUM_THREADS) void paged_attention_ll4mi_QKV_mfma16_ // tmp += dout[t]; //} for (int h=0; h(&tmp16); *exp_sums_ptr = laneid%2 == 0 ? tmpf.x : tmpf.y; } - +#endif } ///////////////////////////////////////////////////////////// // grid (num_seqs, num_partitions,num_heads/gqa_ratio) @@ -1823,6 +1948,7 @@ void paged_attention_custom_launcher( dim3 reduce_block(head_size); const int npar_loops = DIVIDE_ROUND_UP(max_num_partitions, WARP_SIZE); // support upto 8*64*256=128K context length +#if 1 switch (npar_loops) { case 1: LAUNCH_CUSTOM_REDUCTION(1); @@ -1852,6 +1978,7 @@ void paged_attention_custom_launcher( TORCH_CHECK(false, "Unsupported npar_loops: ", npar_loops); break; } +#endif } } From eb63b1f48eafec4b2fa6c692a6849774037bdd1a Mon Sep 17 00:00:00 2001 From: sanyalington Date: Fri, 29 Nov 2024 15:20:43 +0000 Subject: [PATCH 06/10] functional 16 bit dtype kernel with ctx len handling --- csrc/rocm/attention.cu | 85 ++++++++++++++++++++++++++++++------------ 1 file changed, 61 insertions(+), 24 deletions(-) diff --git a/csrc/rocm/attention.cu b/csrc/rocm/attention.cu index c2cd7cd4fa62..21361e0e50fb 100644 --- a/csrc/rocm/attention.cu +++ b/csrc/rocm/attention.cu @@ -420,10 +420,17 @@ __global__ __launch_bounds__(NUM_THREADS) void paged_attention_ll4mi_QKV_mfma16_ const int64_t seq_idx64 = static_cast(seq_idx); const scalar_t* q_ptr = q + seq_idx64 * q_stride + global_qhead_idx * HEAD_SIZE + rowid * CONTIGUOUS_KV_ELEMS_16B_LOAD; - for (int qkhe_depth = 0; qkhe_depth < QKHELOOP; qkhe_depth++) { - const scalar_t* q_fetch_ptr = q_ptr + qkhe_depth * QKHE_PER_FETCH; - const _B16x8* q_fetch_ptr_16B = reinterpret_cast(q_fetch_ptr); - Qlocal[qkhe_depth] = *q_fetch_ptr_16B; + if (lane16id < GQA_RATIO) { + for (int qkhe_depth = 0; qkhe_depth < QKHELOOP; qkhe_depth++) { + const scalar_t* q_fetch_ptr = q_ptr + qkhe_depth * QKHE_PER_FETCH; + const _B16x8* q_fetch_ptr_16B = reinterpret_cast(q_fetch_ptr); + Qlocal[qkhe_depth] = *q_fetch_ptr_16B; + } + } else { + for (int qkhe_depth = 0; qkhe_depth < QKHELOOP; qkhe_depth++) { + Qlocal[qkhe_depth].xy[0] = {0}; + Qlocal[qkhe_depth].xy[1] = {0}; + } } constexpr int KX = 16 / sizeof(cache_t); @@ -511,9 +518,13 @@ __global__ __launch_bounds__(NUM_THREADS) void paged_attention_ll4mi_QKV_mfma16_ float qk_max = -FLT_MAX; + const int qkout_token_idx = partition_start_token_idx + TOKENS_PER_WARP * warpid + rowid * 4; + for (int token_depth = 0; token_depth < TLOOP; token_depth++) { + const int local_token_idx = qkout_token_idx + token_depth * 16; for (int i=0; i<4; i++) { - qk_max = fmaxf(qk_max, dout[token_depth][i]); + const float tmp = (local_token_idx + i < context_len) ? dout[token_depth][i] : -FLT_MAX; + qk_max = fmaxf(qk_max, tmp); } } @@ -524,9 +535,11 @@ __global__ __launch_bounds__(NUM_THREADS) void paged_attention_ll4mi_QKV_mfma16_ float exp_sum = 0.0f; for (int token_depth = 0; token_depth < TLOOP; token_depth++) { + const int local_token_idx = qkout_token_idx + token_depth * 16; for (int i=0; i<4; i++) { - dout[token_depth][i] = __expf(dout[token_depth][i] - qk_max); - exp_sum += dout[token_depth][i]; + const float tmp = (local_token_idx + i < context_len) ? __expf(dout[token_depth][i] - qk_max) : 0.0f; + dout[token_depth][i] = tmp; + exp_sum += tmp; } } @@ -595,14 +608,14 @@ __global__ __launch_bounds__(NUM_THREADS) void paged_attention_ll4mi_QKV_mfma16_ } #endif - floatx4 partition_out[VHELOOP]; + //floatx4 partition_out[VHELOOP]; _B16x4 outelems[VHELOOP]; //floatx4 vout[VHELOOP][VTLOOP]; //this could be B8x16 too //16he across lanes x 16 tokens per lane //const int total_num_heads = gridDim.z * GQA_RATIO; for (int vhe_depth = 0; vhe_depth < VHELOOP; vhe_depth++) { - partition_out[vhe_depth] = {0}; + floatx4 tmp_out = {0}; for (int vtoken_depth = 0; vtoken_depth < VTLOOP; vtoken_depth++) { @@ -614,20 +627,25 @@ __global__ __launch_bounds__(NUM_THREADS) void paged_attention_ll4mi_QKV_mfma16_ //partition_out[vhe_depth] = __builtin_amdgcn_mfma_f32_16x16x16f16(shared_tokens[vtoken_depth][offset2][lane16id][offset1], // Vlocal[vtoken_depth][vhe_depth][vfetch_depth].xy[i], // partition_out[vhe_depth], 0, 0, 0); - partition_out[vhe_depth] = gcn_mfma16x16x16_instr(shared_tokens[vtoken_depth][offset2][lane16id][offset1], - Vlocal[vtoken_depth][vhe_depth][vfetch_depth].xy[i], partition_out[vhe_depth]); + + //output format is 16 head elems across 16 lanes, 16 qheads spread across 4 rows + tmp_out = gcn_mfma16x16x16_instr(shared_tokens[vtoken_depth][offset2][lane16id][offset1], + Vlocal[vtoken_depth][vhe_depth][vfetch_depth].xy[i], tmp_out); + + //output format is 16 qheads across 16 lanes, 16 head elems spread across 4 rows + //partition_out[vhe_depth] = gcn_mfma16x16x16_instr(Vlocal[vtoken_depth][vhe_depth][vfetch_depth].xy[i], + // shared_tokens[vtoken_depth][offset2][lane16id][offset1], + // partition_out[vhe_depth]); } } - //vout *= inv_sum_scale[vtoken_depth]; - //partition_out[vhe_depth] += vout; }//sub partition loop - outelems[vhe_depth] = from_floatx4(partition_out[vhe_depth]); - //output format is 16 he across 16 lanes, 16 qheads spread across 4 rows + outelems[vhe_depth] = from_floatx4(tmp_out); } //scalar_t* out_ptr = final_out + // seq_idx * total_num_heads * HEAD_SIZE; - +#if 1 + //if output format is 16 he across 16 lanes, 16 qheads spread across 4 rows const int hsz_maxp_mult = HEAD_SIZE * max_num_partitions; scalar_t* out_ptr = out + seq_idx * total_num_heads * hsz_maxp_mult + partition_idx * HEAD_SIZE; @@ -648,8 +666,25 @@ __global__ __launch_bounds__(NUM_THREADS) void paged_attention_ll4mi_QKV_mfma16_ } } } - - +#endif +#if 0 + //if output format is 16 qheads across 16 lanes, 16 he spread across 4 rows + if (lane16id < GQA_RATIO) { + const int hsz_maxp_mult = HEAD_SIZE * max_num_partitions; + scalar_t* out_ptr = out + + seq_idx * total_num_heads * hsz_maxp_mult + partition_idx * HEAD_SIZE; + const int local_head_idx = lane16id; + const int out_head_idx = wg_start_head_idx + local_head_idx; + scalar_t* out_ptr2 = out_ptr + out_head_idx * hsz_maxp_mult; + const int vhe_offset = warpid * 16 + rowid * 4; + for (int vhe_depth = 0; vhe_depth < VHELOOP; vhe_depth++) { + const int vhead_elem = vhe_depth * NWARPS * 16 + vhe_offset; + scalar_t* out_ptr3 = out_ptr2 + vhead_elem; + _B16x4* out_ptr_B16x4 = reinterpret_cast<_B16x4*>(out_ptr3); + *out_ptr_B16x4 = outelems[vhe_depth]; + } + } +#endif #if 0 floatx4 partition_out[VHELOOP]; for (int vhe_depth = 0; vhe_depth < VHELOOP; vhe_depth++) { @@ -1921,18 +1956,20 @@ void paged_attention_custom_launcher( //case 12: // LAUNCH_CUSTOM_ATTENTION(12); // break; - //case 13: - // LAUNCH_CUSTOM_ATTENTION(13); - // break; + case 13: + //LAUNCH_CUSTOM_ATTENTION(13); + LAUNCH_CUSTOM_ATTENTION_MFMA16(13); + break; //case 14: // LAUNCH_CUSTOM_ATTENTION(14); // break; //case 15: // LAUNCH_CUSTOM_ATTENTION(15); // break; - //case 16: - // LAUNCH_CUSTOM_ATTENTION(16); - // break; + case 16: + //LAUNCH_CUSTOM_ATTENTION(16); + LAUNCH_CUSTOM_ATTENTION_MFMA16(16); + break; default: TORCH_CHECK(false, "Unsupported gqa ratio: ", gqa_ratio); break; From ae66314ef2720c3af921051d5531b91f820217a2 Mon Sep 17 00:00:00 2001 From: sanyalington Date: Fri, 29 Nov 2024 16:12:12 +0000 Subject: [PATCH 07/10] mfma16 PA kernel clean up; add tests for supported configs only --- .../kernels/benchmark_paged_attention.py | 4 +- csrc/rocm/attention.cu | 167 +++++++++--------- tests/kernels/test_attention.py | 78 +++++--- 3 files changed, 145 insertions(+), 104 deletions(-) diff --git a/benchmarks/kernels/benchmark_paged_attention.py b/benchmarks/kernels/benchmark_paged_attention.py index 1090754d1566..c56f7d9ef629 100644 --- a/benchmarks/kernels/benchmark_paged_attention.py +++ b/benchmarks/kernels/benchmark_paged_attention.py @@ -9,7 +9,7 @@ from vllm.utils import (STR_DTYPE_TO_TORCH_DTYPE, FlexibleArgumentParser, create_kv_caches_with_random) -NUM_BLOCKS = 1024 * 1024 +NUM_BLOCKS = 256 * 1024 PARTITION_SIZE = 256 @torch.inference_mode() @@ -181,7 +181,7 @@ def run_cuda_benchmark(num_iters: int, profile: bool = False) -> float: if do_profile: latency = run_benchmark(num_iters=1, profile=True) else: - latency = run_benchmark(num_iters=5000, profile=False) + latency = run_benchmark(num_iters=10000, profile=False) print(f"Kernel running time: {latency * 1000000:.3f} us") diff --git a/csrc/rocm/attention.cu b/csrc/rocm/attention.cu index 21361e0e50fb..635a6a48fd5e 100644 --- a/csrc/rocm/attention.cu +++ b/csrc/rocm/attention.cu @@ -365,7 +365,7 @@ __global__ __launch_bounds__(NUM_THREADS) void paged_attention_ll4mi_QKV_mfma16_ __shared__ float shared_qk_max[NWARPS][16 + 1]; __shared__ float shared_exp_sum[NWARPS][16 + 1]; - //for QK mfma16x16, layout is QHead/Tokenx16 across every 16 lanes, 16 Bytes HeadElements in each lane, 4x16B HeadElements across warp + //for QK mfma16x16, layout is QHead/Tokenx16 across every 16 lanes, 16 Bytes HeadElements in each lane, 4x16B HeadElements across 4 rows of warp constexpr int ROWS_PER_WARP = WARP_SIZE / 16; //rows refers to 16 lanes; refer dpp terminology constexpr int CONTIGUOUS_KV_ELEMS_16B_LOAD = 16 / sizeof(cache_t); constexpr int QKHE_PER_FETCH = CONTIGUOUS_KV_ELEMS_16B_LOAD * ROWS_PER_WARP; //TODO 8B form? @@ -412,7 +412,6 @@ __global__ __launch_bounds__(NUM_THREADS) void paged_attention_ll4mi_QKV_mfma16_ ? kglobal_token_idx / BLOCK_SIZE : last_ctx_block; kphysical_block_number[token_depth] = block_table_seq[kblock_idx]; - //static_cast(block_table_seq[kblock_idx]); } const int local_qhead_idx = lane16id % GQA_RATIO; @@ -471,43 +470,41 @@ __global__ __launch_bounds__(NUM_THREADS) void paged_attention_ll4mi_QKV_mfma16_ : last_ctx_block; vphysical_block_number[vtoken_depth] = block_table_seq[vblock_idx]; - //static_cast(block_table_seq[vblock_idx]); } _B16x8 Vlocal[VTLOOP][VHELOOP][VTLANELOOP]; //this could be B8x16 too const cache_t* v_ptr = v_cache + wg_start_kv_head_idx * kv_head_stride; - //16he across lanes x 16 tokens per lane - for (int vhe_depth = 0; vhe_depth < VHELOOP; vhe_depth++) { - const int vhead_elem = vhe_depth * NWARPS * 16 + warpid * 16 + lane16id; - const cache_t* v_ptr2 = v_ptr + vhead_elem * BLOCK_SIZE; + //v fetches are 16head elems across lanes x 16 tokens per lane + for (int vhe_depth = 0; vhe_depth < VHELOOP; vhe_depth++) { + const int vhead_elem = vhe_depth * NWARPS * 16 + warpid * 16 + lane16id; + const cache_t* v_ptr2 = v_ptr + vhead_elem * BLOCK_SIZE; - for (int vtoken_depth = 0; vtoken_depth < VTLOOP; vtoken_depth++) { - const int64_t vblock_number = static_cast(vphysical_block_number[vtoken_depth]); - const cache_t* v_ptr3 = v_ptr2 + (vblock_number * kv_block_stride); + for (int vtoken_depth = 0; vtoken_depth < VTLOOP; vtoken_depth++) { + const int64_t vblock_number = static_cast(vphysical_block_number[vtoken_depth]); + const cache_t* v_ptr3 = v_ptr2 + (vblock_number * kv_block_stride); for (int vfetch_depth = 0; vfetch_depth < VTLANELOOP; vfetch_depth++) { const cache_t* v_fetch_ptr = v_ptr3 + vfetch_depth * CONTIGUOUS_KV_ELEMS_16B_LOAD; const _B16x8* v_fetch_ptr_16B = reinterpret_cast(v_fetch_ptr); Vlocal[vtoken_depth][vhe_depth][vfetch_depth] = *v_fetch_ptr_16B; } - } + } } floatx4 dout[TLOOP]; - __shared__ _B16x4 shared_tokens[NWARPS][TLOOP][16][VTOKENS_PER_LANE/4 + 1]; + __shared__ _B16x4 shared_logits[NWARPS][TLOOP][16][VTOKENS_PER_LANE/4 + 1]; for (int token_depth = 0; token_depth < TLOOP; token_depth++) { dout[token_depth] = {0}; for (int qkhe_depth = 0; qkhe_depth < QKHELOOP; qkhe_depth++) { for (int i=0; i<2; i++) { - //dout[token_depth] = __builtin_amdgcn_mfma_f32_16x16x16f16(Klocal[token_depth][qkhe_depth].xy[i], Qlocal[qkhe_depth].xy[i], dout[token_depth], 0, 0, 0); dout[token_depth] = gcn_mfma16x16x16_instr(Klocal[token_depth][qkhe_depth].xy[i], Qlocal[qkhe_depth].xy[i], dout[token_depth]); } } dout[token_depth] *= scale; } -#if 0 //qk * scale +#if 0 //DEBUG ONLY qk * scale for (int token_depth = 0; token_depth < TLOOP; token_depth++) { auto qkout_ptr2 = qkout_ptr + warpid * TLOOP * 16 + token_depth * 16 + rowid * 4; auto qkout_write_ptr = reinterpret_cast<_B16x4 *>(qkout_ptr2); @@ -552,7 +549,7 @@ __global__ __launch_bounds__(NUM_THREADS) void paged_attention_ll4mi_QKV_mfma16_ shared_exp_sum[warpid][lane16id] = exp_sum; } -#if 0 +#if 0 //DEBUG ONLY //scalar_t* qkout_ptr = out + // seq_idx * total_num_heads * T_PAR_SIZE + lane16id * T_PAR_SIZE; for (int token_depth = 0; token_depth < TLOOP; token_depth++) { @@ -583,7 +580,7 @@ __global__ __launch_bounds__(NUM_THREADS) void paged_attention_ll4mi_QKV_mfma16_ for (int token_depth = 0; token_depth < TLOOP; token_depth++) { dout[token_depth] *= inv_sum_scale; - shared_tokens[warpid][token_depth][lane16id][rowid] = from_floatx4(dout[token_depth]); + shared_logits[warpid][token_depth][lane16id][rowid] = from_floatx4(dout[token_depth]); } if (threadIdx.x < GQA_RATIO) { @@ -595,7 +592,7 @@ __global__ __launch_bounds__(NUM_THREADS) void paged_attention_ll4mi_QKV_mfma16_ __syncthreads(); -#if 0 +#if 0 //DEBUG ONLY scalar_t* qkout_ptr = out + seq_idx * total_num_heads * T_PAR_SIZE + lane16id * T_PAR_SIZE; for (int token_depth = 0; token_depth < TLOOP; token_depth++) { @@ -608,11 +605,8 @@ __global__ __launch_bounds__(NUM_THREADS) void paged_attention_ll4mi_QKV_mfma16_ } #endif - //floatx4 partition_out[VHELOOP]; _B16x4 outelems[VHELOOP]; - //floatx4 vout[VHELOOP][VTLOOP]; //this could be B8x16 too - //16he across lanes x 16 tokens per lane - //const int total_num_heads = gridDim.z * GQA_RATIO; + //v layout: 16he across lanes x 16 tokens per lane for (int vhe_depth = 0; vhe_depth < VHELOOP; vhe_depth++) { floatx4 tmp_out = {0}; @@ -621,29 +615,30 @@ __global__ __launch_bounds__(NUM_THREADS) void paged_attention_ll4mi_QKV_mfma16_ for (int vfetch_depth = 0; vfetch_depth < VTLANELOOP; vfetch_depth++) { for (int i=0; i<2; i++) { - const int offset = 4*rowid + 2*vfetch_depth + i; //8=num elems fetched per load + //TODO generalize this for 8 bit dtypes: each lane needs 2*vfetch_depth + 2 _B16x4 K/token dimension elems; each row is multiplied by a factor of 4 + //layout: lane in depth dimension | row across -> + //0 4 8 12 + //1 5 9 13 + //2 6 10 14 + //3 7 11 15 + const int offset = 4*rowid + 2*vfetch_depth + i; const int offset1 = offset % 4; const int offset2 = offset / 4; - //partition_out[vhe_depth] = __builtin_amdgcn_mfma_f32_16x16x16f16(shared_tokens[vtoken_depth][offset2][lane16id][offset1], - // Vlocal[vtoken_depth][vhe_depth][vfetch_depth].xy[i], - // partition_out[vhe_depth], 0, 0, 0); - //output format is 16 head elems across 16 lanes, 16 qheads spread across 4 rows - tmp_out = gcn_mfma16x16x16_instr(shared_tokens[vtoken_depth][offset2][lane16id][offset1], + //if output format is 16 head elems across 16 lanes, 16 qheads spread across 4 rows + tmp_out = gcn_mfma16x16x16_instr(shared_logits[vtoken_depth][offset2][lane16id][offset1], Vlocal[vtoken_depth][vhe_depth][vfetch_depth].xy[i], tmp_out); - //output format is 16 qheads across 16 lanes, 16 head elems spread across 4 rows + //if output format is 16 qheads across 16 lanes, 16 head elems spread across 4 rows //partition_out[vhe_depth] = gcn_mfma16x16x16_instr(Vlocal[vtoken_depth][vhe_depth][vfetch_depth].xy[i], // shared_tokens[vtoken_depth][offset2][lane16id][offset1], // partition_out[vhe_depth]); } } - }//sub partition loop + } outelems[vhe_depth] = from_floatx4(tmp_out); } - //scalar_t* out_ptr = final_out + - // seq_idx * total_num_heads * HEAD_SIZE; #if 1 //if output format is 16 he across 16 lanes, 16 qheads spread across 4 rows const int hsz_maxp_mult = HEAD_SIZE * max_num_partitions; @@ -666,8 +661,7 @@ __global__ __launch_bounds__(NUM_THREADS) void paged_attention_ll4mi_QKV_mfma16_ } } } -#endif -#if 0 +#else //if output format is 16 qheads across 16 lanes, 16 he spread across 4 rows if (lane16id < GQA_RATIO) { const int hsz_maxp_mult = HEAD_SIZE * max_num_partitions; @@ -685,7 +679,7 @@ __global__ __launch_bounds__(NUM_THREADS) void paged_attention_ll4mi_QKV_mfma16_ } } #endif -#if 0 +#if 0 //DEBUG ONLY floatx4 partition_out[VHELOOP]; for (int vhe_depth = 0; vhe_depth < VHELOOP; vhe_depth++) { partition_out[vhe_depth] = {0}; @@ -694,7 +688,7 @@ __global__ __launch_bounds__(NUM_THREADS) void paged_attention_ll4mi_QKV_mfma16_ } } #endif -#if 0 +#if 0 //DEBUG ONLY if (laneid < GQA_RATIO) { auto* exp_sums_ptr = exp_sums + seq_idx * 8 * max_num_partitions + partition_idx; floatx4 tmp = {0}; @@ -1581,12 +1575,13 @@ __launch_bounds__(NUM_THREADS) void paged_attention_ll4mi_reduce_kernel( const int seq_idx = blockIdx.y; const int context_len = context_lens[seq_idx]; const int num_partitions = DIVIDE_ROUND_UP(context_len, PARTITION_SIZE); +#if 0 //disable this as mfma16 kernel does not support this optimization yet if (num_partitions == 1) { // if num_partitions==1, main kernel will write to out directly, no work in // reduction kernel return; } - +#endif constexpr int NUM_WARPS = NUM_THREADS / WARP_SIZE; const int warpid = threadIdx.x / WARP_SIZE; const int laneid = threadIdx.x % WARP_SIZE; @@ -1919,53 +1914,66 @@ void paged_attention_custom_launcher( const at::cuda::OptionalCUDAGuard device_guard(device_of(query)); const cudaStream_t stream = at::cuda::getCurrentCUDAStream(); switch (gqa_ratio) { - //case 1: - // LAUNCH_CUSTOM_ATTENTION(1); - // break; - //case 2: - // LAUNCH_CUSTOM_ATTENTION(2); - // break; - //case 3: - // LAUNCH_CUSTOM_ATTENTION(3); - // break; - //case 4: - // LAUNCH_CUSTOM_ATTENTION(4); - // break; - //case 5: - // LAUNCH_CUSTOM_ATTENTION(5); - // break; - //case 6: - // LAUNCH_CUSTOM_ATTENTION(6); - // break; - //case 7: - // LAUNCH_CUSTOM_ATTENTION(7); - // break; + case 1: + //LAUNCH_CUSTOM_ATTENTION(1); + LAUNCH_CUSTOM_ATTENTION_MFMA16(1); + break; + case 2: + //LAUNCH_CUSTOM_ATTENTION(2); + LAUNCH_CUSTOM_ATTENTION_MFMA16(2); + break; + case 3: + //LAUNCH_CUSTOM_ATTENTION(3); + LAUNCH_CUSTOM_ATTENTION_MFMA16(3); + break; + case 4: + //LAUNCH_CUSTOM_ATTENTION(4); + LAUNCH_CUSTOM_ATTENTION_MFMA16(4); + break; + case 5: + //LAUNCH_CUSTOM_ATTENTION(5); + LAUNCH_CUSTOM_ATTENTION_MFMA16(5); + break; + case 6: + //LAUNCH_CUSTOM_ATTENTION(6); + LAUNCH_CUSTOM_ATTENTION_MFMA16(6); + break; + case 7: + //LAUNCH_CUSTOM_ATTENTION(7); + LAUNCH_CUSTOM_ATTENTION_MFMA16(7); + break; case 8: //LAUNCH_CUSTOM_ATTENTION(8); LAUNCH_CUSTOM_ATTENTION_MFMA16(8); break; - //case 9: - // LAUNCH_CUSTOM_ATTENTION(9); - // break; - //case 10: - // LAUNCH_CUSTOM_ATTENTION(10); - // break; - //case 11: - // LAUNCH_CUSTOM_ATTENTION(11); - // break; - //case 12: - // LAUNCH_CUSTOM_ATTENTION(12); - // break; + case 9: + //LAUNCH_CUSTOM_ATTENTION(9); + LAUNCH_CUSTOM_ATTENTION_MFMA16(9); + break; + case 10: + //LAUNCH_CUSTOM_ATTENTION(10); + LAUNCH_CUSTOM_ATTENTION_MFMA16(10); + break; + case 11: + //LAUNCH_CUSTOM_ATTENTION(11); + LAUNCH_CUSTOM_ATTENTION_MFMA16(11); + break; + case 12: + //LAUNCH_CUSTOM_ATTENTION(12); + LAUNCH_CUSTOM_ATTENTION_MFMA16(12); + break; case 13: //LAUNCH_CUSTOM_ATTENTION(13); LAUNCH_CUSTOM_ATTENTION_MFMA16(13); break; - //case 14: - // LAUNCH_CUSTOM_ATTENTION(14); - // break; - //case 15: - // LAUNCH_CUSTOM_ATTENTION(15); - // break; + case 14: + //LAUNCH_CUSTOM_ATTENTION(14); + LAUNCH_CUSTOM_ATTENTION_MFMA16(14); + break; + case 15: + //LAUNCH_CUSTOM_ATTENTION(15); + LAUNCH_CUSTOM_ATTENTION_MFMA16(15); + break; case 16: //LAUNCH_CUSTOM_ATTENTION(16); LAUNCH_CUSTOM_ATTENTION_MFMA16(16); @@ -1980,7 +1988,9 @@ void paged_attention_custom_launcher( // note there are cases with graphing where max_context_len is the max // supported by graphing, not the actual max among all the sequences: in that // case reduction kernel will still run but return immediately - if (max_context_len > PARTITION_SIZE) { + + //above optimization is not yet implemented in mfma16 kernel + //if (max_context_len > PARTITION_SIZE) { dim3 reduce_grid(num_heads, num_seqs); dim3 reduce_block(head_size); const int npar_loops = DIVIDE_ROUND_UP(max_num_partitions, WARP_SIZE); @@ -2016,7 +2026,7 @@ void paged_attention_custom_launcher( break; } #endif - } + //} //if max_context_len > partition_size } #define CALL_CUSTOM_LAUNCHER(T, KVT, KV_DTYPE, BLK_SIZE, HEAD_SIZE, OUTT, \ @@ -2033,9 +2043,6 @@ void paged_attention_custom_launcher( case 256: \ CALL_CUSTOM_LAUNCHER(T, KVT, KV_DTYPE, BLK_SIZE, HEAD_SIZE, OUTT, 256); \ break; \ - case 512: \ - CALL_CUSTOM_LAUNCHER(T, KVT, KV_DTYPE, BLK_SIZE, HEAD_SIZE, OUTT, 512); \ - break; \ default: \ TORCH_CHECK(false, "Unsupported partition size: ", partition_size); \ break; \ diff --git a/tests/kernels/test_attention.py b/tests/kernels/test_attention.py index 9983aa80d20f..1e3481471ceb 100644 --- a/tests/kernels/test_attention.py +++ b/tests/kernels/test_attention.py @@ -18,31 +18,36 @@ FLOAT32_BYTES = torch.finfo(torch.float).bits // 8 # This will change depending on the compute capability. # - 512 as a buffer -MAX_SEQ_LEN = get_max_shared_memory_bytes() // FLOAT32_BYTES - 512 +#MAX_SEQ_LEN = get_max_shared_memory_bytes() // FLOAT32_BYTES - 512 +MAX_SEQ_LEN = 32768 # There may not be enough gpu memory due to large NUM_BLOCKS. # Reduce NUM_BLOCKS when it happens. -NUM_BLOCKS = 4321 # Arbitrary values for testing +NUM_BLOCKS = 128*1024+4321 # Arbitrary values for testing PARTITION_SIZE = 512 +PARTITION_SIZE_ROCM = 256 # flshattF and tritonflashattF supported: {torch.float16, torch.bfloat16} DTYPES = [ torch.half, torch.bfloat16, torch.float -] if not current_platform.is_rocm() else [torch.half, torch.bfloat16] -NUM_GEN_SEQS = [7] # Arbitrary values for testing +] if not current_platform.is_rocm() else [torch.half,torch.bfloat16] +NUM_GEN_SEQS = [17] # Arbitrary values for testing NUM_PREFILL_SEQS = [3] # Arbitrary values for testing -NUM_HEADS = [(40, 40), (64, 8)] # Arbitrary values for testing +NUM_HEADS = [(64, 8), (26,2), (16,1), (32,32)] # Arbitrary values for testing # FlashAttention forward only supports head dimension at most 128 # https://github.com/ROCmSoftwarePlatform/flash-attention/blob/3d2b6f5d037782cc2c906909a46fb7e2e1b48b25/csrc/flash_attn_rocm/flash_api.cpp#L62 HEAD_SIZES = [64, 80, 96, 112, 120, 128, 192, 256] +HEAD_SIZES = [128] -BLOCK_SIZES = [16, 32] -USE_ALIBI = [False, True] -KV_CACHE_DTYPE = ["auto", "fp8"] +BLOCK_SIZES = [16] +USE_ALIBI = [False] +KV_CACHE_DTYPE = ["auto"] SEEDS = [0] CUDA_DEVICES = [ - f"cuda:{i}" for i in range(1 if torch.cuda.device_count() == 1 else 2) + f"cuda:{i}" for i in range(1 if torch.cuda.device_count() == 1 else 1) ] +REF_TENSOR = None +CMP_TENSOR = None def ref_masked_attention( query: torch.Tensor, @@ -51,10 +56,15 @@ def ref_masked_attention( scale: float, attn_mask: Optional[torch.Tensor] = None, ) -> torch.Tensor: - attn_weights = scale * torch.einsum("qhd,khd->hqk", query, key).float() + qkout = torch.einsum("qhd,khd->hqk", query, key).float() + attn_weights = scale * qkout if attn_mask is not None: attn_weights = attn_weights + attn_mask.float() attn_weights = torch.softmax(attn_weights, dim=-1).to(value.dtype) + #print('>>> ref qkout shape',attn_weights.shape) + #print('>>> ref qkout',attn_weights) + #global REF_TENSOR + #REF_TENSOR = attn_weights out = torch.einsum("hqk,khd->qhd", attn_weights, value) return out @@ -117,7 +127,7 @@ def ref_single_query_cached_kv_attention( @pytest.mark.parametrize( "version", - ["v1", "v2"] if not current_platform.is_rocm() else ["v1", "v2", "rocm"]) + ["v1", "v2"] if not current_platform.is_rocm() else ["rocm"]) @pytest.mark.parametrize("num_seqs", NUM_GEN_SEQS) @pytest.mark.parametrize("num_heads", NUM_HEADS) @pytest.mark.parametrize("head_size", HEAD_SIZES) @@ -150,6 +160,8 @@ def test_paged_attention( num_query_heads, num_kv_heads = num_heads query = torch.empty(num_seqs, num_query_heads, head_size, dtype=dtype) query.uniform_(-scale, scale) + #query = torch.ones_like(query) + query = torch.randn_like(query) assert num_query_heads % num_kv_heads == 0 num_queries_per_kv = num_query_heads // num_kv_heads @@ -158,8 +170,11 @@ def test_paged_attention( alibi_slopes = torch.randn(num_query_heads, dtype=torch.float) seq_lens = [random.randint(1, MAX_SEQ_LEN) for _ in range(num_seqs)] + #seq_lens = [MAX_SEQ_LEN for _ in range(num_seqs)] seq_lens[-1] = MAX_SEQ_LEN max_seq_len = max(seq_lens) + #max_seq_len = 512 + print('>>>', seq_lens, max_seq_len) seq_lens = torch.tensor(seq_lens, dtype=torch.int) # Create the block tables. @@ -181,8 +196,11 @@ def test_paged_attention( device) key_cache, value_cache = key_caches[0], value_caches[0] + #value_cache = torch.ones_like(value_cache) + #key_cache = torch.ones_like(key_cache) + # Using default kv_scale - k_scale = v_scale = 1.0 + k_scale = v_scale = 0.1 # Call the paged attention kernel. output = torch.empty_like(query) @@ -213,7 +231,7 @@ def test_paged_attention( elif version in ("v2", "rocm"): if current_platform.is_rocm(): - PARTITION_SIZE = 1024 if version == "v2" else 512 + PARTITION_SIZE = 256 if version == "v2" else PARTITION_SIZE_ROCM num_partitions = ((max_seq_len + PARTITION_SIZE - 1) // PARTITION_SIZE) assert PARTITION_SIZE % block_size == 0 num_seqs, num_heads, head_size = output.shape @@ -248,13 +266,13 @@ def test_paged_attention( v_scale, ) - opcheck(torch.ops._C.paged_attention_v2, + '''opcheck(torch.ops._C.paged_attention_v2, (output, exp_sums, max_logits, tmp_output, query, key_cache, value_cache, num_kv_heads, scale, block_tables, seq_lens, block_size, max_seq_len, alibi_slopes, kv_cache_dtype, k_scale, v_scale, 0, 0, 0, 64, 0), cond=(head_size == HEAD_SIZES[0] - and block_size == BLOCK_SIZES[0])) + and block_size == BLOCK_SIZES[0]))''' else: ops.paged_attention_rocm( @@ -275,15 +293,17 @@ def test_paged_attention( kv_cache_dtype, k_scale, v_scale, + None, + PARTITION_SIZE, ) - opcheck(torch.ops._rocm_C.paged_attention, + '''opcheck(torch.ops._rocm_C.paged_attention, (output, exp_sums, max_logits, tmp_output, query, key_cache, value_cache, num_kv_heads, scale, block_tables, seq_lens, block_size, max_seq_len, alibi_slopes, - kv_cache_dtype, k_scale, v_scale), + kv_cache_dtype, k_scale, v_scale, None, PARTITION_SIZE), cond=(head_size == HEAD_SIZES[0] - and block_size == BLOCK_SIZES[0])) + and block_size == BLOCK_SIZES[0]))''' else: raise AssertionError(f"Unknown version: {version}") @@ -298,14 +318,14 @@ def test_paged_attention( dtype=dtype, device=device) ops.convert_fp8(dequantized_key_cache, key_cache) - key_cache = dequantized_key_cache + key_cache = k_scale * dequantized_key_cache value_cache_shape = value_cache.shape dequantized_value_cache = torch.empty(size=value_cache_shape, dtype=dtype, device=device) ops.convert_fp8(dequantized_value_cache, value_cache) - value_cache = dequantized_value_cache + value_cache = v_scale * dequantized_value_cache ref_output = torch.empty_like(query) ref_single_query_cached_kv_attention( @@ -328,9 +348,23 @@ def test_paged_attention( # NOTE(zhaoyang): FP8 KV Cache will introduce quantization error, # so we use a relaxed tolerance for the test. - atol, rtol = 1e-3, 1e-5 + atol, rtol = 1e-4, 1e-5 if kv_cache_dtype == "fp8": - atol, rtol = 1e-2, 1e-5 + atol, rtol = 5e-4, 1e-5 + #bf16 rounding is handled via truncation in new kernel, this increses error + if dtype == torch.bfloat16: + atol = 1e-3 + #print('>>>tmpout shape', tmp_output.shape) + #print('>>>tmpout', tmp_output.view(8,1,256)) + #global REF_TENSOR + #torch.testing.assert_close(tmp_output.view(8,1,256), REF_TENSOR, atol=atol, rtol=rtol) + + #print('>>> ref out shape', ref_output.shape) + #print('>>> ref out', ref_output) + #print('>>> out shape', output.shape) + #print('>>> out', output) + #print('>>>', exp_sums) + #print('>>>', max_logits) torch.testing.assert_close(output, ref_output, atol=atol, rtol=rtol) From cf863a75a15199db5d5b66a5500e64ac7eaa7465 Mon Sep 17 00:00:00 2001 From: sanyalington Date: Fri, 29 Nov 2024 19:16:09 +0000 Subject: [PATCH 08/10] optimize output writes --- csrc/rocm/attention.cu | 56 ++++++++++++++++++++++++++++++++++++------ 1 file changed, 48 insertions(+), 8 deletions(-) diff --git a/csrc/rocm/attention.cu b/csrc/rocm/attention.cu index 635a6a48fd5e..1b96b73fd8f2 100644 --- a/csrc/rocm/attention.cu +++ b/csrc/rocm/attention.cu @@ -543,7 +543,7 @@ __global__ __launch_bounds__(NUM_THREADS) void paged_attention_ll4mi_QKV_mfma16_ for (int mask = WARP_SIZE/2; mask >= 16; mask/=2) { exp_sum += __shfl_xor(exp_sum,mask); } - + if (laneid < 16) { shared_qk_max[warpid][lane16id] = qk_max; shared_exp_sum[warpid][lane16id] = exp_sum; @@ -626,20 +626,59 @@ __global__ __launch_bounds__(NUM_THREADS) void paged_attention_ll4mi_QKV_mfma16_ const int offset2 = offset / 4; //if output format is 16 head elems across 16 lanes, 16 qheads spread across 4 rows - tmp_out = gcn_mfma16x16x16_instr(shared_logits[vtoken_depth][offset2][lane16id][offset1], - Vlocal[vtoken_depth][vhe_depth][vfetch_depth].xy[i], tmp_out); + //tmp_out = gcn_mfma16x16x16_instr(shared_logits[vtoken_depth][offset2][lane16id][offset1], + // Vlocal[vtoken_depth][vhe_depth][vfetch_depth].xy[i], tmp_out); //if output format is 16 qheads across 16 lanes, 16 head elems spread across 4 rows - //partition_out[vhe_depth] = gcn_mfma16x16x16_instr(Vlocal[vtoken_depth][vhe_depth][vfetch_depth].xy[i], - // shared_tokens[vtoken_depth][offset2][lane16id][offset1], - // partition_out[vhe_depth]); + tmp_out = gcn_mfma16x16x16_instr(Vlocal[vtoken_depth][vhe_depth][vfetch_depth].xy[i], + shared_logits[vtoken_depth][offset2][lane16id][offset1], + tmp_out); } } } outelems[vhe_depth] = from_floatx4(tmp_out); } -#if 1 + __syncthreads(); + + for (int vhe_depth = 0; vhe_depth < VHELOOP; vhe_depth++) { + shared_logits[warpid][vhe_depth][lane16id][rowid] = outelems[vhe_depth]; //lane16 id head dimension; rowid head element dimension + } + + __syncthreads(); + + if (warpid == 0) { + _B16x8 vout[GQA_RATIO4]; + for (int h = 0; h < GQA_RATIO4; h++) { + const int local_head_idx = 4 * h + rowid; + const int head_elem_idx = lane16id * 8; + const int offset1 = (head_elem_idx / 16)%4; + const int offset2 = head_elem_idx / 16 / NWARPS; + const int offset3 = (head_elem_idx / 4)%4; + for (int i=0; i<2; i++) { + vout[h].xy[i] = shared_logits[offset1][offset2][local_head_idx][offset3+i]; + } + } + + const int hsz_maxp_mult = HEAD_SIZE * max_num_partitions; + scalar_t* out_ptr = out + + seq_idx * total_num_heads * hsz_maxp_mult + partition_idx * HEAD_SIZE; + for (int h = 0; h < GQA_RATIO4; h++) { + const int local_head_idx = 4 * h + rowid; + if (local_head_idx < GQA_RATIO) { + const int out_head_idx = wg_start_head_idx + local_head_idx; + scalar_t* out_ptr2 = out_ptr + out_head_idx * hsz_maxp_mult; + const int head_elem_idx = lane16id * 8; + scalar_t* out_ptr3 = out_ptr2 + head_elem_idx; + _B16x8* out_ptr_B16x8 = reinterpret_cast<_B16x8*>(out_ptr3); + *out_ptr_B16x8 = vout[h]; + } + } + + } + + +#if 0 //if output format is 16 he across 16 lanes, 16 qheads spread across 4 rows const int hsz_maxp_mult = HEAD_SIZE * max_num_partitions; scalar_t* out_ptr = out + @@ -661,7 +700,8 @@ __global__ __launch_bounds__(NUM_THREADS) void paged_attention_ll4mi_QKV_mfma16_ } } } -#else +#endif +#if 0 //if output format is 16 qheads across 16 lanes, 16 he spread across 4 rows if (lane16id < GQA_RATIO) { const int hsz_maxp_mult = HEAD_SIZE * max_num_partitions; From 402fdded47031ff5badff7c65351da2511370a54 Mon Sep 17 00:00:00 2001 From: sanyalington Date: Sun, 1 Dec 2024 14:02:48 +0000 Subject: [PATCH 09/10] fetch q in shared mem for better address patterns --- csrc/rocm/attention.cu | 122 ++++++++++++++++++++++++++++++++--------- 1 file changed, 96 insertions(+), 26 deletions(-) diff --git a/csrc/rocm/attention.cu b/csrc/rocm/attention.cu index 1b96b73fd8f2..97afd3cbf0d0 100644 --- a/csrc/rocm/attention.cu +++ b/csrc/rocm/attention.cu @@ -364,33 +364,33 @@ __global__ __launch_bounds__(NUM_THREADS) void paged_attention_ll4mi_QKV_mfma16_ __shared__ float shared_qk_max[NWARPS][16 + 1]; __shared__ float shared_exp_sum[NWARPS][16 + 1]; + //shared_logits is used for multiple purposes + __shared__ _B16x4 shared_logits[NWARPS][4][16][4 + 1]; //for QK mfma16x16, layout is QHead/Tokenx16 across every 16 lanes, 16 Bytes HeadElements in each lane, 4x16B HeadElements across 4 rows of warp constexpr int ROWS_PER_WARP = WARP_SIZE / 16; //rows refers to 16 lanes; refer dpp terminology - constexpr int CONTIGUOUS_KV_ELEMS_16B_LOAD = 16 / sizeof(cache_t); - constexpr int QKHE_PER_FETCH = CONTIGUOUS_KV_ELEMS_16B_LOAD * ROWS_PER_WARP; //TODO 8B form? + constexpr int CONTIGUOUS_KV_ELEMS_16B_LOAD = 16 / sizeof(cache_t); //8 for 16 bit cache type, 16 for 8 bit types + constexpr int QKHE_PER_FETCH = CONTIGUOUS_KV_ELEMS_16B_LOAD * ROWS_PER_WARP; //each fetch across a warp fetches these many elements + constexpr int QK_SIZE_RATIO = sizeof(scalar_t) / sizeof(cache_t); //1 for 16bit types, 2 for 8bit types constexpr int QKHELOOP = HEAD_SIZE / QKHE_PER_FETCH; //4xQKHE_16B across warp - _B16x8 Qlocal[QKHELOOP]; //this could be B8x16 too + _B16x8 Qlocal[QKHELOOP][QK_SIZE_RATIO]; //note that 16 contiguous elements of Q should be fetched per lane for 8 bit cache types : QK_SIZE_RATIO changes for this constexpr int CONTIGUOUS_SCALAR_ELEMS_16B = 16 / sizeof(scalar_t); - constexpr int x = CONTIGUOUS_SCALAR_ELEMS_16B; //x is defined by vLLM as 16Bytes + //constexpr int x = CONTIGUOUS_SCALAR_ELEMS_16B; //x is defined by vLLM as 16Bytes - constexpr int TLOOP1 = CONTIGUOUS_KV_ELEMS_16B_LOAD / 4; //mfma16x16x16 outputs 4 elements per lane: will be moved to match layout for V dwordx4 loads - constexpr int TOKENS_PER_WARP1 = 16 * TLOOP1; //16 tokens across lanes * TLOOP factor - constexpr int T_PAR_SIZE = 256; - constexpr int T_PAR_LOOP = T_PAR_SIZE / TOKENS_PER_WARP1 / NWARPS; - constexpr int TLOOP = TLOOP1 * T_PAR_LOOP; - constexpr int TOKENS_PER_WARP = T_PAR_SIZE / NWARPS; //TOKENS_PER_WARP1 * T_PAR_LOOP; + constexpr int T_PAR_SIZE = 256; //partition size set to 256 TODO move to template param + //constexpr int TLOOP1 = CONTIGUOUS_KV_ELEMS_16B_LOAD / 4; //mfma16x16x16 outputs 4 elements per lane: will be moved to match layout for V dwordx4 loads + //constexpr int TOKENS_PER_WARP1 = 16 * TLOOP1; //16 tokens across lanes * TLOOP factor + //constexpr int T_PAR_LOOP = T_PAR_SIZE / TOKENS_PER_WARP1 / NWARPS; + constexpr int TOKENS_PER_WARP = T_PAR_SIZE / NWARPS; //sub partition of tokens per warp for qk calculation + constexpr int TLOOP = TOKENS_PER_WARP / 16; //each mfma16x16x16 instruction processes 16 tokens _B16x8 Klocal[TLOOP][QKHELOOP]; //this could be B8x16 too const int wg_start_head_idx = blockIdx.z * GQA_RATIO; const int wg_start_kv_head_idx = blockIdx.z; const int total_num_heads = gridDim.z * GQA_RATIO; - const bool warp_in_context = (partition_start_token_idx + warpid * TOKENS_PER_WARP) < context_len; - - //TODO implement warp out of context logic //for QK mfma, tokens in multiples of TOKENS_PER_WARP are spread across warps //each mfma takes QH16xT16x16HE across warp @@ -414,6 +414,8 @@ __global__ __launch_bounds__(NUM_THREADS) void paged_attention_ll4mi_QKV_mfma16_ kphysical_block_number[token_depth] = block_table_seq[kblock_idx]; } +#if 0 //fetch Q into registers + const int local_qhead_idx = lane16id % GQA_RATIO; const int global_qhead_idx = wg_start_head_idx + local_qhead_idx; const int64_t seq_idx64 = static_cast(seq_idx); @@ -421,16 +423,54 @@ __global__ __launch_bounds__(NUM_THREADS) void paged_attention_ll4mi_QKV_mfma16_ if (lane16id < GQA_RATIO) { for (int qkhe_depth = 0; qkhe_depth < QKHELOOP; qkhe_depth++) { - const scalar_t* q_fetch_ptr = q_ptr + qkhe_depth * QKHE_PER_FETCH; - const _B16x8* q_fetch_ptr_16B = reinterpret_cast(q_fetch_ptr); - Qlocal[qkhe_depth] = *q_fetch_ptr_16B; + const scalar_t* q_ptr2 = q_ptr + qkhe_depth * QKHE_PER_FETCH; + for (int qkratio = 0; qkratio < QK_SIZE_RATIO; qkratio++) { + const scalar_t* q_fetch_ptr = q_ptr2 + qkratio * CONTIGUOUS_SCALAR_ELEMS_16B; + const _B16x8* q_fetch_ptr_16B = reinterpret_cast(q_fetch_ptr); + Qlocal[qkhe_depth][qkratio] = *q_fetch_ptr_16B; + } } } else { for (int qkhe_depth = 0; qkhe_depth < QKHELOOP; qkhe_depth++) { - Qlocal[qkhe_depth].xy[0] = {0}; - Qlocal[qkhe_depth].xy[1] = {0}; + for (int qkratio = 0; qkratio < QK_SIZE_RATIO; qkratio++) { + Qlocal[qkhe_depth][qkratio].xy[0] = {0}; + Qlocal[qkhe_depth][qkratio].xy[1] = {0}; + } } } +#else //fetch Q in shared + const int local_qhead_idx = 4 * warpid + rowid; + const int global_qhead_idx = wg_start_head_idx + local_qhead_idx; + const int64_t seq_idx64 = static_cast(seq_idx); + const scalar_t* q_ptr = q + seq_idx64 * q_stride + global_qhead_idx * HEAD_SIZE; //+ rowid * CONTIGUOUS_KV_ELEMS_16B_LOAD; + + if (local_qhead_idx < GQA_RATIO) { + const scalar_t* q_fetch_ptr = q_ptr + lane16id * CONTIGUOUS_SCALAR_ELEMS_16B; //this works for head size 128 : 16 lanes x 8 elems = 128 elems + const _B16x8* q_fetch_ptr_16B = reinterpret_cast(q_fetch_ptr); + _B16x8 tmp = *q_fetch_ptr_16B; + const int offset1 = lane16id/4; //16 contiguous chunks of head elems are spread across 4x4lanes + shared_logits[offset1][lane4id][local_qhead_idx][0] = tmp.xy[0]; + shared_logits[offset1][lane4id][local_qhead_idx][1] = tmp.xy[1]; + } + //else { //TODO: is this part needed? + // const int offset1 = lane16id/4; //16 contiguous chunks of head elems are spread across 4x4lanes + // shared_logits[offset1][lane4id][local_qhead_idx][0] = {0}; + // shared_logits[offset1][lane4id][local_qhead_idx][1] = {0}; + //} + __syncthreads(); + //if (lane16id < GQA_RATIO) { + for (int qkhe_depth = 0; qkhe_depth < QKHELOOP; qkhe_depth++) { + Qlocal[qkhe_depth][0].xy[0] = shared_logits[qkhe_depth][rowid][lane16id % GQA_RATIO][0]; + Qlocal[qkhe_depth][0].xy[1] = shared_logits[qkhe_depth][rowid][lane16id % GQA_RATIO][1]; + } + //} + //else { + // for (int qkhe_depth = 0; qkhe_depth < QKHELOOP; qkhe_depth++) { + // Qlocal[qkhe_depth][0].xy[0] = {0}; + // Qlocal[qkhe_depth][0].xy[1] = {0}; + // } + //} +#endif constexpr int KX = 16 / sizeof(cache_t); const cache_t* k_ptr = k_cache + wg_start_kv_head_idx * kv_head_stride; @@ -493,17 +533,44 @@ __global__ __launch_bounds__(NUM_THREADS) void paged_attention_ll4mi_QKV_mfma16_ } } + //__syncthreads(); //if using shared Q + floatx4 dout[TLOOP]; - __shared__ _B16x4 shared_logits[NWARPS][TLOOP][16][VTOKENS_PER_LANE/4 + 1]; +#if 1 //Q stored in registers for (int token_depth = 0; token_depth < TLOOP; token_depth++) { dout[token_depth] = {0}; for (int qkhe_depth = 0; qkhe_depth < QKHELOOP; qkhe_depth++) { + for (int qkratio = 0; qkratio < QK_SIZE_RATIO; qkratio++) { for (int i=0; i<2; i++) { - dout[token_depth] = gcn_mfma16x16x16_instr(Klocal[token_depth][qkhe_depth].xy[i], Qlocal[qkhe_depth].xy[i], dout[token_depth]); + dout[token_depth] = gcn_mfma16x16x16_instr(Klocal[token_depth][qkhe_depth].xy[i], Qlocal[qkhe_depth][qkratio].xy[i], dout[token_depth]); } + } } dout[token_depth] *= scale; } + +#else //Q in shared + _B16x4 tmpQ[QKHELOOP][2]; + for (int qkhe_depth = 0; qkhe_depth < QKHELOOP; qkhe_depth++) { + tmpQ[qkhe_depth][0] = shared_logits[qkhe_depth][rowid][lane16id][0]; + tmpQ[qkhe_depth][1] = shared_logits[qkhe_depth][rowid][lane16id][1]; + } + + for (int token_depth = 0; token_depth < TLOOP; token_depth++) { + dout[token_depth] = {0}; + for (int qkhe_depth = 0; qkhe_depth < QKHELOOP; qkhe_depth++) { + //for (int qkratio = 0; qkratio < QK_SIZE_RATIO; qkratio++) { + for (int i=0; i<2; i++) { + dout[token_depth] = gcn_mfma16x16x16_instr(Klocal[token_depth][qkhe_depth].xy[i], + tmpQ[qkhe_depth][i], //shared_logits[qkhe_depth][rowid][lane16id][i], + dout[token_depth]); + } + //} + } + dout[token_depth] *= scale; + } +#endif + #if 0 //DEBUG ONLY qk * scale for (int token_depth = 0; token_depth < TLOOP; token_depth++) { auto qkout_ptr2 = qkout_ptr + warpid * TLOOP * 16 + token_depth * 16 + rowid * 4; @@ -514,6 +581,7 @@ __global__ __launch_bounds__(NUM_THREADS) void paged_attention_ll4mi_QKV_mfma16_ #endif float qk_max = -FLT_MAX; + float exp_sum = 0.0f; const int qkout_token_idx = partition_start_token_idx + TOKENS_PER_WARP * warpid + rowid * 4; @@ -529,7 +597,6 @@ __global__ __launch_bounds__(NUM_THREADS) void paged_attention_ll4mi_QKV_mfma16_ qk_max = fmaxf(qk_max, __shfl_xor(qk_max,mask)); } - float exp_sum = 0.0f; for (int token_depth = 0; token_depth < TLOOP; token_depth++) { const int local_token_idx = qkout_token_idx + token_depth * 16; @@ -578,6 +645,7 @@ __global__ __launch_bounds__(NUM_THREADS) void paged_attention_ll4mi_QKV_mfma16_ const float inv_sum_scale = __fdividef(1.f, partition_exp_sum + 1e-6f) * warp_qk_max_exp[warpid]; + //__shared__ _B16x4 shared_logits[NWARPS][TLOOP][16][VTOKENS_PER_LANE/4 + 1]; for (int token_depth = 0; token_depth < TLOOP; token_depth++) { dout[token_depth] *= inv_sum_scale; shared_logits[warpid][token_depth][lane16id][rowid] = from_floatx4(dout[token_depth]); @@ -624,21 +692,23 @@ __global__ __launch_bounds__(NUM_THREADS) void paged_attention_ll4mi_QKV_mfma16_ const int offset = 4*rowid + 2*vfetch_depth + i; const int offset1 = offset % 4; const int offset2 = offset / 4; - +#if 0 //if output format is 16 head elems across 16 lanes, 16 qheads spread across 4 rows - //tmp_out = gcn_mfma16x16x16_instr(shared_logits[vtoken_depth][offset2][lane16id][offset1], - // Vlocal[vtoken_depth][vhe_depth][vfetch_depth].xy[i], tmp_out); - + tmp_out = gcn_mfma16x16x16_instr(shared_logits[vtoken_depth][offset2][lane16id][offset1], + Vlocal[vtoken_depth][vhe_depth][vfetch_depth].xy[i], tmp_out); +#else //if output format is 16 qheads across 16 lanes, 16 head elems spread across 4 rows tmp_out = gcn_mfma16x16x16_instr(Vlocal[vtoken_depth][vhe_depth][vfetch_depth].xy[i], shared_logits[vtoken_depth][offset2][lane16id][offset1], tmp_out); +#endif } } } outelems[vhe_depth] = from_floatx4(tmp_out); } +#if 1 __syncthreads(); for (int vhe_depth = 0; vhe_depth < VHELOOP; vhe_depth++) { @@ -676,7 +746,7 @@ __global__ __launch_bounds__(NUM_THREADS) void paged_attention_ll4mi_QKV_mfma16_ } } - +#endif #if 0 //if output format is 16 he across 16 lanes, 16 qheads spread across 4 rows From 8f7cbba4fe44f5db927f0a6ef799ce1035f38831 Mon Sep 17 00:00:00 2001 From: sanyalington Date: Mon, 9 Dec 2024 11:01:01 +0000 Subject: [PATCH 10/10] fp8 conversion optimizations; fast f32 -> bf16 rne conversion without inf/nan check --- csrc/rocm/attention.cu | 259 +++++++++++++++++++++++++++++++++-------- 1 file changed, 208 insertions(+), 51 deletions(-) diff --git a/csrc/rocm/attention.cu b/csrc/rocm/attention.cu index 97afd3cbf0d0..9d347053aac7 100644 --- a/csrc/rocm/attention.cu +++ b/csrc/rocm/attention.cu @@ -51,6 +51,9 @@ using floatx4 = __attribute__((__vector_size__(4 * sizeof(float)))) float; using float16x4 = __attribute__((__vector_size__(4 * sizeof(_Float16)))) _Float16; typedef float16x4 _Half4; +using float16x2 = + __attribute__((__vector_size__(2 * sizeof(_Float16)))) _Float16; +typedef float16x2 _Half2; typedef struct _Half8 { _Half4 xy[2]; } _Half8; @@ -63,8 +66,13 @@ typedef struct _B16x8 { } _B16x8; using _B8x8 = uint2; +using _B8x4 = int32_t; //used in builtins using bit8_t = uint8_t; +typedef struct _B8x16 { + _B8x8 xy[2]; +} _B8x16; + ////// Non temporal load stores /////// template @@ -183,11 +191,12 @@ __device__ __forceinline__ _B16x4 from_floatx4(const floatx4& inp) { #pragma unroll for (int i = 0; i < 4; i++) { union fcvt { - uint32_t i32; + uint32_t u32; float f32; } u; u.f32 = inp[i]; - ret[i] = uint16_t(u.i32 >> 16); + u.u32 += 0x7fff + ((u.u32 >> 16) & 1); //RNE with no nan/inf check + ret[i] = uint16_t(u.u32 >> 16); //t16.b = __float2bfloat16(inp[i]); //ret[i] = t16.u; } @@ -299,16 +308,66 @@ __device__ __forceinline__ _B16x8 scaled_convert_b8x8_custom(const _B8x8 input, return ret; } +__device__ __forceinline__ floatx4 to_float_fp8x4(const _B8x4& inp) { + const auto f0 = __builtin_amdgcn_cvt_pk_f32_fp8(inp, false); + const auto f1 = __builtin_amdgcn_cvt_pk_f32_fp8(inp, true); + floatx4 ret; + ret[0] = f0[0]; + ret[1] = f0[1]; + ret[2] = f1[0]; + ret[3] = f1[1]; + return ret; +} + +template +__device__ __forceinline__ _B16x4 from_floatx4_rtz(const floatx4& inp) { + _B16x4 ret; + if constexpr (std::is_same::value) { + union h2cvt { + _Half2 h2[2]; + _B16x4 b16x4; + } u; + u.h2[0] = __builtin_amdgcn_cvt_pkrtz(inp[0],inp[1]); + u.h2[1] = __builtin_amdgcn_cvt_pkrtz(inp[2],inp[3]); + return u.b16x4; + } else if constexpr (std::is_same::value) { + for (int i = 0; i < 4; i++) { + union fcvt { + uint32_t i32; + float f32; + } u; + u.f32 = inp[i]; + ret[i] = uint16_t(u.i32 >> 16); + } + return ret; + } else { + static_assert(false, "unsupported 16b dtype"); + } +} + template __device__ __forceinline__ _B16x8 convert_b8x8_custom(const _B8x8 input) { +#if 0 union { floatx4 f32x4[2]; vllm::Float8_ f32x8; + _B8x8 b8x8[2]; } tmpf8; tmpf8.f32x8 = vllm::fp8::vec_conversion(*reinterpret_cast(&input)); + //tmpf8.b8x8[0] = input; + //tmpf8.b8x8[1] = input; +#endif + union { + _B8x8 b8x8; + _B8x4 b8x4[2]; + } tmp; + tmp.b8x8 = input; _B16x8 ret; - ret.xy[0] = from_floatx4(tmpf8.f32x4[0]); - ret.xy[1] = from_floatx4(tmpf8.f32x4[1]); + for (int i=0; i<2; i++) { + ret.xy[i] = from_floatx4_rtz( to_float_fp8x4(tmp.b8x4[i]) ); + } + //ret.xy[0] = from_floatx4(tmpf8.f32x4[0]); + //ret.xy[1] = from_floatx4(tmpf8.f32x4[1]); return ret; } /////////////////////////////////////// @@ -318,7 +377,7 @@ template -__global__ __launch_bounds__(NUM_THREADS) void paged_attention_ll4mi_QKV_mfma16_kernel( +__global__ __launch_bounds__(NUM_THREADS,5) void paged_attention_ll4mi_QKV_mfma16_kernel( const scalar_t* __restrict__ q, // [num_seqs, num_heads, head_size] const cache_t* __restrict__ k_cache, // [num_blocks, num_kv_heads, // head_size/x, block_size, x] @@ -348,13 +407,14 @@ __global__ __launch_bounds__(NUM_THREADS) void paged_attention_ll4mi_QKV_mfma16_ const int seq_idx = blockIdx.x; const int partition_idx = blockIdx.y; - const int partition_size = 256; //blockDim.x; //TODO this could be head_size or partition_size + constexpr int T_PAR_SIZE = 256; //partition size set to 256 TODO move to template param + //const int partition_size = 256; //blockDim.x; //TODO this could be head_size or partition_size const int max_num_partitions = gridDim.y; const int context_len = context_lens[seq_idx]; - const int partition_start_token_idx = partition_idx * partition_size; + const int partition_start_token_idx = partition_idx * T_PAR_SIZE; //partition_size; // exit if partition is out of context for seq if (partition_start_token_idx >= context_len) { return; @@ -365,7 +425,8 @@ __global__ __launch_bounds__(NUM_THREADS) void paged_attention_ll4mi_QKV_mfma16_ __shared__ float shared_qk_max[NWARPS][16 + 1]; __shared__ float shared_exp_sum[NWARPS][16 + 1]; //shared_logits is used for multiple purposes - __shared__ _B16x4 shared_logits[NWARPS][4][16][4 + 1]; + //__shared__ _B16x4 shared_logits[NWARPS][4][16][4 + 1]; + __shared__ _B16x4 shared_logits[NWARPS][4][16][4]; //for QK mfma16x16, layout is QHead/Tokenx16 across every 16 lanes, 16 Bytes HeadElements in each lane, 4x16B HeadElements across 4 rows of warp constexpr int ROWS_PER_WARP = WARP_SIZE / 16; //rows refers to 16 lanes; refer dpp terminology @@ -379,7 +440,6 @@ __global__ __launch_bounds__(NUM_THREADS) void paged_attention_ll4mi_QKV_mfma16_ constexpr int CONTIGUOUS_SCALAR_ELEMS_16B = 16 / sizeof(scalar_t); //constexpr int x = CONTIGUOUS_SCALAR_ELEMS_16B; //x is defined by vLLM as 16Bytes - constexpr int T_PAR_SIZE = 256; //partition size set to 256 TODO move to template param //constexpr int TLOOP1 = CONTIGUOUS_KV_ELEMS_16B_LOAD / 4; //mfma16x16x16 outputs 4 elements per lane: will be moved to match layout for V dwordx4 loads //constexpr int TOKENS_PER_WARP1 = 16 * TLOOP1; //16 tokens across lanes * TLOOP factor //constexpr int T_PAR_LOOP = T_PAR_SIZE / TOKENS_PER_WARP1 / NWARPS; @@ -448,28 +508,28 @@ __global__ __launch_bounds__(NUM_THREADS) void paged_attention_ll4mi_QKV_mfma16_ const scalar_t* q_fetch_ptr = q_ptr + lane16id * CONTIGUOUS_SCALAR_ELEMS_16B; //this works for head size 128 : 16 lanes x 8 elems = 128 elems const _B16x8* q_fetch_ptr_16B = reinterpret_cast(q_fetch_ptr); _B16x8 tmp = *q_fetch_ptr_16B; - const int offset1 = lane16id/4; //16 contiguous chunks of head elems are spread across 4x4lanes - shared_logits[offset1][lane4id][local_qhead_idx][0] = tmp.xy[0]; - shared_logits[offset1][lane4id][local_qhead_idx][1] = tmp.xy[1]; + if constexpr (KV_DTYPE == vllm::Fp8KVCacheDataType::kAuto) { + const int offset1 = lane16id/4; //16 contiguous chunks of head elems are spread across 4x4lanes + shared_logits[offset1][lane4id][local_qhead_idx][0] = tmp.xy[0]; + shared_logits[offset1][lane4id][local_qhead_idx][1] = tmp.xy[1]; + } else { + for (int i=0; i<2; i++) { + const int head_elem = lane16id * 2 + i; //element id in _B16x4 terms + const int offset3 = head_elem % 4; + const int offset2 = (head_elem / 4) % 4; + const int offset1 = head_elem /4/4; + shared_logits[offset1][offset2][local_qhead_idx][offset3] = tmp.xy[i]; + } + } } - //else { //TODO: is this part needed? - // const int offset1 = lane16id/4; //16 contiguous chunks of head elems are spread across 4x4lanes - // shared_logits[offset1][lane4id][local_qhead_idx][0] = {0}; - // shared_logits[offset1][lane4id][local_qhead_idx][1] = {0}; - //} __syncthreads(); - //if (lane16id < GQA_RATIO) { - for (int qkhe_depth = 0; qkhe_depth < QKHELOOP; qkhe_depth++) { - Qlocal[qkhe_depth][0].xy[0] = shared_logits[qkhe_depth][rowid][lane16id % GQA_RATIO][0]; - Qlocal[qkhe_depth][0].xy[1] = shared_logits[qkhe_depth][rowid][lane16id % GQA_RATIO][1]; + for (int qkhe_depth = 0; qkhe_depth < QKHELOOP; qkhe_depth++) { + for (int qkratio = 0; qkratio < QK_SIZE_RATIO; qkratio++) { + for (int i=0; i<2; i++) { + Qlocal[qkhe_depth][qkratio].xy[i] = shared_logits[qkhe_depth][rowid][lane16id % GQA_RATIO][2*qkratio + i]; + } } - //} - //else { - // for (int qkhe_depth = 0; qkhe_depth < QKHELOOP; qkhe_depth++) { - // Qlocal[qkhe_depth][0].xy[0] = {0}; - // Qlocal[qkhe_depth][0].xy[1] = {0}; - // } - //} + } #endif constexpr int KX = 16 / sizeof(cache_t); @@ -495,21 +555,25 @@ __global__ __launch_bounds__(NUM_THREADS) void paged_attention_ll4mi_QKV_mfma16_ } } - constexpr int VTOKENS_PER_LANE = 16; + constexpr int VTOKENS_PER_LANE = TOKENS_PER_WARP / ROWS_PER_WARP;// 16 * T_PAR_SIZE / 256; + constexpr int VBLOCKS_PER_LANE = DIVIDE_ROUND_UP(VTOKENS_PER_LANE,BLOCK_SIZE); constexpr int VTLOOP = NWARPS; //was * TOKENS_PER_WARP / ROWS_PER_WARP / VTOKENS_PER_LANE; - constexpr int VTLANELOOP = VTOKENS_PER_LANE / CONTIGUOUS_KV_ELEMS_16B_LOAD; //optimized for 16B fetches; assumes minimum block size is 16 + constexpr int VTLANELOOP = DIVIDE_ROUND_UP(VTOKENS_PER_LANE , CONTIGUOUS_KV_ELEMS_16B_LOAD); //optimized for 16B fetches; assumes minimum block size is 16 constexpr int VHELOOP = HEAD_SIZE / 16 / NWARPS; - int vphysical_block_number[VTLOOP]; + + int vphysical_block_number[VTLOOP][VBLOCKS_PER_LANE]; //fetch v physical block numbers for (int vtoken_depth = 0; vtoken_depth < VTLOOP; vtoken_depth++) { - const int vlocal_token_idx = vtoken_depth * VTOKENS_PER_LANE * ROWS_PER_WARP + rowid * VTOKENS_PER_LANE; + for (int vblock_depth = 0; vblock_depth < VBLOCKS_PER_LANE; vblock_depth++) { + const int vlocal_token_idx = vtoken_depth * VTOKENS_PER_LANE * ROWS_PER_WARP + rowid * VTOKENS_PER_LANE + vblock_depth * BLOCK_SIZE; const int vglobal_token_idx = partition_start_token_idx + vlocal_token_idx; const int vblock_idx = (vglobal_token_idx < context_len) ? vglobal_token_idx / BLOCK_SIZE : last_ctx_block; - vphysical_block_number[vtoken_depth] = + vphysical_block_number[vtoken_depth][vblock_depth] = block_table_seq[vblock_idx]; + } } _B16x8 Vlocal[VTLOOP][VHELOOP][VTLANELOOP]; //this could be B8x16 too @@ -522,10 +586,12 @@ __global__ __launch_bounds__(NUM_THREADS) void paged_attention_ll4mi_QKV_mfma16_ const cache_t* v_ptr2 = v_ptr + vhead_elem * BLOCK_SIZE; for (int vtoken_depth = 0; vtoken_depth < VTLOOP; vtoken_depth++) { - const int64_t vblock_number = static_cast(vphysical_block_number[vtoken_depth]); + for (int vfetch_depth = 0; vfetch_depth < VTLANELOOP; vfetch_depth++) { + const int vblock_depth = vfetch_depth * CONTIGUOUS_KV_ELEMS_16B_LOAD / BLOCK_SIZE; + //const int token_depth = vtoken_depth * VBLOCKS_PER_LANE + vblock_depth; + const int64_t vblock_number = static_cast(vphysical_block_number[vtoken_depth][vblock_depth]); const cache_t* v_ptr3 = v_ptr2 + (vblock_number * kv_block_stride); - for (int vfetch_depth = 0; vfetch_depth < VTLANELOOP; vfetch_depth++) { const cache_t* v_fetch_ptr = v_ptr3 + vfetch_depth * CONTIGUOUS_KV_ELEMS_16B_LOAD; const _B16x8* v_fetch_ptr_16B = reinterpret_cast(v_fetch_ptr); Vlocal[vtoken_depth][vhe_depth][vfetch_depth] = *v_fetch_ptr_16B; @@ -534,19 +600,37 @@ __global__ __launch_bounds__(NUM_THREADS) void paged_attention_ll4mi_QKV_mfma16_ } //__syncthreads(); //if using shared Q + float scale2 = scale; + if constexpr (KV_DTYPE != vllm::Fp8KVCacheDataType::kAuto) { + scale2 *= k_scale; + } floatx4 dout[TLOOP]; #if 1 //Q stored in registers for (int token_depth = 0; token_depth < TLOOP; token_depth++) { dout[token_depth] = {0}; for (int qkhe_depth = 0; qkhe_depth < QKHELOOP; qkhe_depth++) { - for (int qkratio = 0; qkratio < QK_SIZE_RATIO; qkratio++) { - for (int i=0; i<2; i++) { - dout[token_depth] = gcn_mfma16x16x16_instr(Klocal[token_depth][qkhe_depth].xy[i], Qlocal[qkhe_depth][qkratio].xy[i], dout[token_depth]); - } + if constexpr (KV_DTYPE == vllm::Fp8KVCacheDataType::kAuto) { + for (int qkratio = 0; qkratio < QK_SIZE_RATIO; qkratio++) { + for (int i=0; i<2; i++) { + dout[token_depth] = gcn_mfma16x16x16_instr(Klocal[token_depth][qkhe_depth].xy[i], + Qlocal[qkhe_depth][qkratio].xy[i], dout[token_depth]); + } + } + } else { //kv cache dtype fp8 + auto Ktmp = Klocal[token_depth][qkhe_depth]; + _B8x16 Ktmp8x16 = *reinterpret_cast<_B8x16*>(&Ktmp); + for (int qkratio = 0; qkratio < QK_SIZE_RATIO; qkratio++) { + _B8x8 Ktmp8x8 = Ktmp8x16.xy[qkratio]; + _B16x8 Klocaltmp = convert_b8x8_custom(Ktmp8x8); + for (int i=0; i<2; i++) { + dout[token_depth] = gcn_mfma16x16x16_instr(Klocaltmp.xy[i], + Qlocal[qkhe_depth][qkratio].xy[i], dout[token_depth]); + } + } } } - dout[token_depth] *= scale; + dout[token_depth] *= scale2; } #else //Q in shared @@ -610,10 +694,17 @@ __global__ __launch_bounds__(NUM_THREADS) void paged_attention_ll4mi_QKV_mfma16_ for (int mask = WARP_SIZE/2; mask >= 16; mask/=2) { exp_sum += __shfl_xor(exp_sum,mask); } - + + __syncthreads(); //sync before writing to shared mem + + float* shared_mem = reinterpret_cast(shared_logits); if (laneid < 16) { - shared_qk_max[warpid][lane16id] = qk_max; - shared_exp_sum[warpid][lane16id] = exp_sum; + //shared_qk_max[warpid][lane16id] = qk_max; + //shared_exp_sum[warpid][lane16id] = exp_sum; + const int qk_max_offset = warpid*16 + lane16id; + shared_mem[qk_max_offset] = qk_max; + const int exp_sum_offset = NWARPS*16 + qk_max_offset; + shared_mem[exp_sum_offset] = exp_sum; } #if 0 //DEBUG ONLY @@ -634,17 +725,21 @@ __global__ __launch_bounds__(NUM_THREADS) void paged_attention_ll4mi_QKV_mfma16_ float partition_exp_sum = 0.0f; for (int w=0; w(&Vtmp); + for (int j=0; j<2; j++) { + _B8x8 Vtmp8x8 = Vtmp8x16.xy[j]; + _B16x8 Vlocaltmp = convert_b8x8_custom(Vtmp8x8); + for (int i=0; i<2; i++) { + const int offset = 4*rowid + 2*j + i; + const int offset1 = offset % 4; + const int offset2 = offset / 4; + tmp_out = gcn_mfma16x16x16_instr(Vlocaltmp.xy[i], + shared_logits[vtoken_depth][offset2][lane16id][offset1], + tmp_out); + } + } + } + } +#endif _B16x4 outelems[VHELOOP]; + _B16x4 S_local[VTLOOP][2][2]; + if constexpr (KV_DTYPE != vllm::Fp8KVCacheDataType::kAuto) { + for (int vtoken_depth = 0; vtoken_depth < VTLOOP; vtoken_depth++) { + //for (int vfetch_depth = 0; vfetch_depth < VTLANELOOP; vfetch_depth++) { + for (int j=0; j<2; j++) { + for (int i=0; i<2; i++) { + const int offset = 4*rowid + 2*j + i; + const int offset1 = offset % 4; + const int offset2 = offset / 4; + S_local[vtoken_depth][j][i] = shared_logits[vtoken_depth][offset2][lane16id][offset1]; + } + } + //} + } + } //v layout: 16he across lanes x 16 tokens per lane for (int vhe_depth = 0; vhe_depth < VHELOOP; vhe_depth++) { @@ -681,6 +812,7 @@ __global__ __launch_bounds__(NUM_THREADS) void paged_attention_ll4mi_QKV_mfma16_ for (int vtoken_depth = 0; vtoken_depth < VTLOOP; vtoken_depth++) { + if constexpr (KV_DTYPE == vllm::Fp8KVCacheDataType::kAuto) { for (int vfetch_depth = 0; vfetch_depth < VTLANELOOP; vfetch_depth++) { for (int i=0; i<2; i++) { //TODO generalize this for 8 bit dtypes: each lane needs 2*vfetch_depth + 2 _B16x4 K/token dimension elems; each row is multiplied by a factor of 4 @@ -689,8 +821,8 @@ __global__ __launch_bounds__(NUM_THREADS) void paged_attention_ll4mi_QKV_mfma16_ //1 5 9 13 //2 6 10 14 //3 7 11 15 - const int offset = 4*rowid + 2*vfetch_depth + i; - const int offset1 = offset % 4; + const int offset = rowid * VTLANELOOP * 2 + 2*vfetch_depth + i; + const int offset1 = offset % 4; //4 corresponds to ROWS_PER_WARP const int offset2 = offset / 4; #if 0 //if output format is 16 head elems across 16 lanes, 16 qheads spread across 4 rows @@ -704,6 +836,30 @@ __global__ __launch_bounds__(NUM_THREADS) void paged_attention_ll4mi_QKV_mfma16_ #endif } } + } else { + for (int vfetch_depth = 0; vfetch_depth < VTLANELOOP; vfetch_depth++) { + _B16x8 Vtmp = Vlocal[vtoken_depth][vhe_depth][vfetch_depth]; + _B8x16 Vtmp8x16 = *reinterpret_cast<_B8x16*>(&Vtmp); + for (int j=0; j<2; j++) { + _B8x8 Vtmp8x8 = Vtmp8x16.xy[j]; + _B16x8 Vlocaltmp = convert_b8x8_custom(Vtmp8x8); + for (int i=0; i<2; i++) { + const int offset = 4*rowid + 2*j + i; + const int offset1 = offset % 4; + const int offset2 = offset / 4; + tmp_out = gcn_mfma16x16x16_instr(Vlocaltmp.xy[i], + S_local[vtoken_depth][j][i], + tmp_out); + //shared_logits[vtoken_depth][offset2][lane16id][offset1], + //tmp_out); + } + } + } + + } + } + if constexpr (KV_DTYPE != vllm::Fp8KVCacheDataType::kAuto) { + tmp_out *= v_scale; } outelems[vhe_depth] = from_floatx4(tmp_out); } @@ -1972,7 +2128,7 @@ __launch_bounds__(NUM_THREADS) void paged_attention_ll4mi_reduce_kernel( context_lens_ptr, max_num_partitions, fp8_out_scale_ptr); template + int BLOCK_SIZE, int HEAD_SIZE, typename OUTT, int PARTITION_SIZE_OLD> void paged_attention_custom_launcher( torch::Tensor& out, torch::Tensor& exp_sums, torch::Tensor& max_logits, torch::Tensor& tmp_out, torch::Tensor& query, torch::Tensor& key_cache, @@ -2012,13 +2168,14 @@ void paged_attention_custom_launcher( OUTT* out_ptr = reinterpret_cast(out.data_ptr()); const int max_ctx_blocks = DIVIDE_ROUND_UP(max_context_len, BLOCK_SIZE); + constexpr int PARTITION_SIZE = 256; const int max_num_partitions = - DIVIDE_ROUND_UP(max_context_len, 256); //PARTITION_SIZE); + DIVIDE_ROUND_UP(max_context_len, PARTITION_SIZE); const int gqa_ratio = num_heads / num_kv_heads; assert(num_heads % num_kv_heads == 0); assert(head_size == HEAD_SIZE); - constexpr int NTHR = PARTITION_SIZE; + constexpr int NTHR = 256; //PARTITION_SIZE; dim3 grid(num_seqs, max_num_partitions, num_kv_heads); dim3 block(NTHR); const at::cuda::OptionalCUDAGuard device_guard(device_of(query));