From f0cd590d62ea8190a4f00b49e981f5f9972a8d56 Mon Sep 17 00:00:00 2001 From: Jie Fang Date: Thu, 4 Jun 2026 11:22:57 +0800 Subject: [PATCH] optimize the compressor 128 split cutedsl kernel (#44230) Signed-off-by: Jie Fang Co-authored-by: Yongye Zhu --- .../ops/sparse_attn_compress_cutedsl.py | 467 +++++++++--------- 1 file changed, 236 insertions(+), 231 deletions(-) diff --git a/vllm/models/deepseek_v4/nvidia/ops/sparse_attn_compress_cutedsl.py b/vllm/models/deepseek_v4/nvidia/ops/sparse_attn_compress_cutedsl.py index ed16ca6d3b..18e596029f 100644 --- a/vllm/models/deepseek_v4/nvidia/ops/sparse_attn_compress_cutedsl.py +++ b/vllm/models/deepseek_v4/nvidia/ops/sparse_attn_compress_cutedsl.py @@ -508,30 +508,28 @@ class SparseAttnCompressNormRopeStoreC4Kernel: ) -class SparseAttnCompressKernel: +class SparseAttnCompressC128Block8Kernel: head_tile = 64 rows_per_warp = 16 - row_pairs_per_warp = rows_per_warp // 2 - elems_per_lane = 4 + elems_per_lane = 2 lanes_per_row = head_tile // elems_per_lane num_warps = 8 - stats_warp_stride = num_warps + 1 + stats_lane_stride = lanes_per_row + 1 + final_reduce_steps = 3 + final_reduce_initial_offset = 4 tb_size = num_warps * 32 + compress_ratio = 128 + state_block_size = 8 rcp_ln2 = 1.4426950408889634 def __init__( self, head_size: int, state_width: int, - compress_ratio: int, - overlap: bool, ): self.head_dim = head_size self.num_splits = head_size // self.head_tile self.state_width = state_width - self.compress_ratio = compress_ratio - self.overlap = overlap - self.window = (1 + int(overlap)) * compress_ratio @cute.jit def __call__( @@ -541,7 +539,6 @@ class SparseAttnCompressKernel: positions: cute.Tensor, slot_mapping: cute.Tensor, block_table: cute.Tensor, - block_size: Int64, compressed_kv: cute.Tensor, stream: CUstream, ): @@ -552,7 +549,6 @@ class SparseAttnCompressKernel: positions, slot_mapping, block_table, - block_size, compressed_kv, ).launch(grid=grid, block=(self.tb_size, 1, 1), stream=stream) @@ -564,29 +560,36 @@ class SparseAttnCompressKernel: positions: cute.Tensor, slot_mapping: cute.Tensor, block_table: cute.Tensor, - block_size: Int64, compressed_kv: cute.Tensor, ): block_id, _, _ = cute.arch.block_idx() tid, _, _ = cute.arch.thread_idx() warp_id = cute.arch.make_warp_uniform(tid // 32) lane_id = tid % 32 - row_lane = lane_id // self.lanes_per_row col_group = lane_id % self.lanes_per_row token_idx = block_id // self.num_splits split_idx = block_id - token_idx * self.num_splits col_base = split_idx * self.head_tile + col_group * self.elems_per_lane - slot_id = slot_mapping[token_idx] - has_position = token_idx < positions.shape[0] position = Int64(0) - if has_position: + req_idx = Int32(0) + slot_id = Int64(-1) + has_position = token_idx < positions.shape[0] + has_req_idx = token_idx < token_to_req_indices.shape[0] + if lane_id == 0: + slot_id = slot_mapping[token_idx] + if lane_id == 0 and has_position: position = positions[token_idx] + if lane_id == 0 and has_req_idx: + req_idx = token_to_req_indices[token_idx] + slot_id = cute.arch.shuffle_sync(slot_id, offset=0) + position = cute.arch.shuffle_sync(position, offset=0) + req_idx = cute.arch.shuffle_sync(req_idx, offset=0) boundary = has_position and ( (position + Int64(1)) % Int64(self.compress_ratio) == Int64(0) ) - has_req_idx = token_idx < token_to_req_indices.shape[0] + start = position - Int64(self.compress_ratio - 1) active = slot_id >= Int64(0) and has_req_idx and boundary if active: @@ -595,13 +598,13 @@ class SparseAttnCompressKernel: Float32, cute.make_layout( ( + self.num_warps, self.lanes_per_row, self.elems_per_lane, - self.stats_warp_stride, ), stride=( - self.elems_per_lane * self.stats_warp_stride, - self.stats_warp_stride, + self.stats_lane_stride * self.elems_per_lane, + self.elems_per_lane, 1, ), ), @@ -611,13 +614,13 @@ class SparseAttnCompressKernel: Float32, cute.make_layout( ( + self.num_warps, self.lanes_per_row, self.elems_per_lane, - self.stats_warp_stride, ), stride=( - self.elems_per_lane * self.stats_warp_stride, - self.stats_warp_stride, + self.stats_lane_stride * self.elems_per_lane, + self.elems_per_lane, 1, ), ), @@ -627,25 +630,25 @@ class SparseAttnCompressKernel: Float32, cute.make_layout( ( + self.num_warps, self.lanes_per_row, self.elems_per_lane, - self.stats_warp_stride, ), stride=( - self.elems_per_lane * self.stats_warp_stride, - self.stats_warp_stride, + self.stats_lane_stride * self.elems_per_lane, + self.elems_per_lane, 1, ), ), byte_alignment=4, ) - row_pair_layout = cute.make_layout( - (self.row_pairs_per_warp, self.elems_per_lane), + row_layout = cute.make_layout( + (self.rows_per_warp, self.elems_per_lane), stride=(self.elems_per_lane, 1), ) - kv_vals = cute.make_rmem_tensor(row_pair_layout, Float32) - score_vals = cute.make_rmem_tensor(row_pair_layout, Float32) + kv_vals = cute.make_rmem_tensor(row_layout, Float32) + score_vals = cute.make_rmem_tensor(row_layout, Float32) local_max = cute.make_rmem_tensor((self.elems_per_lane,), Float32) local_sum = cute.make_rmem_tensor((self.elems_per_lane,), Float32) local_product = cute.make_rmem_tensor((self.elems_per_lane,), Float32) @@ -655,100 +658,61 @@ class SparseAttnCompressKernel: local_sum[e] = Float32(0.0) local_product[e] = Float32(0.0) - req_idx = token_to_req_indices[token_idx] - start = position - Int64(self.window - 1) - cp_f32x4 = cute.make_copy_atom( - cute.nvgpu.CopyUniversalOp(), Float32, num_bits_per_copy=128 + first_block_index = start // Int64(self.state_block_size) + warp_block_index = first_block_index + (warp_id * 2).to(Int64) + block0_i32 = Int32(0) + block1_i32 = Int32(0) + if lane_id == 0: + block0_i32 = block_table[req_idx, warp_block_index] + block1_i32 = block_table[req_idx, warp_block_index + Int64(1)] + block0_i32 = cute.arch.shuffle_sync(block0_i32, offset=0) + block1_i32 = cute.arch.shuffle_sync(block1_i32, offset=0) + + cp_f32x2 = cute.make_copy_atom( + cute.nvgpu.CopyUniversalOp(), Float32, num_bits_per_copy=64 ) - row_mask_and_clamp = const_expr( - (cute.arch.WARP_SIZE - self.lanes_per_row) << 8 - | (cute.arch.WARP_SIZE - 1) + final_mask_and_clamp = const_expr( + (cute.arch.WARP_SIZE - self.num_warps) << 8 | (cute.arch.WARP_SIZE - 1) ) + col_tile = col_base.to(Int64) // Int64(self.elems_per_lane) + score_col_tile = col_tile + Int64(self.state_width // self.elems_per_lane) - for i in cutlass.range_constexpr(self.row_pairs_per_warp): - row = warp_id * self.rows_per_warp + i * 2 + row_lane - pos = start + row.to(Int64) - valid = row < self.window and pos >= Int64(0) - head_offset = ((row // self.compress_ratio) * self.head_dim).to(Int64) - - for e in cutlass.range_constexpr(self.elems_per_lane): - kv = Float32(0.0) - score = -Float32.inf - kv_vals[i, e] = kv - score_vals[i, e] = score - - block_index = Int64(0) - block_offset = Int64(0) - block_number_i32 = Int32(0) - if valid: - block_index = pos // block_size - block_offset = pos - block_index * block_size - if col_group == 0: - block_number_i32 = block_table[req_idx, block_index] - block_number_i32 = cute.arch.shuffle_sync( - block_number_i32, - offset=0, - mask_and_clamp=row_mask_and_clamp, + for i in cutlass.range_constexpr(self.rows_per_warp): + block_number_i32 = block0_i32 + block_offset = Int64(i) + if const_expr(i >= self.state_block_size): + block_number_i32 = block1_i32 + block_offset = Int64(i - self.state_block_size) + row_tensor = state_cache[block_number_i32.to(Int64), block_offset, None] + kv_src = cute.local_tile( + row_tensor, + tiler=(self.elems_per_lane,), + coord=(col_tile,), ) - - if valid: - block_number = block_number_i32.to(Int64) - row_tensor = state_cache[block_number, block_offset, None] - col_tile = (head_offset + col_base.to(Int64)) // Int64( - self.elems_per_lane - ) - kv_src = cute.local_tile( - row_tensor, - tiler=(self.elems_per_lane,), - coord=(col_tile,), - ) - score_src = cute.local_tile( - row_tensor, - tiler=(self.elems_per_lane,), - coord=( - col_tile + Int64(self.state_width // self.elems_per_lane), - ), - ) - cute.copy(cp_f32x4, kv_src, kv_vals[i, None]) - cute.copy(cp_f32x4, score_src, score_vals[i, None]) + score_src = cute.local_tile( + row_tensor, + tiler=(self.elems_per_lane,), + coord=(score_col_tile,), + ) + cute.copy(cp_f32x2, kv_src, kv_vals[i, None]) + cute.copy(cp_f32x2, score_src, score_vals[i, None]) for e in cutlass.range_constexpr(self.elems_per_lane): local_max[e] = cute.arch.fmax(local_max[e], score_vals[i, e]) for e in cutlass.range_constexpr(self.elems_per_lane): - if local_max[e] > -Float32.inf: - for i in cutlass.range_constexpr(self.row_pairs_per_warp): - exp_score = cute.math.exp2( - (score_vals[i, e] - local_max[e]) * Float32(self.rcp_ln2), - fastmath=True, - ) - local_sum[e] += exp_score - local_product[e] += kv_vals[i, e] * exp_score + for i in cutlass.range_constexpr(self.rows_per_warp): + exp_score = cute.math.exp2( + (score_vals[i, e] - local_max[e]) * Float32(self.rcp_ln2), + fastmath=True, + ) + local_sum[e] += exp_score + local_product[e] += kv_vals[i, e] * exp_score for e in cutlass.range_constexpr(self.elems_per_lane): - pair_max = cute.arch.shuffle_sync_bfly(local_max[e], offset=16) - pair_sum = cute.arch.shuffle_sync_bfly(local_sum[e], offset=16) - pair_product = cute.arch.shuffle_sync_bfly(local_product[e], offset=16) - warp_max = cute.arch.fmax(local_max[e], pair_max) - warp_sum = Float32(0.0) - warp_product = Float32(0.0) - if warp_max > -Float32.inf: - local_scale = cute.math.exp2( - (local_max[e] - warp_max) * Float32(self.rcp_ln2), - fastmath=True, - ) - pair_scale = cute.math.exp2( - (pair_max - warp_max) * Float32(self.rcp_ln2), - fastmath=True, - ) - warp_sum = local_sum[e] * local_scale + pair_sum * pair_scale - warp_product = ( - local_product[e] * local_scale + pair_product * pair_scale - ) - if lane_id < self.lanes_per_row: - s_max[col_group, e, warp_id] = warp_max - s_sum[col_group, e, warp_id] = warp_sum - s_product[col_group, e, warp_id] = warp_product + s_max[warp_id, col_group, e] = local_max[e] + s_sum[warp_id, col_group, e] = local_sum[e] + s_product[warp_id, col_group, e] = local_product[e] cute.arch.sync_threads() out_group = tid // self.num_warps @@ -761,16 +725,16 @@ class SparseAttnCompressKernel: out_lane = out_idx // self.elems_per_lane out_elem = out_idx % self.elems_per_lane - local_warp_max = s_max[out_lane, out_elem, final_lane] + local_warp_max = s_max[final_lane, out_lane, out_elem] global_max = local_warp_max - for step in cutlass.range_constexpr(3): - offset = const_expr(4 >> step) + for step in cutlass.range_constexpr(self.final_reduce_steps): + offset = const_expr(self.final_reduce_initial_offset >> step) global_max = cute.arch.fmax( global_max, cute.arch.shuffle_sync_bfly( global_max, offset=offset, - mask_and_clamp=row_mask_and_clamp, + mask_and_clamp=final_mask_and_clamp, ), ) @@ -778,19 +742,19 @@ class SparseAttnCompressKernel: (local_warp_max - global_max) * Float32(self.rcp_ln2), fastmath=True, ) - global_sum = s_sum[out_lane, out_elem, final_lane] * scale - global_product = s_product[out_lane, out_elem, final_lane] * scale - for step in cutlass.range_constexpr(3): - offset = const_expr(4 >> step) + global_sum = s_sum[final_lane, out_lane, out_elem] * scale + global_product = s_product[final_lane, out_lane, out_elem] * scale + for step in cutlass.range_constexpr(self.final_reduce_steps): + offset = const_expr(self.final_reduce_initial_offset >> step) global_sum += cute.arch.shuffle_sync_bfly( global_sum, offset=offset, - mask_and_clamp=row_mask_and_clamp, + mask_and_clamp=final_mask_and_clamp, ) global_product += cute.arch.shuffle_sync_bfly( global_product, offset=offset, - mask_and_clamp=row_mask_and_clamp, + mask_and_clamp=final_mask_and_clamp, ) if final_lane == 0: @@ -804,10 +768,8 @@ class SparseAttnCompressKernel: def compile( head_size: int = 512, state_width: int = 512, - compress_ratio: int = 128, - overlap: bool = False, ): - if head_size % SparseAttnCompressKernel.head_tile != 0: + if head_size % SparseAttnCompressC128Block8Kernel.head_tile != 0: raise ValueError("head_size must be divisible by the 64-wide head tile.") num_positions = cute.sym_int() num_slots = cute.sym_int() @@ -838,15 +800,13 @@ class SparseAttnCompressKernel: compressed_kv = cute.runtime.make_fake_tensor( Float32, (num_slots, head_size), - stride=(cute.sym_int64(divisibility=4), 1), + stride=(head_size, 1), assumed_align=4, ) - kernel = SparseAttnCompressKernel( + kernel = SparseAttnCompressC128Block8Kernel( head_size, state_width, - compress_ratio, - overlap, ) stream = cute.runtime.make_fake_stream(use_tvm_ffi_env_stream=True) return cute.compile( @@ -856,7 +816,6 @@ class SparseAttnCompressKernel: positions, slot_mapping, block_table, - Int64(0), compressed_kv, stream, options="--enable-tvm-ffi", @@ -875,6 +834,7 @@ class SparseAttnNormRopeStoreKernel: token_stride: int, scale_dim: int, compress_ratio: int, + static_kv_cache_block_size: int, ): self.head_dim = head_size self.rope_dim = rope_head_dim @@ -887,6 +847,7 @@ class SparseAttnNormRopeStoreKernel: self.nope_blocks = self.nope_dim // quant_block self.tb_size = head_size // 2 self.compress_ratio = compress_ratio + self.static_kv_cache_block_size = static_kv_cache_block_size @cute.jit def __call__( @@ -899,7 +860,6 @@ class SparseAttnNormRopeStoreKernel: cos_sin_cache: cute.Tensor, k_cache: cute.Tensor, kv_slot_mapping: cute.Tensor, - kv_cache_block_size: Int64, stream: CUstream, ): grid = (slot_mapping.shape[0], 1, 1) @@ -912,7 +872,6 @@ class SparseAttnNormRopeStoreKernel: cos_sin_cache, k_cache, kv_slot_mapping, - kv_cache_block_size, ).launch(grid=grid, block=(self.tb_size, 1, 1), stream=stream) @cute.kernel @@ -926,7 +885,6 @@ class SparseAttnNormRopeStoreKernel: cos_sin_cache: cute.Tensor, k_cache: cute.Tensor, kv_slot_mapping: cute.Tensor, - kv_cache_block_size: Int64, ): token_idx, _, _ = cute.arch.block_idx() tid, _, _ = cute.arch.thread_idx() @@ -934,21 +892,41 @@ class SparseAttnNormRopeStoreKernel: lane_id = tid % 32 elem0 = tid * 2 - slot_id = slot_mapping[token_idx] - has_position = token_idx < positions.shape[0] position = Int64(0) - if has_position: + kv_slot_idx = Int64(-1) + has_position = token_idx < positions.shape[0] + slot_id = Int64(-1) + if lane_id == 0: + slot_id = slot_mapping[token_idx] + if lane_id == 0 and has_position: position = positions[token_idx] + has_kv_slot_idx = token_idx < kv_slot_mapping.shape[0] + if lane_id == 0 and has_kv_slot_idx: + kv_slot_idx = kv_slot_mapping[token_idx] + slot_id = cute.arch.shuffle_sync(slot_id, offset=0) + position = cute.arch.shuffle_sync(position, offset=0) + kv_slot_idx = cute.arch.shuffle_sync(kv_slot_idx, offset=0) boundary = has_position and ( (position + Int64(1)) % Int64(self.compress_ratio) == Int64(0) ) - has_kv_slot_idx = token_idx < kv_slot_mapping.shape[0] - kv_slot_idx = Int64(-1) - if has_kv_slot_idx: - kv_slot_idx = kv_slot_mapping[token_idx] active = slot_id >= Int64(0) and boundary and kv_slot_idx >= Int64(0) if active: + k_cache_u16 = cute.recast_tensor(k_cache, Uint16) + k_cache_u32 = cute.recast_tensor(k_cache, Uint32) + static_block_size = Int64(self.static_kv_cache_block_size) + page = kv_slot_idx // static_block_size + kv_offset = kv_slot_idx - page * static_block_size + scale_row_offset = static_block_size * Int64(self.token_stride) + value_base = page * k_cache.stride[0] + kv_offset * Int64(self.token_stride) + scale_base = ( + page * k_cache.stride[0] + + scale_row_offset + + kv_offset * Int64(self.scale_dim) + ) + weight0 = rms_norm_weight[elem0].to(Float32) + weight1 = rms_norm_weight[elem0 + 1].to(Float32) + base = token_idx.to(Int64) * compressed_kv.stride[0] + elem0.to(Int64) x0 = compressed_kv.iterator[base] x1 = compressed_kv.iterator[base + Int64(1)] @@ -963,42 +941,32 @@ class SparseAttnNormRopeStoreKernel: partial_sums = smem.allocate_tensor( Float32, cute.make_layout((self.num_warps,)), byte_alignment=4 ) - rrms_shared = smem.allocate_tensor( - Float32, cute.make_layout((1,)), byte_alignment=4 - ) if lane_id == 0: partial_sums[warp_id] = warp_sum cute.arch.sync_threads() - if tid == 0: - total = Float32(0.0) - for i in cutlass.range_constexpr(self.num_warps): - total += partial_sums[i] - rrms_shared[0] = cute.math.rsqrt( - total / Float32(self.head_dim) + rms_norm_eps, fastmath=True - ) - cute.arch.sync_threads() - rrms = rrms_shared[0] - x0 = x0 * rrms * rms_norm_weight[elem0].to(Float32) - x1 = x1 * rrms * rms_norm_weight[elem0 + 1].to(Float32) - - k_cache_u16 = cute.recast_tensor(k_cache, Uint16) - k_cache_u32 = cute.recast_tensor(k_cache, Uint32) - page = kv_slot_idx // kv_cache_block_size - kv_offset = kv_slot_idx - page * kv_cache_block_size - value_base = page * k_cache.stride[0] + kv_offset * Int64(self.token_stride) - scale_base = ( - page * k_cache.stride[0] - + kv_cache_block_size * Int64(self.token_stride) - + kv_offset * Int64(self.scale_dim) + total = partial_sums[lane_id % self.num_warps] + sum_mask_and_clamp = const_expr( + (cute.arch.WARP_SIZE - self.num_warps) << 8 | (cute.arch.WARP_SIZE - 1) ) + for step in cutlass.range_constexpr(3): + offset = const_expr(4 >> step) + total += cute.arch.shuffle_sync_bfly( + total, + offset, + mask_and_clamp=sum_mask_and_clamp, + ) + + rrms = cute.math.rsqrt( + total / Float32(self.head_dim) + rms_norm_eps, fastmath=True + ) + x0 = x0 * rrms * weight0 + x1 = x1 * rrms * weight1 if warp_id == self.nope_blocks: pair_idx = lane_id - compressed_pos = (position // Int64(self.compress_ratio)) * Int64( - self.compress_ratio - ) + compressed_pos = position - Int64(self.compress_ratio - 1) cs_base = compressed_pos * cos_sin_cache.stride[0] + pair_idx.to(Int64) cos_v = cos_sin_cache.iterator[cs_base] sin_v = cos_sin_cache.iterator[cs_base + Int64(self.rope_dim // 2)] @@ -1058,6 +1026,7 @@ class SparseAttnNormRopeStoreKernel: kv_block_stride: int = 74752, compress_ratio: int = 128, norm_weight_dtype: type[cutlass.Numeric] = Float32, + static_kv_cache_block_size: int = 0, ): if quant_block != 64: raise ValueError( @@ -1074,6 +1043,11 @@ class SparseAttnNormRopeStoreKernel: expected_scale_dim = (head_size - rope_head_dim) // quant_block + 1 if scale_dim < expected_scale_dim: raise ValueError("scale_dim is too small for the UE8M0 scale row.") + if static_kv_cache_block_size <= 0: + raise ValueError( + "CuTe DSL sparse-attn store requires a positive static " + "kv_cache_block_size." + ) num_positions = cute.sym_int() num_slots = cute.sym_int() num_kv_slots = cute.sym_int() @@ -1083,7 +1057,7 @@ class SparseAttnNormRopeStoreKernel: compressed_kv = cute.runtime.make_fake_tensor( Float32, (num_slots, head_size), - stride=(cute.sym_int64(divisibility=4), 1), + stride=(head_size, 1), assumed_align=4, ) positions = make_fake_tensor(Int64, (num_positions,), divisibility=8) @@ -1094,14 +1068,14 @@ class SparseAttnNormRopeStoreKernel: cos_sin_cache = cute.runtime.make_fake_tensor( Float32, (max_pos, rope_head_dim), - stride=(cute.sym_int64(divisibility=4), 1), + stride=(rope_head_dim, 1), assumed_align=4, ) k_cache = cute.runtime.make_fake_tensor( Uint8, (num_blocks, cute.sym_int(), cute.sym_int()), stride=( - cute.sym_int64(divisibility=16), + kv_block_stride, cute.sym_int64(divisibility=8), 1, ), @@ -1117,6 +1091,7 @@ class SparseAttnNormRopeStoreKernel: token_stride, scale_dim, compress_ratio, + static_kv_cache_block_size, ) stream = cute.runtime.make_fake_stream(use_tvm_ffi_env_stream=True) return cute.compile( @@ -1129,13 +1104,59 @@ class SparseAttnNormRopeStoreKernel: cos_sin_cache, k_cache, kv_slot_mapping, - Int64(0), stream, options="--enable-tvm-ffi", ) -def compress_kv_sparse_attn_cutedsl( +def compile_split_sparse_attn_cutedsl( + head_size: int, + state_width: int, + block_size: int, + rope_head_dim: int, + fp8_max: float, + quant_block: int, + token_stride: int, + scale_dim: int, + kv_cache_block_size: int, + kv_block_stride: int, + compress_ratio: int, + overlap: bool, + rms_norm_weight_dtype: torch.dtype, +): + if not ( + head_size == 512 + and state_width == head_size + and compress_ratio == 128 + and not overlap + and block_size == 8 + ): + raise ValueError( + "CuTe DSL split sparse-attn wrapper only supports the real " + "DeepSeek V4 C128 layout: head_size=512, state_width=512, " + "compress_ratio=128, overlap=False, block_size=8." + ) + compress = SparseAttnCompressC128Block8Kernel.compile( + head_size=head_size, + state_width=state_width, + ) + norm_weight_dtype = _TORCH_TO_CUTE[rms_norm_weight_dtype] + store = SparseAttnNormRopeStoreKernel.compile( + head_size, + rope_head_dim, + fp8_max, + quant_block, + token_stride, + scale_dim, + kv_block_stride, + compress_ratio, + norm_weight_dtype, + kv_cache_block_size, + ) + return compress, store + + +def split_kv_compress_norm_rope_insert_sparse_attn_cutedsl( state_cache: torch.Tensor, token_to_req_indices: torch.Tensor, positions: torch.Tensor, @@ -1143,34 +1164,6 @@ def compress_kv_sparse_attn_cutedsl( block_table: torch.Tensor, block_size: int, compressed_kv: torch.Tensor, - head_size: int = 512, - state_width: int = 512, - compress_ratio: int = 128, - overlap: bool = False, -) -> None: - if positions.numel() == 0: - return - compiled = SparseAttnCompressKernel.compile( - head_size=head_size, - state_width=state_width, - compress_ratio=compress_ratio, - overlap=overlap, - ) - compiled( - state_cache, - token_to_req_indices, - positions, - slot_mapping, - block_table, - block_size, - compressed_kv, - ) - - -def norm_rope_insert_sparse_attn_cutedsl( - compressed_kv: torch.Tensor, - positions: torch.Tensor, - slot_mapping: torch.Tensor, rms_norm_weight: torch.Tensor, rms_norm_eps: float, cos_sin_cache: torch.Tensor, @@ -1179,38 +1172,58 @@ def norm_rope_insert_sparse_attn_cutedsl( kv_cache_block_size: int, kv_block_stride: int, head_size: int = 512, + state_width: int = 512, rope_head_dim: int = 64, fp8_max: float = 448.0, quant_block: int = 64, token_stride: int = 576, scale_dim: int = 8, compress_ratio: int = 128, + overlap: bool = False, ) -> None: - if positions.numel() == 0: - return - norm_weight_dtype = _TORCH_TO_CUTE.get(rms_norm_weight.dtype) - if norm_weight_dtype is None: - raise ValueError( - "CuTe DSL sparse-attn store supports rms_norm_weight dtype " - f"bf16/fp32, got {rms_norm_weight.dtype}." - ) if k_cache.ndim != 3: raise ValueError( "CuTe DSL sparse-attn store expects the real DeepSeek V4 " f"3D k_cache layout [num_blocks, block_size, 584], got ndim={k_cache.ndim}." ) - compiled = SparseAttnNormRopeStoreKernel.compile( - head_size=head_size, - rope_head_dim=rope_head_dim, - fp8_max=fp8_max, - quant_block=quant_block, - token_stride=token_stride, - scale_dim=scale_dim, - kv_block_stride=kv_block_stride, - compress_ratio=compress_ratio, - norm_weight_dtype=norm_weight_dtype, + if kv_cache_block_size != k_cache.shape[1]: + raise ValueError( + "CuTe DSL split sparse-attn wrapper expected kv_cache_block_size " + f"to match k_cache.shape[1], got {kv_cache_block_size} and " + f"{k_cache.shape[1]}." + ) + if positions.numel() == 0: + return + if rms_norm_weight.dtype not in _TORCH_TO_CUTE: + raise ValueError( + "CuTe DSL sparse-attn store supports rms_norm_weight dtype " + f"bf16/fp32, got {rms_norm_weight.dtype}." + ) + compress, store = compile_split_sparse_attn_cutedsl( + head_size, + state_width, + block_size, + rope_head_dim, + fp8_max, + quant_block, + token_stride, + scale_dim, + kv_cache_block_size, + kv_block_stride, + compress_ratio, + overlap, + rms_norm_weight.dtype, ) - compiled( + compress( + state_cache, + token_to_req_indices, + positions, + slot_mapping, + block_table, + compressed_kv, + ) + + store( compressed_kv, positions, slot_mapping, @@ -1219,7 +1232,6 @@ def norm_rope_insert_sparse_attn_cutedsl( cos_sin_cache, k_cache, kv_slot_mapping, - kv_cache_block_size, ) @@ -1346,7 +1358,7 @@ def compress_norm_rope_store_cutedsl( dtype=torch.float32, device=state_cache.device, ) - compress_kv_sparse_attn_cutedsl( + split_kv_compress_norm_rope_insert_sparse_attn_cutedsl( state_cache, token_to_req_indices, positions, @@ -1354,15 +1366,6 @@ def compress_norm_rope_store_cutedsl( block_table, block_size, compressed_kv, - head_size=head_dim, - state_width=state_width, - compress_ratio=compress_ratio, - overlap=overlap, - ) - norm_rope_insert_sparse_attn_cutedsl( - compressed_kv, - positions, - slot_mapping, rms_norm_weight, rms_norm_eps, cos_sin_cache, @@ -1371,10 +1374,12 @@ def compress_norm_rope_store_cutedsl( kv_cache.shape[1], # paged KV cache block size kv_cache.stride(0), head_size=head_dim, + state_width=state_width, rope_head_dim=rope_head_dim, fp8_max=448.0, quant_block=quant_block, token_stride=token_stride, scale_dim=scale_dim, compress_ratio=compress_ratio, + overlap=overlap, )