optimize the compressor 128 split cutedsl kernel (#44230)

Signed-off-by: Jie Fang <jief@nvidia.com>
Co-authored-by: Yongye Zhu <zyy1102000@gmail.com>
This commit is contained in:
Jie Fang
2026-06-03 20:22:57 -07:00
committed by GitHub
co-authored by Yongye Zhu
parent e6018c644a
commit f0cd590d62
@@ -508,30 +508,28 @@ class SparseAttnCompressNormRopeStoreC4Kernel:
)
class SparseAttnCompressKernel:
class SparseAttnCompressC128Block8Kernel:
head_tile = 64
rows_per_warp = 16
row_pairs_per_warp = rows_per_warp // 2
elems_per_lane = 4
elems_per_lane = 2
lanes_per_row = head_tile // elems_per_lane
num_warps = 8
stats_warp_stride = num_warps + 1
stats_lane_stride = lanes_per_row + 1
final_reduce_steps = 3
final_reduce_initial_offset = 4
tb_size = num_warps * 32
compress_ratio = 128
state_block_size = 8
rcp_ln2 = 1.4426950408889634
def __init__(
self,
head_size: int,
state_width: int,
compress_ratio: int,
overlap: bool,
):
self.head_dim = head_size
self.num_splits = head_size // self.head_tile
self.state_width = state_width
self.compress_ratio = compress_ratio
self.overlap = overlap
self.window = (1 + int(overlap)) * compress_ratio
@cute.jit
def __call__(
@@ -541,7 +539,6 @@ class SparseAttnCompressKernel:
positions: cute.Tensor,
slot_mapping: cute.Tensor,
block_table: cute.Tensor,
block_size: Int64,
compressed_kv: cute.Tensor,
stream: CUstream,
):
@@ -552,7 +549,6 @@ class SparseAttnCompressKernel:
positions,
slot_mapping,
block_table,
block_size,
compressed_kv,
).launch(grid=grid, block=(self.tb_size, 1, 1), stream=stream)
@@ -564,29 +560,36 @@ class SparseAttnCompressKernel:
positions: cute.Tensor,
slot_mapping: cute.Tensor,
block_table: cute.Tensor,
block_size: Int64,
compressed_kv: cute.Tensor,
):
block_id, _, _ = cute.arch.block_idx()
tid, _, _ = cute.arch.thread_idx()
warp_id = cute.arch.make_warp_uniform(tid // 32)
lane_id = tid % 32
row_lane = lane_id // self.lanes_per_row
col_group = lane_id % self.lanes_per_row
token_idx = block_id // self.num_splits
split_idx = block_id - token_idx * self.num_splits
col_base = split_idx * self.head_tile + col_group * self.elems_per_lane
slot_id = slot_mapping[token_idx]
has_position = token_idx < positions.shape[0]
position = Int64(0)
if has_position:
req_idx = Int32(0)
slot_id = Int64(-1)
has_position = token_idx < positions.shape[0]
has_req_idx = token_idx < token_to_req_indices.shape[0]
if lane_id == 0:
slot_id = slot_mapping[token_idx]
if lane_id == 0 and has_position:
position = positions[token_idx]
if lane_id == 0 and has_req_idx:
req_idx = token_to_req_indices[token_idx]
slot_id = cute.arch.shuffle_sync(slot_id, offset=0)
position = cute.arch.shuffle_sync(position, offset=0)
req_idx = cute.arch.shuffle_sync(req_idx, offset=0)
boundary = has_position and (
(position + Int64(1)) % Int64(self.compress_ratio) == Int64(0)
)
has_req_idx = token_idx < token_to_req_indices.shape[0]
start = position - Int64(self.compress_ratio - 1)
active = slot_id >= Int64(0) and has_req_idx and boundary
if active:
@@ -595,13 +598,13 @@ class SparseAttnCompressKernel:
Float32,
cute.make_layout(
(
self.num_warps,
self.lanes_per_row,
self.elems_per_lane,
self.stats_warp_stride,
),
stride=(
self.elems_per_lane * self.stats_warp_stride,
self.stats_warp_stride,
self.stats_lane_stride * self.elems_per_lane,
self.elems_per_lane,
1,
),
),
@@ -611,13 +614,13 @@ class SparseAttnCompressKernel:
Float32,
cute.make_layout(
(
self.num_warps,
self.lanes_per_row,
self.elems_per_lane,
self.stats_warp_stride,
),
stride=(
self.elems_per_lane * self.stats_warp_stride,
self.stats_warp_stride,
self.stats_lane_stride * self.elems_per_lane,
self.elems_per_lane,
1,
),
),
@@ -627,25 +630,25 @@ class SparseAttnCompressKernel:
Float32,
cute.make_layout(
(
self.num_warps,
self.lanes_per_row,
self.elems_per_lane,
self.stats_warp_stride,
),
stride=(
self.elems_per_lane * self.stats_warp_stride,
self.stats_warp_stride,
self.stats_lane_stride * self.elems_per_lane,
self.elems_per_lane,
1,
),
),
byte_alignment=4,
)
row_pair_layout = cute.make_layout(
(self.row_pairs_per_warp, self.elems_per_lane),
row_layout = cute.make_layout(
(self.rows_per_warp, self.elems_per_lane),
stride=(self.elems_per_lane, 1),
)
kv_vals = cute.make_rmem_tensor(row_pair_layout, Float32)
score_vals = cute.make_rmem_tensor(row_pair_layout, Float32)
kv_vals = cute.make_rmem_tensor(row_layout, Float32)
score_vals = cute.make_rmem_tensor(row_layout, Float32)
local_max = cute.make_rmem_tensor((self.elems_per_lane,), Float32)
local_sum = cute.make_rmem_tensor((self.elems_per_lane,), Float32)
local_product = cute.make_rmem_tensor((self.elems_per_lane,), Float32)
@@ -655,100 +658,61 @@ class SparseAttnCompressKernel:
local_sum[e] = Float32(0.0)
local_product[e] = Float32(0.0)
req_idx = token_to_req_indices[token_idx]
start = position - Int64(self.window - 1)
cp_f32x4 = cute.make_copy_atom(
cute.nvgpu.CopyUniversalOp(), Float32, num_bits_per_copy=128
first_block_index = start // Int64(self.state_block_size)
warp_block_index = first_block_index + (warp_id * 2).to(Int64)
block0_i32 = Int32(0)
block1_i32 = Int32(0)
if lane_id == 0:
block0_i32 = block_table[req_idx, warp_block_index]
block1_i32 = block_table[req_idx, warp_block_index + Int64(1)]
block0_i32 = cute.arch.shuffle_sync(block0_i32, offset=0)
block1_i32 = cute.arch.shuffle_sync(block1_i32, offset=0)
cp_f32x2 = cute.make_copy_atom(
cute.nvgpu.CopyUniversalOp(), Float32, num_bits_per_copy=64
)
row_mask_and_clamp = const_expr(
(cute.arch.WARP_SIZE - self.lanes_per_row) << 8
| (cute.arch.WARP_SIZE - 1)
final_mask_and_clamp = const_expr(
(cute.arch.WARP_SIZE - self.num_warps) << 8 | (cute.arch.WARP_SIZE - 1)
)
col_tile = col_base.to(Int64) // Int64(self.elems_per_lane)
score_col_tile = col_tile + Int64(self.state_width // self.elems_per_lane)
for i in cutlass.range_constexpr(self.row_pairs_per_warp):
row = warp_id * self.rows_per_warp + i * 2 + row_lane
pos = start + row.to(Int64)
valid = row < self.window and pos >= Int64(0)
head_offset = ((row // self.compress_ratio) * self.head_dim).to(Int64)
for e in cutlass.range_constexpr(self.elems_per_lane):
kv = Float32(0.0)
score = -Float32.inf
kv_vals[i, e] = kv
score_vals[i, e] = score
block_index = Int64(0)
block_offset = Int64(0)
block_number_i32 = Int32(0)
if valid:
block_index = pos // block_size
block_offset = pos - block_index * block_size
if col_group == 0:
block_number_i32 = block_table[req_idx, block_index]
block_number_i32 = cute.arch.shuffle_sync(
block_number_i32,
offset=0,
mask_and_clamp=row_mask_and_clamp,
for i in cutlass.range_constexpr(self.rows_per_warp):
block_number_i32 = block0_i32
block_offset = Int64(i)
if const_expr(i >= self.state_block_size):
block_number_i32 = block1_i32
block_offset = Int64(i - self.state_block_size)
row_tensor = state_cache[block_number_i32.to(Int64), block_offset, None]
kv_src = cute.local_tile(
row_tensor,
tiler=(self.elems_per_lane,),
coord=(col_tile,),
)
if valid:
block_number = block_number_i32.to(Int64)
row_tensor = state_cache[block_number, block_offset, None]
col_tile = (head_offset + col_base.to(Int64)) // Int64(
self.elems_per_lane
)
kv_src = cute.local_tile(
row_tensor,
tiler=(self.elems_per_lane,),
coord=(col_tile,),
)
score_src = cute.local_tile(
row_tensor,
tiler=(self.elems_per_lane,),
coord=(
col_tile + Int64(self.state_width // self.elems_per_lane),
),
)
cute.copy(cp_f32x4, kv_src, kv_vals[i, None])
cute.copy(cp_f32x4, score_src, score_vals[i, None])
score_src = cute.local_tile(
row_tensor,
tiler=(self.elems_per_lane,),
coord=(score_col_tile,),
)
cute.copy(cp_f32x2, kv_src, kv_vals[i, None])
cute.copy(cp_f32x2, score_src, score_vals[i, None])
for e in cutlass.range_constexpr(self.elems_per_lane):
local_max[e] = cute.arch.fmax(local_max[e], score_vals[i, e])
for e in cutlass.range_constexpr(self.elems_per_lane):
if local_max[e] > -Float32.inf:
for i in cutlass.range_constexpr(self.row_pairs_per_warp):
exp_score = cute.math.exp2(
(score_vals[i, e] - local_max[e]) * Float32(self.rcp_ln2),
fastmath=True,
)
local_sum[e] += exp_score
local_product[e] += kv_vals[i, e] * exp_score
for i in cutlass.range_constexpr(self.rows_per_warp):
exp_score = cute.math.exp2(
(score_vals[i, e] - local_max[e]) * Float32(self.rcp_ln2),
fastmath=True,
)
local_sum[e] += exp_score
local_product[e] += kv_vals[i, e] * exp_score
for e in cutlass.range_constexpr(self.elems_per_lane):
pair_max = cute.arch.shuffle_sync_bfly(local_max[e], offset=16)
pair_sum = cute.arch.shuffle_sync_bfly(local_sum[e], offset=16)
pair_product = cute.arch.shuffle_sync_bfly(local_product[e], offset=16)
warp_max = cute.arch.fmax(local_max[e], pair_max)
warp_sum = Float32(0.0)
warp_product = Float32(0.0)
if warp_max > -Float32.inf:
local_scale = cute.math.exp2(
(local_max[e] - warp_max) * Float32(self.rcp_ln2),
fastmath=True,
)
pair_scale = cute.math.exp2(
(pair_max - warp_max) * Float32(self.rcp_ln2),
fastmath=True,
)
warp_sum = local_sum[e] * local_scale + pair_sum * pair_scale
warp_product = (
local_product[e] * local_scale + pair_product * pair_scale
)
if lane_id < self.lanes_per_row:
s_max[col_group, e, warp_id] = warp_max
s_sum[col_group, e, warp_id] = warp_sum
s_product[col_group, e, warp_id] = warp_product
s_max[warp_id, col_group, e] = local_max[e]
s_sum[warp_id, col_group, e] = local_sum[e]
s_product[warp_id, col_group, e] = local_product[e]
cute.arch.sync_threads()
out_group = tid // self.num_warps
@@ -761,16 +725,16 @@ class SparseAttnCompressKernel:
out_lane = out_idx // self.elems_per_lane
out_elem = out_idx % self.elems_per_lane
local_warp_max = s_max[out_lane, out_elem, final_lane]
local_warp_max = s_max[final_lane, out_lane, out_elem]
global_max = local_warp_max
for step in cutlass.range_constexpr(3):
offset = const_expr(4 >> step)
for step in cutlass.range_constexpr(self.final_reduce_steps):
offset = const_expr(self.final_reduce_initial_offset >> step)
global_max = cute.arch.fmax(
global_max,
cute.arch.shuffle_sync_bfly(
global_max,
offset=offset,
mask_and_clamp=row_mask_and_clamp,
mask_and_clamp=final_mask_and_clamp,
),
)
@@ -778,19 +742,19 @@ class SparseAttnCompressKernel:
(local_warp_max - global_max) * Float32(self.rcp_ln2),
fastmath=True,
)
global_sum = s_sum[out_lane, out_elem, final_lane] * scale
global_product = s_product[out_lane, out_elem, final_lane] * scale
for step in cutlass.range_constexpr(3):
offset = const_expr(4 >> step)
global_sum = s_sum[final_lane, out_lane, out_elem] * scale
global_product = s_product[final_lane, out_lane, out_elem] * scale
for step in cutlass.range_constexpr(self.final_reduce_steps):
offset = const_expr(self.final_reduce_initial_offset >> step)
global_sum += cute.arch.shuffle_sync_bfly(
global_sum,
offset=offset,
mask_and_clamp=row_mask_and_clamp,
mask_and_clamp=final_mask_and_clamp,
)
global_product += cute.arch.shuffle_sync_bfly(
global_product,
offset=offset,
mask_and_clamp=row_mask_and_clamp,
mask_and_clamp=final_mask_and_clamp,
)
if final_lane == 0:
@@ -804,10 +768,8 @@ class SparseAttnCompressKernel:
def compile(
head_size: int = 512,
state_width: int = 512,
compress_ratio: int = 128,
overlap: bool = False,
):
if head_size % SparseAttnCompressKernel.head_tile != 0:
if head_size % SparseAttnCompressC128Block8Kernel.head_tile != 0:
raise ValueError("head_size must be divisible by the 64-wide head tile.")
num_positions = cute.sym_int()
num_slots = cute.sym_int()
@@ -838,15 +800,13 @@ class SparseAttnCompressKernel:
compressed_kv = cute.runtime.make_fake_tensor(
Float32,
(num_slots, head_size),
stride=(cute.sym_int64(divisibility=4), 1),
stride=(head_size, 1),
assumed_align=4,
)
kernel = SparseAttnCompressKernel(
kernel = SparseAttnCompressC128Block8Kernel(
head_size,
state_width,
compress_ratio,
overlap,
)
stream = cute.runtime.make_fake_stream(use_tvm_ffi_env_stream=True)
return cute.compile(
@@ -856,7 +816,6 @@ class SparseAttnCompressKernel:
positions,
slot_mapping,
block_table,
Int64(0),
compressed_kv,
stream,
options="--enable-tvm-ffi",
@@ -875,6 +834,7 @@ class SparseAttnNormRopeStoreKernel:
token_stride: int,
scale_dim: int,
compress_ratio: int,
static_kv_cache_block_size: int,
):
self.head_dim = head_size
self.rope_dim = rope_head_dim
@@ -887,6 +847,7 @@ class SparseAttnNormRopeStoreKernel:
self.nope_blocks = self.nope_dim // quant_block
self.tb_size = head_size // 2
self.compress_ratio = compress_ratio
self.static_kv_cache_block_size = static_kv_cache_block_size
@cute.jit
def __call__(
@@ -899,7 +860,6 @@ class SparseAttnNormRopeStoreKernel:
cos_sin_cache: cute.Tensor,
k_cache: cute.Tensor,
kv_slot_mapping: cute.Tensor,
kv_cache_block_size: Int64,
stream: CUstream,
):
grid = (slot_mapping.shape[0], 1, 1)
@@ -912,7 +872,6 @@ class SparseAttnNormRopeStoreKernel:
cos_sin_cache,
k_cache,
kv_slot_mapping,
kv_cache_block_size,
).launch(grid=grid, block=(self.tb_size, 1, 1), stream=stream)
@cute.kernel
@@ -926,7 +885,6 @@ class SparseAttnNormRopeStoreKernel:
cos_sin_cache: cute.Tensor,
k_cache: cute.Tensor,
kv_slot_mapping: cute.Tensor,
kv_cache_block_size: Int64,
):
token_idx, _, _ = cute.arch.block_idx()
tid, _, _ = cute.arch.thread_idx()
@@ -934,21 +892,41 @@ class SparseAttnNormRopeStoreKernel:
lane_id = tid % 32
elem0 = tid * 2
slot_id = slot_mapping[token_idx]
has_position = token_idx < positions.shape[0]
position = Int64(0)
if has_position:
kv_slot_idx = Int64(-1)
has_position = token_idx < positions.shape[0]
slot_id = Int64(-1)
if lane_id == 0:
slot_id = slot_mapping[token_idx]
if lane_id == 0 and has_position:
position = positions[token_idx]
has_kv_slot_idx = token_idx < kv_slot_mapping.shape[0]
if lane_id == 0 and has_kv_slot_idx:
kv_slot_idx = kv_slot_mapping[token_idx]
slot_id = cute.arch.shuffle_sync(slot_id, offset=0)
position = cute.arch.shuffle_sync(position, offset=0)
kv_slot_idx = cute.arch.shuffle_sync(kv_slot_idx, offset=0)
boundary = has_position and (
(position + Int64(1)) % Int64(self.compress_ratio) == Int64(0)
)
has_kv_slot_idx = token_idx < kv_slot_mapping.shape[0]
kv_slot_idx = Int64(-1)
if has_kv_slot_idx:
kv_slot_idx = kv_slot_mapping[token_idx]
active = slot_id >= Int64(0) and boundary and kv_slot_idx >= Int64(0)
if active:
k_cache_u16 = cute.recast_tensor(k_cache, Uint16)
k_cache_u32 = cute.recast_tensor(k_cache, Uint32)
static_block_size = Int64(self.static_kv_cache_block_size)
page = kv_slot_idx // static_block_size
kv_offset = kv_slot_idx - page * static_block_size
scale_row_offset = static_block_size * Int64(self.token_stride)
value_base = page * k_cache.stride[0] + kv_offset * Int64(self.token_stride)
scale_base = (
page * k_cache.stride[0]
+ scale_row_offset
+ kv_offset * Int64(self.scale_dim)
)
weight0 = rms_norm_weight[elem0].to(Float32)
weight1 = rms_norm_weight[elem0 + 1].to(Float32)
base = token_idx.to(Int64) * compressed_kv.stride[0] + elem0.to(Int64)
x0 = compressed_kv.iterator[base]
x1 = compressed_kv.iterator[base + Int64(1)]
@@ -963,42 +941,32 @@ class SparseAttnNormRopeStoreKernel:
partial_sums = smem.allocate_tensor(
Float32, cute.make_layout((self.num_warps,)), byte_alignment=4
)
rrms_shared = smem.allocate_tensor(
Float32, cute.make_layout((1,)), byte_alignment=4
)
if lane_id == 0:
partial_sums[warp_id] = warp_sum
cute.arch.sync_threads()
if tid == 0:
total = Float32(0.0)
for i in cutlass.range_constexpr(self.num_warps):
total += partial_sums[i]
rrms_shared[0] = cute.math.rsqrt(
total / Float32(self.head_dim) + rms_norm_eps, fastmath=True
)
cute.arch.sync_threads()
rrms = rrms_shared[0]
x0 = x0 * rrms * rms_norm_weight[elem0].to(Float32)
x1 = x1 * rrms * rms_norm_weight[elem0 + 1].to(Float32)
k_cache_u16 = cute.recast_tensor(k_cache, Uint16)
k_cache_u32 = cute.recast_tensor(k_cache, Uint32)
page = kv_slot_idx // kv_cache_block_size
kv_offset = kv_slot_idx - page * kv_cache_block_size
value_base = page * k_cache.stride[0] + kv_offset * Int64(self.token_stride)
scale_base = (
page * k_cache.stride[0]
+ kv_cache_block_size * Int64(self.token_stride)
+ kv_offset * Int64(self.scale_dim)
total = partial_sums[lane_id % self.num_warps]
sum_mask_and_clamp = const_expr(
(cute.arch.WARP_SIZE - self.num_warps) << 8 | (cute.arch.WARP_SIZE - 1)
)
for step in cutlass.range_constexpr(3):
offset = const_expr(4 >> step)
total += cute.arch.shuffle_sync_bfly(
total,
offset,
mask_and_clamp=sum_mask_and_clamp,
)
rrms = cute.math.rsqrt(
total / Float32(self.head_dim) + rms_norm_eps, fastmath=True
)
x0 = x0 * rrms * weight0
x1 = x1 * rrms * weight1
if warp_id == self.nope_blocks:
pair_idx = lane_id
compressed_pos = (position // Int64(self.compress_ratio)) * Int64(
self.compress_ratio
)
compressed_pos = position - Int64(self.compress_ratio - 1)
cs_base = compressed_pos * cos_sin_cache.stride[0] + pair_idx.to(Int64)
cos_v = cos_sin_cache.iterator[cs_base]
sin_v = cos_sin_cache.iterator[cs_base + Int64(self.rope_dim // 2)]
@@ -1058,6 +1026,7 @@ class SparseAttnNormRopeStoreKernel:
kv_block_stride: int = 74752,
compress_ratio: int = 128,
norm_weight_dtype: type[cutlass.Numeric] = Float32,
static_kv_cache_block_size: int = 0,
):
if quant_block != 64:
raise ValueError(
@@ -1074,6 +1043,11 @@ class SparseAttnNormRopeStoreKernel:
expected_scale_dim = (head_size - rope_head_dim) // quant_block + 1
if scale_dim < expected_scale_dim:
raise ValueError("scale_dim is too small for the UE8M0 scale row.")
if static_kv_cache_block_size <= 0:
raise ValueError(
"CuTe DSL sparse-attn store requires a positive static "
"kv_cache_block_size."
)
num_positions = cute.sym_int()
num_slots = cute.sym_int()
num_kv_slots = cute.sym_int()
@@ -1083,7 +1057,7 @@ class SparseAttnNormRopeStoreKernel:
compressed_kv = cute.runtime.make_fake_tensor(
Float32,
(num_slots, head_size),
stride=(cute.sym_int64(divisibility=4), 1),
stride=(head_size, 1),
assumed_align=4,
)
positions = make_fake_tensor(Int64, (num_positions,), divisibility=8)
@@ -1094,14 +1068,14 @@ class SparseAttnNormRopeStoreKernel:
cos_sin_cache = cute.runtime.make_fake_tensor(
Float32,
(max_pos, rope_head_dim),
stride=(cute.sym_int64(divisibility=4), 1),
stride=(rope_head_dim, 1),
assumed_align=4,
)
k_cache = cute.runtime.make_fake_tensor(
Uint8,
(num_blocks, cute.sym_int(), cute.sym_int()),
stride=(
cute.sym_int64(divisibility=16),
kv_block_stride,
cute.sym_int64(divisibility=8),
1,
),
@@ -1117,6 +1091,7 @@ class SparseAttnNormRopeStoreKernel:
token_stride,
scale_dim,
compress_ratio,
static_kv_cache_block_size,
)
stream = cute.runtime.make_fake_stream(use_tvm_ffi_env_stream=True)
return cute.compile(
@@ -1129,13 +1104,59 @@ class SparseAttnNormRopeStoreKernel:
cos_sin_cache,
k_cache,
kv_slot_mapping,
Int64(0),
stream,
options="--enable-tvm-ffi",
)
def compress_kv_sparse_attn_cutedsl(
def compile_split_sparse_attn_cutedsl(
head_size: int,
state_width: int,
block_size: int,
rope_head_dim: int,
fp8_max: float,
quant_block: int,
token_stride: int,
scale_dim: int,
kv_cache_block_size: int,
kv_block_stride: int,
compress_ratio: int,
overlap: bool,
rms_norm_weight_dtype: torch.dtype,
):
if not (
head_size == 512
and state_width == head_size
and compress_ratio == 128
and not overlap
and block_size == 8
):
raise ValueError(
"CuTe DSL split sparse-attn wrapper only supports the real "
"DeepSeek V4 C128 layout: head_size=512, state_width=512, "
"compress_ratio=128, overlap=False, block_size=8."
)
compress = SparseAttnCompressC128Block8Kernel.compile(
head_size=head_size,
state_width=state_width,
)
norm_weight_dtype = _TORCH_TO_CUTE[rms_norm_weight_dtype]
store = SparseAttnNormRopeStoreKernel.compile(
head_size,
rope_head_dim,
fp8_max,
quant_block,
token_stride,
scale_dim,
kv_block_stride,
compress_ratio,
norm_weight_dtype,
kv_cache_block_size,
)
return compress, store
def split_kv_compress_norm_rope_insert_sparse_attn_cutedsl(
state_cache: torch.Tensor,
token_to_req_indices: torch.Tensor,
positions: torch.Tensor,
@@ -1143,34 +1164,6 @@ def compress_kv_sparse_attn_cutedsl(
block_table: torch.Tensor,
block_size: int,
compressed_kv: torch.Tensor,
head_size: int = 512,
state_width: int = 512,
compress_ratio: int = 128,
overlap: bool = False,
) -> None:
if positions.numel() == 0:
return
compiled = SparseAttnCompressKernel.compile(
head_size=head_size,
state_width=state_width,
compress_ratio=compress_ratio,
overlap=overlap,
)
compiled(
state_cache,
token_to_req_indices,
positions,
slot_mapping,
block_table,
block_size,
compressed_kv,
)
def norm_rope_insert_sparse_attn_cutedsl(
compressed_kv: torch.Tensor,
positions: torch.Tensor,
slot_mapping: torch.Tensor,
rms_norm_weight: torch.Tensor,
rms_norm_eps: float,
cos_sin_cache: torch.Tensor,
@@ -1179,38 +1172,58 @@ def norm_rope_insert_sparse_attn_cutedsl(
kv_cache_block_size: int,
kv_block_stride: int,
head_size: int = 512,
state_width: int = 512,
rope_head_dim: int = 64,
fp8_max: float = 448.0,
quant_block: int = 64,
token_stride: int = 576,
scale_dim: int = 8,
compress_ratio: int = 128,
overlap: bool = False,
) -> None:
if positions.numel() == 0:
return
norm_weight_dtype = _TORCH_TO_CUTE.get(rms_norm_weight.dtype)
if norm_weight_dtype is None:
raise ValueError(
"CuTe DSL sparse-attn store supports rms_norm_weight dtype "
f"bf16/fp32, got {rms_norm_weight.dtype}."
)
if k_cache.ndim != 3:
raise ValueError(
"CuTe DSL sparse-attn store expects the real DeepSeek V4 "
f"3D k_cache layout [num_blocks, block_size, 584], got ndim={k_cache.ndim}."
)
compiled = SparseAttnNormRopeStoreKernel.compile(
head_size=head_size,
rope_head_dim=rope_head_dim,
fp8_max=fp8_max,
quant_block=quant_block,
token_stride=token_stride,
scale_dim=scale_dim,
kv_block_stride=kv_block_stride,
compress_ratio=compress_ratio,
norm_weight_dtype=norm_weight_dtype,
if kv_cache_block_size != k_cache.shape[1]:
raise ValueError(
"CuTe DSL split sparse-attn wrapper expected kv_cache_block_size "
f"to match k_cache.shape[1], got {kv_cache_block_size} and "
f"{k_cache.shape[1]}."
)
if positions.numel() == 0:
return
if rms_norm_weight.dtype not in _TORCH_TO_CUTE:
raise ValueError(
"CuTe DSL sparse-attn store supports rms_norm_weight dtype "
f"bf16/fp32, got {rms_norm_weight.dtype}."
)
compress, store = compile_split_sparse_attn_cutedsl(
head_size,
state_width,
block_size,
rope_head_dim,
fp8_max,
quant_block,
token_stride,
scale_dim,
kv_cache_block_size,
kv_block_stride,
compress_ratio,
overlap,
rms_norm_weight.dtype,
)
compiled(
compress(
state_cache,
token_to_req_indices,
positions,
slot_mapping,
block_table,
compressed_kv,
)
store(
compressed_kv,
positions,
slot_mapping,
@@ -1219,7 +1232,6 @@ def norm_rope_insert_sparse_attn_cutedsl(
cos_sin_cache,
k_cache,
kv_slot_mapping,
kv_cache_block_size,
)
@@ -1346,7 +1358,7 @@ def compress_norm_rope_store_cutedsl(
dtype=torch.float32,
device=state_cache.device,
)
compress_kv_sparse_attn_cutedsl(
split_kv_compress_norm_rope_insert_sparse_attn_cutedsl(
state_cache,
token_to_req_indices,
positions,
@@ -1354,15 +1366,6 @@ def compress_norm_rope_store_cutedsl(
block_table,
block_size,
compressed_kv,
head_size=head_dim,
state_width=state_width,
compress_ratio=compress_ratio,
overlap=overlap,
)
norm_rope_insert_sparse_attn_cutedsl(
compressed_kv,
positions,
slot_mapping,
rms_norm_weight,
rms_norm_eps,
cos_sin_cache,
@@ -1371,10 +1374,12 @@ def compress_norm_rope_store_cutedsl(
kv_cache.shape[1], # paged KV cache block size
kv_cache.stride(0),
head_size=head_dim,
state_width=state_width,
rope_head_dim=rope_head_dim,
fp8_max=448.0,
quant_block=quant_block,
token_stride=token_stride,
scale_dim=scale_dim,
compress_ratio=compress_ratio,
overlap=overlap,
)