optimize the compressor 128 split cutedsl kernel (#44230)
Signed-off-by: Jie Fang <jief@nvidia.com> Co-authored-by: Yongye Zhu <zyy1102000@gmail.com>
This commit is contained in:
@@ -508,30 +508,28 @@ class SparseAttnCompressNormRopeStoreC4Kernel:
|
||||
)
|
||||
|
||||
|
||||
class SparseAttnCompressKernel:
|
||||
class SparseAttnCompressC128Block8Kernel:
|
||||
head_tile = 64
|
||||
rows_per_warp = 16
|
||||
row_pairs_per_warp = rows_per_warp // 2
|
||||
elems_per_lane = 4
|
||||
elems_per_lane = 2
|
||||
lanes_per_row = head_tile // elems_per_lane
|
||||
num_warps = 8
|
||||
stats_warp_stride = num_warps + 1
|
||||
stats_lane_stride = lanes_per_row + 1
|
||||
final_reduce_steps = 3
|
||||
final_reduce_initial_offset = 4
|
||||
tb_size = num_warps * 32
|
||||
compress_ratio = 128
|
||||
state_block_size = 8
|
||||
rcp_ln2 = 1.4426950408889634
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
head_size: int,
|
||||
state_width: int,
|
||||
compress_ratio: int,
|
||||
overlap: bool,
|
||||
):
|
||||
self.head_dim = head_size
|
||||
self.num_splits = head_size // self.head_tile
|
||||
self.state_width = state_width
|
||||
self.compress_ratio = compress_ratio
|
||||
self.overlap = overlap
|
||||
self.window = (1 + int(overlap)) * compress_ratio
|
||||
|
||||
@cute.jit
|
||||
def __call__(
|
||||
@@ -541,7 +539,6 @@ class SparseAttnCompressKernel:
|
||||
positions: cute.Tensor,
|
||||
slot_mapping: cute.Tensor,
|
||||
block_table: cute.Tensor,
|
||||
block_size: Int64,
|
||||
compressed_kv: cute.Tensor,
|
||||
stream: CUstream,
|
||||
):
|
||||
@@ -552,7 +549,6 @@ class SparseAttnCompressKernel:
|
||||
positions,
|
||||
slot_mapping,
|
||||
block_table,
|
||||
block_size,
|
||||
compressed_kv,
|
||||
).launch(grid=grid, block=(self.tb_size, 1, 1), stream=stream)
|
||||
|
||||
@@ -564,29 +560,36 @@ class SparseAttnCompressKernel:
|
||||
positions: cute.Tensor,
|
||||
slot_mapping: cute.Tensor,
|
||||
block_table: cute.Tensor,
|
||||
block_size: Int64,
|
||||
compressed_kv: cute.Tensor,
|
||||
):
|
||||
block_id, _, _ = cute.arch.block_idx()
|
||||
tid, _, _ = cute.arch.thread_idx()
|
||||
warp_id = cute.arch.make_warp_uniform(tid // 32)
|
||||
lane_id = tid % 32
|
||||
row_lane = lane_id // self.lanes_per_row
|
||||
col_group = lane_id % self.lanes_per_row
|
||||
|
||||
token_idx = block_id // self.num_splits
|
||||
split_idx = block_id - token_idx * self.num_splits
|
||||
col_base = split_idx * self.head_tile + col_group * self.elems_per_lane
|
||||
|
||||
slot_id = slot_mapping[token_idx]
|
||||
has_position = token_idx < positions.shape[0]
|
||||
position = Int64(0)
|
||||
if has_position:
|
||||
req_idx = Int32(0)
|
||||
slot_id = Int64(-1)
|
||||
has_position = token_idx < positions.shape[0]
|
||||
has_req_idx = token_idx < token_to_req_indices.shape[0]
|
||||
if lane_id == 0:
|
||||
slot_id = slot_mapping[token_idx]
|
||||
if lane_id == 0 and has_position:
|
||||
position = positions[token_idx]
|
||||
if lane_id == 0 and has_req_idx:
|
||||
req_idx = token_to_req_indices[token_idx]
|
||||
slot_id = cute.arch.shuffle_sync(slot_id, offset=0)
|
||||
position = cute.arch.shuffle_sync(position, offset=0)
|
||||
req_idx = cute.arch.shuffle_sync(req_idx, offset=0)
|
||||
boundary = has_position and (
|
||||
(position + Int64(1)) % Int64(self.compress_ratio) == Int64(0)
|
||||
)
|
||||
has_req_idx = token_idx < token_to_req_indices.shape[0]
|
||||
start = position - Int64(self.compress_ratio - 1)
|
||||
active = slot_id >= Int64(0) and has_req_idx and boundary
|
||||
|
||||
if active:
|
||||
@@ -595,13 +598,13 @@ class SparseAttnCompressKernel:
|
||||
Float32,
|
||||
cute.make_layout(
|
||||
(
|
||||
self.num_warps,
|
||||
self.lanes_per_row,
|
||||
self.elems_per_lane,
|
||||
self.stats_warp_stride,
|
||||
),
|
||||
stride=(
|
||||
self.elems_per_lane * self.stats_warp_stride,
|
||||
self.stats_warp_stride,
|
||||
self.stats_lane_stride * self.elems_per_lane,
|
||||
self.elems_per_lane,
|
||||
1,
|
||||
),
|
||||
),
|
||||
@@ -611,13 +614,13 @@ class SparseAttnCompressKernel:
|
||||
Float32,
|
||||
cute.make_layout(
|
||||
(
|
||||
self.num_warps,
|
||||
self.lanes_per_row,
|
||||
self.elems_per_lane,
|
||||
self.stats_warp_stride,
|
||||
),
|
||||
stride=(
|
||||
self.elems_per_lane * self.stats_warp_stride,
|
||||
self.stats_warp_stride,
|
||||
self.stats_lane_stride * self.elems_per_lane,
|
||||
self.elems_per_lane,
|
||||
1,
|
||||
),
|
||||
),
|
||||
@@ -627,25 +630,25 @@ class SparseAttnCompressKernel:
|
||||
Float32,
|
||||
cute.make_layout(
|
||||
(
|
||||
self.num_warps,
|
||||
self.lanes_per_row,
|
||||
self.elems_per_lane,
|
||||
self.stats_warp_stride,
|
||||
),
|
||||
stride=(
|
||||
self.elems_per_lane * self.stats_warp_stride,
|
||||
self.stats_warp_stride,
|
||||
self.stats_lane_stride * self.elems_per_lane,
|
||||
self.elems_per_lane,
|
||||
1,
|
||||
),
|
||||
),
|
||||
byte_alignment=4,
|
||||
)
|
||||
|
||||
row_pair_layout = cute.make_layout(
|
||||
(self.row_pairs_per_warp, self.elems_per_lane),
|
||||
row_layout = cute.make_layout(
|
||||
(self.rows_per_warp, self.elems_per_lane),
|
||||
stride=(self.elems_per_lane, 1),
|
||||
)
|
||||
kv_vals = cute.make_rmem_tensor(row_pair_layout, Float32)
|
||||
score_vals = cute.make_rmem_tensor(row_pair_layout, Float32)
|
||||
kv_vals = cute.make_rmem_tensor(row_layout, Float32)
|
||||
score_vals = cute.make_rmem_tensor(row_layout, Float32)
|
||||
local_max = cute.make_rmem_tensor((self.elems_per_lane,), Float32)
|
||||
local_sum = cute.make_rmem_tensor((self.elems_per_lane,), Float32)
|
||||
local_product = cute.make_rmem_tensor((self.elems_per_lane,), Float32)
|
||||
@@ -655,100 +658,61 @@ class SparseAttnCompressKernel:
|
||||
local_sum[e] = Float32(0.0)
|
||||
local_product[e] = Float32(0.0)
|
||||
|
||||
req_idx = token_to_req_indices[token_idx]
|
||||
start = position - Int64(self.window - 1)
|
||||
cp_f32x4 = cute.make_copy_atom(
|
||||
cute.nvgpu.CopyUniversalOp(), Float32, num_bits_per_copy=128
|
||||
first_block_index = start // Int64(self.state_block_size)
|
||||
warp_block_index = first_block_index + (warp_id * 2).to(Int64)
|
||||
block0_i32 = Int32(0)
|
||||
block1_i32 = Int32(0)
|
||||
if lane_id == 0:
|
||||
block0_i32 = block_table[req_idx, warp_block_index]
|
||||
block1_i32 = block_table[req_idx, warp_block_index + Int64(1)]
|
||||
block0_i32 = cute.arch.shuffle_sync(block0_i32, offset=0)
|
||||
block1_i32 = cute.arch.shuffle_sync(block1_i32, offset=0)
|
||||
|
||||
cp_f32x2 = cute.make_copy_atom(
|
||||
cute.nvgpu.CopyUniversalOp(), Float32, num_bits_per_copy=64
|
||||
)
|
||||
row_mask_and_clamp = const_expr(
|
||||
(cute.arch.WARP_SIZE - self.lanes_per_row) << 8
|
||||
| (cute.arch.WARP_SIZE - 1)
|
||||
final_mask_and_clamp = const_expr(
|
||||
(cute.arch.WARP_SIZE - self.num_warps) << 8 | (cute.arch.WARP_SIZE - 1)
|
||||
)
|
||||
col_tile = col_base.to(Int64) // Int64(self.elems_per_lane)
|
||||
score_col_tile = col_tile + Int64(self.state_width // self.elems_per_lane)
|
||||
|
||||
for i in cutlass.range_constexpr(self.row_pairs_per_warp):
|
||||
row = warp_id * self.rows_per_warp + i * 2 + row_lane
|
||||
pos = start + row.to(Int64)
|
||||
valid = row < self.window and pos >= Int64(0)
|
||||
head_offset = ((row // self.compress_ratio) * self.head_dim).to(Int64)
|
||||
|
||||
for e in cutlass.range_constexpr(self.elems_per_lane):
|
||||
kv = Float32(0.0)
|
||||
score = -Float32.inf
|
||||
kv_vals[i, e] = kv
|
||||
score_vals[i, e] = score
|
||||
|
||||
block_index = Int64(0)
|
||||
block_offset = Int64(0)
|
||||
block_number_i32 = Int32(0)
|
||||
if valid:
|
||||
block_index = pos // block_size
|
||||
block_offset = pos - block_index * block_size
|
||||
if col_group == 0:
|
||||
block_number_i32 = block_table[req_idx, block_index]
|
||||
block_number_i32 = cute.arch.shuffle_sync(
|
||||
block_number_i32,
|
||||
offset=0,
|
||||
mask_and_clamp=row_mask_and_clamp,
|
||||
for i in cutlass.range_constexpr(self.rows_per_warp):
|
||||
block_number_i32 = block0_i32
|
||||
block_offset = Int64(i)
|
||||
if const_expr(i >= self.state_block_size):
|
||||
block_number_i32 = block1_i32
|
||||
block_offset = Int64(i - self.state_block_size)
|
||||
row_tensor = state_cache[block_number_i32.to(Int64), block_offset, None]
|
||||
kv_src = cute.local_tile(
|
||||
row_tensor,
|
||||
tiler=(self.elems_per_lane,),
|
||||
coord=(col_tile,),
|
||||
)
|
||||
|
||||
if valid:
|
||||
block_number = block_number_i32.to(Int64)
|
||||
row_tensor = state_cache[block_number, block_offset, None]
|
||||
col_tile = (head_offset + col_base.to(Int64)) // Int64(
|
||||
self.elems_per_lane
|
||||
)
|
||||
kv_src = cute.local_tile(
|
||||
row_tensor,
|
||||
tiler=(self.elems_per_lane,),
|
||||
coord=(col_tile,),
|
||||
)
|
||||
score_src = cute.local_tile(
|
||||
row_tensor,
|
||||
tiler=(self.elems_per_lane,),
|
||||
coord=(
|
||||
col_tile + Int64(self.state_width // self.elems_per_lane),
|
||||
),
|
||||
)
|
||||
cute.copy(cp_f32x4, kv_src, kv_vals[i, None])
|
||||
cute.copy(cp_f32x4, score_src, score_vals[i, None])
|
||||
score_src = cute.local_tile(
|
||||
row_tensor,
|
||||
tiler=(self.elems_per_lane,),
|
||||
coord=(score_col_tile,),
|
||||
)
|
||||
cute.copy(cp_f32x2, kv_src, kv_vals[i, None])
|
||||
cute.copy(cp_f32x2, score_src, score_vals[i, None])
|
||||
|
||||
for e in cutlass.range_constexpr(self.elems_per_lane):
|
||||
local_max[e] = cute.arch.fmax(local_max[e], score_vals[i, e])
|
||||
|
||||
for e in cutlass.range_constexpr(self.elems_per_lane):
|
||||
if local_max[e] > -Float32.inf:
|
||||
for i in cutlass.range_constexpr(self.row_pairs_per_warp):
|
||||
exp_score = cute.math.exp2(
|
||||
(score_vals[i, e] - local_max[e]) * Float32(self.rcp_ln2),
|
||||
fastmath=True,
|
||||
)
|
||||
local_sum[e] += exp_score
|
||||
local_product[e] += kv_vals[i, e] * exp_score
|
||||
for i in cutlass.range_constexpr(self.rows_per_warp):
|
||||
exp_score = cute.math.exp2(
|
||||
(score_vals[i, e] - local_max[e]) * Float32(self.rcp_ln2),
|
||||
fastmath=True,
|
||||
)
|
||||
local_sum[e] += exp_score
|
||||
local_product[e] += kv_vals[i, e] * exp_score
|
||||
|
||||
for e in cutlass.range_constexpr(self.elems_per_lane):
|
||||
pair_max = cute.arch.shuffle_sync_bfly(local_max[e], offset=16)
|
||||
pair_sum = cute.arch.shuffle_sync_bfly(local_sum[e], offset=16)
|
||||
pair_product = cute.arch.shuffle_sync_bfly(local_product[e], offset=16)
|
||||
warp_max = cute.arch.fmax(local_max[e], pair_max)
|
||||
warp_sum = Float32(0.0)
|
||||
warp_product = Float32(0.0)
|
||||
if warp_max > -Float32.inf:
|
||||
local_scale = cute.math.exp2(
|
||||
(local_max[e] - warp_max) * Float32(self.rcp_ln2),
|
||||
fastmath=True,
|
||||
)
|
||||
pair_scale = cute.math.exp2(
|
||||
(pair_max - warp_max) * Float32(self.rcp_ln2),
|
||||
fastmath=True,
|
||||
)
|
||||
warp_sum = local_sum[e] * local_scale + pair_sum * pair_scale
|
||||
warp_product = (
|
||||
local_product[e] * local_scale + pair_product * pair_scale
|
||||
)
|
||||
if lane_id < self.lanes_per_row:
|
||||
s_max[col_group, e, warp_id] = warp_max
|
||||
s_sum[col_group, e, warp_id] = warp_sum
|
||||
s_product[col_group, e, warp_id] = warp_product
|
||||
s_max[warp_id, col_group, e] = local_max[e]
|
||||
s_sum[warp_id, col_group, e] = local_sum[e]
|
||||
s_product[warp_id, col_group, e] = local_product[e]
|
||||
cute.arch.sync_threads()
|
||||
|
||||
out_group = tid // self.num_warps
|
||||
@@ -761,16 +725,16 @@ class SparseAttnCompressKernel:
|
||||
out_lane = out_idx // self.elems_per_lane
|
||||
out_elem = out_idx % self.elems_per_lane
|
||||
|
||||
local_warp_max = s_max[out_lane, out_elem, final_lane]
|
||||
local_warp_max = s_max[final_lane, out_lane, out_elem]
|
||||
global_max = local_warp_max
|
||||
for step in cutlass.range_constexpr(3):
|
||||
offset = const_expr(4 >> step)
|
||||
for step in cutlass.range_constexpr(self.final_reduce_steps):
|
||||
offset = const_expr(self.final_reduce_initial_offset >> step)
|
||||
global_max = cute.arch.fmax(
|
||||
global_max,
|
||||
cute.arch.shuffle_sync_bfly(
|
||||
global_max,
|
||||
offset=offset,
|
||||
mask_and_clamp=row_mask_and_clamp,
|
||||
mask_and_clamp=final_mask_and_clamp,
|
||||
),
|
||||
)
|
||||
|
||||
@@ -778,19 +742,19 @@ class SparseAttnCompressKernel:
|
||||
(local_warp_max - global_max) * Float32(self.rcp_ln2),
|
||||
fastmath=True,
|
||||
)
|
||||
global_sum = s_sum[out_lane, out_elem, final_lane] * scale
|
||||
global_product = s_product[out_lane, out_elem, final_lane] * scale
|
||||
for step in cutlass.range_constexpr(3):
|
||||
offset = const_expr(4 >> step)
|
||||
global_sum = s_sum[final_lane, out_lane, out_elem] * scale
|
||||
global_product = s_product[final_lane, out_lane, out_elem] * scale
|
||||
for step in cutlass.range_constexpr(self.final_reduce_steps):
|
||||
offset = const_expr(self.final_reduce_initial_offset >> step)
|
||||
global_sum += cute.arch.shuffle_sync_bfly(
|
||||
global_sum,
|
||||
offset=offset,
|
||||
mask_and_clamp=row_mask_and_clamp,
|
||||
mask_and_clamp=final_mask_and_clamp,
|
||||
)
|
||||
global_product += cute.arch.shuffle_sync_bfly(
|
||||
global_product,
|
||||
offset=offset,
|
||||
mask_and_clamp=row_mask_and_clamp,
|
||||
mask_and_clamp=final_mask_and_clamp,
|
||||
)
|
||||
|
||||
if final_lane == 0:
|
||||
@@ -804,10 +768,8 @@ class SparseAttnCompressKernel:
|
||||
def compile(
|
||||
head_size: int = 512,
|
||||
state_width: int = 512,
|
||||
compress_ratio: int = 128,
|
||||
overlap: bool = False,
|
||||
):
|
||||
if head_size % SparseAttnCompressKernel.head_tile != 0:
|
||||
if head_size % SparseAttnCompressC128Block8Kernel.head_tile != 0:
|
||||
raise ValueError("head_size must be divisible by the 64-wide head tile.")
|
||||
num_positions = cute.sym_int()
|
||||
num_slots = cute.sym_int()
|
||||
@@ -838,15 +800,13 @@ class SparseAttnCompressKernel:
|
||||
compressed_kv = cute.runtime.make_fake_tensor(
|
||||
Float32,
|
||||
(num_slots, head_size),
|
||||
stride=(cute.sym_int64(divisibility=4), 1),
|
||||
stride=(head_size, 1),
|
||||
assumed_align=4,
|
||||
)
|
||||
|
||||
kernel = SparseAttnCompressKernel(
|
||||
kernel = SparseAttnCompressC128Block8Kernel(
|
||||
head_size,
|
||||
state_width,
|
||||
compress_ratio,
|
||||
overlap,
|
||||
)
|
||||
stream = cute.runtime.make_fake_stream(use_tvm_ffi_env_stream=True)
|
||||
return cute.compile(
|
||||
@@ -856,7 +816,6 @@ class SparseAttnCompressKernel:
|
||||
positions,
|
||||
slot_mapping,
|
||||
block_table,
|
||||
Int64(0),
|
||||
compressed_kv,
|
||||
stream,
|
||||
options="--enable-tvm-ffi",
|
||||
@@ -875,6 +834,7 @@ class SparseAttnNormRopeStoreKernel:
|
||||
token_stride: int,
|
||||
scale_dim: int,
|
||||
compress_ratio: int,
|
||||
static_kv_cache_block_size: int,
|
||||
):
|
||||
self.head_dim = head_size
|
||||
self.rope_dim = rope_head_dim
|
||||
@@ -887,6 +847,7 @@ class SparseAttnNormRopeStoreKernel:
|
||||
self.nope_blocks = self.nope_dim // quant_block
|
||||
self.tb_size = head_size // 2
|
||||
self.compress_ratio = compress_ratio
|
||||
self.static_kv_cache_block_size = static_kv_cache_block_size
|
||||
|
||||
@cute.jit
|
||||
def __call__(
|
||||
@@ -899,7 +860,6 @@ class SparseAttnNormRopeStoreKernel:
|
||||
cos_sin_cache: cute.Tensor,
|
||||
k_cache: cute.Tensor,
|
||||
kv_slot_mapping: cute.Tensor,
|
||||
kv_cache_block_size: Int64,
|
||||
stream: CUstream,
|
||||
):
|
||||
grid = (slot_mapping.shape[0], 1, 1)
|
||||
@@ -912,7 +872,6 @@ class SparseAttnNormRopeStoreKernel:
|
||||
cos_sin_cache,
|
||||
k_cache,
|
||||
kv_slot_mapping,
|
||||
kv_cache_block_size,
|
||||
).launch(grid=grid, block=(self.tb_size, 1, 1), stream=stream)
|
||||
|
||||
@cute.kernel
|
||||
@@ -926,7 +885,6 @@ class SparseAttnNormRopeStoreKernel:
|
||||
cos_sin_cache: cute.Tensor,
|
||||
k_cache: cute.Tensor,
|
||||
kv_slot_mapping: cute.Tensor,
|
||||
kv_cache_block_size: Int64,
|
||||
):
|
||||
token_idx, _, _ = cute.arch.block_idx()
|
||||
tid, _, _ = cute.arch.thread_idx()
|
||||
@@ -934,21 +892,41 @@ class SparseAttnNormRopeStoreKernel:
|
||||
lane_id = tid % 32
|
||||
elem0 = tid * 2
|
||||
|
||||
slot_id = slot_mapping[token_idx]
|
||||
has_position = token_idx < positions.shape[0]
|
||||
position = Int64(0)
|
||||
if has_position:
|
||||
kv_slot_idx = Int64(-1)
|
||||
has_position = token_idx < positions.shape[0]
|
||||
slot_id = Int64(-1)
|
||||
if lane_id == 0:
|
||||
slot_id = slot_mapping[token_idx]
|
||||
if lane_id == 0 and has_position:
|
||||
position = positions[token_idx]
|
||||
has_kv_slot_idx = token_idx < kv_slot_mapping.shape[0]
|
||||
if lane_id == 0 and has_kv_slot_idx:
|
||||
kv_slot_idx = kv_slot_mapping[token_idx]
|
||||
slot_id = cute.arch.shuffle_sync(slot_id, offset=0)
|
||||
position = cute.arch.shuffle_sync(position, offset=0)
|
||||
kv_slot_idx = cute.arch.shuffle_sync(kv_slot_idx, offset=0)
|
||||
boundary = has_position and (
|
||||
(position + Int64(1)) % Int64(self.compress_ratio) == Int64(0)
|
||||
)
|
||||
has_kv_slot_idx = token_idx < kv_slot_mapping.shape[0]
|
||||
kv_slot_idx = Int64(-1)
|
||||
if has_kv_slot_idx:
|
||||
kv_slot_idx = kv_slot_mapping[token_idx]
|
||||
active = slot_id >= Int64(0) and boundary and kv_slot_idx >= Int64(0)
|
||||
|
||||
if active:
|
||||
k_cache_u16 = cute.recast_tensor(k_cache, Uint16)
|
||||
k_cache_u32 = cute.recast_tensor(k_cache, Uint32)
|
||||
static_block_size = Int64(self.static_kv_cache_block_size)
|
||||
page = kv_slot_idx // static_block_size
|
||||
kv_offset = kv_slot_idx - page * static_block_size
|
||||
scale_row_offset = static_block_size * Int64(self.token_stride)
|
||||
value_base = page * k_cache.stride[0] + kv_offset * Int64(self.token_stride)
|
||||
scale_base = (
|
||||
page * k_cache.stride[0]
|
||||
+ scale_row_offset
|
||||
+ kv_offset * Int64(self.scale_dim)
|
||||
)
|
||||
weight0 = rms_norm_weight[elem0].to(Float32)
|
||||
weight1 = rms_norm_weight[elem0 + 1].to(Float32)
|
||||
|
||||
base = token_idx.to(Int64) * compressed_kv.stride[0] + elem0.to(Int64)
|
||||
x0 = compressed_kv.iterator[base]
|
||||
x1 = compressed_kv.iterator[base + Int64(1)]
|
||||
@@ -963,42 +941,32 @@ class SparseAttnNormRopeStoreKernel:
|
||||
partial_sums = smem.allocate_tensor(
|
||||
Float32, cute.make_layout((self.num_warps,)), byte_alignment=4
|
||||
)
|
||||
rrms_shared = smem.allocate_tensor(
|
||||
Float32, cute.make_layout((1,)), byte_alignment=4
|
||||
)
|
||||
|
||||
if lane_id == 0:
|
||||
partial_sums[warp_id] = warp_sum
|
||||
cute.arch.sync_threads()
|
||||
if tid == 0:
|
||||
total = Float32(0.0)
|
||||
for i in cutlass.range_constexpr(self.num_warps):
|
||||
total += partial_sums[i]
|
||||
rrms_shared[0] = cute.math.rsqrt(
|
||||
total / Float32(self.head_dim) + rms_norm_eps, fastmath=True
|
||||
)
|
||||
cute.arch.sync_threads()
|
||||
|
||||
rrms = rrms_shared[0]
|
||||
x0 = x0 * rrms * rms_norm_weight[elem0].to(Float32)
|
||||
x1 = x1 * rrms * rms_norm_weight[elem0 + 1].to(Float32)
|
||||
|
||||
k_cache_u16 = cute.recast_tensor(k_cache, Uint16)
|
||||
k_cache_u32 = cute.recast_tensor(k_cache, Uint32)
|
||||
page = kv_slot_idx // kv_cache_block_size
|
||||
kv_offset = kv_slot_idx - page * kv_cache_block_size
|
||||
value_base = page * k_cache.stride[0] + kv_offset * Int64(self.token_stride)
|
||||
scale_base = (
|
||||
page * k_cache.stride[0]
|
||||
+ kv_cache_block_size * Int64(self.token_stride)
|
||||
+ kv_offset * Int64(self.scale_dim)
|
||||
total = partial_sums[lane_id % self.num_warps]
|
||||
sum_mask_and_clamp = const_expr(
|
||||
(cute.arch.WARP_SIZE - self.num_warps) << 8 | (cute.arch.WARP_SIZE - 1)
|
||||
)
|
||||
for step in cutlass.range_constexpr(3):
|
||||
offset = const_expr(4 >> step)
|
||||
total += cute.arch.shuffle_sync_bfly(
|
||||
total,
|
||||
offset,
|
||||
mask_and_clamp=sum_mask_and_clamp,
|
||||
)
|
||||
|
||||
rrms = cute.math.rsqrt(
|
||||
total / Float32(self.head_dim) + rms_norm_eps, fastmath=True
|
||||
)
|
||||
x0 = x0 * rrms * weight0
|
||||
x1 = x1 * rrms * weight1
|
||||
|
||||
if warp_id == self.nope_blocks:
|
||||
pair_idx = lane_id
|
||||
compressed_pos = (position // Int64(self.compress_ratio)) * Int64(
|
||||
self.compress_ratio
|
||||
)
|
||||
compressed_pos = position - Int64(self.compress_ratio - 1)
|
||||
cs_base = compressed_pos * cos_sin_cache.stride[0] + pair_idx.to(Int64)
|
||||
cos_v = cos_sin_cache.iterator[cs_base]
|
||||
sin_v = cos_sin_cache.iterator[cs_base + Int64(self.rope_dim // 2)]
|
||||
@@ -1058,6 +1026,7 @@ class SparseAttnNormRopeStoreKernel:
|
||||
kv_block_stride: int = 74752,
|
||||
compress_ratio: int = 128,
|
||||
norm_weight_dtype: type[cutlass.Numeric] = Float32,
|
||||
static_kv_cache_block_size: int = 0,
|
||||
):
|
||||
if quant_block != 64:
|
||||
raise ValueError(
|
||||
@@ -1074,6 +1043,11 @@ class SparseAttnNormRopeStoreKernel:
|
||||
expected_scale_dim = (head_size - rope_head_dim) // quant_block + 1
|
||||
if scale_dim < expected_scale_dim:
|
||||
raise ValueError("scale_dim is too small for the UE8M0 scale row.")
|
||||
if static_kv_cache_block_size <= 0:
|
||||
raise ValueError(
|
||||
"CuTe DSL sparse-attn store requires a positive static "
|
||||
"kv_cache_block_size."
|
||||
)
|
||||
num_positions = cute.sym_int()
|
||||
num_slots = cute.sym_int()
|
||||
num_kv_slots = cute.sym_int()
|
||||
@@ -1083,7 +1057,7 @@ class SparseAttnNormRopeStoreKernel:
|
||||
compressed_kv = cute.runtime.make_fake_tensor(
|
||||
Float32,
|
||||
(num_slots, head_size),
|
||||
stride=(cute.sym_int64(divisibility=4), 1),
|
||||
stride=(head_size, 1),
|
||||
assumed_align=4,
|
||||
)
|
||||
positions = make_fake_tensor(Int64, (num_positions,), divisibility=8)
|
||||
@@ -1094,14 +1068,14 @@ class SparseAttnNormRopeStoreKernel:
|
||||
cos_sin_cache = cute.runtime.make_fake_tensor(
|
||||
Float32,
|
||||
(max_pos, rope_head_dim),
|
||||
stride=(cute.sym_int64(divisibility=4), 1),
|
||||
stride=(rope_head_dim, 1),
|
||||
assumed_align=4,
|
||||
)
|
||||
k_cache = cute.runtime.make_fake_tensor(
|
||||
Uint8,
|
||||
(num_blocks, cute.sym_int(), cute.sym_int()),
|
||||
stride=(
|
||||
cute.sym_int64(divisibility=16),
|
||||
kv_block_stride,
|
||||
cute.sym_int64(divisibility=8),
|
||||
1,
|
||||
),
|
||||
@@ -1117,6 +1091,7 @@ class SparseAttnNormRopeStoreKernel:
|
||||
token_stride,
|
||||
scale_dim,
|
||||
compress_ratio,
|
||||
static_kv_cache_block_size,
|
||||
)
|
||||
stream = cute.runtime.make_fake_stream(use_tvm_ffi_env_stream=True)
|
||||
return cute.compile(
|
||||
@@ -1129,13 +1104,59 @@ class SparseAttnNormRopeStoreKernel:
|
||||
cos_sin_cache,
|
||||
k_cache,
|
||||
kv_slot_mapping,
|
||||
Int64(0),
|
||||
stream,
|
||||
options="--enable-tvm-ffi",
|
||||
)
|
||||
|
||||
|
||||
def compress_kv_sparse_attn_cutedsl(
|
||||
def compile_split_sparse_attn_cutedsl(
|
||||
head_size: int,
|
||||
state_width: int,
|
||||
block_size: int,
|
||||
rope_head_dim: int,
|
||||
fp8_max: float,
|
||||
quant_block: int,
|
||||
token_stride: int,
|
||||
scale_dim: int,
|
||||
kv_cache_block_size: int,
|
||||
kv_block_stride: int,
|
||||
compress_ratio: int,
|
||||
overlap: bool,
|
||||
rms_norm_weight_dtype: torch.dtype,
|
||||
):
|
||||
if not (
|
||||
head_size == 512
|
||||
and state_width == head_size
|
||||
and compress_ratio == 128
|
||||
and not overlap
|
||||
and block_size == 8
|
||||
):
|
||||
raise ValueError(
|
||||
"CuTe DSL split sparse-attn wrapper only supports the real "
|
||||
"DeepSeek V4 C128 layout: head_size=512, state_width=512, "
|
||||
"compress_ratio=128, overlap=False, block_size=8."
|
||||
)
|
||||
compress = SparseAttnCompressC128Block8Kernel.compile(
|
||||
head_size=head_size,
|
||||
state_width=state_width,
|
||||
)
|
||||
norm_weight_dtype = _TORCH_TO_CUTE[rms_norm_weight_dtype]
|
||||
store = SparseAttnNormRopeStoreKernel.compile(
|
||||
head_size,
|
||||
rope_head_dim,
|
||||
fp8_max,
|
||||
quant_block,
|
||||
token_stride,
|
||||
scale_dim,
|
||||
kv_block_stride,
|
||||
compress_ratio,
|
||||
norm_weight_dtype,
|
||||
kv_cache_block_size,
|
||||
)
|
||||
return compress, store
|
||||
|
||||
|
||||
def split_kv_compress_norm_rope_insert_sparse_attn_cutedsl(
|
||||
state_cache: torch.Tensor,
|
||||
token_to_req_indices: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
@@ -1143,34 +1164,6 @@ def compress_kv_sparse_attn_cutedsl(
|
||||
block_table: torch.Tensor,
|
||||
block_size: int,
|
||||
compressed_kv: torch.Tensor,
|
||||
head_size: int = 512,
|
||||
state_width: int = 512,
|
||||
compress_ratio: int = 128,
|
||||
overlap: bool = False,
|
||||
) -> None:
|
||||
if positions.numel() == 0:
|
||||
return
|
||||
compiled = SparseAttnCompressKernel.compile(
|
||||
head_size=head_size,
|
||||
state_width=state_width,
|
||||
compress_ratio=compress_ratio,
|
||||
overlap=overlap,
|
||||
)
|
||||
compiled(
|
||||
state_cache,
|
||||
token_to_req_indices,
|
||||
positions,
|
||||
slot_mapping,
|
||||
block_table,
|
||||
block_size,
|
||||
compressed_kv,
|
||||
)
|
||||
|
||||
|
||||
def norm_rope_insert_sparse_attn_cutedsl(
|
||||
compressed_kv: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
slot_mapping: torch.Tensor,
|
||||
rms_norm_weight: torch.Tensor,
|
||||
rms_norm_eps: float,
|
||||
cos_sin_cache: torch.Tensor,
|
||||
@@ -1179,38 +1172,58 @@ def norm_rope_insert_sparse_attn_cutedsl(
|
||||
kv_cache_block_size: int,
|
||||
kv_block_stride: int,
|
||||
head_size: int = 512,
|
||||
state_width: int = 512,
|
||||
rope_head_dim: int = 64,
|
||||
fp8_max: float = 448.0,
|
||||
quant_block: int = 64,
|
||||
token_stride: int = 576,
|
||||
scale_dim: int = 8,
|
||||
compress_ratio: int = 128,
|
||||
overlap: bool = False,
|
||||
) -> None:
|
||||
if positions.numel() == 0:
|
||||
return
|
||||
norm_weight_dtype = _TORCH_TO_CUTE.get(rms_norm_weight.dtype)
|
||||
if norm_weight_dtype is None:
|
||||
raise ValueError(
|
||||
"CuTe DSL sparse-attn store supports rms_norm_weight dtype "
|
||||
f"bf16/fp32, got {rms_norm_weight.dtype}."
|
||||
)
|
||||
if k_cache.ndim != 3:
|
||||
raise ValueError(
|
||||
"CuTe DSL sparse-attn store expects the real DeepSeek V4 "
|
||||
f"3D k_cache layout [num_blocks, block_size, 584], got ndim={k_cache.ndim}."
|
||||
)
|
||||
compiled = SparseAttnNormRopeStoreKernel.compile(
|
||||
head_size=head_size,
|
||||
rope_head_dim=rope_head_dim,
|
||||
fp8_max=fp8_max,
|
||||
quant_block=quant_block,
|
||||
token_stride=token_stride,
|
||||
scale_dim=scale_dim,
|
||||
kv_block_stride=kv_block_stride,
|
||||
compress_ratio=compress_ratio,
|
||||
norm_weight_dtype=norm_weight_dtype,
|
||||
if kv_cache_block_size != k_cache.shape[1]:
|
||||
raise ValueError(
|
||||
"CuTe DSL split sparse-attn wrapper expected kv_cache_block_size "
|
||||
f"to match k_cache.shape[1], got {kv_cache_block_size} and "
|
||||
f"{k_cache.shape[1]}."
|
||||
)
|
||||
if positions.numel() == 0:
|
||||
return
|
||||
if rms_norm_weight.dtype not in _TORCH_TO_CUTE:
|
||||
raise ValueError(
|
||||
"CuTe DSL sparse-attn store supports rms_norm_weight dtype "
|
||||
f"bf16/fp32, got {rms_norm_weight.dtype}."
|
||||
)
|
||||
compress, store = compile_split_sparse_attn_cutedsl(
|
||||
head_size,
|
||||
state_width,
|
||||
block_size,
|
||||
rope_head_dim,
|
||||
fp8_max,
|
||||
quant_block,
|
||||
token_stride,
|
||||
scale_dim,
|
||||
kv_cache_block_size,
|
||||
kv_block_stride,
|
||||
compress_ratio,
|
||||
overlap,
|
||||
rms_norm_weight.dtype,
|
||||
)
|
||||
compiled(
|
||||
compress(
|
||||
state_cache,
|
||||
token_to_req_indices,
|
||||
positions,
|
||||
slot_mapping,
|
||||
block_table,
|
||||
compressed_kv,
|
||||
)
|
||||
|
||||
store(
|
||||
compressed_kv,
|
||||
positions,
|
||||
slot_mapping,
|
||||
@@ -1219,7 +1232,6 @@ def norm_rope_insert_sparse_attn_cutedsl(
|
||||
cos_sin_cache,
|
||||
k_cache,
|
||||
kv_slot_mapping,
|
||||
kv_cache_block_size,
|
||||
)
|
||||
|
||||
|
||||
@@ -1346,7 +1358,7 @@ def compress_norm_rope_store_cutedsl(
|
||||
dtype=torch.float32,
|
||||
device=state_cache.device,
|
||||
)
|
||||
compress_kv_sparse_attn_cutedsl(
|
||||
split_kv_compress_norm_rope_insert_sparse_attn_cutedsl(
|
||||
state_cache,
|
||||
token_to_req_indices,
|
||||
positions,
|
||||
@@ -1354,15 +1366,6 @@ def compress_norm_rope_store_cutedsl(
|
||||
block_table,
|
||||
block_size,
|
||||
compressed_kv,
|
||||
head_size=head_dim,
|
||||
state_width=state_width,
|
||||
compress_ratio=compress_ratio,
|
||||
overlap=overlap,
|
||||
)
|
||||
norm_rope_insert_sparse_attn_cutedsl(
|
||||
compressed_kv,
|
||||
positions,
|
||||
slot_mapping,
|
||||
rms_norm_weight,
|
||||
rms_norm_eps,
|
||||
cos_sin_cache,
|
||||
@@ -1371,10 +1374,12 @@ def compress_norm_rope_store_cutedsl(
|
||||
kv_cache.shape[1], # paged KV cache block size
|
||||
kv_cache.stride(0),
|
||||
head_size=head_dim,
|
||||
state_width=state_width,
|
||||
rope_head_dim=rope_head_dim,
|
||||
fp8_max=448.0,
|
||||
quant_block=quant_block,
|
||||
token_stride=token_stride,
|
||||
scale_dim=scale_dim,
|
||||
compress_ratio=compress_ratio,
|
||||
overlap=overlap,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user