[DSv4] Adding TRTLLM gen attention kernel (#43827)
This commit is contained in:
@@ -468,6 +468,7 @@ def _reference_kv_compress_norm_rope(
|
||||
use_fp4: bool = False,
|
||||
rms_eps: float = 1e-6,
|
||||
fp8_max: float = 448.0,
|
||||
return_full_cache: bool = False,
|
||||
):
|
||||
"""Compress → RMSNorm → GPT-J RoPE → quantize.
|
||||
|
||||
@@ -521,6 +522,12 @@ def _reference_kv_compress_norm_rope(
|
||||
results.append(torch.cat([nope, rope]).to(state_cache.dtype))
|
||||
result = torch.stack(results)
|
||||
|
||||
if return_full_cache:
|
||||
# Contiguous 512-wide bf16 row (nope unrotated + rope rotated), matching
|
||||
# the FlashInfer full-cache layout before any per-tensor fp8 quant. The
|
||||
# kernel rounds the fp32 result to bf16 once at the store.
|
||||
return result.to(torch.bfloat16)
|
||||
|
||||
if use_fp4:
|
||||
return quantize_to_mxfp4(result)
|
||||
else:
|
||||
@@ -667,3 +674,145 @@ def test_fused_kv_insert_indexer(num_tokens: int, kv_block_size: int, use_fp4: b
|
||||
assert torch.equal(actual_scale, scale[i : i + 1]), (
|
||||
f"token {i}: scale {actual_scale.item()} != {scale[i].item()}"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("compress_ratio", [4, 128])
|
||||
@pytest.mark.parametrize("store_fp8", [False, True])
|
||||
def test_cutedsl_full_cache_store(compress_ratio: int, store_fp8: bool):
|
||||
"""CuTeDSL compressor full-cache (FlashInfer) store parity for head=512.
|
||||
|
||||
Exercises the contiguous bf16 / per-tensor fp8 store branch of both the C4
|
||||
fused kernel and the C128 split kernel against the PyTorch reference.
|
||||
"""
|
||||
cutedsl = pytest.importorskip("cutlass") # noqa: F841
|
||||
from vllm.models.deepseek_v4.nvidia.ops.sparse_attn_compress_cutedsl import (
|
||||
fused_kv_compress_norm_rope_insert_sparse_attn_cutedsl,
|
||||
split_kv_compress_norm_rope_insert_sparse_attn_cutedsl,
|
||||
)
|
||||
|
||||
HEAD_DIM = 512
|
||||
ROPE_DIM = 64
|
||||
RMS_EPS = 1e-6
|
||||
FP8_MAX = 448.0
|
||||
# C128 compress (Block8 kernel) requires state-cache block_size=8; C4 uses 16.
|
||||
BLOCK_SIZE = 8 if compress_ratio == 128 else 16
|
||||
KV_BLOCK_SIZE = 64
|
||||
device = "cuda"
|
||||
torch.manual_seed(7)
|
||||
|
||||
overlap = 1 if compress_ratio == 4 else 0
|
||||
coff = 1 + overlap
|
||||
num_tokens = 8
|
||||
|
||||
num_pages = (compress_ratio * num_tokens - 1) // BLOCK_SIZE + 2
|
||||
# The production CompressorStateCache is fp32.
|
||||
state_cache = torch.randn(
|
||||
num_pages, BLOCK_SIZE, 2 * coff * HEAD_DIM, dtype=torch.float32, device=device
|
||||
)
|
||||
block_table = torch.arange(num_pages, dtype=torch.int32, device=device).unsqueeze(0)
|
||||
token_to_req = torch.zeros(num_tokens, dtype=torch.int32, device=device)
|
||||
slot_mapping = torch.arange(num_tokens, dtype=torch.int64, device=device)
|
||||
positions = torch.arange(
|
||||
compress_ratio - 1,
|
||||
compress_ratio * num_tokens,
|
||||
compress_ratio,
|
||||
dtype=torch.int64,
|
||||
device=device,
|
||||
)
|
||||
rms_weight = torch.randn(HEAD_DIM, dtype=torch.bfloat16, device=device)
|
||||
cos_sin_cache = torch.randn(
|
||||
compress_ratio * num_tokens, ROPE_DIM, dtype=torch.float32, device=device
|
||||
)
|
||||
|
||||
dtype = torch.float8_e4m3fn if store_fp8 else torch.bfloat16
|
||||
kv_n_blocks = (num_tokens + KV_BLOCK_SIZE - 1) // KV_BLOCK_SIZE + 1
|
||||
k_cache = torch.zeros(
|
||||
kv_n_blocks, KV_BLOCK_SIZE, HEAD_DIM, dtype=dtype, device=device
|
||||
)
|
||||
fp8_scale = torch.tensor(
|
||||
[0.5 if store_fp8 else 1.0], dtype=torch.float32, device=device
|
||||
)
|
||||
|
||||
if compress_ratio == 4:
|
||||
fused_kv_compress_norm_rope_insert_sparse_attn_cutedsl(
|
||||
state_cache,
|
||||
token_to_req,
|
||||
positions,
|
||||
slot_mapping,
|
||||
block_table,
|
||||
BLOCK_SIZE,
|
||||
rms_weight,
|
||||
RMS_EPS,
|
||||
cos_sin_cache,
|
||||
k_cache,
|
||||
slot_mapping,
|
||||
KV_BLOCK_SIZE,
|
||||
k_cache.stride(0),
|
||||
head_size=HEAD_DIM,
|
||||
state_width=coff * HEAD_DIM,
|
||||
rope_head_dim=ROPE_DIM,
|
||||
fp8_max=FP8_MAX,
|
||||
quant_block=64,
|
||||
token_stride=576,
|
||||
scale_dim=8,
|
||||
compress_ratio=compress_ratio,
|
||||
overlap=True,
|
||||
store_full_kv=True,
|
||||
store_full_fp8=store_fp8,
|
||||
fp8_scale=fp8_scale,
|
||||
)
|
||||
else:
|
||||
compressed_kv = torch.empty(
|
||||
(num_tokens, HEAD_DIM), dtype=torch.float32, device=device
|
||||
)
|
||||
split_kv_compress_norm_rope_insert_sparse_attn_cutedsl(
|
||||
state_cache,
|
||||
token_to_req,
|
||||
positions,
|
||||
slot_mapping,
|
||||
block_table,
|
||||
BLOCK_SIZE,
|
||||
compressed_kv,
|
||||
rms_weight,
|
||||
RMS_EPS,
|
||||
cos_sin_cache,
|
||||
k_cache,
|
||||
slot_mapping,
|
||||
KV_BLOCK_SIZE,
|
||||
k_cache.stride(0),
|
||||
head_size=HEAD_DIM,
|
||||
state_width=coff * HEAD_DIM,
|
||||
rope_head_dim=ROPE_DIM,
|
||||
fp8_max=FP8_MAX,
|
||||
quant_block=64,
|
||||
token_stride=576,
|
||||
scale_dim=8,
|
||||
compress_ratio=compress_ratio,
|
||||
overlap=bool(overlap),
|
||||
store_full_kv=True,
|
||||
store_full_fp8=store_fp8,
|
||||
fp8_scale=fp8_scale,
|
||||
)
|
||||
|
||||
ref = _reference_kv_compress_norm_rope(
|
||||
state_cache,
|
||||
block_table,
|
||||
positions,
|
||||
rms_weight,
|
||||
cos_sin_cache,
|
||||
compress_ratio,
|
||||
overlap,
|
||||
rms_eps=RMS_EPS,
|
||||
return_full_cache=True,
|
||||
) # [num_tokens, HEAD_DIM] bf16
|
||||
|
||||
actual = torch.stack(
|
||||
[k_cache[i // KV_BLOCK_SIZE, i % KV_BLOCK_SIZE] for i in range(num_tokens)]
|
||||
)
|
||||
if store_fp8:
|
||||
ref_fp8 = torch.clamp(ref.float() / fp8_scale, -FP8_MAX, FP8_MAX).to(
|
||||
torch.float8_e4m3fn
|
||||
)
|
||||
torch.testing.assert_close(actual.float(), ref_fp8.float(), rtol=0.0, atol=0.3)
|
||||
else:
|
||||
torch.testing.assert_close(actual.float(), ref.float(), rtol=3e-2, atol=3e-2)
|
||||
|
||||
@@ -67,7 +67,7 @@ def apply_rope_gptj_last_k(
|
||||
head_dim = x.shape[-1]
|
||||
nope_dim = head_dim - rope_dim
|
||||
|
||||
cs = cos_sin_cache[positions].to(torch.float32)
|
||||
cs = cos_sin_cache[positions.long()].to(torch.float32)
|
||||
cos = cs[..., :half]
|
||||
sin = cs[..., half:]
|
||||
|
||||
@@ -114,6 +114,18 @@ def _op_available() -> bool:
|
||||
return hasattr(torch.ops._C, "fused_deepseek_v4_qnorm_rope_kv_rope_quant_insert")
|
||||
|
||||
|
||||
def _full_cache_fp8_op_available() -> bool:
|
||||
return hasattr(
|
||||
torch.ops._C, "fused_deepseek_v4_qnorm_rope_kv_rope_full_cache_fp8_insert"
|
||||
)
|
||||
|
||||
|
||||
def _full_cache_bf16_op_available() -> bool:
|
||||
return hasattr(
|
||||
torch.ops._C, "fused_deepseek_v4_qnorm_rope_kv_rope_full_cache_bf16_insert"
|
||||
)
|
||||
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not torch.cuda.is_available() or not _op_available(),
|
||||
reason="CUDA not available or fused DeepseekV4 op not built in",
|
||||
@@ -415,3 +427,238 @@ def test_combined_q_and_kv(
|
||||
"padded head slots must be exact zero"
|
||||
)
|
||||
torch.testing.assert_close(k_cache_fused, k_cache_ref, rtol=0, atol=0)
|
||||
|
||||
|
||||
# ── Full-cache (FlashInfer) path parity ──────────────────────────────────────
|
||||
|
||||
|
||||
def _call_full_cache_fp8_fused(
|
||||
q,
|
||||
kv,
|
||||
q_fp8,
|
||||
k_cache,
|
||||
slot_mapping,
|
||||
positions,
|
||||
cos_sin_cache,
|
||||
fp8_scale,
|
||||
q_fp8_scale_inv,
|
||||
eps,
|
||||
bs,
|
||||
):
|
||||
torch.ops._C.fused_deepseek_v4_qnorm_rope_kv_rope_full_cache_fp8_insert(
|
||||
q,
|
||||
kv,
|
||||
q_fp8,
|
||||
k_cache,
|
||||
slot_mapping,
|
||||
positions.long(),
|
||||
cos_sin_cache,
|
||||
fp8_scale,
|
||||
q_fp8_scale_inv,
|
||||
eps,
|
||||
bs,
|
||||
)
|
||||
|
||||
|
||||
def _call_full_cache_bf16_fused(
|
||||
q,
|
||||
kv,
|
||||
k_cache,
|
||||
slot_mapping,
|
||||
positions,
|
||||
cos_sin_cache,
|
||||
eps,
|
||||
bs,
|
||||
):
|
||||
torch.ops._C.fused_deepseek_v4_qnorm_rope_kv_rope_full_cache_bf16_insert(
|
||||
q,
|
||||
kv,
|
||||
k_cache,
|
||||
slot_mapping,
|
||||
positions.long(),
|
||||
cos_sin_cache,
|
||||
eps,
|
||||
bs,
|
||||
)
|
||||
|
||||
|
||||
def _fp8_full_cache_reference(
|
||||
q,
|
||||
kv,
|
||||
k_cache,
|
||||
q_fp8,
|
||||
slot_mapping,
|
||||
positions,
|
||||
cos_sin_cache,
|
||||
eps,
|
||||
block_size,
|
||||
fp8_scale,
|
||||
q_fp8_scale_inv,
|
||||
):
|
||||
q_ref = rmsnorm_no_weight(q, eps)
|
||||
q_ref = apply_rope_gptj_last_k(q_ref, positions, cos_sin_cache)
|
||||
q_fp8.copy_(
|
||||
torch.clamp(q_ref.float() * q_fp8_scale_inv, -FP8_MAX, FP8_MAX).to(
|
||||
torch.float8_e4m3fn
|
||||
)
|
||||
)
|
||||
|
||||
kv_ref = apply_rope_gptj_last_k(kv, positions, cos_sin_cache)
|
||||
valid = slot_mapping >= 0
|
||||
slots = slot_mapping[valid]
|
||||
block_idx = slots // block_size
|
||||
pos_in_block = slots % block_size
|
||||
k_cache[block_idx, pos_in_block] = torch.clamp(
|
||||
kv_ref[valid].float() / fp8_scale, -FP8_MAX, FP8_MAX
|
||||
).to(torch.float8_e4m3fn)
|
||||
|
||||
|
||||
def _bf16_full_cache_reference(
|
||||
q,
|
||||
kv,
|
||||
k_cache,
|
||||
slot_mapping,
|
||||
positions,
|
||||
cos_sin_cache,
|
||||
eps,
|
||||
block_size,
|
||||
):
|
||||
q_ref = rmsnorm_no_weight(q, eps)
|
||||
# Kernel keeps RMSNorm+RoPE in fp32 and rounds to bf16 once at the store.
|
||||
q_ref = apply_rope_gptj_last_k(q_ref, positions, cos_sin_cache).to(q.dtype)
|
||||
|
||||
kv_ref = apply_rope_gptj_last_k(kv, positions, cos_sin_cache)
|
||||
valid = slot_mapping >= 0
|
||||
slots = slot_mapping[valid]
|
||||
block_idx = slots // block_size
|
||||
pos_in_block = slots % block_size
|
||||
k_cache[block_idx, pos_in_block] = kv_ref[valid]
|
||||
return q_ref
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not _full_cache_fp8_op_available(),
|
||||
reason="full-cache per-tensor FP8 DeepseekV4 op not built in",
|
||||
)
|
||||
@pytest.mark.parametrize("num_tokens", [4, 17])
|
||||
@pytest.mark.parametrize("n_heads", [8, 17])
|
||||
@pytest.mark.parametrize("positions_dtype", [torch.int32, torch.int64])
|
||||
def test_full_cache_per_tensor_fp8_matches_reference(
|
||||
num_tokens: int,
|
||||
n_heads: int,
|
||||
positions_dtype: torch.dtype,
|
||||
):
|
||||
torch.manual_seed(4)
|
||||
device = "cuda"
|
||||
dtype = torch.bfloat16
|
||||
eps = 1e-6
|
||||
block_size = 16
|
||||
max_pos = 4096
|
||||
|
||||
q = torch.randn(num_tokens, n_heads, HEAD_DIM, dtype=dtype, device=device)
|
||||
kv = torch.randn(num_tokens, HEAD_DIM, dtype=dtype, device=device)
|
||||
positions = torch.arange(num_tokens, dtype=positions_dtype, device=device)
|
||||
cos_sin_cache = make_cos_sin_cache(max_pos, ROPE_DIM, torch.float32, device)
|
||||
|
||||
num_blocks = (num_tokens + block_size - 1) // block_size + 1
|
||||
slot_mapping = torch.arange(num_tokens, dtype=torch.int64, device=device)
|
||||
fp8_scale = torch.tensor([1.0], dtype=torch.float32, device=device)
|
||||
q_fp8_scale_inv = torch.tensor([1.0], dtype=torch.float32, device=device)
|
||||
|
||||
q_fp8_ref = torch.empty_like(q, dtype=torch.float8_e4m3fn)
|
||||
q_fp8_fused = torch.empty_like(q, dtype=torch.float8_e4m3fn)
|
||||
k_cache_ref = torch.zeros(
|
||||
num_blocks, block_size, HEAD_DIM, dtype=torch.float8_e4m3fn, device=device
|
||||
)
|
||||
k_cache_fused = torch.zeros_like(k_cache_ref)
|
||||
|
||||
_fp8_full_cache_reference(
|
||||
q,
|
||||
kv,
|
||||
k_cache_ref,
|
||||
q_fp8_ref,
|
||||
slot_mapping,
|
||||
positions,
|
||||
cos_sin_cache,
|
||||
eps,
|
||||
block_size,
|
||||
fp8_scale,
|
||||
q_fp8_scale_inv,
|
||||
)
|
||||
_call_full_cache_fp8_fused(
|
||||
q.clone(),
|
||||
kv,
|
||||
q_fp8_fused,
|
||||
k_cache_fused,
|
||||
slot_mapping,
|
||||
positions,
|
||||
cos_sin_cache,
|
||||
fp8_scale,
|
||||
q_fp8_scale_inv,
|
||||
eps,
|
||||
block_size,
|
||||
)
|
||||
|
||||
torch.testing.assert_close(
|
||||
q_fp8_fused.float(), q_fp8_ref.float(), rtol=0, atol=0.25
|
||||
)
|
||||
torch.testing.assert_close(
|
||||
k_cache_fused.float(), k_cache_ref.float(), rtol=0, atol=0.25
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not _full_cache_bf16_op_available(),
|
||||
reason="full-cache BF16 DeepseekV4 op not built in",
|
||||
)
|
||||
@pytest.mark.parametrize("num_tokens", [4, 17])
|
||||
@pytest.mark.parametrize("n_heads", [8, 17])
|
||||
@pytest.mark.parametrize("positions_dtype", [torch.int32, torch.int64])
|
||||
def test_full_cache_bf16_matches_reference(
|
||||
num_tokens: int,
|
||||
n_heads: int,
|
||||
positions_dtype: torch.dtype,
|
||||
):
|
||||
torch.manual_seed(5)
|
||||
device = "cuda"
|
||||
dtype = torch.bfloat16
|
||||
eps = 1e-6
|
||||
block_size = 16
|
||||
max_pos = 4096
|
||||
|
||||
q = torch.randn(num_tokens, n_heads, HEAD_DIM, dtype=dtype, device=device)
|
||||
kv = torch.randn(num_tokens, HEAD_DIM, dtype=dtype, device=device)
|
||||
positions = torch.arange(num_tokens, dtype=positions_dtype, device=device)
|
||||
cos_sin_cache = make_cos_sin_cache(max_pos, ROPE_DIM, torch.float32, device)
|
||||
|
||||
num_blocks = (num_tokens + block_size - 1) // block_size + 1
|
||||
slot_mapping = torch.arange(num_tokens, dtype=torch.int64, device=device)
|
||||
|
||||
q_fused = q.clone()
|
||||
k_cache_ref = torch.zeros(
|
||||
num_blocks, block_size, HEAD_DIM, dtype=torch.bfloat16, device=device
|
||||
)
|
||||
k_cache_fused = torch.zeros_like(k_cache_ref)
|
||||
q_ref = _bf16_full_cache_reference(
|
||||
q,
|
||||
kv,
|
||||
k_cache_ref,
|
||||
slot_mapping,
|
||||
positions,
|
||||
cos_sin_cache,
|
||||
eps,
|
||||
block_size,
|
||||
)
|
||||
_call_full_cache_bf16_fused(
|
||||
q_fused,
|
||||
kv,
|
||||
k_cache_fused,
|
||||
slot_mapping,
|
||||
positions,
|
||||
cos_sin_cache,
|
||||
eps,
|
||||
block_size,
|
||||
)
|
||||
|
||||
torch.testing.assert_close(q_fused, q_ref, rtol=1e-2, atol=1e-2)
|
||||
torch.testing.assert_close(k_cache_fused, k_cache_ref, rtol=0, atol=0)
|
||||
|
||||
Reference in New Issue
Block a user