[Attention][DSA] Enable W4A16 DSA (#51724)

Signed-off-by: Shiyang Chen <shiychen@nvidia.com>
This commit is contained in:
sychen52
2026-09-01 12:37:47 +08:00
committed by GitHub
parent ce2e343be1
commit 7c5dc571cb
17 changed files with 824 additions and 22 deletions
+12 -1
View File
@@ -505,7 +505,10 @@ def create_and_prepopulate_kv_cache(
block_table = common_attn_metadata.block_table_tensor
slot_mapping = common_attn_metadata.slot_mapping
fp8_attention = kv_cache_dtype and kv_cache_dtype.startswith("fp8")
use_nvfp4_ds_mla = kv_cache_dtype == "nvfp4_ds_mla"
fp8_attention = (
bool(kv_cache_dtype and kv_cache_dtype.startswith("fp8")) or use_nvfp4_ds_mla
)
use_fp8_ds_mla = kv_cache_dtype == "fp8_ds_mla"
if fp8_attention:
@@ -515,6 +518,14 @@ def create_and_prepopulate_kv_cache(
# 4 * 4: 4 float32 scale values for 128-element tiles
# 2 * rope_dim: 16-bit RoPE values
kv_entry_size = kv_lora_rank + 4 * 4 + 2 * rope_dim
elif use_nvfp4_ds_mla:
kv_lora_rank = kv_c_contexts[0].shape[-1]
rope_dim = k_pe_contexts[0].shape[-1]
# e2m1-packed NoPE + unscaled e4m3 rope + per-16 e4m3 NoPE SFs,
# padded to 16B
kv_entry_size = (
cdiv(kv_lora_rank // 2 + rope_dim + kv_lora_rank // 16, 16) * 16
)
else:
kv_entry_size = head_size
+101 -1
View File
@@ -40,6 +40,9 @@ if not current_platform.is_cuda():
allow_module_level=True,
)
from vllm.model_executor.layers.attention.mla_attention import (
_canonicalize_sparse_mla_kv_cache_dtype,
)
from vllm.utils.math_utils import cdiv
from vllm.v1.attention.backends.mla.flashinfer_mla_sparse import (
FlashInferMLASparseTRTLLMBackend,
@@ -184,13 +187,89 @@ def _quantize_dequantize_fp8_ds_mla(
return dequant_kv_c, dequant_k_pe
_E2M1_TABLE = [0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0]
def _quantize_dequantize_nvfp4_ds_mla(
kv_c: torch.Tensor,
k_pe: torch.Tensor,
block_size: int,
kv_cache_dtype: str,
scale: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Round-trip kv_c/k_pe through the nvfp4_ds_mla cache layout.
Layout (per token, uint8): [e2m1-packed NoPE | unscaled e4m3 RoPE |
per-16 e4m3 NoPE SFs]. The SF bytes are stored permuted (an 8x4 -> 4x8
transpose: the scale for element block s lives at byte 8 * (s & 3) +
(s >> 2)) so that the 8 scales one FlashMLA dequant thread needs are
contiguous. Dequant: x = value * float(sf), which is exact in bf16, so
this python dequant matches the kernels bit-for-bit.
"""
if kv_c.numel() == 0:
return kv_c.clone(), k_pe.clone()
kv_lora_rank = kv_c.shape[-1]
rope_dim = k_pe.shape[-1]
num_tokens = kv_c.shape[0]
num_blocks = max(1, math.ceil(num_tokens / block_size))
entry_size = (
math.ceil((kv_lora_rank // 2 + rope_dim + kv_lora_rank // 16) / 16) * 16
)
sf_nope_off = kv_lora_rank // 2 + rope_dim
tmp_cache = torch.zeros(
num_blocks, block_size, entry_size, dtype=torch.uint8, device=kv_c.device
)
slot_mapping = torch.arange(num_tokens, dtype=torch.long, device=kv_c.device)
ops.concat_and_cache_mla(
kv_c, k_pe, tmp_cache, slot_mapping, kv_cache_dtype=kv_cache_dtype, scale=scale
)
tokens = tmp_cache.view(-1, entry_size)[:num_tokens]
table = torch.tensor(
_E2M1_TABLE + [-v for v in _E2M1_TABLE],
dtype=torch.float32,
device=kv_c.device,
)
def unpack_e2m1(packed: torch.Tensor) -> torch.Tensor:
lo = (packed & 0xF).long()
hi = (packed >> 4).long()
return table[torch.stack([lo, hi], dim=-1).flatten(-2)]
nope_vals = unpack_e2m1(tokens[:, : kv_lora_rank // 2])
num_nope_sf = kv_lora_rank // 16
nope_sf = (
# undo the on-wire SF permutation -> element-block order
tokens[:, sf_nope_off : sf_nope_off + num_nope_sf]
.unflatten(-1, (4, num_nope_sf // 4))
.transpose(-1, -2)
.flatten(-2)
.view(torch.float8_e4m3fn)
.float()
)
dequant_kv_c = (
(nope_vals.unflatten(-1, (-1, 16)) * nope_sf.unsqueeze(-1)).flatten(-2)
).to(kv_c.dtype)
# RoPE: plain e4m3, no scale factor
rope_raw = tokens[:, kv_lora_rank // 2 : sf_nope_off]
dequant_k_pe = rope_raw.view(torch.float8_e4m3fn).to(k_pe.dtype)
return dequant_kv_c, dequant_k_pe
@pytest.mark.parametrize(
"backend_cls",
[FlashMLASparseBackend, FlashInferMLASparseTRTLLMBackend],
ids=["FlashMLA", "FlashInferTRTLLM"],
)
@pytest.mark.parametrize("batch_name", list(SPARSE_BACKEND_BATCH_SPECS.keys()))
@pytest.mark.parametrize("kv_cache_dtype", ["auto", "fp8", "fp8_ds_mla"])
@pytest.mark.parametrize(
"kv_cache_dtype",
["auto", "fp8", "fp8_ds_mla", "nvfp4_ds_mla"],
)
@pytest.mark.parametrize("tensor_parallel_size", [1, 2, 4])
@pytest.mark.parametrize("block_size", [32, 64])
@pytest.mark.parametrize(("q_scale", "k_scale"), [(1.0, 1.0), (2.0, 3.0)])
@@ -219,6 +298,17 @@ def test_sparse_backend_decode_correctness(
"fp8_ds_mla kv-cache dtype"
)
if kv_cache_dtype == "nvfp4_ds_mla":
# Unlike "fp8" (an alias canonicalized to "fp8_ds_mla" above), the
# NVFP4 DS-MLA dtype must reach the backend unchanged.
assert (
_canonicalize_sparse_mla_kv_cache_dtype(backend_cls, kv_cache_dtype)
== kv_cache_dtype
)
device_capability = current_platform.get_device_capability()
if device_capability is None or device_capability.major != 10:
pytest.skip("The NVFP4 DS-MLA kv-cache dtype requires SM 10.x")
supported_block_sizes = backend_cls.get_supported_kernel_block_sizes()
if block_size not in supported_block_sizes:
pytest.skip(
@@ -238,6 +328,7 @@ def test_sparse_backend_decode_correctness(
batch_spec = SPARSE_BACKEND_BATCH_SPECS[batch_name]
use_fp8_ds_mla_quantization = kv_cache_dtype == "fp8_ds_mla"
use_nvfp4_ds_mla_quantization = kv_cache_dtype == "nvfp4_ds_mla"
device = torch.device(DEVICE_TYPE)
dtype = torch.bfloat16
@@ -381,6 +472,15 @@ def test_sparse_backend_decode_correctness(
simulate_sm100_e8m0_scales=is_sm100,
)
k_pe_full = k_pe_squeezed.unsqueeze(1)
elif use_nvfp4_ds_mla_quantization:
kv_c_full, k_pe_squeezed = _quantize_dequantize_nvfp4_ds_mla(
kv_c_full,
k_pe_full.squeeze(1),
block_size=block_size,
kv_cache_dtype=kv_cache_dtype,
scale=kv_cache_scale,
)
k_pe_full = k_pe_squeezed.unsqueeze(1)
q_nope, q_pe = q_c.split([qk_nope_head_dim, qk_rope_head_dim], dim=-1)
ql_nope = torch.einsum("qnh,lnh->qnl", q_nope, W_UK)