[Attention][DSA] Enable W4A16 DSA (#51724)

Signed-off-by: Shiyang Chen <shiychen@nvidia.com>
This commit is contained in:
sychen52
2026-09-01 12:37:47 +08:00
committed by GitHub
parent ce2e343be1
commit 7c5dc571cb
17 changed files with 824 additions and 22 deletions
+151
View File
@@ -907,6 +907,157 @@ def test_concat_and_cache_ds_mla(
torch.testing.assert_close(kv_rope, ref_rope, atol=0.001, rtol=0.1)
# Bytes per token for the nvfp4_ds_mla cache layout (see flashmla_sparse.py).
NVFP4_DS_MLA_ENTRY_SIZE = 352
# e2m1 magnitude table; the sign lives in bit 3 of each 4-bit code.
E2M1_VALUES = [0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0]
def _unpack_e2m1(packed: torch.Tensor) -> torch.Tensor:
"""Unpack uint8 bytes holding two e2m1 codes each into float32 values.
The low nibble holds the even element, the high nibble the odd one.
"""
low = packed & 0x0F
high = packed >> 4
codes = torch.stack([low, high], dim=-1).reshape(-1).long()
table = torch.tensor(E2M1_VALUES, dtype=torch.float32, device=packed.device)
magnitude = table[codes & 0x7]
sign = torch.where((codes & 0x8) != 0, -1.0, 1.0)
return sign * magnitude
def _ref_nvfp4_sf_bytes(vals: torch.Tensor, divisor: float) -> torch.Tensor:
"""Reference e4m3 scale-factor bytes: e4m3(max(amax_per_16 / divisor, 2^-9)).
In element-block order (byte s scales elements [16s, 16s + 16)).
"""
tiles = vals.to(torch.float32).reshape(-1, 16)
sf = torch.clamp(tiles.abs().amax(dim=1) / divisor, min=2.0**-9)
return sf.to(torch.float8_e4m3fn).view(torch.uint8)
def _nvfp4_sf_element_order(wire_sf_bytes: torch.Tensor) -> torch.Tensor:
"""Stored NVFP4 scale-factor byte order -> element-block order.
The 32 NoPE scale-factor bytes of an nvfp4_ds_mla entry are stored
permuted (an 8x4 -> 4x8 transpose): the scale for element block s lives at
byte 8 * (s & 3) + (s >> 2), so that the 8 scales one FlashMLA dequant
thread needs are contiguous. This undoes that permutation.
"""
num_sf = wire_sf_bytes.shape[-1]
return wire_sf_bytes.unflatten(-1, (4, num_sf // 4)).transpose(-1, -2).flatten(-2)
@pytest.mark.parametrize("kv_lora_rank", KV_LORA_RANKS)
@pytest.mark.parametrize("qk_rope_head_dim", QK_ROPE_HEAD_DIMS)
@pytest.mark.parametrize("num_tokens", NUM_TOKENS_MLA)
@pytest.mark.parametrize("block_size", BLOCK_SIZES_MLA)
@pytest.mark.parametrize("num_blocks", NUM_BLOCKS_MLA)
@pytest.mark.parametrize("seed", SEEDS)
@pytest.mark.parametrize("device", CUDA_DEVICES)
@torch.inference_mode()
def test_concat_and_cache_nvfp4_ds_mla(
kv_lora_rank: int,
qk_rope_head_dim: int,
num_tokens: int,
block_size: int,
num_blocks: int,
seed: int,
device: str,
) -> None:
if current_platform.is_rocm():
pytest.skip("concat_and_cache_mla doesn't support NVFP4 DS-MLA on ROCm")
device_capability = current_platform.get_device_capability()
if device_capability is None or device_capability.major != 10:
pytest.skip("The NVFP4 DS-MLA kv-cache dtype requires SM 10.x")
if kv_lora_rank != 512:
pytest.skip("The NVFP4 DS-MLA layout requires kv_lora_rank == 512")
dtype = torch.bfloat16
set_random_seed(seed)
torch.set_default_device(device)
torch.accelerator.set_device_index(device)
kv_cache_dtype = "nvfp4_ds_mla"
entry_size = NVFP4_DS_MLA_ENTRY_SIZE
total_slots = num_blocks * block_size
slot_mapping_lst = random.sample(range(total_slots), num_tokens)
slot_mapping = torch.tensor(slot_mapping_lst, dtype=torch.long, device=device)
kv_c = torch.randn(num_tokens, kv_lora_rank, dtype=dtype, device=device)
k_pe = torch.randn(num_tokens, qk_rope_head_dim, dtype=dtype, device=device)
scale = torch.tensor(1.0, dtype=torch.float32, device=device)
kv_cache = _create_mla_cache(
num_blocks,
block_size,
entry_size,
dtype=torch.uint8,
kv_cache_dtype=kv_cache_dtype,
device=device,
)
opcheck(
torch.ops._C_cache_ops.concat_and_cache_mla,
(kv_c, k_pe, kv_cache, slot_mapping, kv_cache_dtype, scale),
test_utils=DEFAULT_OPCHECK_TEST_UTILS,
)
ops.concat_and_cache_mla(kv_c, k_pe, kv_cache, slot_mapping, kv_cache_dtype, scale)
# Byte layout of a single cache entry:
# [0, 256) 512 e2m1 NoPE values packed 2/byte
# [256, 320) 64 unscaled e4m3 RoPE values
# [nope_sf_off, 352) 32 e4m3 NoPE SFs (one per 16 elements), stored
# permuted (see _nvfp4_sf_element_order)
nope_bytes = kv_lora_rank // 2
rope_bytes = qk_rope_head_dim
nope_sf_off = nope_bytes + rope_bytes
num_nope_sf = kv_lora_rank // 16
assert nope_sf_off + num_nope_sf == entry_size
for i in range(num_tokens):
slot = slot_mapping[i].item()
block_idx = slot // block_size
block_offset = slot % block_size
entry = kv_cache[block_idx, block_offset]
kv_c_ref = kv_c[i].to(torch.float32)
k_pe_ref = k_pe[i].to(torch.float32)
# Scale-factor bytes must match e4m3(max(amax_per_16 / divisor, 2^-9))
# computed in python, allowing a 1-ulp e4m3 difference (all SFs are
# positive, so e4m3 bit patterns are monotonic and adjacent codes
# differ by 1).
nope_sf_bytes = _nvfp4_sf_element_order(
entry[nope_sf_off : nope_sf_off + num_nope_sf]
)
ref_nope_sf_bytes = _ref_nvfp4_sf_bytes(kv_c_ref, 6.0)
assert (
(nope_sf_bytes.to(torch.int16) - ref_nope_sf_bytes.to(torch.int16)).abs()
<= 1
).all()
# Dequantize with the *stored* scale factors: x = float(code) * float(sf).
nope_sf = nope_sf_bytes.view(torch.float8_e4m3fn).to(torch.float32)
nope_sf_per_elem = nope_sf.repeat_interleave(16)
nope_vals = _unpack_e2m1(entry[:nope_bytes]) * nope_sf_per_elem
# e2m1 error bound: the widest grid spacing is 2 (between codes 4 and
# 6), so round-to-nearest is off by at most 1.0*sf; the e4m3 rounding
# of the SF itself adds at most ~2^-4 relative, so 1.5*sf is safe for
# either encode convention (quantized or unquantized SF).
assert ((nope_vals - kv_c_ref).abs() <= 1.5 * nope_sf_per_elem).all()
# Plain unscaled e4m3: ~2^-4 relative rounding, with a small absolute
# floor for values in the subnormal range.
rope_payload = entry[nope_bytes:nope_sf_off]
rope_vals = rope_payload.view(torch.float8_e4m3fn).to(torch.float32)
rope_tol = k_pe_ref.abs() * 2.0**-3 + 2.0**-9
assert ((rope_vals - k_pe_ref).abs() <= rope_tol).all()
@pytest.mark.parametrize("kv_lora_rank", KV_LORA_RANKS)
@pytest.mark.parametrize("qk_rope_head_dim", QK_ROPE_HEAD_DIMS)
@pytest.mark.parametrize("block_size", BLOCK_SIZES_MLA)
+12 -1
View File
@@ -505,7 +505,10 @@ def create_and_prepopulate_kv_cache(
block_table = common_attn_metadata.block_table_tensor
slot_mapping = common_attn_metadata.slot_mapping
fp8_attention = kv_cache_dtype and kv_cache_dtype.startswith("fp8")
use_nvfp4_ds_mla = kv_cache_dtype == "nvfp4_ds_mla"
fp8_attention = (
bool(kv_cache_dtype and kv_cache_dtype.startswith("fp8")) or use_nvfp4_ds_mla
)
use_fp8_ds_mla = kv_cache_dtype == "fp8_ds_mla"
if fp8_attention:
@@ -515,6 +518,14 @@ def create_and_prepopulate_kv_cache(
# 4 * 4: 4 float32 scale values for 128-element tiles
# 2 * rope_dim: 16-bit RoPE values
kv_entry_size = kv_lora_rank + 4 * 4 + 2 * rope_dim
elif use_nvfp4_ds_mla:
kv_lora_rank = kv_c_contexts[0].shape[-1]
rope_dim = k_pe_contexts[0].shape[-1]
# e2m1-packed NoPE + unscaled e4m3 rope + per-16 e4m3 NoPE SFs,
# padded to 16B
kv_entry_size = (
cdiv(kv_lora_rank // 2 + rope_dim + kv_lora_rank // 16, 16) * 16
)
else:
kv_entry_size = head_size
+101 -1
View File
@@ -40,6 +40,9 @@ if not current_platform.is_cuda():
allow_module_level=True,
)
from vllm.model_executor.layers.attention.mla_attention import (
_canonicalize_sparse_mla_kv_cache_dtype,
)
from vllm.utils.math_utils import cdiv
from vllm.v1.attention.backends.mla.flashinfer_mla_sparse import (
FlashInferMLASparseTRTLLMBackend,
@@ -184,13 +187,89 @@ def _quantize_dequantize_fp8_ds_mla(
return dequant_kv_c, dequant_k_pe
_E2M1_TABLE = [0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0]
def _quantize_dequantize_nvfp4_ds_mla(
kv_c: torch.Tensor,
k_pe: torch.Tensor,
block_size: int,
kv_cache_dtype: str,
scale: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Round-trip kv_c/k_pe through the nvfp4_ds_mla cache layout.
Layout (per token, uint8): [e2m1-packed NoPE | unscaled e4m3 RoPE |
per-16 e4m3 NoPE SFs]. The SF bytes are stored permuted (an 8x4 -> 4x8
transpose: the scale for element block s lives at byte 8 * (s & 3) +
(s >> 2)) so that the 8 scales one FlashMLA dequant thread needs are
contiguous. Dequant: x = value * float(sf), which is exact in bf16, so
this python dequant matches the kernels bit-for-bit.
"""
if kv_c.numel() == 0:
return kv_c.clone(), k_pe.clone()
kv_lora_rank = kv_c.shape[-1]
rope_dim = k_pe.shape[-1]
num_tokens = kv_c.shape[0]
num_blocks = max(1, math.ceil(num_tokens / block_size))
entry_size = (
math.ceil((kv_lora_rank // 2 + rope_dim + kv_lora_rank // 16) / 16) * 16
)
sf_nope_off = kv_lora_rank // 2 + rope_dim
tmp_cache = torch.zeros(
num_blocks, block_size, entry_size, dtype=torch.uint8, device=kv_c.device
)
slot_mapping = torch.arange(num_tokens, dtype=torch.long, device=kv_c.device)
ops.concat_and_cache_mla(
kv_c, k_pe, tmp_cache, slot_mapping, kv_cache_dtype=kv_cache_dtype, scale=scale
)
tokens = tmp_cache.view(-1, entry_size)[:num_tokens]
table = torch.tensor(
_E2M1_TABLE + [-v for v in _E2M1_TABLE],
dtype=torch.float32,
device=kv_c.device,
)
def unpack_e2m1(packed: torch.Tensor) -> torch.Tensor:
lo = (packed & 0xF).long()
hi = (packed >> 4).long()
return table[torch.stack([lo, hi], dim=-1).flatten(-2)]
nope_vals = unpack_e2m1(tokens[:, : kv_lora_rank // 2])
num_nope_sf = kv_lora_rank // 16
nope_sf = (
# undo the on-wire SF permutation -> element-block order
tokens[:, sf_nope_off : sf_nope_off + num_nope_sf]
.unflatten(-1, (4, num_nope_sf // 4))
.transpose(-1, -2)
.flatten(-2)
.view(torch.float8_e4m3fn)
.float()
)
dequant_kv_c = (
(nope_vals.unflatten(-1, (-1, 16)) * nope_sf.unsqueeze(-1)).flatten(-2)
).to(kv_c.dtype)
# RoPE: plain e4m3, no scale factor
rope_raw = tokens[:, kv_lora_rank // 2 : sf_nope_off]
dequant_k_pe = rope_raw.view(torch.float8_e4m3fn).to(k_pe.dtype)
return dequant_kv_c, dequant_k_pe
@pytest.mark.parametrize(
"backend_cls",
[FlashMLASparseBackend, FlashInferMLASparseTRTLLMBackend],
ids=["FlashMLA", "FlashInferTRTLLM"],
)
@pytest.mark.parametrize("batch_name", list(SPARSE_BACKEND_BATCH_SPECS.keys()))
@pytest.mark.parametrize("kv_cache_dtype", ["auto", "fp8", "fp8_ds_mla"])
@pytest.mark.parametrize(
"kv_cache_dtype",
["auto", "fp8", "fp8_ds_mla", "nvfp4_ds_mla"],
)
@pytest.mark.parametrize("tensor_parallel_size", [1, 2, 4])
@pytest.mark.parametrize("block_size", [32, 64])
@pytest.mark.parametrize(("q_scale", "k_scale"), [(1.0, 1.0), (2.0, 3.0)])
@@ -219,6 +298,17 @@ def test_sparse_backend_decode_correctness(
"fp8_ds_mla kv-cache dtype"
)
if kv_cache_dtype == "nvfp4_ds_mla":
# Unlike "fp8" (an alias canonicalized to "fp8_ds_mla" above), the
# NVFP4 DS-MLA dtype must reach the backend unchanged.
assert (
_canonicalize_sparse_mla_kv_cache_dtype(backend_cls, kv_cache_dtype)
== kv_cache_dtype
)
device_capability = current_platform.get_device_capability()
if device_capability is None or device_capability.major != 10:
pytest.skip("The NVFP4 DS-MLA kv-cache dtype requires SM 10.x")
supported_block_sizes = backend_cls.get_supported_kernel_block_sizes()
if block_size not in supported_block_sizes:
pytest.skip(
@@ -238,6 +328,7 @@ def test_sparse_backend_decode_correctness(
batch_spec = SPARSE_BACKEND_BATCH_SPECS[batch_name]
use_fp8_ds_mla_quantization = kv_cache_dtype == "fp8_ds_mla"
use_nvfp4_ds_mla_quantization = kv_cache_dtype == "nvfp4_ds_mla"
device = torch.device(DEVICE_TYPE)
dtype = torch.bfloat16
@@ -381,6 +472,15 @@ def test_sparse_backend_decode_correctness(
simulate_sm100_e8m0_scales=is_sm100,
)
k_pe_full = k_pe_squeezed.unsqueeze(1)
elif use_nvfp4_ds_mla_quantization:
kv_c_full, k_pe_squeezed = _quantize_dequantize_nvfp4_ds_mla(
kv_c_full,
k_pe_full.squeeze(1),
block_size=block_size,
kv_cache_dtype=kv_cache_dtype,
scale=kv_cache_scale,
)
k_pe_full = k_pe_squeezed.unsqueeze(1)
q_nope, q_pe = q_c.split([qk_nope_head_dim, qk_rope_head_dim], dim=-1)
ql_nope = torch.einsum("qnh,lnh->qnl", q_nope, W_UK)