[Attention][DSA] Enable W4A16 DSA (#51724)
Signed-off-by: Shiyang Chen <shiychen@nvidia.com>
This commit is contained in:
@@ -907,6 +907,157 @@ def test_concat_and_cache_ds_mla(
|
||||
torch.testing.assert_close(kv_rope, ref_rope, atol=0.001, rtol=0.1)
|
||||
|
||||
|
||||
# Bytes per token for the nvfp4_ds_mla cache layout (see flashmla_sparse.py).
|
||||
NVFP4_DS_MLA_ENTRY_SIZE = 352
|
||||
|
||||
# e2m1 magnitude table; the sign lives in bit 3 of each 4-bit code.
|
||||
E2M1_VALUES = [0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0]
|
||||
|
||||
|
||||
def _unpack_e2m1(packed: torch.Tensor) -> torch.Tensor:
|
||||
"""Unpack uint8 bytes holding two e2m1 codes each into float32 values.
|
||||
|
||||
The low nibble holds the even element, the high nibble the odd one.
|
||||
"""
|
||||
low = packed & 0x0F
|
||||
high = packed >> 4
|
||||
codes = torch.stack([low, high], dim=-1).reshape(-1).long()
|
||||
table = torch.tensor(E2M1_VALUES, dtype=torch.float32, device=packed.device)
|
||||
magnitude = table[codes & 0x7]
|
||||
sign = torch.where((codes & 0x8) != 0, -1.0, 1.0)
|
||||
return sign * magnitude
|
||||
|
||||
|
||||
def _ref_nvfp4_sf_bytes(vals: torch.Tensor, divisor: float) -> torch.Tensor:
|
||||
"""Reference e4m3 scale-factor bytes: e4m3(max(amax_per_16 / divisor, 2^-9)).
|
||||
|
||||
In element-block order (byte s scales elements [16s, 16s + 16)).
|
||||
"""
|
||||
tiles = vals.to(torch.float32).reshape(-1, 16)
|
||||
sf = torch.clamp(tiles.abs().amax(dim=1) / divisor, min=2.0**-9)
|
||||
return sf.to(torch.float8_e4m3fn).view(torch.uint8)
|
||||
|
||||
|
||||
def _nvfp4_sf_element_order(wire_sf_bytes: torch.Tensor) -> torch.Tensor:
|
||||
"""Stored NVFP4 scale-factor byte order -> element-block order.
|
||||
|
||||
The 32 NoPE scale-factor bytes of an nvfp4_ds_mla entry are stored
|
||||
permuted (an 8x4 -> 4x8 transpose): the scale for element block s lives at
|
||||
byte 8 * (s & 3) + (s >> 2), so that the 8 scales one FlashMLA dequant
|
||||
thread needs are contiguous. This undoes that permutation.
|
||||
"""
|
||||
num_sf = wire_sf_bytes.shape[-1]
|
||||
return wire_sf_bytes.unflatten(-1, (4, num_sf // 4)).transpose(-1, -2).flatten(-2)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("kv_lora_rank", KV_LORA_RANKS)
|
||||
@pytest.mark.parametrize("qk_rope_head_dim", QK_ROPE_HEAD_DIMS)
|
||||
@pytest.mark.parametrize("num_tokens", NUM_TOKENS_MLA)
|
||||
@pytest.mark.parametrize("block_size", BLOCK_SIZES_MLA)
|
||||
@pytest.mark.parametrize("num_blocks", NUM_BLOCKS_MLA)
|
||||
@pytest.mark.parametrize("seed", SEEDS)
|
||||
@pytest.mark.parametrize("device", CUDA_DEVICES)
|
||||
@torch.inference_mode()
|
||||
def test_concat_and_cache_nvfp4_ds_mla(
|
||||
kv_lora_rank: int,
|
||||
qk_rope_head_dim: int,
|
||||
num_tokens: int,
|
||||
block_size: int,
|
||||
num_blocks: int,
|
||||
seed: int,
|
||||
device: str,
|
||||
) -> None:
|
||||
if current_platform.is_rocm():
|
||||
pytest.skip("concat_and_cache_mla doesn't support NVFP4 DS-MLA on ROCm")
|
||||
device_capability = current_platform.get_device_capability()
|
||||
if device_capability is None or device_capability.major != 10:
|
||||
pytest.skip("The NVFP4 DS-MLA kv-cache dtype requires SM 10.x")
|
||||
if kv_lora_rank != 512:
|
||||
pytest.skip("The NVFP4 DS-MLA layout requires kv_lora_rank == 512")
|
||||
dtype = torch.bfloat16
|
||||
set_random_seed(seed)
|
||||
torch.set_default_device(device)
|
||||
torch.accelerator.set_device_index(device)
|
||||
|
||||
kv_cache_dtype = "nvfp4_ds_mla"
|
||||
entry_size = NVFP4_DS_MLA_ENTRY_SIZE
|
||||
|
||||
total_slots = num_blocks * block_size
|
||||
slot_mapping_lst = random.sample(range(total_slots), num_tokens)
|
||||
slot_mapping = torch.tensor(slot_mapping_lst, dtype=torch.long, device=device)
|
||||
|
||||
kv_c = torch.randn(num_tokens, kv_lora_rank, dtype=dtype, device=device)
|
||||
k_pe = torch.randn(num_tokens, qk_rope_head_dim, dtype=dtype, device=device)
|
||||
|
||||
scale = torch.tensor(1.0, dtype=torch.float32, device=device)
|
||||
kv_cache = _create_mla_cache(
|
||||
num_blocks,
|
||||
block_size,
|
||||
entry_size,
|
||||
dtype=torch.uint8,
|
||||
kv_cache_dtype=kv_cache_dtype,
|
||||
device=device,
|
||||
)
|
||||
|
||||
opcheck(
|
||||
torch.ops._C_cache_ops.concat_and_cache_mla,
|
||||
(kv_c, k_pe, kv_cache, slot_mapping, kv_cache_dtype, scale),
|
||||
test_utils=DEFAULT_OPCHECK_TEST_UTILS,
|
||||
)
|
||||
|
||||
ops.concat_and_cache_mla(kv_c, k_pe, kv_cache, slot_mapping, kv_cache_dtype, scale)
|
||||
|
||||
# Byte layout of a single cache entry:
|
||||
# [0, 256) 512 e2m1 NoPE values packed 2/byte
|
||||
# [256, 320) 64 unscaled e4m3 RoPE values
|
||||
# [nope_sf_off, 352) 32 e4m3 NoPE SFs (one per 16 elements), stored
|
||||
# permuted (see _nvfp4_sf_element_order)
|
||||
nope_bytes = kv_lora_rank // 2
|
||||
rope_bytes = qk_rope_head_dim
|
||||
nope_sf_off = nope_bytes + rope_bytes
|
||||
num_nope_sf = kv_lora_rank // 16
|
||||
assert nope_sf_off + num_nope_sf == entry_size
|
||||
|
||||
for i in range(num_tokens):
|
||||
slot = slot_mapping[i].item()
|
||||
block_idx = slot // block_size
|
||||
block_offset = slot % block_size
|
||||
entry = kv_cache[block_idx, block_offset]
|
||||
|
||||
kv_c_ref = kv_c[i].to(torch.float32)
|
||||
k_pe_ref = k_pe[i].to(torch.float32)
|
||||
|
||||
# Scale-factor bytes must match e4m3(max(amax_per_16 / divisor, 2^-9))
|
||||
# computed in python, allowing a 1-ulp e4m3 difference (all SFs are
|
||||
# positive, so e4m3 bit patterns are monotonic and adjacent codes
|
||||
# differ by 1).
|
||||
nope_sf_bytes = _nvfp4_sf_element_order(
|
||||
entry[nope_sf_off : nope_sf_off + num_nope_sf]
|
||||
)
|
||||
ref_nope_sf_bytes = _ref_nvfp4_sf_bytes(kv_c_ref, 6.0)
|
||||
assert (
|
||||
(nope_sf_bytes.to(torch.int16) - ref_nope_sf_bytes.to(torch.int16)).abs()
|
||||
<= 1
|
||||
).all()
|
||||
|
||||
# Dequantize with the *stored* scale factors: x = float(code) * float(sf).
|
||||
nope_sf = nope_sf_bytes.view(torch.float8_e4m3fn).to(torch.float32)
|
||||
nope_sf_per_elem = nope_sf.repeat_interleave(16)
|
||||
nope_vals = _unpack_e2m1(entry[:nope_bytes]) * nope_sf_per_elem
|
||||
# e2m1 error bound: the widest grid spacing is 2 (between codes 4 and
|
||||
# 6), so round-to-nearest is off by at most 1.0*sf; the e4m3 rounding
|
||||
# of the SF itself adds at most ~2^-4 relative, so 1.5*sf is safe for
|
||||
# either encode convention (quantized or unquantized SF).
|
||||
assert ((nope_vals - kv_c_ref).abs() <= 1.5 * nope_sf_per_elem).all()
|
||||
|
||||
# Plain unscaled e4m3: ~2^-4 relative rounding, with a small absolute
|
||||
# floor for values in the subnormal range.
|
||||
rope_payload = entry[nope_bytes:nope_sf_off]
|
||||
rope_vals = rope_payload.view(torch.float8_e4m3fn).to(torch.float32)
|
||||
rope_tol = k_pe_ref.abs() * 2.0**-3 + 2.0**-9
|
||||
assert ((rope_vals - k_pe_ref).abs() <= rope_tol).all()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("kv_lora_rank", KV_LORA_RANKS)
|
||||
@pytest.mark.parametrize("qk_rope_head_dim", QK_ROPE_HEAD_DIMS)
|
||||
@pytest.mark.parametrize("block_size", BLOCK_SIZES_MLA)
|
||||
|
||||
@@ -505,7 +505,10 @@ def create_and_prepopulate_kv_cache(
|
||||
block_table = common_attn_metadata.block_table_tensor
|
||||
slot_mapping = common_attn_metadata.slot_mapping
|
||||
|
||||
fp8_attention = kv_cache_dtype and kv_cache_dtype.startswith("fp8")
|
||||
use_nvfp4_ds_mla = kv_cache_dtype == "nvfp4_ds_mla"
|
||||
fp8_attention = (
|
||||
bool(kv_cache_dtype and kv_cache_dtype.startswith("fp8")) or use_nvfp4_ds_mla
|
||||
)
|
||||
use_fp8_ds_mla = kv_cache_dtype == "fp8_ds_mla"
|
||||
|
||||
if fp8_attention:
|
||||
@@ -515,6 +518,14 @@ def create_and_prepopulate_kv_cache(
|
||||
# 4 * 4: 4 float32 scale values for 128-element tiles
|
||||
# 2 * rope_dim: 16-bit RoPE values
|
||||
kv_entry_size = kv_lora_rank + 4 * 4 + 2 * rope_dim
|
||||
elif use_nvfp4_ds_mla:
|
||||
kv_lora_rank = kv_c_contexts[0].shape[-1]
|
||||
rope_dim = k_pe_contexts[0].shape[-1]
|
||||
# e2m1-packed NoPE + unscaled e4m3 rope + per-16 e4m3 NoPE SFs,
|
||||
# padded to 16B
|
||||
kv_entry_size = (
|
||||
cdiv(kv_lora_rank // 2 + rope_dim + kv_lora_rank // 16, 16) * 16
|
||||
)
|
||||
else:
|
||||
kv_entry_size = head_size
|
||||
|
||||
|
||||
@@ -40,6 +40,9 @@ if not current_platform.is_cuda():
|
||||
allow_module_level=True,
|
||||
)
|
||||
|
||||
from vllm.model_executor.layers.attention.mla_attention import (
|
||||
_canonicalize_sparse_mla_kv_cache_dtype,
|
||||
)
|
||||
from vllm.utils.math_utils import cdiv
|
||||
from vllm.v1.attention.backends.mla.flashinfer_mla_sparse import (
|
||||
FlashInferMLASparseTRTLLMBackend,
|
||||
@@ -184,13 +187,89 @@ def _quantize_dequantize_fp8_ds_mla(
|
||||
return dequant_kv_c, dequant_k_pe
|
||||
|
||||
|
||||
_E2M1_TABLE = [0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0]
|
||||
|
||||
|
||||
def _quantize_dequantize_nvfp4_ds_mla(
|
||||
kv_c: torch.Tensor,
|
||||
k_pe: torch.Tensor,
|
||||
block_size: int,
|
||||
kv_cache_dtype: str,
|
||||
scale: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Round-trip kv_c/k_pe through the nvfp4_ds_mla cache layout.
|
||||
|
||||
Layout (per token, uint8): [e2m1-packed NoPE | unscaled e4m3 RoPE |
|
||||
per-16 e4m3 NoPE SFs]. The SF bytes are stored permuted (an 8x4 -> 4x8
|
||||
transpose: the scale for element block s lives at byte 8 * (s & 3) +
|
||||
(s >> 2)) so that the 8 scales one FlashMLA dequant thread needs are
|
||||
contiguous. Dequant: x = value * float(sf), which is exact in bf16, so
|
||||
this python dequant matches the kernels bit-for-bit.
|
||||
"""
|
||||
if kv_c.numel() == 0:
|
||||
return kv_c.clone(), k_pe.clone()
|
||||
|
||||
kv_lora_rank = kv_c.shape[-1]
|
||||
rope_dim = k_pe.shape[-1]
|
||||
num_tokens = kv_c.shape[0]
|
||||
num_blocks = max(1, math.ceil(num_tokens / block_size))
|
||||
entry_size = (
|
||||
math.ceil((kv_lora_rank // 2 + rope_dim + kv_lora_rank // 16) / 16) * 16
|
||||
)
|
||||
sf_nope_off = kv_lora_rank // 2 + rope_dim
|
||||
|
||||
tmp_cache = torch.zeros(
|
||||
num_blocks, block_size, entry_size, dtype=torch.uint8, device=kv_c.device
|
||||
)
|
||||
slot_mapping = torch.arange(num_tokens, dtype=torch.long, device=kv_c.device)
|
||||
ops.concat_and_cache_mla(
|
||||
kv_c, k_pe, tmp_cache, slot_mapping, kv_cache_dtype=kv_cache_dtype, scale=scale
|
||||
)
|
||||
|
||||
tokens = tmp_cache.view(-1, entry_size)[:num_tokens]
|
||||
table = torch.tensor(
|
||||
_E2M1_TABLE + [-v for v in _E2M1_TABLE],
|
||||
dtype=torch.float32,
|
||||
device=kv_c.device,
|
||||
)
|
||||
|
||||
def unpack_e2m1(packed: torch.Tensor) -> torch.Tensor:
|
||||
lo = (packed & 0xF).long()
|
||||
hi = (packed >> 4).long()
|
||||
return table[torch.stack([lo, hi], dim=-1).flatten(-2)]
|
||||
|
||||
nope_vals = unpack_e2m1(tokens[:, : kv_lora_rank // 2])
|
||||
num_nope_sf = kv_lora_rank // 16
|
||||
nope_sf = (
|
||||
# undo the on-wire SF permutation -> element-block order
|
||||
tokens[:, sf_nope_off : sf_nope_off + num_nope_sf]
|
||||
.unflatten(-1, (4, num_nope_sf // 4))
|
||||
.transpose(-1, -2)
|
||||
.flatten(-2)
|
||||
.view(torch.float8_e4m3fn)
|
||||
.float()
|
||||
)
|
||||
dequant_kv_c = (
|
||||
(nope_vals.unflatten(-1, (-1, 16)) * nope_sf.unsqueeze(-1)).flatten(-2)
|
||||
).to(kv_c.dtype)
|
||||
|
||||
# RoPE: plain e4m3, no scale factor
|
||||
rope_raw = tokens[:, kv_lora_rank // 2 : sf_nope_off]
|
||||
dequant_k_pe = rope_raw.view(torch.float8_e4m3fn).to(k_pe.dtype)
|
||||
|
||||
return dequant_kv_c, dequant_k_pe
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"backend_cls",
|
||||
[FlashMLASparseBackend, FlashInferMLASparseTRTLLMBackend],
|
||||
ids=["FlashMLA", "FlashInferTRTLLM"],
|
||||
)
|
||||
@pytest.mark.parametrize("batch_name", list(SPARSE_BACKEND_BATCH_SPECS.keys()))
|
||||
@pytest.mark.parametrize("kv_cache_dtype", ["auto", "fp8", "fp8_ds_mla"])
|
||||
@pytest.mark.parametrize(
|
||||
"kv_cache_dtype",
|
||||
["auto", "fp8", "fp8_ds_mla", "nvfp4_ds_mla"],
|
||||
)
|
||||
@pytest.mark.parametrize("tensor_parallel_size", [1, 2, 4])
|
||||
@pytest.mark.parametrize("block_size", [32, 64])
|
||||
@pytest.mark.parametrize(("q_scale", "k_scale"), [(1.0, 1.0), (2.0, 3.0)])
|
||||
@@ -219,6 +298,17 @@ def test_sparse_backend_decode_correctness(
|
||||
"fp8_ds_mla kv-cache dtype"
|
||||
)
|
||||
|
||||
if kv_cache_dtype == "nvfp4_ds_mla":
|
||||
# Unlike "fp8" (an alias canonicalized to "fp8_ds_mla" above), the
|
||||
# NVFP4 DS-MLA dtype must reach the backend unchanged.
|
||||
assert (
|
||||
_canonicalize_sparse_mla_kv_cache_dtype(backend_cls, kv_cache_dtype)
|
||||
== kv_cache_dtype
|
||||
)
|
||||
device_capability = current_platform.get_device_capability()
|
||||
if device_capability is None or device_capability.major != 10:
|
||||
pytest.skip("The NVFP4 DS-MLA kv-cache dtype requires SM 10.x")
|
||||
|
||||
supported_block_sizes = backend_cls.get_supported_kernel_block_sizes()
|
||||
if block_size not in supported_block_sizes:
|
||||
pytest.skip(
|
||||
@@ -238,6 +328,7 @@ def test_sparse_backend_decode_correctness(
|
||||
|
||||
batch_spec = SPARSE_BACKEND_BATCH_SPECS[batch_name]
|
||||
use_fp8_ds_mla_quantization = kv_cache_dtype == "fp8_ds_mla"
|
||||
use_nvfp4_ds_mla_quantization = kv_cache_dtype == "nvfp4_ds_mla"
|
||||
|
||||
device = torch.device(DEVICE_TYPE)
|
||||
dtype = torch.bfloat16
|
||||
@@ -381,6 +472,15 @@ def test_sparse_backend_decode_correctness(
|
||||
simulate_sm100_e8m0_scales=is_sm100,
|
||||
)
|
||||
k_pe_full = k_pe_squeezed.unsqueeze(1)
|
||||
elif use_nvfp4_ds_mla_quantization:
|
||||
kv_c_full, k_pe_squeezed = _quantize_dequantize_nvfp4_ds_mla(
|
||||
kv_c_full,
|
||||
k_pe_full.squeeze(1),
|
||||
block_size=block_size,
|
||||
kv_cache_dtype=kv_cache_dtype,
|
||||
scale=kv_cache_scale,
|
||||
)
|
||||
k_pe_full = k_pe_squeezed.unsqueeze(1)
|
||||
|
||||
q_nope, q_pe = q_c.split([qk_nope_head_dim, qk_rope_head_dim], dim=-1)
|
||||
ql_nope = torch.einsum("qnh,lnh->qnl", q_nope, W_UK)
|
||||
|
||||
Reference in New Issue
Block a user