[Feature] Support DCP with FP8 KV cache in MLA decode path (#44044)

Signed-off-by: shivampr <shivampr.dev@gmail.com>
Signed-off-by: Shivam <shivamprasad91@gmail.com>
Co-authored-by: Matthew Bonanni <mbonanni@redhat.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
shivampr
2026-06-24 19:28:17 +00:00
committed by GitHub
co-authored by Matthew Bonanni mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent d511b5bae9
commit 49f2104c53
4 changed files with 359 additions and 15 deletions
+72
View File
@@ -1015,6 +1015,78 @@ def test_gather_and_maybe_dequant_cache_mla(
torch.testing.assert_close(dst, expected)
@pytest.mark.parametrize("kv_lora_rank", [512])
@pytest.mark.parametrize("qk_rope_head_dim", [64])
@pytest.mark.parametrize("block_size", [16])
@pytest.mark.parametrize("num_blocks", [128])
@pytest.mark.parametrize("dtype", [torch.float32])
@pytest.mark.parametrize("kv_cache_dtype", ["auto", "fp8"])
@pytest.mark.parametrize("device", CUDA_DEVICES)
@torch.inference_mode()
def test_gather_and_maybe_dequant_cache_mla_with_seq_starts(
kv_lora_rank,
qk_rope_head_dim,
block_size,
num_blocks,
dtype,
kv_cache_dtype,
device,
):
entry_size = kv_lora_rank + qk_rope_head_dim
scale = torch.tensor(0.1, dtype=torch.float32, device=device)
src_cache = _create_mla_cache(
num_blocks, block_size, entry_size, dtype, kv_cache_dtype, device
)
_fill_mla_cache(src_cache, kv_cache_dtype=kv_cache_dtype)
seq_starts = torch.tensor([3, 17, 5], dtype=torch.int32, device=device)
seq_lens = torch.tensor([20, 10, 16], dtype=torch.int32, device=device)
batch_size = seq_lens.shape[0]
total_tokens = seq_lens.sum().item()
cu_seq_lens = torch.empty((batch_size + 1), dtype=torch.int32, device=device)
cu_seq_lens[0] = 0
cu_seq_lens[1:] = seq_lens.cumsum(dim=0)
token_to_seq = torch.repeat_interleave(
torch.arange(batch_size, dtype=torch.int32, device=device), seq_lens
)
block_table = torch.empty(
(batch_size, num_blocks), dtype=torch.int32, device=device
)
for b in range(batch_size):
block_table[b, :] = torch.randperm(num_blocks, device=device)
if kv_cache_dtype == "fp8":
dequant_src_cache = torch.empty_like(src_cache, dtype=dtype)
ops.convert_fp8(dequant_src_cache, src_cache, scale.item())
else:
dequant_src_cache = src_cache
expected_rows = []
for b in range(batch_size):
start = seq_starts[b].item()
length = seq_lens[b].item()
for offset in range(start, start + length):
block_id = block_table[b, offset // block_size]
slot = offset % block_size
expected_rows.append(dequant_src_cache[block_id, slot])
expected = torch.stack(expected_rows)
dst = torch.zeros((total_tokens, entry_size), dtype=dtype, device=device)
ops.gather_and_maybe_dequant_cache(
src_cache,
dst,
block_table,
cu_seq_lens,
token_to_seq,
total_tokens,
kv_cache_dtype,
scale,
seq_starts,
)
torch.testing.assert_close(dst, expected)
@pytest.mark.parametrize("kv_lora_rank", [512])
@pytest.mark.parametrize("qk_rope_head_dim", [64])
@pytest.mark.parametrize("block_size", [16])
+215
View File
@@ -19,6 +19,7 @@ from tests.v1.attention.utils import (
)
from vllm import _custom_ops as ops
from vllm.config.vllm import set_current_vllm_config
from vllm.model_executor.layers.attention import mla_attention as mla_attention_module
from vllm.model_executor.layers.attention.mla_attention import (
MLAAttention,
QueryLenSupport,
@@ -30,6 +31,7 @@ from vllm.utils.math_utils import cdiv
from vllm.utils.torch_utils import STR_DTYPE_TO_TORCH_DTYPE
from vllm.v1.attention.backend import CommonAttentionMetadata
from vllm.v1.attention.backends.fa_utils import flash_attn_supports_mla
from vllm.v1.attention.backends.mla import flashmla as flashmla_module
from vllm.v1.attention.backends.mla.prefill import (
MLAPrefillBackendEnum,
get_mla_prefill_backend,
@@ -552,6 +554,10 @@ class MockMLAAttentionLayer(MLAAttention):
)
else:
mqa_q = (mqa_ql_nope, mqa_q_pe)
if self.impl.dcp_world_size > 1:
if isinstance(mqa_q, tuple):
mqa_q = torch.cat(mqa_q, dim=-1)
mqa_q = mla_attention_module.get_dcp_group().all_gather(mqa_q, dim=1)
attn_out, _ = self.impl.forward_mqa(mqa_q, kv_cache, attn_metadata, self)
@@ -569,6 +575,215 @@ class MockMLAAttentionLayer(MLAAttention):
return output
def test_mock_mla_dcp_fp8_decode_gathers_quantized_query(
monkeypatch, default_vllm_config
):
if not torch.cuda.is_available():
pytest.skip("CUDA is required for FP8 decode query quantization path.")
device = torch.device(f"{DEVICE_TYPE}:0")
num_tokens = 2
num_heads = 2
qk_nope_head_dim = 4
qk_rope_head_dim = 2
v_head_dim = 3
kv_lora_rank = 5
class _DummyKVProj:
def __init__(self):
# Shape expected by MockMLAAttentionLayer.__init__
self.weight = torch.randn(
num_heads * (qk_nope_head_dim + v_head_dim),
kv_lora_rank,
device=device,
dtype=torch.float32,
)
class _FakeImpl:
def __init__(self):
self.kv_cache_dtype = "fp8"
self.supports_quant_query_input = True
self.dcp_world_size = 2
self.forward_q = None
def forward_mha(self, *args, **kwargs):
return None
def forward_mqa(self, q, kv_cache, attn_metadata, layer):
self.forward_q = q
assert isinstance(q, torch.Tensor)
bsz, _, _ = q.shape
return (
torch.zeros(
bsz,
num_heads,
kv_lora_rank,
device=q.device,
dtype=torch.float32,
),
None,
)
class _FakeDCPGroup:
def __init__(self):
self.calls = 0
self.input_dtype = None
self.input_shape = None
def all_gather(self, x, dim=1):
self.calls += 1
self.input_dtype = x.dtype
self.input_shape = tuple(x.shape)
return torch.cat([x, x], dim=dim)
fake_group = _FakeDCPGroup()
monkeypatch.setattr(mla_attention_module, "get_dcp_group", lambda: fake_group)
impl = _FakeImpl()
with set_current_vllm_config(default_vllm_config):
layer = MockMLAAttentionLayer(
impl=impl,
num_heads=num_heads,
qk_nope_head_dim=qk_nope_head_dim,
qk_rope_head_dim=qk_rope_head_dim,
v_head_dim=v_head_dim,
kv_lora_rank=kv_lora_rank,
device=device,
kv_b_proj=_DummyKVProj(),
q_scale=1.0,
k_scale=1.0,
)
q = torch.randn(
num_tokens,
num_heads,
qk_nope_head_dim + qk_rope_head_dim,
device=device,
dtype=torch.float32,
)
kv_c = torch.randn(num_tokens, kv_lora_rank, device=device, dtype=torch.float32)
k_pe = torch.randn(
num_tokens, 1, qk_rope_head_dim, device=device, dtype=torch.float32
)
kv_cache = torch.empty(0, device=device, dtype=torch.float32)
output = torch.empty(
num_tokens, num_heads * v_head_dim, device=device, dtype=torch.float32
)
class _AttnMeta:
num_decode_tokens = num_tokens
num_decodes = 1
num_prefills = 0
slot_mapping = torch.empty(0, dtype=torch.long, device=device)
layer.forward_impl(q, kv_c, k_pe, kv_cache, _AttnMeta(), output)
assert fake_group.calls == 1
assert fake_group.input_dtype == current_platform.fp8_dtype()
assert fake_group.input_shape == (
num_tokens,
num_heads,
kv_lora_rank + qk_rope_head_dim,
)
assert isinstance(impl.forward_q, torch.Tensor)
assert tuple(impl.forward_q.shape) == (
num_tokens,
num_heads * impl.dcp_world_size,
kv_lora_rank + qk_rope_head_dim,
)
@pytest.mark.parametrize("is_fp8_kvcache", [False, True], ids=["bf16", "fp8"])
def test_flashmla_dcp_decode_metadata_uses_gathered_query_heads(
monkeypatch, is_fp8_kvcache
):
class _FakeSchedulerMetadata:
tile_scheduler_metadata = None
num_splits = None
base_call: tuple[torch.Tensor, int, int, bool] | None = None
fp8_call: tuple[torch.Tensor, int, int] | None = None
def fake_get_mla_metadata(
seq_lens_device,
num_q_tokens_per_head_k,
num_heads_k,
is_fp8_kvcache=False,
):
nonlocal base_call
base_call = (
seq_lens_device,
num_q_tokens_per_head_k,
num_heads_k,
is_fp8_kvcache,
)
return _FakeSchedulerMetadata(), None
def fake_get_mla_metadata_dense_fp8(
seq_lens_device, num_q_tokens_per_head_k, num_heads_k
):
nonlocal fp8_call
fp8_call = (
seq_lens_device,
num_q_tokens_per_head_k,
num_heads_k,
)
return (
torch.empty((0, 8), dtype=torch.int32),
torch.empty((0,), dtype=torch.int32),
)
monkeypatch.setattr(flashmla_module, "get_mla_metadata", fake_get_mla_metadata)
monkeypatch.setattr(
flashmla_module,
"get_mla_metadata_dense_fp8",
fake_get_mla_metadata_dense_fp8,
)
builder = object.__new__(flashmla_module.FlashMLAMetadataBuilder)
builder.num_q_heads = 4
builder.dcp_world_size = 2
builder.is_fp8_kvcache = is_fp8_kvcache
builder.compilation_config = type(
"_CompilationConfig",
(),
{
"cudagraph_mode": type(
"_CudaGraphMode",
(),
{"has_full_cudagraphs": lambda self: False},
)()
},
)()
seq_lens = torch.tensor([16, 24], dtype=torch.int32)
query_start_loc = torch.tensor([0, 1, 2], dtype=torch.int32)
metadata = builder._build_decode(
block_table_tensor=torch.empty((2, 1), dtype=torch.int32),
seq_lens_device=seq_lens,
max_seq_len=24,
query_start_loc_cpu=query_start_loc,
query_start_loc_device=query_start_loc,
num_decode_tokens=2,
dcp_tot_seq_lens_device=None,
)
assert base_call is not None
assert base_call[0] is seq_lens
assert base_call[1:] == (8, 1, is_fp8_kvcache)
if is_fp8_kvcache:
assert metadata.scheduler_metadata.tile_scheduler_metadata is not None
assert metadata.scheduler_metadata.num_splits is not None
assert fp8_call is not None
assert fp8_call[0] is seq_lens
assert fp8_call[1:] == (8, 1)
else:
assert metadata.scheduler_metadata.tile_scheduler_metadata is None
assert metadata.scheduler_metadata.num_splits is None
assert fp8_call is None
def run_attention_backend(
backend: AttentionBackendEnum,
kv_cache_spec: MLAAttentionSpec,
@@ -800,9 +800,9 @@ class MLAAttention(nn.Module, AttentionLayerBase):
else:
mqa_q = (mqa_ql_nope, mqa_q_pe)
if self.impl.dcp_world_size > 1:
assert not fp8_attention, "DCP not support fp8 kvcache now."
# concatenate mqa_ql_nope and mqa_q_pe -> (B, N, L + P)
mqa_q = torch.cat(mqa_q, dim=-1)
if isinstance(mqa_q, tuple):
# concatenate mqa_ql_nope and mqa_q_pe -> (B, N, L + P)
mqa_q = torch.cat(mqa_q, dim=-1)
# mqa_q do allgather in head dim.
mqa_q = get_dcp_group().all_gather(mqa_q, dim=1)
@@ -1263,6 +1263,7 @@ class MLACommonPrefillMetadata:
padded_local_chunk_seq_lens: list[list[int]] | None = None
local_context_lens_allranks: list[list[int]] | None = None
padded_local_cu_seq_lens: torch.Tensor | None = None
padded_local_token_to_seq: torch.Tensor | None = None
cu_seq_lens_lst: list[list[int]] | None = None
chunk_size: int | None = None
prefill_tokens_with_context: int | None = None
@@ -1787,6 +1788,21 @@ class MLACommonMetadataBuilder(AttentionMetadataBuilder[M]):
out=padded_local_cu_chunk_seq_lens_cpu[:, 1:],
dtype=torch.int32,
)
max_padded_local_tokens_over_chunk = (
padded_local_cu_chunk_seq_lens_cpu[:, -1].max().item()
)
padded_local_token_to_seq_tensor_cpu = torch.zeros(
[num_chunks, max_padded_local_tokens_over_chunk],
dtype=torch.int32,
)
for i in range(num_chunks):
chunk_token_to_seq_tensor = torch.repeat_interleave(
range_idx, padded_local_chunk_seq_lens[i]
)
chunk_len = chunk_token_to_seq_tensor.shape[0]
padded_local_token_to_seq_tensor_cpu[i, :chunk_len] = (
chunk_token_to_seq_tensor
)
prefill_tokens_with_context = None
if num_prefills_with_context_cpu > 0:
@@ -1811,6 +1827,9 @@ class MLACommonMetadataBuilder(AttentionMetadataBuilder[M]):
padded_local_cu_seq_lens=padded_local_cu_chunk_seq_lens_cpu.to(
device, non_blocking=True
),
padded_local_token_to_seq=padded_local_token_to_seq_tensor_cpu.to(
device, non_blocking=True
),
cu_seq_lens_lst=cu_seq_lens_cpu.tolist(),
chunk_size=padded_local_max_context_chunk_across_ranks,
prefill_tokens_with_context=prefill_tokens_with_context,
@@ -2187,7 +2206,6 @@ class MLACommonImpl(MLAAttentionImpl[M], Generic[M]):
k_scale: torch.Tensor,
dcp_world_size: int,
):
assert k_scale is None, "DCP not support scaled kvcache now."
assert attn_metadata.prefill is not None
prefill_metadata = attn_metadata.prefill
assert prefill_metadata.prefill_backend is not None
@@ -2195,9 +2213,11 @@ class MLACommonImpl(MLAAttentionImpl[M], Generic[M]):
assert prefill_metadata.chunked_context.padded_local_chunk_seq_lens is not None
assert prefill_metadata.chunked_context.local_context_lens_allranks is not None
assert prefill_metadata.chunked_context.padded_local_cu_seq_lens is not None
assert prefill_metadata.chunked_context.padded_local_token_to_seq is not None
assert prefill_metadata.chunked_context.cu_seq_lens_lst is not None
assert prefill_metadata.chunked_context.chunk_size is not None
use_fp8_prefill = prefill_metadata.q_data_type == current_platform.fp8_dtype()
output = None
merge_output = None
iters = len(prefill_metadata.chunked_context.seq_tot)
@@ -2205,16 +2225,37 @@ class MLACommonImpl(MLAAttentionImpl[M], Generic[M]):
for i in range(iters):
toks = prefill_metadata.chunked_context.seq_tot[i]
ops.cp_gather_cache(
src_cache=kv_c_and_k_pe_cache,
dst=workspace,
block_table=prefill_metadata.block_table,
cu_seq_lens=prefill_metadata.chunked_context.padded_local_cu_seq_lens[
i
],
batch_size=attn_metadata.num_prefills,
seq_starts=prefill_metadata.chunked_context.starts[i],
if toks == 0:
continue
padded_local_cu_seq_lens = (
prefill_metadata.chunked_context.padded_local_cu_seq_lens[i]
)
if is_quantized_kv_cache(self.kv_cache_dtype) and (
self.kv_cache_dtype != "fp8_ds_mla"
):
assert k_scale is not None
ops.gather_and_maybe_dequant_cache(
src_cache=kv_c_and_k_pe_cache,
dst=workspace,
block_table=prefill_metadata.block_table,
cu_seq_lens=padded_local_cu_seq_lens,
token_to_seq=prefill_metadata.chunked_context.padded_local_token_to_seq[
i
],
num_tokens=toks,
kv_cache_dtype=self.kv_cache_dtype,
scale=k_scale,
seq_starts=prefill_metadata.chunked_context.starts[i],
)
else:
ops.cp_gather_cache(
src_cache=kv_c_and_k_pe_cache,
dst=workspace,
block_table=prefill_metadata.block_table,
cu_seq_lens=padded_local_cu_seq_lens,
batch_size=attn_metadata.num_prefills,
seq_starts=prefill_metadata.chunked_context.starts[i],
)
# workspace
# |------- N tokens --------|--------- N*dcp_size tokens ----------|
# |<- use for local_gather ->|<--------- use for allgather -------->|
@@ -2252,9 +2293,22 @@ class MLACommonImpl(MLAAttentionImpl[M], Generic[M]):
toks=toks,
)
kv_b_proj_w_dtype = (
self.kv_b_proj.weight.dtype
if hasattr(self.kv_b_proj, "weight")
else self.kv_b_proj.params_dtype
)
if (
use_fp8_prefill or kv_b_proj_w_dtype != current_platform.fp8_dtype()
) and kv_b_proj_w_dtype != torch.uint8:
kv_c_normed = kv_c_normed.to(kv_b_proj_w_dtype)
kv_nope = self.kv_b_proj(kv_c_normed)[0].view(
-1, self.num_heads, self.qk_nope_head_dim + self.v_head_dim
)
if use_fp8_prefill:
kv_nope = kv_nope.to(prefill_metadata.q_data_type)
k_pe = k_pe.to(prefill_metadata.q_data_type)
k_nope, v = kv_nope.split([self.qk_nope_head_dim, self.v_head_dim], dim=-1)
k = self._concat_k_nope_k_pe(k_nope, k_pe)
@@ -2346,7 +2400,7 @@ class MLACommonImpl(MLAAttentionImpl[M], Generic[M]):
q,
kv_c_and_k_pe_cache,
attn_metadata,
k_scale=None,
k_scale=k_scale,
dcp_world_size=self.dcp_world_size,
)
)
+4 -1
View File
@@ -171,7 +171,10 @@ class FlashMLAMetadataBuilder(MLACommonMetadataBuilder[FlashMLAMetadata]):
query_lens_cpu = query_start_loc_cpu[1:] - query_start_loc_cpu[:-1]
# we use the max but all should be the same due to uniform length requirement
max_query_len = query_lens_cpu.max().item()
num_q_tokens_per_head_k = max_query_len * self.num_q_heads // 1
num_q_heads = self.num_q_heads
if self.dcp_world_size > 1:
num_q_heads *= self.dcp_world_size
num_q_tokens_per_head_k = max_query_len * num_q_heads // 1
scheduler_metadata, _ = get_mla_metadata(
seq_lens_device,
num_q_tokens_per_head_k,