[Feature] Support DCP with FP8 KV cache in MLA decode path (#44044)

Signed-off-by: shivampr <shivampr.dev@gmail.com>
Signed-off-by: Shivam <shivamprasad91@gmail.com>
Co-authored-by: Matthew Bonanni <mbonanni@redhat.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
shivampr
2026-06-24 19:28:17 +00:00
committed by GitHub
co-authored by Matthew Bonanni mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent d511b5bae9
commit 49f2104c53
4 changed files with 359 additions and 15 deletions
+72
View File
@@ -1015,6 +1015,78 @@ def test_gather_and_maybe_dequant_cache_mla(
torch.testing.assert_close(dst, expected)
@pytest.mark.parametrize("kv_lora_rank", [512])
@pytest.mark.parametrize("qk_rope_head_dim", [64])
@pytest.mark.parametrize("block_size", [16])
@pytest.mark.parametrize("num_blocks", [128])
@pytest.mark.parametrize("dtype", [torch.float32])
@pytest.mark.parametrize("kv_cache_dtype", ["auto", "fp8"])
@pytest.mark.parametrize("device", CUDA_DEVICES)
@torch.inference_mode()
def test_gather_and_maybe_dequant_cache_mla_with_seq_starts(
kv_lora_rank,
qk_rope_head_dim,
block_size,
num_blocks,
dtype,
kv_cache_dtype,
device,
):
entry_size = kv_lora_rank + qk_rope_head_dim
scale = torch.tensor(0.1, dtype=torch.float32, device=device)
src_cache = _create_mla_cache(
num_blocks, block_size, entry_size, dtype, kv_cache_dtype, device
)
_fill_mla_cache(src_cache, kv_cache_dtype=kv_cache_dtype)
seq_starts = torch.tensor([3, 17, 5], dtype=torch.int32, device=device)
seq_lens = torch.tensor([20, 10, 16], dtype=torch.int32, device=device)
batch_size = seq_lens.shape[0]
total_tokens = seq_lens.sum().item()
cu_seq_lens = torch.empty((batch_size + 1), dtype=torch.int32, device=device)
cu_seq_lens[0] = 0
cu_seq_lens[1:] = seq_lens.cumsum(dim=0)
token_to_seq = torch.repeat_interleave(
torch.arange(batch_size, dtype=torch.int32, device=device), seq_lens
)
block_table = torch.empty(
(batch_size, num_blocks), dtype=torch.int32, device=device
)
for b in range(batch_size):
block_table[b, :] = torch.randperm(num_blocks, device=device)
if kv_cache_dtype == "fp8":
dequant_src_cache = torch.empty_like(src_cache, dtype=dtype)
ops.convert_fp8(dequant_src_cache, src_cache, scale.item())
else:
dequant_src_cache = src_cache
expected_rows = []
for b in range(batch_size):
start = seq_starts[b].item()
length = seq_lens[b].item()
for offset in range(start, start + length):
block_id = block_table[b, offset // block_size]
slot = offset % block_size
expected_rows.append(dequant_src_cache[block_id, slot])
expected = torch.stack(expected_rows)
dst = torch.zeros((total_tokens, entry_size), dtype=dtype, device=device)
ops.gather_and_maybe_dequant_cache(
src_cache,
dst,
block_table,
cu_seq_lens,
token_to_seq,
total_tokens,
kv_cache_dtype,
scale,
seq_starts,
)
torch.testing.assert_close(dst, expected)
@pytest.mark.parametrize("kv_lora_rank", [512])
@pytest.mark.parametrize("qk_rope_head_dim", [64])
@pytest.mark.parametrize("block_size", [16])