[Feature] Support DCP with FP8 KV cache in MLA decode path (#44044)
Signed-off-by: shivampr <shivampr.dev@gmail.com> Signed-off-by: Shivam <shivamprasad91@gmail.com> Co-authored-by: Matthew Bonanni <mbonanni@redhat.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
Matthew Bonanni
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
d511b5bae9
commit
49f2104c53
@@ -1015,6 +1015,78 @@ def test_gather_and_maybe_dequant_cache_mla(
|
||||
torch.testing.assert_close(dst, expected)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("kv_lora_rank", [512])
|
||||
@pytest.mark.parametrize("qk_rope_head_dim", [64])
|
||||
@pytest.mark.parametrize("block_size", [16])
|
||||
@pytest.mark.parametrize("num_blocks", [128])
|
||||
@pytest.mark.parametrize("dtype", [torch.float32])
|
||||
@pytest.mark.parametrize("kv_cache_dtype", ["auto", "fp8"])
|
||||
@pytest.mark.parametrize("device", CUDA_DEVICES)
|
||||
@torch.inference_mode()
|
||||
def test_gather_and_maybe_dequant_cache_mla_with_seq_starts(
|
||||
kv_lora_rank,
|
||||
qk_rope_head_dim,
|
||||
block_size,
|
||||
num_blocks,
|
||||
dtype,
|
||||
kv_cache_dtype,
|
||||
device,
|
||||
):
|
||||
entry_size = kv_lora_rank + qk_rope_head_dim
|
||||
scale = torch.tensor(0.1, dtype=torch.float32, device=device)
|
||||
src_cache = _create_mla_cache(
|
||||
num_blocks, block_size, entry_size, dtype, kv_cache_dtype, device
|
||||
)
|
||||
_fill_mla_cache(src_cache, kv_cache_dtype=kv_cache_dtype)
|
||||
|
||||
seq_starts = torch.tensor([3, 17, 5], dtype=torch.int32, device=device)
|
||||
seq_lens = torch.tensor([20, 10, 16], dtype=torch.int32, device=device)
|
||||
batch_size = seq_lens.shape[0]
|
||||
total_tokens = seq_lens.sum().item()
|
||||
cu_seq_lens = torch.empty((batch_size + 1), dtype=torch.int32, device=device)
|
||||
cu_seq_lens[0] = 0
|
||||
cu_seq_lens[1:] = seq_lens.cumsum(dim=0)
|
||||
token_to_seq = torch.repeat_interleave(
|
||||
torch.arange(batch_size, dtype=torch.int32, device=device), seq_lens
|
||||
)
|
||||
|
||||
block_table = torch.empty(
|
||||
(batch_size, num_blocks), dtype=torch.int32, device=device
|
||||
)
|
||||
for b in range(batch_size):
|
||||
block_table[b, :] = torch.randperm(num_blocks, device=device)
|
||||
|
||||
if kv_cache_dtype == "fp8":
|
||||
dequant_src_cache = torch.empty_like(src_cache, dtype=dtype)
|
||||
ops.convert_fp8(dequant_src_cache, src_cache, scale.item())
|
||||
else:
|
||||
dequant_src_cache = src_cache
|
||||
|
||||
expected_rows = []
|
||||
for b in range(batch_size):
|
||||
start = seq_starts[b].item()
|
||||
length = seq_lens[b].item()
|
||||
for offset in range(start, start + length):
|
||||
block_id = block_table[b, offset // block_size]
|
||||
slot = offset % block_size
|
||||
expected_rows.append(dequant_src_cache[block_id, slot])
|
||||
expected = torch.stack(expected_rows)
|
||||
|
||||
dst = torch.zeros((total_tokens, entry_size), dtype=dtype, device=device)
|
||||
ops.gather_and_maybe_dequant_cache(
|
||||
src_cache,
|
||||
dst,
|
||||
block_table,
|
||||
cu_seq_lens,
|
||||
token_to_seq,
|
||||
total_tokens,
|
||||
kv_cache_dtype,
|
||||
scale,
|
||||
seq_starts,
|
||||
)
|
||||
torch.testing.assert_close(dst, expected)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("kv_lora_rank", [512])
|
||||
@pytest.mark.parametrize("qk_rope_head_dim", [64])
|
||||
@pytest.mark.parametrize("block_size", [16])
|
||||
|
||||
Reference in New Issue
Block a user