[Attention][MLA] Fuse Kimi-K3 chunked-context K/V packing (#51772)
Signed-off-by: Yongye Zhu <zyy1102000@gmail.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
Claude Opus 5
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
63344913a6
commit
903d2efe7e
@@ -9,6 +9,8 @@ from vllm.models.kimi_k3.nvidia.ops.fused_mla_key_concat_kv_cache import (
|
||||
fused_mla_decode_q_concat_kv_cache_insert,
|
||||
fused_mla_key_concat_ds_mla_insert,
|
||||
fused_mla_key_concat_kv_cache_insert,
|
||||
fused_mla_kv_concat,
|
||||
fused_mla_kv_concat_quant_fp8,
|
||||
fused_mla_qkv_quant_kv_cache_fp8_insert,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
@@ -25,8 +27,8 @@ _POSITIONS = (1, 7, 13)
|
||||
_SLOTS = (0, 3, 9)
|
||||
|
||||
|
||||
def _randn(*shape: int) -> torch.Tensor:
|
||||
return torch.randn(*shape, device="cuda", dtype=_DTYPE) * 0.2
|
||||
def _randn(*shape: int, dtype: torch.dtype = _DTYPE) -> torch.Tensor:
|
||||
return torch.randn(*shape, device="cuda", dtype=dtype) * 0.2
|
||||
|
||||
|
||||
def _rope_cache(max_position: int = 32) -> torch.Tensor:
|
||||
@@ -66,6 +68,70 @@ def _assert_fp8_close(actual: torch.Tensor, expected: torch.Tensor) -> None:
|
||||
)
|
||||
|
||||
|
||||
def _strided_context_inputs(
|
||||
num_tokens: int, num_heads: int, dtype: torch.dtype, k_pe_fp8: bool
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
"""The real prefill-context layouts: ``k_nope``/``v`` as the two strided
|
||||
halves of one kv_b_proj output and ``k_pe`` as a column slice of the gather
|
||||
workspace (left in the fp8 cache layout for a plain fp8 cache)."""
|
||||
kv_nope = _randn(num_tokens, num_heads, 256, dtype=dtype)
|
||||
k_nope, v = kv_nope.split((128, 128), dim=-1)
|
||||
workspace = _randn(num_tokens, 576, dtype=dtype)
|
||||
if k_pe_fp8:
|
||||
workspace = workspace.to(torch.float8_e4m3fn)
|
||||
return k_nope, workspace[:, 512:].unsqueeze(1), v
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_tokens", [0, _NUM_TOKENS])
|
||||
@pytest.mark.parametrize("num_heads", [3, _NUM_HEADS])
|
||||
@pytest.mark.parametrize("input_dtype", [torch.bfloat16, torch.float16])
|
||||
@torch.inference_mode()
|
||||
def test_context_kv_concat_accepts_strided_inputs(
|
||||
num_tokens: int, num_heads: int, input_dtype: torch.dtype
|
||||
) -> None:
|
||||
k_nope, k_pe, _ = _strided_context_inputs(
|
||||
num_tokens, num_heads, input_dtype, k_pe_fp8=False
|
||||
)
|
||||
|
||||
k = fused_mla_kv_concat(k_nope, k_pe)
|
||||
|
||||
assert k.is_contiguous()
|
||||
assert k.dtype == input_dtype
|
||||
torch.testing.assert_close(k[..., :128], k_nope, atol=0, rtol=0)
|
||||
torch.testing.assert_close(
|
||||
k[..., 128:], k_pe.expand(-1, num_heads, -1), atol=0, rtol=0
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("k_pe_dtype", ["input", "fp8"])
|
||||
@pytest.mark.parametrize("num_tokens", [0, _NUM_TOKENS])
|
||||
@pytest.mark.parametrize("num_heads", [3, _NUM_HEADS])
|
||||
@pytest.mark.parametrize("input_dtype", [torch.bfloat16, torch.float16])
|
||||
@torch.inference_mode()
|
||||
def test_context_kv_pack_quantizes_strided_inputs(
|
||||
k_pe_dtype: str,
|
||||
num_tokens: int,
|
||||
num_heads: int,
|
||||
input_dtype: torch.dtype,
|
||||
) -> None:
|
||||
k_nope, k_pe, v = _strided_context_inputs(
|
||||
num_tokens, num_heads, input_dtype, k_pe_fp8=k_pe_dtype == "fp8"
|
||||
)
|
||||
|
||||
k_actual, v_actual = fused_mla_kv_concat_quant_fp8(k_nope, k_pe, v)
|
||||
|
||||
fp8 = torch.float8_e4m3fn
|
||||
k_expected = torch.empty_like(k_actual)
|
||||
k_expected[..., :128] = k_nope.to(fp8)
|
||||
k_expected[..., 128:] = k_pe.to(fp8)
|
||||
assert k_actual.is_contiguous()
|
||||
assert v_actual.is_contiguous()
|
||||
# The pack casts with the native pairwise converters, so it must be
|
||||
# bit-identical to torch's `.to(fp8)` rather than merely close.
|
||||
torch.testing.assert_close(k_actual.float(), k_expected.float(), atol=0, rtol=0)
|
||||
torch.testing.assert_close(v_actual.float(), v.to(fp8).float(), atol=0, rtol=0)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("cache_kind", ["bf16", "fp8", "fp8_ds_mla"])
|
||||
@torch.inference_mode()
|
||||
def test_prefill_epilogue_fuses_gptj_rope(cache_kind: str) -> None:
|
||||
|
||||
Reference in New Issue
Block a user