[Attention][MLA] Fuse Kimi-K3 chunked-context K/V packing (#51772)

Signed-off-by: Yongye Zhu <zyy1102000@gmail.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Yongye Zhu
2026-08-12 22:42:57 -07:00
committed by GitHub
co-authored by Claude Opus 5 mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 63344913a6
commit 903d2efe7e
15 changed files with 996 additions and 35 deletions
@@ -9,6 +9,8 @@ from vllm.models.kimi_k3.nvidia.ops.fused_mla_key_concat_kv_cache import (
fused_mla_decode_q_concat_kv_cache_insert,
fused_mla_key_concat_ds_mla_insert,
fused_mla_key_concat_kv_cache_insert,
fused_mla_kv_concat,
fused_mla_kv_concat_quant_fp8,
fused_mla_qkv_quant_kv_cache_fp8_insert,
)
from vllm.platforms import current_platform
@@ -25,8 +27,8 @@ _POSITIONS = (1, 7, 13)
_SLOTS = (0, 3, 9)
def _randn(*shape: int) -> torch.Tensor:
return torch.randn(*shape, device="cuda", dtype=_DTYPE) * 0.2
def _randn(*shape: int, dtype: torch.dtype = _DTYPE) -> torch.Tensor:
return torch.randn(*shape, device="cuda", dtype=dtype) * 0.2
def _rope_cache(max_position: int = 32) -> torch.Tensor:
@@ -66,6 +68,70 @@ def _assert_fp8_close(actual: torch.Tensor, expected: torch.Tensor) -> None:
)
def _strided_context_inputs(
num_tokens: int, num_heads: int, dtype: torch.dtype, k_pe_fp8: bool
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
"""The real prefill-context layouts: ``k_nope``/``v`` as the two strided
halves of one kv_b_proj output and ``k_pe`` as a column slice of the gather
workspace (left in the fp8 cache layout for a plain fp8 cache)."""
kv_nope = _randn(num_tokens, num_heads, 256, dtype=dtype)
k_nope, v = kv_nope.split((128, 128), dim=-1)
workspace = _randn(num_tokens, 576, dtype=dtype)
if k_pe_fp8:
workspace = workspace.to(torch.float8_e4m3fn)
return k_nope, workspace[:, 512:].unsqueeze(1), v
@pytest.mark.parametrize("num_tokens", [0, _NUM_TOKENS])
@pytest.mark.parametrize("num_heads", [3, _NUM_HEADS])
@pytest.mark.parametrize("input_dtype", [torch.bfloat16, torch.float16])
@torch.inference_mode()
def test_context_kv_concat_accepts_strided_inputs(
num_tokens: int, num_heads: int, input_dtype: torch.dtype
) -> None:
k_nope, k_pe, _ = _strided_context_inputs(
num_tokens, num_heads, input_dtype, k_pe_fp8=False
)
k = fused_mla_kv_concat(k_nope, k_pe)
assert k.is_contiguous()
assert k.dtype == input_dtype
torch.testing.assert_close(k[..., :128], k_nope, atol=0, rtol=0)
torch.testing.assert_close(
k[..., 128:], k_pe.expand(-1, num_heads, -1), atol=0, rtol=0
)
@pytest.mark.parametrize("k_pe_dtype", ["input", "fp8"])
@pytest.mark.parametrize("num_tokens", [0, _NUM_TOKENS])
@pytest.mark.parametrize("num_heads", [3, _NUM_HEADS])
@pytest.mark.parametrize("input_dtype", [torch.bfloat16, torch.float16])
@torch.inference_mode()
def test_context_kv_pack_quantizes_strided_inputs(
k_pe_dtype: str,
num_tokens: int,
num_heads: int,
input_dtype: torch.dtype,
) -> None:
k_nope, k_pe, v = _strided_context_inputs(
num_tokens, num_heads, input_dtype, k_pe_fp8=k_pe_dtype == "fp8"
)
k_actual, v_actual = fused_mla_kv_concat_quant_fp8(k_nope, k_pe, v)
fp8 = torch.float8_e4m3fn
k_expected = torch.empty_like(k_actual)
k_expected[..., :128] = k_nope.to(fp8)
k_expected[..., 128:] = k_pe.to(fp8)
assert k_actual.is_contiguous()
assert v_actual.is_contiguous()
# The pack casts with the native pairwise converters, so it must be
# bit-identical to torch's `.to(fp8)` rather than merely close.
torch.testing.assert_close(k_actual.float(), k_expected.float(), atol=0, rtol=0)
torch.testing.assert_close(v_actual.float(), v.to(fp8).float(), atol=0, rtol=0)
@pytest.mark.parametrize("cache_kind", ["bf16", "fp8", "fp8_ds_mla"])
@torch.inference_mode()
def test_prefill_epilogue_fuses_gptj_rope(cache_kind: str) -> None: