[Attention][MLA] Fuse Kimi-K3 chunked-context K/V packing (#51772)

Signed-off-by: Yongye Zhu <zyy1102000@gmail.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Yongye Zhu
2026-08-12 22:42:57 -07:00
committed by GitHub
co-authored by Claude Opus 5 mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 63344913a6
commit 903d2efe7e
15 changed files with 996 additions and 35 deletions
+192 -6
View File
@@ -8,8 +8,9 @@ This is a self-contained MLA layer that owns the full attention path:
-> fused pre-attention ops (fused_qkv_a_proj / norms / q_b_proj)
-> explicit prefill / decode split
prefill: fused key-concat + cache-insert kernel -> run_prefill_new_tokens
(+ chunked-context merge); dispatched by cache dtype
(bf16 / plain fp8 / fp8_ds_mla)
(+ chunked-context merge, whose per-chunk gather -> kv_b_proj
-> fused K/V pack loop this layer owns); dispatched by cache
dtype (bf16 / plain fp8 / fp8_ds_mla)
decode : W_UK absorb (BMM1) -> fused q-concat + cache-insert kernel
-> impl.forward_mqa -> W_UV up-proj (MQA)
-> optional output gate
@@ -34,6 +35,7 @@ from typing import TYPE_CHECKING, cast
import torch
from torch import nn
from vllm import _custom_ops as ops
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
from vllm.config import (
CacheConfig,
@@ -48,6 +50,12 @@ from vllm.model_executor.layers.attention.attention import (
set_default_quant_scales,
should_load_quant_weights,
)
from vllm.model_executor.layers.attention.mla_attention import (
_get_kv_b_proj_input_dtype,
accumulate_mla_context_chunk,
init_mla_context_partial,
neutralize_empty_context_partials,
)
from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.linear import (
@@ -67,6 +75,8 @@ from vllm.models.kimi_k3.nvidia.ops.fused_mla_key_concat_kv_cache import (
fused_mla_decode_q_concat_kv_cache_insert,
fused_mla_key_concat_ds_mla_insert,
fused_mla_key_concat_kv_cache_insert,
fused_mla_kv_concat,
fused_mla_kv_concat_quant_fp8,
fused_mla_qkv_quant_kv_cache_fp8_insert,
)
from vllm.platforms import current_platform
@@ -696,6 +706,181 @@ class MultiHeadLatentAttention(nn.Module, AttentionLayerBase):
cos_sin_cache=cos_sin_cache,
)
def _compute_prefill_context(
self,
q: torch.Tensor,
attn_metadata: "MLACommonMetadata",
) -> tuple[torch.Tensor, torch.Tensor]:
"""Chunked-context prefill, K3-fused. Replaces the impl's version.
Per chunk the impl gathers the paged latent, up-projects it, then casts
and concatenates K (and casts V) in two or three more launches. Here that
tail is one fused kernel per chunk -- ``fused_mla_kv_concat`` for a bf16
query, ``fused_mla_kv_concat_quant_fp8`` when the query is fp8 -- reading
the strided ``kv_b_proj`` output in place and writing a contiguous key, so
only the gather and ``kv_b_proj`` remain.
The impl's query cast is gone as well: ``q`` already carries
``prefill.q_data_type`` because the new-token epilogue quantized it. The
gathered latent still gets the impl's cast to whatever ``kv_b_proj``
consumes -- free (a no-op ``.to``) for a checkpoint whose ``kv_b_proj``
takes the fp8 latent directly, and required for a bf16 one, which is
what a stock K3 checkpoint carries. Its output is bf16 either way.
The gathered ``k_pe`` is likewise used as-is (fp8 for a plain fp8 cache)
and needs no RoPE: it was rotated on the way in.
Chunk partials are written straight into the accumulating context partial
when the prefill backend honors ``out``, so only the (64x smaller) lse is
copied per chunk.
Decode context parallelism keeps using
``impl._context_parallel_compute_prefill_context``; its extra allgather
and reorg are not fused here.
"""
prefill = attn_metadata.prefill
assert prefill is not None
prefill_backend = prefill.prefill_backend
assert prefill_backend is not None
chunked_context = prefill.chunked_context
assert chunked_context is not None
assert q.dtype == prefill.q_data_type, (
"Kimi-K3 chunked context expects the new-token epilogue to have "
f"produced a {prefill.q_data_type} query; got {q.dtype}."
)
fp8_prefill = q.dtype == current_platform.fp8_dtype()
workspace = chunked_context.workspace
kv_cache = self._attn_read_kv_cache()
kv_b_proj_input_dtype = _get_kv_b_proj_input_dtype(self.kv_b_proj, fp8_prefill)
def run_chunk(
chunk, out: torch.Tensor | None = None
) -> tuple[torch.Tensor, torch.Tensor]:
self._gather_context_latent(chunk, kv_cache, prefill, fp8_prefill)
gathered = workspace[: chunk.num_context_tokens]
kv_c_normed = gathered[..., : self.kv_lora_rank]
if kv_b_proj_input_dtype is not None:
kv_c_normed = kv_c_normed.to(kv_b_proj_input_dtype)
kv_nope = self.kv_b_proj(kv_c_normed)[0].view(
-1, self.num_local_heads, self.qk_nope_head_dim + self.v_head_dim
)
k_nope, v = kv_nope.split([self.qk_nope_head_dim, self.v_head_dim], dim=-1)
k_pe = gathered[..., self.kv_lora_rank :]
if fp8_prefill:
k, v = fused_mla_kv_concat_quant_fp8(k_nope, k_pe, v)
else:
k = fused_mla_kv_concat(k_nope, k_pe)
attn_output, attn_lse = prefill_backend.run_prefill_context_chunk(
chunk=chunk, q=q[chunk.token_slice], k=k, v=v, out=out
)
assert out is None or attn_output.data_ptr() == out.data_ptr(), (
f"{prefill_backend.get_name()} reports supports_out() but did not "
"write the context chunk into the `out` it was given."
)
return attn_output, attn_lse
chunks = chunked_context.chunks
if len(chunks) == 1 and not chunked_context.empty_token_slices:
# One chunk covering every prefill token: its partial *is* the context
# partial, so it needs neither an accumulator nor a copy.
return run_chunk(chunks[0])
# A backend honoring `out` writes each chunk's partial straight into the
# accumulator, so the per-chunk output copy disappears -- and because that
# contract fixes the trailing shape, the accumulator can be sized before
# any chunk runs. Otherwise the shape is only knowable from a real partial,
# so the first chunk runs ahead of the loop and is copied in.
writes_out = prefill_backend.supports_out()
if writes_out:
assert prefill.output_dtype is not None
output = torch.empty(
(q.shape[0], self.num_local_heads, self.v_head_dim),
dtype=prefill.output_dtype,
device=q.device,
)
output_lse = torch.empty(
(self.num_local_heads, q.shape[0]),
dtype=torch.float32,
device=q.device,
)
neutralize_empty_context_partials(chunked_context, output, output_lse)
else:
attn_output, attn_lse = run_chunk(chunks[0])
output, output_lse = init_mla_context_partial(
chunked_context, attn_output, attn_lse, num_tokens=q.shape[0]
)
accumulate_mla_context_chunk(
chunks[0], attn_output, attn_lse, output, output_lse
)
chunks = chunks[1:]
for chunk in chunks:
# A continuation chunk's leading tokens have to be merged with the
# partial already sitting there, so it cannot write in place.
out = (
output[chunk.token_slice]
if writes_out and not chunk.is_continuation
else None
)
attn_output, attn_lse = run_chunk(chunk, out=out)
accumulate_mla_context_chunk(
chunk,
attn_output,
attn_lse,
output,
output_lse,
output_written=out is not None,
)
return output, output_lse
def _gather_context_latent(
self,
chunk,
kv_cache: torch.Tensor,
prefill,
fp8_prefill: bool,
) -> None:
"""Gather one chunk's paged context latent into the workspace.
Dispatched exactly as in ``impl._compute_prefill_context``: an fp8 query
reads the plain fp8 cache in its stored layout, anything else lands in the
workspace as the model dtype.
"""
workspace = prefill.chunked_context.workspace
toks = chunk.num_context_tokens
block_table = prefill.block_table[chunk.request_slice]
if self.kv_cache_dtype == "fp8_ds_mla":
ops.cp_gather_and_upconvert_fp8_kv_cache(
src_cache=kv_cache,
dst=workspace[:toks],
block_table=block_table,
workspace_starts=chunk.cu_seq_lens,
batch_size=chunk.num_requests,
seq_starts=chunk.starts,
)
elif not fp8_prefill:
ops.gather_and_maybe_dequant_cache(
src_cache=kv_cache,
dst=workspace,
block_table=block_table,
cu_seq_lens=chunk.cu_seq_lens,
token_to_seq=chunk.token_to_seq,
num_tokens=toks,
kv_cache_dtype=self.kv_cache_dtype,
scale=self._k_scale,
seq_starts=chunk.starts,
)
else:
ops.cp_gather_cache(
src_cache=kv_cache,
dst=workspace[:toks],
block_table=block_table,
cu_seq_lens=chunk.cu_seq_lens,
batch_size=chunk.num_requests,
seq_starts=chunk.starts,
)
def _forward_prefill_fused(
self,
q: torch.Tensor,
@@ -710,8 +895,9 @@ class MultiHeadLatentAttention(nn.Module, AttentionLayerBase):
"""Prefill using the fused key-concat + cache-insert kernel.
Replaces ``_concat_k_nope_k_pe`` and the prefill cache write with one
fused kernel launch, dispatched by cache dtype. The chunked context
gather + online-softmax merge are delegated to the impl.
fused kernel launch, dispatched by cache dtype. Chunked context runs
through this layer's ``_compute_prefill_context``, except under DCP where
it is delegated to the impl.
Supported configs (K3 fp8 policy):
- bf16 cache -> bf16 prefill query
@@ -812,8 +998,8 @@ class MultiHeadLatentAttention(nn.Module, AttentionLayerBase):
)
)
else:
context_output, context_lse = self.impl._compute_prefill_context( # type: ignore[attr-defined]
q, self._attn_read_kv_cache(), attn_metadata, self._k_scale
context_output, context_lse = self._compute_prefill_context(
q, attn_metadata
)
suffix_output, suffix_lse = output_prefill
out = out.view(-1, self.num_local_heads, self.v_head_dim)
@@ -12,6 +12,9 @@ mirror ``fused_deepseek_v4_qnorm_rope_kv_rope_full_cache_{bf16,fp8}_insert``.
- ``fused_mla_qkv_quant_kv_cache_fp8_insert`` (fp8): additionally quantize
``q``/``k``/``v`` to E4M3 with ``q_scale`` / ``k_scale`` / ``v_scale`` (the
cache shares ``k_scale``, as in ``concat_and_cache_mla``).
- ``fused_mla_kv_concat`` / ``fused_mla_kv_concat_quant_fp8`` (chunked context):
the same K concat (plus the fp8 K/V cast) without the cache insert, for context
chunks whose latent was gathered back out of the paged cache.
The optional ``positions`` / ``cos_sin_cache`` pair enables GPT-J-style RoPE
inside the epilogue. Omitting both keeps the K3 NoPE fast path. The kernels use
@@ -156,6 +159,59 @@ def fused_mla_qkv_quant_kv_cache_fp8_insert(
return q_fp8, k_fp8, v_fp8
def _empty_full_key(
k_nope: torch.Tensor, k_pe: torch.Tensor, dtype: torch.dtype
) -> torch.Tensor:
num_tokens, num_heads, qk_nope_head_dim = k_nope.shape
return torch.empty(
(num_tokens, num_heads, qk_nope_head_dim + k_pe.shape[1]),
dtype=dtype,
device=k_nope.device,
)
def fused_mla_kv_concat(
k_nope: torch.Tensor, # [T, H, qk_nope_head_dim], may be strided
k_pe: torch.Tensor, # [T, rope] or [T, 1, rope], same dtype as k_nope
) -> torch.Tensor:
"""Concat ``k = [k_nope | k_pe]`` into a contiguous key, in one launch.
The chunked-context counterpart of ``fused_mla_key_concat_kv_cache_insert``:
no query, no cache insert and no RoPE (the gathered ``k_pe`` is already
rotated). ``k_nope`` is a strided half of one ``kv_b_proj`` output and
``k_pe`` a strided view of the gather workspace, so neither has to be made
contiguous first.
"""
k_pe = k_pe.reshape(k_pe.shape[0], k_pe.shape[-1])
k = _empty_full_key(k_nope, k_pe, k_nope.dtype)
if k.shape[0]:
torch.ops._C.fused_kimi_k3_mla_kv_concat(k_nope, k_pe, k)
return k
def fused_mla_kv_concat_quant_fp8(
k_nope: torch.Tensor, # [T, H, qk_nope_head_dim], may be strided
k_pe: torch.Tensor, # [T, rope] or [T, 1, rope], k_nope's dtype or fp8
v: torch.Tensor, # [T, H, v_head_dim], may be strided
) -> tuple[torch.Tensor, torch.Tensor]:
"""``fused_mla_kv_concat`` plus an fp8 cast of the key and of ``v``.
``k_pe`` may already be fp8: a plain fp8 cache is gathered without
dequantizing, and those bytes are copied through as-is.
Returns contiguous ``(k_fp8, v_fp8)``.
"""
k_pe = k_pe.reshape(k_pe.shape[0], k_pe.shape[-1])
fp8 = torch.float8_e4m3fn
k_fp8 = _empty_full_key(k_nope, k_pe, fp8)
v_fp8 = torch.empty(v.shape, dtype=fp8, device=v.device)
if k_fp8.shape[0]:
torch.ops._C.fused_kimi_k3_mla_kv_concat_quant_fp8(
k_nope, k_pe, v, k_fp8, v_fp8
)
return k_fp8, v_fp8
def fused_mla_decode_q_concat_kv_cache_insert(
ql_nope: torch.Tensor, # [B, H, kv_lora_rank] (BMM1 output, absorbed q)
q_pe: torch.Tensor, # [B, H, qk_rope_head_dim]