[Kimi-K3] DCP support (#50484)

Signed-off-by: girasoley <girasoley@inferact.ai>
Signed-off-by: Summer Yang <girasoleyang@gmail.com>
Signed-off-by: 云挚 <ningyunxiao.nyx@antgroup.com>
Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
Co-authored-by: 云挚 <ningyunxiao.nyx@antgroup.com>
Co-authored-by: foraxe <73625538+foraxe@users.noreply.github.com>
Co-authored-by: Woosuk Kwon <woosuk@inferact.ai>
This commit is contained in:
Summer Yang
2026-08-10 12:54:26 -07:00
committed by GitHub
co-authored by Claude Fable 5 云挚 foraxe Woosuk Kwon
parent 3f142bd85e
commit 63ac04a61e
19 changed files with 3529 additions and 82 deletions
@@ -206,6 +206,7 @@ from abc import abstractmethod
from collections.abc import Callable
from dataclasses import dataclass
from enum import Enum
from math import lcm
from typing import ClassVar, Generic, TypeVar, cast
import numpy as np
@@ -222,7 +223,6 @@ from vllm.config import (
ModelConfig,
VllmConfig,
get_current_vllm_config,
get_current_vllm_config_or_none,
)
from vllm.config.cache import CacheDType
from vllm.distributed.parallel_state import (
@@ -264,7 +264,7 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
from vllm.model_executor.utils import replace_parameter
from vllm.platforms import current_platform
from vllm.utils.flashinfer import has_flashinfer
from vllm.utils.math_utils import cdiv, round_down
from vllm.utils.math_utils import cdiv, round_down, round_up
from vllm.utils.torch_utils import (
LayerNameType,
_encode_layer_name,
@@ -292,8 +292,7 @@ from vllm.v1.attention.backends.utils import (
get_num_attention_heads_from_layers,
split_decodes_and_prefills,
)
from vllm.v1.attention.ops.common import cp_lse_ag_out_ar, cp_lse_ag_out_rs
from vllm.v1.attention.ops.dcp_alltoall import dcp_a2a_lse_reduce
from vllm.v1.attention.ops.dcp_utils import MLADCPManager
from vllm.v1.attention.ops.merge_attn_states import merge_attn_states
from vllm.v1.attention.selector import get_attn_backend
from vllm.v1.kv_cache_interface import (
@@ -579,12 +578,27 @@ class MLAAttention(nn.Module, AttentionLayerBase):
self.use_sparse = use_sparse
_vllm_config = get_current_vllm_config_or_none()
self.dcp_a2a = (
_vllm_config is not None
and _vllm_config.parallel_config.decode_context_parallel_size > 1
and _vllm_config.parallel_config.dcp_comm_backend == "a2a"
)
self.dcp_manager: MLADCPManager | None = None
if self.impl.dcp_world_size > 1:
query_dtype = (
current_platform.fp8_dtype()
if is_quantized_kv_cache(self.kv_cache_dtype)
and self.kv_cache_dtype != "fp8_ds_mla"
and self.impl.supports_quant_query_input
else dtype
)
self.dcp_manager = MLADCPManager(
vllm_config=vllm_config,
device=next(kv_b_proj.parameters()).device,
num_heads=self.num_heads,
query_head_dim=self.kv_lora_rank + self.qk_rope_head_dim,
output_head_dim=self.kv_lora_rank,
query_dtype=query_dtype,
output_dtype=dtype,
padded_num_heads=self.q_pad_num_heads,
is_lse_base_on_e=self.impl.lse_base_on_e,
use_pcp=self.use_pcp,
)
self.is_aiter_triton_fp8_bmm_enabled = rocm_aiter_ops.is_fp8bmm_enabled()
@@ -753,9 +767,6 @@ class MLAAttention(nn.Module, AttentionLayerBase):
return quant_output.fill_(0)
return output.fill_(0)
if self.impl.dcp_world_size == -1:
self.impl.dcp_world_size = get_dcp_group().world_size
fp8_attention = is_quantized_kv_cache(self.kv_cache_dtype)
num_actual_toks = attn_metadata.num_actual_tokens
@@ -914,6 +925,7 @@ class MLAAttention(nn.Module, AttentionLayerBase):
mqa_q = (mqa_ql_nope, mqa_q_pe)
# concatenate nope + pe -> (B, N, L + P) (fp8 op above may have fused)
if self.impl.dcp_world_size > 1:
assert self.dcp_manager is not None
if self.use_pcp:
if self.impl.dcp_world_size > self.impl.pcp_world_size:
if isinstance(mqa_q, tuple):
@@ -924,8 +936,8 @@ class MLAAttention(nn.Module, AttentionLayerBase):
# concatenate mqa_ql_nope and mqa_q_pe -> (B, N, L + P)
mqa_q = torch.cat(mqa_q, dim=-1)
if not qrep_decode:
# mqa_q do allgather in head dim.
mqa_q = get_dcp_group().all_gather(mqa_q, dim=1)
assert self.dcp_manager.query_gather is not None
mqa_q = self.dcp_manager.query_gather(mqa_q)
# call decode attn
if not self.impl.is_sparse:
@@ -935,27 +947,23 @@ class MLAAttention(nn.Module, AttentionLayerBase):
# correct dcp attn_out with lse.
if self.impl.dcp_world_size > 1:
assert lse is not None
if self.dcp_a2a:
attn_out = dcp_a2a_lse_reduce(
attn_out,
lse,
get_dcp_group(),
is_lse_base_on_e=self.impl.lse_base_on_e,
)
elif self.use_pcp:
attn_out = cp_lse_ag_out_ar(
attn_out,
lse,
get_dcp_group(),
is_lse_base_on_e=self.impl.lse_base_on_e,
)
else:
attn_out = cp_lse_ag_out_rs(
attn_out,
lse,
get_dcp_group(),
is_lse_base_on_e=self.impl.lse_base_on_e,
)
assert self.dcp_manager is not None
seq_lens = (
attn_metadata.decode.seq_lens
if attn_metadata.decode is not None
else cast(torch.Tensor, attn_metadata.seq_lens)[ # type: ignore[attr-defined]
: attn_metadata.num_decodes
]
)
query_start_loc = attn_metadata.query_start_loc[
: attn_metadata.num_decodes + 1
]
attn_out = self.dcp_manager.combine(
attn_out,
lse,
seq_lens=seq_lens,
query_start_loc=query_start_loc,
)
if self.use_pcp:
attn_out = finalize_mla_pcp_decode(attn_out, self.num_heads)
@@ -1427,6 +1435,7 @@ class MLACommonPrefillMetadata:
chunks: "list[MLACommonPrefillMetadata.ContextChunk]"
context_lens_list: list[int]
empty_token_slices: list[slice]
dcp_manager: MLADCPManager | None = None
block_table: torch.Tensor
query_start_loc: torch.Tensor
@@ -1684,6 +1693,25 @@ def _flat_int32(values: list[int] | np.ndarray) -> torch.Tensor:
return np_to_pinned_tensor(np.asarray(values, dtype=np.int32))
def align_mla_chunked_context_workspace_size(
vllm_config: VllmConfig,
workspace_size: int,
) -> int:
parallel_config = vllm_config.parallel_config
alignment = vllm_config.cache_config.block_size
if parallel_config.decode_context_parallel_size > 1:
alignment = lcm(
alignment,
parallel_config.decode_context_parallel_size
* parallel_config.cp_kv_cache_interleave_size,
)
workspace_size = max(
workspace_size,
vllm_config.scheduler_config.max_num_seqs * alignment,
)
return round_up(workspace_size, alignment)
def build_mla_chunked_context_metadata(
*,
context_lens_cpu: torch.Tensor,
@@ -1696,6 +1724,7 @@ def build_mla_chunked_context_metadata(
dcp_world_size: int,
dcp_local_block_size: int,
dcp_virtual_block_size: int,
dcp_manager: MLADCPManager | None = None,
) -> "MLACommonPrefillMetadata.ChunkedContextMetadata | None":
"""Build chunked-context metadata for an MLA prefill.
@@ -1714,6 +1743,7 @@ def build_mla_chunked_context_metadata(
dcp_world_size: Decode-context-parallel world size (1 if disabled).
dcp_local_block_size: Per-rank interleave block size for DCP.
dcp_virtual_block_size: ``dcp_local_block_size * dcp_world_size``.
dcp_manager: Shared MLA DCP collective manager.
Returns:
The chunked-context metadata, or None when no prefill has any context.
@@ -1902,6 +1932,7 @@ def build_mla_chunked_context_metadata(
chunks=chunks,
context_lens_list=context_lens,
empty_token_slices=empty_token_slices,
dcp_manager=dcp_manager,
)
@@ -1954,7 +1985,10 @@ class MLACommonMetadataBuilder(AttentionMetadataBuilder[M]):
64 * 1024,
)
return max(chunked_prefill_workspace_size, cache_config.block_size)
return align_mla_chunked_context_workspace_size(
vllm_config,
chunked_prefill_workspace_size,
)
@staticmethod
def determine_prefill_query_data_type(
@@ -2033,6 +2067,7 @@ class MLACommonMetadataBuilder(AttentionMetadataBuilder[M]):
self.q_data_type = self.determine_prefill_query_data_type(
vllm_config, self.model_config.dtype
)
attention_layer = self.compilation_config.static_forward_context[layer_names[0]]
try:
self.dcp_world_size = get_dcp_group().world_size
@@ -2050,6 +2085,7 @@ class MLACommonMetadataBuilder(AttentionMetadataBuilder[M]):
)
use_packed_fp8_cache = vllm_config.cache_config.cache_dtype == "fp8_ds_mla"
self.dcp_manager: MLADCPManager | None = None
if self.dcp_world_size > 1:
# Note(hc): The local kvcache is incomplete when DCP is triggered,
# an additional kvcache allgather across the DCP group is therefore
@@ -2067,6 +2103,12 @@ class MLACommonMetadataBuilder(AttentionMetadataBuilder[M]):
else self.model_config.dtype,
device=device,
)
self.dcp_manager = getattr(attention_layer, "dcp_manager", None)
assert isinstance(self.dcp_manager, MLADCPManager)
self.dcp_manager.init_kv_gather(
self.chunked_prefill_workspace,
self.chunked_prefill_workspace_size,
)
else:
self.chunked_prefill_workspace = torch.empty(
(
@@ -2080,9 +2122,7 @@ class MLACommonMetadataBuilder(AttentionMetadataBuilder[M]):
# Metadata builders are created per ubatch when DBO is enabled. MLA
# prefill backends keep the prepared metadata on the backend object, so
# each builder needs its own backend instance to avoid cross-ubatch races.
self._prefill_backend = self.compilation_config.static_forward_context[
layer_names[0]
].prefill_backend.clone()
self._prefill_backend = attention_layer.prefill_backend.clone()
supports_spec_decode = self.query_len_support != QueryLenSupport.SINGLE_ONLY
self._init_reorder_batch_threshold(
@@ -2223,6 +2263,7 @@ class MLACommonMetadataBuilder(AttentionMetadataBuilder[M]):
dcp_world_size=self.dcp_world_size,
dcp_local_block_size=self.dcp_local_block_size,
dcp_virtual_block_size=self.dcp_virtual_block_size,
dcp_manager=self.dcp_manager,
)
prefill_metadata = MLACommonPrefillMetadata(
@@ -2660,9 +2701,8 @@ class MLACommonBaseImpl(MLAAttentionImpl[A], Generic[A]):
]
assert toks * dcp_world_size <= cur_allgather_workspace.shape[0]
cur_allgather_kvcache = cur_allgather_workspace[: toks * dcp_world_size]
cur_allgather_kvcache.copy_(
get_dcp_group().all_gather(local_gathered_kvcache, dim=0)
)
dcp_manager = cast(MLADCPManager, chunked_context.dcp_manager)
dcp_manager.kv_gather(cur_allgather_kvcache, local_gathered_kvcache)
assert (
cur_allgather_kvcache.shape[-1]
== self.kv_lora_rank + self.qk_rope_head_dim
@@ -2732,8 +2772,6 @@ class MLACommonBaseImpl(MLAAttentionImpl[A], Generic[A]):
output_scale: torch.Tensor | None = None,
) -> None:
assert attn_metadata.prefill is not None
assert self.dcp_world_size != -1
prefill_metadata = attn_metadata.prefill
assert prefill_metadata.prefill_backend is not None
use_fp8_prefill = prefill_metadata.q_data_type == current_platform.fp8_dtype()
@@ -2877,10 +2915,11 @@ class MLACommonImpl(MLACommonBaseImpl[M], Generic[M]):
and (self.qk_rope_head_dim == 64)
)
self.dcp_world_size: int = -1
parallel_config = get_current_vllm_config().parallel_config
# Avoid requiring an initialized DCP group in tests.
self.dcp_world_size: int = parallel_config.decode_context_parallel_size
self.cp_kv_cache_interleave_size: int = (
get_current_vllm_config().parallel_config.cp_kv_cache_interleave_size
parallel_config.cp_kv_cache_interleave_size
)
@abstractmethod
@@ -20,6 +20,7 @@ from vllm.model_executor.layers.attention.mla_attention import (
MLACommonMetadata,
MLACommonPrefillMetadata,
accumulate_mla_context_chunk,
align_mla_chunked_context_workspace_size,
build_mla_chunked_context_metadata,
get_mla_dims,
init_mla_context_partial,
@@ -31,6 +32,7 @@ from vllm.utils.torch_utils import is_quantized_kv_cache, np_to_pinned_tensor
from vllm.v1.attention.backend import AttentionMetadata, AttentionMetadataBuilder
from vllm.v1.attention.backends.fa_utils import get_flash_attn_version
from vllm.v1.attention.backends.utils import split_decodes_and_prefills
from vllm.v1.attention.ops.dcp_utils import MLADCPManager
from vllm.v1.attention.ops.merge_attn_states import merge_attn_states
if TYPE_CHECKING:
@@ -162,9 +164,19 @@ class SparseMLACommonMetadataBuilder(AttentionMetadataBuilder[T]):
dtype=torch.int32,
device=device,
)
layer_prefill_backend = vllm_config.compilation_config.static_forward_context[
attention_layer = vllm_config.compilation_config.static_forward_context[
layer_names[0]
].prefill_backend
]
layer_prefill_backend = attention_layer.prefill_backend
self.dcp_manager: MLADCPManager | None = None
if self.dcp_world_size > 1:
self.dcp_manager = getattr(attention_layer, "dcp_manager", None)
assert isinstance(self.dcp_manager, MLADCPManager)
if layer_prefill_backend is not None:
self.dcp_manager.init_kv_gather(
self.chunked_prefill_workspace,
self.chunked_prefill_workspace_size,
)
self._prefill_backend = (
layer_prefill_backend.clone() if layer_prefill_backend is not None else None
)
@@ -184,7 +196,13 @@ class SparseMLACommonMetadataBuilder(AttentionMetadataBuilder[T]):
64 * 1024,
scheduler_config.max_num_seqs * topk_tokens,
)
return max(workspace_size, cache_config.block_size)
workspace_size = max(
workspace_size,
scheduler_config.max_num_seqs * cache_config.block_size,
)
if vllm_config.parallel_config.decode_context_parallel_size > 1:
return align_mla_chunked_context_workspace_size(vllm_config, workspace_size)
return workspace_size
def _build_req_id_per_token(
self,
@@ -232,6 +250,7 @@ class SparseMLACommonMetadataBuilder(AttentionMetadataBuilder[T]):
dcp_world_size=self.dcp_world_size,
dcp_local_block_size=self.dcp_local_block_size,
dcp_virtual_block_size=self.dcp_virtual_block_size,
dcp_manager=self.dcp_manager,
)
def build(