[Kimi-K3] DCP support (#50484)
Signed-off-by: girasoley <girasoley@inferact.ai> Signed-off-by: Summer Yang <girasoleyang@gmail.com> Signed-off-by: 云挚 <ningyunxiao.nyx@antgroup.com> Co-authored-by: Claude Fable 5 <noreply@anthropic.com> Co-authored-by: 云挚 <ningyunxiao.nyx@antgroup.com> Co-authored-by: foraxe <73625538+foraxe@users.noreply.github.com> Co-authored-by: Woosuk Kwon <woosuk@inferact.ai>
This commit is contained in:
co-authored by
Claude Fable 5
云挚
foraxe
Woosuk Kwon
parent
3f142bd85e
commit
63ac04a61e
@@ -206,6 +206,7 @@ from abc import abstractmethod
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass
|
||||
from enum import Enum
|
||||
from math import lcm
|
||||
from typing import ClassVar, Generic, TypeVar, cast
|
||||
|
||||
import numpy as np
|
||||
@@ -222,7 +223,6 @@ from vllm.config import (
|
||||
ModelConfig,
|
||||
VllmConfig,
|
||||
get_current_vllm_config,
|
||||
get_current_vllm_config_or_none,
|
||||
)
|
||||
from vllm.config.cache import CacheDType
|
||||
from vllm.distributed.parallel_state import (
|
||||
@@ -264,7 +264,7 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
from vllm.model_executor.utils import replace_parameter
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.flashinfer import has_flashinfer
|
||||
from vllm.utils.math_utils import cdiv, round_down
|
||||
from vllm.utils.math_utils import cdiv, round_down, round_up
|
||||
from vllm.utils.torch_utils import (
|
||||
LayerNameType,
|
||||
_encode_layer_name,
|
||||
@@ -292,8 +292,7 @@ from vllm.v1.attention.backends.utils import (
|
||||
get_num_attention_heads_from_layers,
|
||||
split_decodes_and_prefills,
|
||||
)
|
||||
from vllm.v1.attention.ops.common import cp_lse_ag_out_ar, cp_lse_ag_out_rs
|
||||
from vllm.v1.attention.ops.dcp_alltoall import dcp_a2a_lse_reduce
|
||||
from vllm.v1.attention.ops.dcp_utils import MLADCPManager
|
||||
from vllm.v1.attention.ops.merge_attn_states import merge_attn_states
|
||||
from vllm.v1.attention.selector import get_attn_backend
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
@@ -579,12 +578,27 @@ class MLAAttention(nn.Module, AttentionLayerBase):
|
||||
|
||||
self.use_sparse = use_sparse
|
||||
|
||||
_vllm_config = get_current_vllm_config_or_none()
|
||||
self.dcp_a2a = (
|
||||
_vllm_config is not None
|
||||
and _vllm_config.parallel_config.decode_context_parallel_size > 1
|
||||
and _vllm_config.parallel_config.dcp_comm_backend == "a2a"
|
||||
)
|
||||
self.dcp_manager: MLADCPManager | None = None
|
||||
if self.impl.dcp_world_size > 1:
|
||||
query_dtype = (
|
||||
current_platform.fp8_dtype()
|
||||
if is_quantized_kv_cache(self.kv_cache_dtype)
|
||||
and self.kv_cache_dtype != "fp8_ds_mla"
|
||||
and self.impl.supports_quant_query_input
|
||||
else dtype
|
||||
)
|
||||
self.dcp_manager = MLADCPManager(
|
||||
vllm_config=vllm_config,
|
||||
device=next(kv_b_proj.parameters()).device,
|
||||
num_heads=self.num_heads,
|
||||
query_head_dim=self.kv_lora_rank + self.qk_rope_head_dim,
|
||||
output_head_dim=self.kv_lora_rank,
|
||||
query_dtype=query_dtype,
|
||||
output_dtype=dtype,
|
||||
padded_num_heads=self.q_pad_num_heads,
|
||||
is_lse_base_on_e=self.impl.lse_base_on_e,
|
||||
use_pcp=self.use_pcp,
|
||||
)
|
||||
|
||||
self.is_aiter_triton_fp8_bmm_enabled = rocm_aiter_ops.is_fp8bmm_enabled()
|
||||
|
||||
@@ -753,9 +767,6 @@ class MLAAttention(nn.Module, AttentionLayerBase):
|
||||
return quant_output.fill_(0)
|
||||
return output.fill_(0)
|
||||
|
||||
if self.impl.dcp_world_size == -1:
|
||||
self.impl.dcp_world_size = get_dcp_group().world_size
|
||||
|
||||
fp8_attention = is_quantized_kv_cache(self.kv_cache_dtype)
|
||||
|
||||
num_actual_toks = attn_metadata.num_actual_tokens
|
||||
@@ -914,6 +925,7 @@ class MLAAttention(nn.Module, AttentionLayerBase):
|
||||
mqa_q = (mqa_ql_nope, mqa_q_pe)
|
||||
# concatenate nope + pe -> (B, N, L + P) (fp8 op above may have fused)
|
||||
if self.impl.dcp_world_size > 1:
|
||||
assert self.dcp_manager is not None
|
||||
if self.use_pcp:
|
||||
if self.impl.dcp_world_size > self.impl.pcp_world_size:
|
||||
if isinstance(mqa_q, tuple):
|
||||
@@ -924,8 +936,8 @@ class MLAAttention(nn.Module, AttentionLayerBase):
|
||||
# concatenate mqa_ql_nope and mqa_q_pe -> (B, N, L + P)
|
||||
mqa_q = torch.cat(mqa_q, dim=-1)
|
||||
if not qrep_decode:
|
||||
# mqa_q do allgather in head dim.
|
||||
mqa_q = get_dcp_group().all_gather(mqa_q, dim=1)
|
||||
assert self.dcp_manager.query_gather is not None
|
||||
mqa_q = self.dcp_manager.query_gather(mqa_q)
|
||||
|
||||
# call decode attn
|
||||
if not self.impl.is_sparse:
|
||||
@@ -935,27 +947,23 @@ class MLAAttention(nn.Module, AttentionLayerBase):
|
||||
# correct dcp attn_out with lse.
|
||||
if self.impl.dcp_world_size > 1:
|
||||
assert lse is not None
|
||||
if self.dcp_a2a:
|
||||
attn_out = dcp_a2a_lse_reduce(
|
||||
attn_out,
|
||||
lse,
|
||||
get_dcp_group(),
|
||||
is_lse_base_on_e=self.impl.lse_base_on_e,
|
||||
)
|
||||
elif self.use_pcp:
|
||||
attn_out = cp_lse_ag_out_ar(
|
||||
attn_out,
|
||||
lse,
|
||||
get_dcp_group(),
|
||||
is_lse_base_on_e=self.impl.lse_base_on_e,
|
||||
)
|
||||
else:
|
||||
attn_out = cp_lse_ag_out_rs(
|
||||
attn_out,
|
||||
lse,
|
||||
get_dcp_group(),
|
||||
is_lse_base_on_e=self.impl.lse_base_on_e,
|
||||
)
|
||||
assert self.dcp_manager is not None
|
||||
seq_lens = (
|
||||
attn_metadata.decode.seq_lens
|
||||
if attn_metadata.decode is not None
|
||||
else cast(torch.Tensor, attn_metadata.seq_lens)[ # type: ignore[attr-defined]
|
||||
: attn_metadata.num_decodes
|
||||
]
|
||||
)
|
||||
query_start_loc = attn_metadata.query_start_loc[
|
||||
: attn_metadata.num_decodes + 1
|
||||
]
|
||||
attn_out = self.dcp_manager.combine(
|
||||
attn_out,
|
||||
lse,
|
||||
seq_lens=seq_lens,
|
||||
query_start_loc=query_start_loc,
|
||||
)
|
||||
if self.use_pcp:
|
||||
attn_out = finalize_mla_pcp_decode(attn_out, self.num_heads)
|
||||
|
||||
@@ -1427,6 +1435,7 @@ class MLACommonPrefillMetadata:
|
||||
chunks: "list[MLACommonPrefillMetadata.ContextChunk]"
|
||||
context_lens_list: list[int]
|
||||
empty_token_slices: list[slice]
|
||||
dcp_manager: MLADCPManager | None = None
|
||||
|
||||
block_table: torch.Tensor
|
||||
query_start_loc: torch.Tensor
|
||||
@@ -1684,6 +1693,25 @@ def _flat_int32(values: list[int] | np.ndarray) -> torch.Tensor:
|
||||
return np_to_pinned_tensor(np.asarray(values, dtype=np.int32))
|
||||
|
||||
|
||||
def align_mla_chunked_context_workspace_size(
|
||||
vllm_config: VllmConfig,
|
||||
workspace_size: int,
|
||||
) -> int:
|
||||
parallel_config = vllm_config.parallel_config
|
||||
alignment = vllm_config.cache_config.block_size
|
||||
if parallel_config.decode_context_parallel_size > 1:
|
||||
alignment = lcm(
|
||||
alignment,
|
||||
parallel_config.decode_context_parallel_size
|
||||
* parallel_config.cp_kv_cache_interleave_size,
|
||||
)
|
||||
workspace_size = max(
|
||||
workspace_size,
|
||||
vllm_config.scheduler_config.max_num_seqs * alignment,
|
||||
)
|
||||
return round_up(workspace_size, alignment)
|
||||
|
||||
|
||||
def build_mla_chunked_context_metadata(
|
||||
*,
|
||||
context_lens_cpu: torch.Tensor,
|
||||
@@ -1696,6 +1724,7 @@ def build_mla_chunked_context_metadata(
|
||||
dcp_world_size: int,
|
||||
dcp_local_block_size: int,
|
||||
dcp_virtual_block_size: int,
|
||||
dcp_manager: MLADCPManager | None = None,
|
||||
) -> "MLACommonPrefillMetadata.ChunkedContextMetadata | None":
|
||||
"""Build chunked-context metadata for an MLA prefill.
|
||||
|
||||
@@ -1714,6 +1743,7 @@ def build_mla_chunked_context_metadata(
|
||||
dcp_world_size: Decode-context-parallel world size (1 if disabled).
|
||||
dcp_local_block_size: Per-rank interleave block size for DCP.
|
||||
dcp_virtual_block_size: ``dcp_local_block_size * dcp_world_size``.
|
||||
dcp_manager: Shared MLA DCP collective manager.
|
||||
|
||||
Returns:
|
||||
The chunked-context metadata, or None when no prefill has any context.
|
||||
@@ -1902,6 +1932,7 @@ def build_mla_chunked_context_metadata(
|
||||
chunks=chunks,
|
||||
context_lens_list=context_lens,
|
||||
empty_token_slices=empty_token_slices,
|
||||
dcp_manager=dcp_manager,
|
||||
)
|
||||
|
||||
|
||||
@@ -1954,7 +1985,10 @@ class MLACommonMetadataBuilder(AttentionMetadataBuilder[M]):
|
||||
64 * 1024,
|
||||
)
|
||||
|
||||
return max(chunked_prefill_workspace_size, cache_config.block_size)
|
||||
return align_mla_chunked_context_workspace_size(
|
||||
vllm_config,
|
||||
chunked_prefill_workspace_size,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def determine_prefill_query_data_type(
|
||||
@@ -2033,6 +2067,7 @@ class MLACommonMetadataBuilder(AttentionMetadataBuilder[M]):
|
||||
self.q_data_type = self.determine_prefill_query_data_type(
|
||||
vllm_config, self.model_config.dtype
|
||||
)
|
||||
attention_layer = self.compilation_config.static_forward_context[layer_names[0]]
|
||||
|
||||
try:
|
||||
self.dcp_world_size = get_dcp_group().world_size
|
||||
@@ -2050,6 +2085,7 @@ class MLACommonMetadataBuilder(AttentionMetadataBuilder[M]):
|
||||
)
|
||||
|
||||
use_packed_fp8_cache = vllm_config.cache_config.cache_dtype == "fp8_ds_mla"
|
||||
self.dcp_manager: MLADCPManager | None = None
|
||||
if self.dcp_world_size > 1:
|
||||
# Note(hc): The local kvcache is incomplete when DCP is triggered,
|
||||
# an additional kvcache allgather across the DCP group is therefore
|
||||
@@ -2067,6 +2103,12 @@ class MLACommonMetadataBuilder(AttentionMetadataBuilder[M]):
|
||||
else self.model_config.dtype,
|
||||
device=device,
|
||||
)
|
||||
self.dcp_manager = getattr(attention_layer, "dcp_manager", None)
|
||||
assert isinstance(self.dcp_manager, MLADCPManager)
|
||||
self.dcp_manager.init_kv_gather(
|
||||
self.chunked_prefill_workspace,
|
||||
self.chunked_prefill_workspace_size,
|
||||
)
|
||||
else:
|
||||
self.chunked_prefill_workspace = torch.empty(
|
||||
(
|
||||
@@ -2080,9 +2122,7 @@ class MLACommonMetadataBuilder(AttentionMetadataBuilder[M]):
|
||||
# Metadata builders are created per ubatch when DBO is enabled. MLA
|
||||
# prefill backends keep the prepared metadata on the backend object, so
|
||||
# each builder needs its own backend instance to avoid cross-ubatch races.
|
||||
self._prefill_backend = self.compilation_config.static_forward_context[
|
||||
layer_names[0]
|
||||
].prefill_backend.clone()
|
||||
self._prefill_backend = attention_layer.prefill_backend.clone()
|
||||
|
||||
supports_spec_decode = self.query_len_support != QueryLenSupport.SINGLE_ONLY
|
||||
self._init_reorder_batch_threshold(
|
||||
@@ -2223,6 +2263,7 @@ class MLACommonMetadataBuilder(AttentionMetadataBuilder[M]):
|
||||
dcp_world_size=self.dcp_world_size,
|
||||
dcp_local_block_size=self.dcp_local_block_size,
|
||||
dcp_virtual_block_size=self.dcp_virtual_block_size,
|
||||
dcp_manager=self.dcp_manager,
|
||||
)
|
||||
|
||||
prefill_metadata = MLACommonPrefillMetadata(
|
||||
@@ -2660,9 +2701,8 @@ class MLACommonBaseImpl(MLAAttentionImpl[A], Generic[A]):
|
||||
]
|
||||
assert toks * dcp_world_size <= cur_allgather_workspace.shape[0]
|
||||
cur_allgather_kvcache = cur_allgather_workspace[: toks * dcp_world_size]
|
||||
cur_allgather_kvcache.copy_(
|
||||
get_dcp_group().all_gather(local_gathered_kvcache, dim=0)
|
||||
)
|
||||
dcp_manager = cast(MLADCPManager, chunked_context.dcp_manager)
|
||||
dcp_manager.kv_gather(cur_allgather_kvcache, local_gathered_kvcache)
|
||||
assert (
|
||||
cur_allgather_kvcache.shape[-1]
|
||||
== self.kv_lora_rank + self.qk_rope_head_dim
|
||||
@@ -2732,8 +2772,6 @@ class MLACommonBaseImpl(MLAAttentionImpl[A], Generic[A]):
|
||||
output_scale: torch.Tensor | None = None,
|
||||
) -> None:
|
||||
assert attn_metadata.prefill is not None
|
||||
assert self.dcp_world_size != -1
|
||||
|
||||
prefill_metadata = attn_metadata.prefill
|
||||
assert prefill_metadata.prefill_backend is not None
|
||||
use_fp8_prefill = prefill_metadata.q_data_type == current_platform.fp8_dtype()
|
||||
@@ -2877,10 +2915,11 @@ class MLACommonImpl(MLACommonBaseImpl[M], Generic[M]):
|
||||
and (self.qk_rope_head_dim == 64)
|
||||
)
|
||||
|
||||
self.dcp_world_size: int = -1
|
||||
|
||||
parallel_config = get_current_vllm_config().parallel_config
|
||||
# Avoid requiring an initialized DCP group in tests.
|
||||
self.dcp_world_size: int = parallel_config.decode_context_parallel_size
|
||||
self.cp_kv_cache_interleave_size: int = (
|
||||
get_current_vllm_config().parallel_config.cp_kv_cache_interleave_size
|
||||
parallel_config.cp_kv_cache_interleave_size
|
||||
)
|
||||
|
||||
@abstractmethod
|
||||
|
||||
@@ -20,6 +20,7 @@ from vllm.model_executor.layers.attention.mla_attention import (
|
||||
MLACommonMetadata,
|
||||
MLACommonPrefillMetadata,
|
||||
accumulate_mla_context_chunk,
|
||||
align_mla_chunked_context_workspace_size,
|
||||
build_mla_chunked_context_metadata,
|
||||
get_mla_dims,
|
||||
init_mla_context_partial,
|
||||
@@ -31,6 +32,7 @@ from vllm.utils.torch_utils import is_quantized_kv_cache, np_to_pinned_tensor
|
||||
from vllm.v1.attention.backend import AttentionMetadata, AttentionMetadataBuilder
|
||||
from vllm.v1.attention.backends.fa_utils import get_flash_attn_version
|
||||
from vllm.v1.attention.backends.utils import split_decodes_and_prefills
|
||||
from vllm.v1.attention.ops.dcp_utils import MLADCPManager
|
||||
from vllm.v1.attention.ops.merge_attn_states import merge_attn_states
|
||||
|
||||
if TYPE_CHECKING:
|
||||
@@ -162,9 +164,19 @@ class SparseMLACommonMetadataBuilder(AttentionMetadataBuilder[T]):
|
||||
dtype=torch.int32,
|
||||
device=device,
|
||||
)
|
||||
layer_prefill_backend = vllm_config.compilation_config.static_forward_context[
|
||||
attention_layer = vllm_config.compilation_config.static_forward_context[
|
||||
layer_names[0]
|
||||
].prefill_backend
|
||||
]
|
||||
layer_prefill_backend = attention_layer.prefill_backend
|
||||
self.dcp_manager: MLADCPManager | None = None
|
||||
if self.dcp_world_size > 1:
|
||||
self.dcp_manager = getattr(attention_layer, "dcp_manager", None)
|
||||
assert isinstance(self.dcp_manager, MLADCPManager)
|
||||
if layer_prefill_backend is not None:
|
||||
self.dcp_manager.init_kv_gather(
|
||||
self.chunked_prefill_workspace,
|
||||
self.chunked_prefill_workspace_size,
|
||||
)
|
||||
self._prefill_backend = (
|
||||
layer_prefill_backend.clone() if layer_prefill_backend is not None else None
|
||||
)
|
||||
@@ -184,7 +196,13 @@ class SparseMLACommonMetadataBuilder(AttentionMetadataBuilder[T]):
|
||||
64 * 1024,
|
||||
scheduler_config.max_num_seqs * topk_tokens,
|
||||
)
|
||||
return max(workspace_size, cache_config.block_size)
|
||||
workspace_size = max(
|
||||
workspace_size,
|
||||
scheduler_config.max_num_seqs * cache_config.block_size,
|
||||
)
|
||||
if vllm_config.parallel_config.decode_context_parallel_size > 1:
|
||||
return align_mla_chunked_context_workspace_size(vllm_config, workspace_size)
|
||||
return workspace_size
|
||||
|
||||
def _build_req_id_per_token(
|
||||
self,
|
||||
@@ -232,6 +250,7 @@ class SparseMLACommonMetadataBuilder(AttentionMetadataBuilder[T]):
|
||||
dcp_world_size=self.dcp_world_size,
|
||||
dcp_local_block_size=self.dcp_local_block_size,
|
||||
dcp_virtual_block_size=self.dcp_virtual_block_size,
|
||||
dcp_manager=self.dcp_manager,
|
||||
)
|
||||
|
||||
def build(
|
||||
|
||||
Reference in New Issue
Block a user