[refactor] consolidate cp attn ops (#52839)

Signed-off-by: Summer Yang <girasoleyang@gmail.com>
Co-authored-by: Woosuk Kwon <woosuk@inferact.ai>
This commit is contained in:
Summer Yang
2026-08-19 18:04:11 -07:00
committed by GitHub
co-authored by Woosuk Kwon
parent bf2866f8bf
commit 58e5ee0158
18 changed files with 1603 additions and 1585 deletions
@@ -242,10 +242,6 @@ from vllm.model_executor.layers.attention.attention import (
from vllm.model_executor.layers.attention.kv_transfer_utils import (
maybe_transfer_kv_layer,
)
from vllm.model_executor.layers.attention.pcp import (
finalize_mla_pcp_decode,
maybe_gather_mla_latent_cache_inputs,
)
from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase
from vllm.model_executor.layers.linear import (
ColumnParallelLinear,
@@ -293,8 +289,12 @@ from vllm.v1.attention.backends.utils import (
get_num_attention_heads_from_layers,
split_decodes_and_prefills,
)
from vllm.v1.attention.ops.dcp_utils import MLADCPManager
from vllm.v1.attention.ops.dcp import MLADCPManager
from vllm.v1.attention.ops.merge_attn_states import merge_attn_states
from vllm.v1.attention.ops.pcp import (
finalize_mla_pcp_decode,
maybe_gather_mla_latent_cache_inputs,
)
from vllm.v1.attention.selector import get_attn_backend
from vllm.v1.kv_cache_interface import (
AttentionSpec,
@@ -1,92 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
from vllm.distributed.parallel_state import (
get_pcp_group,
get_tp_group,
)
def _gather_prefill_cache_inputs(
tensors: tuple[torch.Tensor, ...],
slot_mapping: torch.Tensor,
num_decode_tokens: int,
) -> tuple[tuple[torch.Tensor, ...], torch.Tensor]:
"""Keep replicated decode writes local and gather partitioned prefills."""
local_num_tokens = tensors[0].shape[0]
assert all(tensor.shape[0] == local_num_tokens for tensor in tensors)
assert 0 <= num_decode_tokens <= local_num_tokens
if num_decode_tokens == local_num_tokens:
return tensors, slot_mapping[:num_decode_tokens]
pcp_group = get_pcp_group()
gathered_prefills = tuple(
pcp_group.all_gather(tensor[num_decode_tokens:].contiguous(), dim=0)
for tensor in tensors
)
pcp_size = pcp_group.world_size
gathered_slot_mapping = slot_mapping[: pcp_size * local_num_tokens]
if num_decode_tokens == 0:
return gathered_prefills, gathered_slot_mapping
cache_inputs = tuple(
torch.cat((tensor[:num_decode_tokens], gathered_prefill), dim=0)
for tensor, gathered_prefill in zip(tensors, gathered_prefills)
)
rank_slot_mappings = gathered_slot_mapping.view(pcp_size, local_num_tokens)
cache_slot_mapping = torch.cat(
(
rank_slot_mappings[0, :num_decode_tokens],
rank_slot_mappings[:, num_decode_tokens:].flatten(),
)
)
return cache_inputs, cache_slot_mapping
def maybe_gather_mla_latent_cache_inputs(
kv_c_normed: torch.Tensor,
k_pe: torch.Tensor,
slot_mapping: torch.Tensor | None,
num_decode_tokens: int | None,
use_pcp: bool,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor | None]:
if not use_pcp or num_decode_tokens is None:
return kv_c_normed, k_pe, slot_mapping
assert slot_mapping is not None
num_tokens = kv_c_normed.shape[0]
k_pe_flat = k_pe.reshape(num_tokens, -1)
(cache_kv_c, cache_k_pe_flat), cache_slot_mapping = _gather_prefill_cache_inputs(
(kv_c_normed, k_pe_flat),
slot_mapping,
num_decode_tokens,
)
cache_k_pe = cache_k_pe_flat.view(-1, *k_pe.shape[1:])
return cache_kv_c, cache_k_pe, cache_slot_mapping
def maybe_gather_indexer_k(
k: torch.Tensor,
slot_mapping: torch.Tensor,
num_decode_tokens: int,
use_pcp: bool,
) -> tuple[torch.Tensor, torch.Tensor]:
if not use_pcp:
return k, slot_mapping
(cache_k,), cache_slot_mapping = _gather_prefill_cache_inputs(
(k,), slot_mapping, num_decode_tokens
)
return cache_k, cache_slot_mapping
def finalize_mla_pcp_decode(
output: torch.Tensor,
num_heads: int,
) -> torch.Tensor:
if output.shape[1] < num_heads:
output = get_pcp_group().all_gather(output, dim=1)
elif output.shape[1] > num_heads:
head_start = get_tp_group().rank_in_group * num_heads
output = output[:, head_start : head_start + num_heads]
return output
@@ -32,7 +32,7 @@ from vllm.utils.torch_utils import is_quantized_kv_cache, np_to_pinned_tensor
from vllm.v1.attention.backend import AttentionMetadata, AttentionMetadataBuilder
from vllm.v1.attention.backends.fa_utils import get_flash_attn_version
from vllm.v1.attention.backends.utils import split_decodes_and_prefills
from vllm.v1.attention.ops.dcp_utils import MLADCPManager
from vllm.v1.attention.ops.dcp import MLADCPManager
from vllm.v1.attention.ops.merge_attn_states import merge_attn_states
if TYPE_CHECKING:
@@ -13,7 +13,6 @@ from vllm.distributed import get_dcp_group, get_pcp_group
from vllm.forward_context import get_forward_context
from vllm.logger import init_logger
from vllm.model_executor.custom_op import CustomOp
from vllm.model_executor.layers.attention.pcp import maybe_gather_indexer_k
from vllm.model_executor.layers.quantization.utils.quant_utils import (
get_fp8_min_max,
)
@@ -35,6 +34,7 @@ from vllm.v1.attention.backends.mla.indexer import (
DeepseekV32IndexerMetadata,
)
from vllm.v1.attention.ops.common import pack_seq_triton, unpack_seq_triton
from vllm.v1.attention.ops.pcp import maybe_gather_indexer_k
from vllm.v1.worker.workspace import current_workspace_manager
logger = init_logger(__name__)