[refactor] consolidate cp attn ops (#52839)
Signed-off-by: Summer Yang <girasoleyang@gmail.com> Co-authored-by: Woosuk Kwon <woosuk@inferact.ai>
This commit is contained in:
co-authored by
Woosuk Kwon
parent
bf2866f8bf
commit
58e5ee0158
@@ -242,10 +242,6 @@ from vllm.model_executor.layers.attention.attention import (
|
||||
from vllm.model_executor.layers.attention.kv_transfer_utils import (
|
||||
maybe_transfer_kv_layer,
|
||||
)
|
||||
from vllm.model_executor.layers.attention.pcp import (
|
||||
finalize_mla_pcp_decode,
|
||||
maybe_gather_mla_latent_cache_inputs,
|
||||
)
|
||||
from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase
|
||||
from vllm.model_executor.layers.linear import (
|
||||
ColumnParallelLinear,
|
||||
@@ -293,8 +289,12 @@ from vllm.v1.attention.backends.utils import (
|
||||
get_num_attention_heads_from_layers,
|
||||
split_decodes_and_prefills,
|
||||
)
|
||||
from vllm.v1.attention.ops.dcp_utils import MLADCPManager
|
||||
from vllm.v1.attention.ops.dcp import MLADCPManager
|
||||
from vllm.v1.attention.ops.merge_attn_states import merge_attn_states
|
||||
from vllm.v1.attention.ops.pcp import (
|
||||
finalize_mla_pcp_decode,
|
||||
maybe_gather_mla_latent_cache_inputs,
|
||||
)
|
||||
from vllm.v1.attention.selector import get_attn_backend
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
AttentionSpec,
|
||||
|
||||
@@ -1,92 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import torch
|
||||
|
||||
from vllm.distributed.parallel_state import (
|
||||
get_pcp_group,
|
||||
get_tp_group,
|
||||
)
|
||||
|
||||
|
||||
def _gather_prefill_cache_inputs(
|
||||
tensors: tuple[torch.Tensor, ...],
|
||||
slot_mapping: torch.Tensor,
|
||||
num_decode_tokens: int,
|
||||
) -> tuple[tuple[torch.Tensor, ...], torch.Tensor]:
|
||||
"""Keep replicated decode writes local and gather partitioned prefills."""
|
||||
local_num_tokens = tensors[0].shape[0]
|
||||
assert all(tensor.shape[0] == local_num_tokens for tensor in tensors)
|
||||
assert 0 <= num_decode_tokens <= local_num_tokens
|
||||
|
||||
if num_decode_tokens == local_num_tokens:
|
||||
return tensors, slot_mapping[:num_decode_tokens]
|
||||
|
||||
pcp_group = get_pcp_group()
|
||||
gathered_prefills = tuple(
|
||||
pcp_group.all_gather(tensor[num_decode_tokens:].contiguous(), dim=0)
|
||||
for tensor in tensors
|
||||
)
|
||||
pcp_size = pcp_group.world_size
|
||||
gathered_slot_mapping = slot_mapping[: pcp_size * local_num_tokens]
|
||||
if num_decode_tokens == 0:
|
||||
return gathered_prefills, gathered_slot_mapping
|
||||
|
||||
cache_inputs = tuple(
|
||||
torch.cat((tensor[:num_decode_tokens], gathered_prefill), dim=0)
|
||||
for tensor, gathered_prefill in zip(tensors, gathered_prefills)
|
||||
)
|
||||
rank_slot_mappings = gathered_slot_mapping.view(pcp_size, local_num_tokens)
|
||||
cache_slot_mapping = torch.cat(
|
||||
(
|
||||
rank_slot_mappings[0, :num_decode_tokens],
|
||||
rank_slot_mappings[:, num_decode_tokens:].flatten(),
|
||||
)
|
||||
)
|
||||
return cache_inputs, cache_slot_mapping
|
||||
|
||||
|
||||
def maybe_gather_mla_latent_cache_inputs(
|
||||
kv_c_normed: torch.Tensor,
|
||||
k_pe: torch.Tensor,
|
||||
slot_mapping: torch.Tensor | None,
|
||||
num_decode_tokens: int | None,
|
||||
use_pcp: bool,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor | None]:
|
||||
if not use_pcp or num_decode_tokens is None:
|
||||
return kv_c_normed, k_pe, slot_mapping
|
||||
assert slot_mapping is not None
|
||||
num_tokens = kv_c_normed.shape[0]
|
||||
k_pe_flat = k_pe.reshape(num_tokens, -1)
|
||||
(cache_kv_c, cache_k_pe_flat), cache_slot_mapping = _gather_prefill_cache_inputs(
|
||||
(kv_c_normed, k_pe_flat),
|
||||
slot_mapping,
|
||||
num_decode_tokens,
|
||||
)
|
||||
cache_k_pe = cache_k_pe_flat.view(-1, *k_pe.shape[1:])
|
||||
return cache_kv_c, cache_k_pe, cache_slot_mapping
|
||||
|
||||
|
||||
def maybe_gather_indexer_k(
|
||||
k: torch.Tensor,
|
||||
slot_mapping: torch.Tensor,
|
||||
num_decode_tokens: int,
|
||||
use_pcp: bool,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
if not use_pcp:
|
||||
return k, slot_mapping
|
||||
(cache_k,), cache_slot_mapping = _gather_prefill_cache_inputs(
|
||||
(k,), slot_mapping, num_decode_tokens
|
||||
)
|
||||
return cache_k, cache_slot_mapping
|
||||
|
||||
|
||||
def finalize_mla_pcp_decode(
|
||||
output: torch.Tensor,
|
||||
num_heads: int,
|
||||
) -> torch.Tensor:
|
||||
if output.shape[1] < num_heads:
|
||||
output = get_pcp_group().all_gather(output, dim=1)
|
||||
elif output.shape[1] > num_heads:
|
||||
head_start = get_tp_group().rank_in_group * num_heads
|
||||
output = output[:, head_start : head_start + num_heads]
|
||||
return output
|
||||
@@ -32,7 +32,7 @@ from vllm.utils.torch_utils import is_quantized_kv_cache, np_to_pinned_tensor
|
||||
from vllm.v1.attention.backend import AttentionMetadata, AttentionMetadataBuilder
|
||||
from vllm.v1.attention.backends.fa_utils import get_flash_attn_version
|
||||
from vllm.v1.attention.backends.utils import split_decodes_and_prefills
|
||||
from vllm.v1.attention.ops.dcp_utils import MLADCPManager
|
||||
from vllm.v1.attention.ops.dcp import MLADCPManager
|
||||
from vllm.v1.attention.ops.merge_attn_states import merge_attn_states
|
||||
|
||||
if TYPE_CHECKING:
|
||||
|
||||
@@ -13,7 +13,6 @@ from vllm.distributed import get_dcp_group, get_pcp_group
|
||||
from vllm.forward_context import get_forward_context
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.custom_op import CustomOp
|
||||
from vllm.model_executor.layers.attention.pcp import maybe_gather_indexer_k
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
get_fp8_min_max,
|
||||
)
|
||||
@@ -35,6 +34,7 @@ from vllm.v1.attention.backends.mla.indexer import (
|
||||
DeepseekV32IndexerMetadata,
|
||||
)
|
||||
from vllm.v1.attention.ops.common import pack_seq_triton, unpack_seq_triton
|
||||
from vllm.v1.attention.ops.pcp import maybe_gather_indexer_k
|
||||
from vllm.v1.worker.workspace import current_workspace_manager
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
Reference in New Issue
Block a user