[Core] Add MRV2 virtual-batch PCP for MLA (#46570)
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Signed-off-by: Matthew Bonanni <mbonanni@redhat.com> Co-authored-by: Codex <noreply@openai.com> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: Matthew Bonanni <mbonanni@redhat.com>
This commit is contained in:
co-authored by
Codex
OpenAI Codex
Matthew Bonanni
parent
9243e0124e
commit
b6ff8a2f50
@@ -211,6 +211,7 @@ from vllm.config import (
|
||||
from vllm.config.cache import CacheDType
|
||||
from vllm.distributed.parallel_state import (
|
||||
get_dcp_group,
|
||||
get_tp_group,
|
||||
is_global_first_rank,
|
||||
)
|
||||
from vllm.forward_context import ForwardContext, get_forward_context
|
||||
@@ -225,6 +226,10 @@ from vllm.model_executor.layers.attention.attention import (
|
||||
from vllm.model_executor.layers.attention.kv_transfer_utils import (
|
||||
maybe_transfer_kv_layer,
|
||||
)
|
||||
from vllm.model_executor.layers.attention.pcp import (
|
||||
finalize_mla_pcp_decode,
|
||||
maybe_gather_mla_latent_cache_inputs,
|
||||
)
|
||||
from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase
|
||||
from vllm.model_executor.layers.linear import (
|
||||
ColumnParallelLinear,
|
||||
@@ -269,7 +274,7 @@ from vllm.v1.attention.backends.utils import (
|
||||
get_dcp_local_seq_lens,
|
||||
split_decodes_and_prefills,
|
||||
)
|
||||
from vllm.v1.attention.ops.common import cp_lse_ag_out_rs
|
||||
from vllm.v1.attention.ops.common import cp_lse_ag_out_ar, cp_lse_ag_out_rs
|
||||
from vllm.v1.attention.ops.dcp_alltoall import dcp_a2a_lse_reduce
|
||||
from vllm.v1.attention.ops.merge_attn_states import merge_attn_states
|
||||
from vllm.v1.attention.selector import get_attn_backend
|
||||
@@ -499,6 +504,8 @@ class MLAAttention(nn.Module, AttentionLayerBase):
|
||||
self.use_direct_call = not current_platform.opaque_attention_op()
|
||||
|
||||
vllm_config = get_current_vllm_config()
|
||||
parallel_config = vllm_config.parallel_config
|
||||
self.use_pcp = parallel_config.prefill_context_parallel_size > 1
|
||||
compilation_config = vllm_config.compilation_config
|
||||
if prefix in compilation_config.static_forward_context:
|
||||
raise ValueError(f"Duplicate layer name: {prefix}")
|
||||
@@ -611,11 +618,23 @@ class MLAAttention(nn.Module, AttentionLayerBase):
|
||||
assert isinstance(slot_mapping, dict), (
|
||||
f"Expected slot_mapping to be a dict, got {type(slot_mapping)}. "
|
||||
)
|
||||
layer_slot_mapping = slot_mapping.get(self.layer_name)
|
||||
kv_for_cache, kpe_for_cache, layer_slot_mapping = (
|
||||
maybe_gather_mla_latent_cache_inputs(
|
||||
kv_c_normed,
|
||||
k_pe,
|
||||
layer_slot_mapping,
|
||||
attn_metadata.num_decode_tokens
|
||||
if attn_metadata is not None
|
||||
else None,
|
||||
self.use_pcp,
|
||||
)
|
||||
)
|
||||
self.impl.do_kv_cache_update( # type: ignore[attr-defined]
|
||||
kv_c_normed,
|
||||
k_pe,
|
||||
kv_for_cache,
|
||||
kpe_for_cache,
|
||||
self_kv_cache,
|
||||
slot_mapping.get(self.layer_name),
|
||||
layer_slot_mapping,
|
||||
self.kv_cache_dtype,
|
||||
self._k_scale,
|
||||
)
|
||||
@@ -710,6 +729,10 @@ class MLAAttention(nn.Module, AttentionLayerBase):
|
||||
fp8_attention = is_quantized_kv_cache(self.kv_cache_dtype)
|
||||
|
||||
num_actual_toks = attn_metadata.num_actual_tokens
|
||||
if self.use_pcp and self.impl.dcp_world_size > 1 and quant_key is not None:
|
||||
raise NotImplementedError(
|
||||
"MRV2 MLA PCP+DCP does not support fused output quantization yet."
|
||||
)
|
||||
|
||||
# Inputs and outputs may be padded for CUDA graphs
|
||||
output_padded = output
|
||||
@@ -833,11 +856,15 @@ class MLAAttention(nn.Module, AttentionLayerBase):
|
||||
else:
|
||||
mqa_q = (mqa_ql_nope, mqa_q_pe)
|
||||
if self.impl.dcp_world_size > 1:
|
||||
if isinstance(mqa_q, tuple):
|
||||
# concatenate mqa_ql_nope and mqa_q_pe -> (B, N, L + P)
|
||||
mqa_q = torch.cat(mqa_q, dim=-1)
|
||||
# mqa_q do allgather in head dim.
|
||||
mqa_q = get_dcp_group().all_gather(mqa_q, dim=1)
|
||||
if self.use_pcp:
|
||||
if self.impl.dcp_world_size > self.impl.pcp_world_size:
|
||||
if isinstance(mqa_q, tuple):
|
||||
mqa_q = torch.cat(mqa_q, dim=-1)
|
||||
mqa_q = get_tp_group().all_gather(mqa_q, dim=1)
|
||||
else:
|
||||
if isinstance(mqa_q, tuple):
|
||||
mqa_q = torch.cat(mqa_q, dim=-1)
|
||||
mqa_q = get_dcp_group().all_gather(mqa_q, dim=1)
|
||||
|
||||
# call decode attn
|
||||
if not self.impl.is_sparse:
|
||||
@@ -846,6 +873,7 @@ class MLAAttention(nn.Module, AttentionLayerBase):
|
||||
|
||||
# correct dcp attn_out with lse.
|
||||
if self.impl.dcp_world_size > 1:
|
||||
assert lse is not None
|
||||
if self.dcp_a2a:
|
||||
attn_out = dcp_a2a_lse_reduce(
|
||||
attn_out,
|
||||
@@ -853,6 +881,13 @@ class MLAAttention(nn.Module, AttentionLayerBase):
|
||||
get_dcp_group(),
|
||||
is_lse_base_on_e=self.impl.lse_base_on_e,
|
||||
)
|
||||
elif self.use_pcp:
|
||||
attn_out = cp_lse_ag_out_ar(
|
||||
attn_out,
|
||||
lse,
|
||||
get_dcp_group(),
|
||||
is_lse_base_on_e=self.impl.lse_base_on_e,
|
||||
)
|
||||
else:
|
||||
attn_out = cp_lse_ag_out_rs(
|
||||
attn_out,
|
||||
@@ -860,6 +895,8 @@ class MLAAttention(nn.Module, AttentionLayerBase):
|
||||
get_dcp_group(),
|
||||
is_lse_base_on_e=self.impl.lse_base_on_e,
|
||||
)
|
||||
if self.use_pcp:
|
||||
attn_out = finalize_mla_pcp_decode(attn_out, self.num_heads)
|
||||
|
||||
# v_up projection
|
||||
self._v_up_proj(attn_out, out=mqa_output_slice)
|
||||
@@ -903,6 +940,8 @@ class MLAAttention(nn.Module, AttentionLayerBase):
|
||||
raise ValueError(f"Unsupported quant_key: {quant_key}")
|
||||
return quant_output
|
||||
|
||||
if self.use_pcp and output_padded.shape[0] > num_actual_toks:
|
||||
output_padded[num_actual_toks:].zero_()
|
||||
return output_padded
|
||||
|
||||
def process_weights_after_loading(self, act_dtype: torch.dtype):
|
||||
@@ -1082,8 +1121,17 @@ def unified_mla_kv_cache_update(
|
||||
the data dependency between them to ensure torch.compile preserves ordering.
|
||||
"""
|
||||
layer_name = _resolve_layer_name(layer_name)
|
||||
_, attn_layer, kv_cache, layer_slot_mapping = get_attention_context(layer_name)
|
||||
attn_metadata, attn_layer, kv_cache, layer_slot_mapping = get_attention_context(
|
||||
layer_name
|
||||
)
|
||||
if layer_slot_mapping is not None:
|
||||
kv_c_normed, k_pe, layer_slot_mapping = maybe_gather_mla_latent_cache_inputs(
|
||||
kv_c_normed,
|
||||
k_pe,
|
||||
layer_slot_mapping,
|
||||
attn_metadata.num_decode_tokens if attn_metadata is not None else None,
|
||||
attn_layer.use_pcp,
|
||||
)
|
||||
attn_layer.impl.do_kv_cache_update( # type: ignore[attr-defined]
|
||||
kv_c_normed,
|
||||
k_pe,
|
||||
@@ -1723,6 +1771,7 @@ class MLACommonMetadataBuilder(AttentionMetadataBuilder[M]):
|
||||
self.compilation_config = vllm_config.compilation_config
|
||||
self.vllm_config = vllm_config
|
||||
self.device = device
|
||||
self.use_pcp = parallel_config.prefill_context_parallel_size > 1
|
||||
|
||||
self.num_heads = self.model_config.get_num_attention_heads(parallel_config)
|
||||
self.mla_dims = get_mla_dims(self.model_config)
|
||||
@@ -1735,11 +1784,9 @@ class MLACommonMetadataBuilder(AttentionMetadataBuilder[M]):
|
||||
|
||||
try:
|
||||
self.dcp_world_size = get_dcp_group().world_size
|
||||
self.dcp_rank = get_dcp_group().rank_in_group
|
||||
except AssertionError:
|
||||
# DCP might not be initialized in testing
|
||||
self.dcp_world_size = 1
|
||||
self.dcp_rank = 0
|
||||
self.dcp_local_block_size = parallel_config.cp_kv_cache_interleave_size
|
||||
self.dcp_virtual_block_size = self.dcp_local_block_size * self.dcp_world_size
|
||||
self.cp_kv_cache_interleave_size = parallel_config.cp_kv_cache_interleave_size
|
||||
@@ -1857,6 +1904,7 @@ class MLACommonMetadataBuilder(AttentionMetadataBuilder[M]):
|
||||
common_attn_metadata,
|
||||
decode_threshold=self.reorder_batch_threshold,
|
||||
require_uniform=(self.query_len_support != QueryLenSupport.VARLEN),
|
||||
treat_short_extends_as_decodes=not self.use_pcp,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@@ -0,0 +1,92 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import torch
|
||||
|
||||
from vllm.distributed.parallel_state import (
|
||||
get_pcp_group,
|
||||
get_tp_group,
|
||||
)
|
||||
|
||||
|
||||
def _gather_prefill_cache_inputs(
|
||||
tensors: tuple[torch.Tensor, ...],
|
||||
slot_mapping: torch.Tensor,
|
||||
num_decode_tokens: int,
|
||||
) -> tuple[tuple[torch.Tensor, ...], torch.Tensor]:
|
||||
"""Keep replicated decode writes local and gather partitioned prefills."""
|
||||
local_num_tokens = tensors[0].shape[0]
|
||||
assert all(tensor.shape[0] == local_num_tokens for tensor in tensors)
|
||||
assert 0 <= num_decode_tokens <= local_num_tokens
|
||||
|
||||
if num_decode_tokens == local_num_tokens:
|
||||
return tensors, slot_mapping[:num_decode_tokens]
|
||||
|
||||
pcp_group = get_pcp_group()
|
||||
gathered_prefills = tuple(
|
||||
pcp_group.all_gather(tensor[num_decode_tokens:].contiguous(), dim=0)
|
||||
for tensor in tensors
|
||||
)
|
||||
pcp_size = pcp_group.world_size
|
||||
gathered_slot_mapping = slot_mapping[: pcp_size * local_num_tokens]
|
||||
if num_decode_tokens == 0:
|
||||
return gathered_prefills, gathered_slot_mapping
|
||||
|
||||
cache_inputs = tuple(
|
||||
torch.cat((tensor[:num_decode_tokens], gathered_prefill), dim=0)
|
||||
for tensor, gathered_prefill in zip(tensors, gathered_prefills)
|
||||
)
|
||||
rank_slot_mappings = gathered_slot_mapping.view(pcp_size, local_num_tokens)
|
||||
cache_slot_mapping = torch.cat(
|
||||
(
|
||||
rank_slot_mappings[0, :num_decode_tokens],
|
||||
rank_slot_mappings[:, num_decode_tokens:].flatten(),
|
||||
)
|
||||
)
|
||||
return cache_inputs, cache_slot_mapping
|
||||
|
||||
|
||||
def maybe_gather_mla_latent_cache_inputs(
|
||||
kv_c_normed: torch.Tensor,
|
||||
k_pe: torch.Tensor,
|
||||
slot_mapping: torch.Tensor | None,
|
||||
num_decode_tokens: int | None,
|
||||
use_pcp: bool,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor | None]:
|
||||
if not use_pcp or num_decode_tokens is None:
|
||||
return kv_c_normed, k_pe, slot_mapping
|
||||
assert slot_mapping is not None
|
||||
num_tokens = kv_c_normed.shape[0]
|
||||
k_pe_flat = k_pe.reshape(num_tokens, -1)
|
||||
(cache_kv_c, cache_k_pe_flat), cache_slot_mapping = _gather_prefill_cache_inputs(
|
||||
(kv_c_normed, k_pe_flat),
|
||||
slot_mapping,
|
||||
num_decode_tokens,
|
||||
)
|
||||
cache_k_pe = cache_k_pe_flat.view(-1, *k_pe.shape[1:])
|
||||
return cache_kv_c, cache_k_pe, cache_slot_mapping
|
||||
|
||||
|
||||
def maybe_gather_indexer_k(
|
||||
k: torch.Tensor,
|
||||
slot_mapping: torch.Tensor,
|
||||
num_decode_tokens: int,
|
||||
use_pcp: bool,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
if not use_pcp:
|
||||
return k, slot_mapping
|
||||
(cache_k,), cache_slot_mapping = _gather_prefill_cache_inputs(
|
||||
(k,), slot_mapping, num_decode_tokens
|
||||
)
|
||||
return cache_k, cache_slot_mapping
|
||||
|
||||
|
||||
def finalize_mla_pcp_decode(
|
||||
output: torch.Tensor,
|
||||
num_heads: int,
|
||||
) -> torch.Tensor:
|
||||
if output.shape[1] < num_heads:
|
||||
output = get_pcp_group().all_gather(output, dim=1)
|
||||
elif output.shape[1] > num_heads:
|
||||
head_start = get_tp_group().rank_in_group * num_heads
|
||||
output = output[:, head_start : head_start + num_heads]
|
||||
return output
|
||||
@@ -59,13 +59,12 @@ class SparseMLACommonMetadataBuilder(AttentionMetadataBuilder[T]):
|
||||
device=device,
|
||||
)
|
||||
parallel_config = vllm_config.parallel_config
|
||||
self.use_pcp = parallel_config.prefill_context_parallel_size > 1
|
||||
try:
|
||||
self.dcp_world_size = get_dcp_group().world_size
|
||||
self.dcp_rank = get_dcp_group().rank_in_group
|
||||
except AssertionError:
|
||||
# DCP might not be initialized in testing
|
||||
self.dcp_world_size = 1
|
||||
self.dcp_rank = 0
|
||||
self.cp_kv_cache_interleave_size = parallel_config.cp_kv_cache_interleave_size
|
||||
self.dcp_local_block_size = self.cp_kv_cache_interleave_size
|
||||
self.dcp_virtual_block_size = self.dcp_local_block_size * self.dcp_world_size
|
||||
@@ -174,6 +173,7 @@ class SparseMLACommonMetadataBuilder(AttentionMetadataBuilder[T]):
|
||||
num_decodes, num_prefills, num_decode_tokens, _ = split_decodes_and_prefills(
|
||||
common_attn_metadata,
|
||||
decode_threshold=self.reorder_batch_threshold or 1,
|
||||
treat_short_extends_as_decodes=not self.use_pcp,
|
||||
require_uniform=self.require_uniform_decodes,
|
||||
)
|
||||
(
|
||||
|
||||
@@ -1050,7 +1050,9 @@ class FusedMoEParallelConfig:
|
||||
|
||||
@property
|
||||
def use_all2all_kernels(self):
|
||||
return self.use_ep and (self.dp_size > 1 or self.is_sequence_parallel)
|
||||
return self.use_ep and (
|
||||
self.dp_size > 1 or self.pcp_size > 1 or self.is_sequence_parallel
|
||||
)
|
||||
|
||||
@property
|
||||
def use_deepep_ht_kernels(self):
|
||||
|
||||
@@ -752,18 +752,12 @@ class MoERunner(MoERunnerInterface):
|
||||
assert len(result) == 2
|
||||
hidden_states, router_logits = result
|
||||
|
||||
# NOTE: Similar with DP, PCP also needs dispatch and combine. For
|
||||
# simplicity, AgRsAll2All was added separately for PCP here. Maybe
|
||||
# we should modify All2AllManager abstraction to better support PCP.
|
||||
if self.moe_config.pcp_size > 1:
|
||||
hidden_states = get_pcp_group().all_gather(
|
||||
hidden_states,
|
||||
dim=0,
|
||||
)
|
||||
router_logits = get_pcp_group().all_gather(
|
||||
router_logits,
|
||||
dim=0,
|
||||
)
|
||||
if (
|
||||
self.moe_config.pcp_size > 1
|
||||
and not self.moe_config.moe_parallel_config.use_all2all_kernels
|
||||
):
|
||||
hidden_states = get_pcp_group().all_gather(hidden_states, dim=0)
|
||||
router_logits = get_pcp_group().all_gather(router_logits, dim=0)
|
||||
|
||||
return hidden_states, router_logits
|
||||
|
||||
@@ -777,11 +771,11 @@ class MoERunner(MoERunnerInterface):
|
||||
hidden_states, self.moe_config.is_sequence_parallel
|
||||
)
|
||||
|
||||
if self.moe_config.pcp_size > 1:
|
||||
hidden_states = get_pcp_group().reduce_scatter(
|
||||
hidden_states,
|
||||
dim=0,
|
||||
)
|
||||
if (
|
||||
self.moe_config.pcp_size > 1
|
||||
and not self.moe_config.moe_parallel_config.use_all2all_kernels
|
||||
):
|
||||
hidden_states = get_pcp_group().reduce_scatter(hidden_states, dim=0)
|
||||
|
||||
if self.shared_experts is not None:
|
||||
assert shared_output is not None
|
||||
|
||||
@@ -9,10 +9,11 @@ from vllm import _custom_ops as ops
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
|
||||
from vllm.config import get_current_vllm_config
|
||||
from vllm.distributed import get_dcp_group
|
||||
from vllm.distributed import get_dcp_group, get_pcp_group
|
||||
from vllm.forward_context import get_forward_context
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.custom_op import CustomOp
|
||||
from vllm.model_executor.layers.attention.pcp import maybe_gather_indexer_k
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
get_fp8_min_max,
|
||||
)
|
||||
@@ -308,6 +309,7 @@ def sparse_attn_indexer(
|
||||
total_seq_lens: int,
|
||||
topk_indices_buffer: torch.Tensor,
|
||||
skip_k_cache_insert: bool,
|
||||
use_pcp: bool,
|
||||
use_fp4_cache: bool = False,
|
||||
dcp_rank: int = 0,
|
||||
dcp_world_size: int = 1,
|
||||
@@ -354,6 +356,7 @@ def sparse_attn_indexer(
|
||||
total_seq_lens,
|
||||
topk_indices_buffer,
|
||||
skip_k_cache_insert,
|
||||
use_pcp,
|
||||
use_fp4_cache,
|
||||
)
|
||||
attn_metadata_narrowed = attn_metadata[k_cache_prefix]
|
||||
@@ -373,18 +376,28 @@ def sparse_attn_indexer(
|
||||
# During speculative decoding, k may be padded to the CUDA graph batch
|
||||
# size while slot_mapping only covers actual tokens. Truncate k to avoid
|
||||
# out-of-bounds reads in the kernel.
|
||||
# Keep PCP padding so every rank contributes the same all-gather shape.
|
||||
num_tokens = slot_mapping.shape[0]
|
||||
if use_pcp:
|
||||
num_tokens //= get_pcp_group().world_size
|
||||
if k is not None:
|
||||
k = k[:num_tokens]
|
||||
|
||||
if not skip_k_cache_insert:
|
||||
assert k is not None
|
||||
k, slot_mapping_for_cache = maybe_gather_indexer_k(
|
||||
k,
|
||||
slot_mapping,
|
||||
num_decode_tokens,
|
||||
use_pcp,
|
||||
)
|
||||
# scale_fmt can be None, but the function expects str
|
||||
assert scale_fmt is not None
|
||||
assert not use_fp4_cache, "Unfused FP4 Insert is not supported yet"
|
||||
ops.indexer_k_quant_and_cache(
|
||||
k,
|
||||
kv_cache,
|
||||
slot_mapping,
|
||||
slot_mapping_for_cache,
|
||||
quant_block_size,
|
||||
scale_fmt,
|
||||
)
|
||||
@@ -498,7 +511,14 @@ def sparse_attn_indexer(
|
||||
assert decode_metadata is not None
|
||||
kv_cache = kv_cache_as_quant_view(kv_cache, head_dim, use_fp4_cache)
|
||||
decode_lens = decode_metadata.decode_lens
|
||||
if decode_metadata.requires_padding:
|
||||
if num_decode_tokens == 0:
|
||||
padded_q_quant_decode_tokens = q_quant[:1].reshape(1, 1, *q_quant.shape[1:])
|
||||
padded_q_scale = (
|
||||
q_scale[:1].reshape(1, 1, *q_scale.shape[1:])
|
||||
if q_scale is not None
|
||||
else None
|
||||
)
|
||||
elif decode_metadata.requires_padding:
|
||||
# pad in edge case where we have short chunked prefill length <
|
||||
# decode_threshold since we unstrictly split
|
||||
# prefill and decode by decode_threshold
|
||||
@@ -663,6 +683,7 @@ def sparse_attn_indexer_fake(
|
||||
total_seq_lens: int,
|
||||
topk_indices_buffer: torch.Tensor | None,
|
||||
skip_k_cache_insert: bool,
|
||||
use_pcp: bool,
|
||||
use_fp4_cache: bool = False,
|
||||
dcp_rank: int = 0,
|
||||
dcp_world_size: int = 1,
|
||||
@@ -725,6 +746,7 @@ class SparseAttnIndexer(CustomOp):
|
||||
self.dcp_world_size = parallel_config.decode_context_parallel_size
|
||||
self.dcp_rank = get_dcp_group().rank_in_group if self.dcp_world_size > 1 else 0
|
||||
self.cp_kv_cache_interleave_size = parallel_config.cp_kv_cache_interleave_size
|
||||
self.use_pcp = parallel_config.prefill_context_parallel_size > 1
|
||||
if current_platform.is_cuda() and not has_deep_gemm():
|
||||
raise RuntimeError(
|
||||
"Sparse Attention Indexer CUDA op requires DeepGEMM support in "
|
||||
@@ -777,6 +799,7 @@ class SparseAttnIndexer(CustomOp):
|
||||
self.max_total_seq_len,
|
||||
self.topk_indices_buffer,
|
||||
self.skip_k_cache_insert,
|
||||
self.use_pcp,
|
||||
self.use_fp4_cache,
|
||||
self.dcp_rank,
|
||||
self.dcp_world_size,
|
||||
|
||||
Reference in New Issue
Block a user