[Core] Add MRV2 virtual-batch PCP for MLA (#46570)
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Signed-off-by: Matthew Bonanni <mbonanni@redhat.com> Co-authored-by: Codex <noreply@openai.com> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: Matthew Bonanni <mbonanni@redhat.com>
This commit is contained in:
co-authored by
Codex
OpenAI Codex
Matthew Bonanni
parent
9243e0124e
commit
b6ff8a2f50
@@ -9,10 +9,11 @@ from vllm import _custom_ops as ops
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
|
||||
from vllm.config import get_current_vllm_config
|
||||
from vllm.distributed import get_dcp_group
|
||||
from vllm.distributed import get_dcp_group, get_pcp_group
|
||||
from vllm.forward_context import get_forward_context
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.custom_op import CustomOp
|
||||
from vllm.model_executor.layers.attention.pcp import maybe_gather_indexer_k
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
get_fp8_min_max,
|
||||
)
|
||||
@@ -308,6 +309,7 @@ def sparse_attn_indexer(
|
||||
total_seq_lens: int,
|
||||
topk_indices_buffer: torch.Tensor,
|
||||
skip_k_cache_insert: bool,
|
||||
use_pcp: bool,
|
||||
use_fp4_cache: bool = False,
|
||||
dcp_rank: int = 0,
|
||||
dcp_world_size: int = 1,
|
||||
@@ -354,6 +356,7 @@ def sparse_attn_indexer(
|
||||
total_seq_lens,
|
||||
topk_indices_buffer,
|
||||
skip_k_cache_insert,
|
||||
use_pcp,
|
||||
use_fp4_cache,
|
||||
)
|
||||
attn_metadata_narrowed = attn_metadata[k_cache_prefix]
|
||||
@@ -373,18 +376,28 @@ def sparse_attn_indexer(
|
||||
# During speculative decoding, k may be padded to the CUDA graph batch
|
||||
# size while slot_mapping only covers actual tokens. Truncate k to avoid
|
||||
# out-of-bounds reads in the kernel.
|
||||
# Keep PCP padding so every rank contributes the same all-gather shape.
|
||||
num_tokens = slot_mapping.shape[0]
|
||||
if use_pcp:
|
||||
num_tokens //= get_pcp_group().world_size
|
||||
if k is not None:
|
||||
k = k[:num_tokens]
|
||||
|
||||
if not skip_k_cache_insert:
|
||||
assert k is not None
|
||||
k, slot_mapping_for_cache = maybe_gather_indexer_k(
|
||||
k,
|
||||
slot_mapping,
|
||||
num_decode_tokens,
|
||||
use_pcp,
|
||||
)
|
||||
# scale_fmt can be None, but the function expects str
|
||||
assert scale_fmt is not None
|
||||
assert not use_fp4_cache, "Unfused FP4 Insert is not supported yet"
|
||||
ops.indexer_k_quant_and_cache(
|
||||
k,
|
||||
kv_cache,
|
||||
slot_mapping,
|
||||
slot_mapping_for_cache,
|
||||
quant_block_size,
|
||||
scale_fmt,
|
||||
)
|
||||
@@ -498,7 +511,14 @@ def sparse_attn_indexer(
|
||||
assert decode_metadata is not None
|
||||
kv_cache = kv_cache_as_quant_view(kv_cache, head_dim, use_fp4_cache)
|
||||
decode_lens = decode_metadata.decode_lens
|
||||
if decode_metadata.requires_padding:
|
||||
if num_decode_tokens == 0:
|
||||
padded_q_quant_decode_tokens = q_quant[:1].reshape(1, 1, *q_quant.shape[1:])
|
||||
padded_q_scale = (
|
||||
q_scale[:1].reshape(1, 1, *q_scale.shape[1:])
|
||||
if q_scale is not None
|
||||
else None
|
||||
)
|
||||
elif decode_metadata.requires_padding:
|
||||
# pad in edge case where we have short chunked prefill length <
|
||||
# decode_threshold since we unstrictly split
|
||||
# prefill and decode by decode_threshold
|
||||
@@ -663,6 +683,7 @@ def sparse_attn_indexer_fake(
|
||||
total_seq_lens: int,
|
||||
topk_indices_buffer: torch.Tensor | None,
|
||||
skip_k_cache_insert: bool,
|
||||
use_pcp: bool,
|
||||
use_fp4_cache: bool = False,
|
||||
dcp_rank: int = 0,
|
||||
dcp_world_size: int = 1,
|
||||
@@ -725,6 +746,7 @@ class SparseAttnIndexer(CustomOp):
|
||||
self.dcp_world_size = parallel_config.decode_context_parallel_size
|
||||
self.dcp_rank = get_dcp_group().rank_in_group if self.dcp_world_size > 1 else 0
|
||||
self.cp_kv_cache_interleave_size = parallel_config.cp_kv_cache_interleave_size
|
||||
self.use_pcp = parallel_config.prefill_context_parallel_size > 1
|
||||
if current_platform.is_cuda() and not has_deep_gemm():
|
||||
raise RuntimeError(
|
||||
"Sparse Attention Indexer CUDA op requires DeepGEMM support in "
|
||||
@@ -777,6 +799,7 @@ class SparseAttnIndexer(CustomOp):
|
||||
self.max_total_seq_len,
|
||||
self.topk_indices_buffer,
|
||||
self.skip_k_cache_insert,
|
||||
self.use_pcp,
|
||||
self.use_fp4_cache,
|
||||
self.dcp_rank,
|
||||
self.dcp_world_size,
|
||||
|
||||
Reference in New Issue
Block a user