[Perf][GLM-5.3-Flash] Shard long-context indexer prefill rows across TP (#54951)
Signed-off-by: Zheng Cai <8370601+zigzagcai@users.noreply.github.com> Signed-off-by: Zheng Cai <caizheng1993@gmail.com> Signed-off-by: Codex <noreply@openai.com> Co-authored-by: root <root@rjob-fb29567b88fec0a9-88c9a4c797eb2ec1-0.caizheng.ailab-sys.svc.pjlab.local> Co-authored-by: OpenAI Codex <noreply@openai.com> Co-authored-by: zigzagcai <zigzagcai@users.noreply.github.com> Co-authored-by: Claude Code <noreply@anthropic.com> Co-authored-by: Wentao Ye <44945378+yewentao256@users.noreply.github.com>
This commit is contained in:
co-authored by
root
OpenAI Codex
zigzagcai
Claude Code
Wentao Ye
parent
bd5bbce6e4
commit
3e8e1364ff
@@ -9,8 +9,17 @@ import vllm.envs as envs
|
||||
from vllm import _custom_ops as ops
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
|
||||
from vllm.config import CUDAGraphMode, get_current_vllm_config
|
||||
from vllm.distributed import get_dcp_group, get_pcp_group
|
||||
from vllm.config import (
|
||||
CUDAGraphMode,
|
||||
get_current_vllm_config,
|
||||
get_current_vllm_config_or_none,
|
||||
)
|
||||
from vllm.distributed import (
|
||||
get_dcp_group,
|
||||
get_pcp_group,
|
||||
get_tensor_model_parallel_rank,
|
||||
get_tp_group,
|
||||
)
|
||||
from vllm.forward_context import get_forward_context
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.custom_op import CustomOp
|
||||
@@ -43,6 +52,7 @@ from vllm.utils.torch_utils import (
|
||||
)
|
||||
from vllm.v1.attention.backends.mla.indexer import (
|
||||
DeepseekV32IndexerMetadata,
|
||||
tp_prefill_row_sharding_supported,
|
||||
)
|
||||
from vllm.v1.attention.ops.common import pack_seq_triton, unpack_seq_triton
|
||||
from vllm.v1.attention.ops.pcp import maybe_gather_indexer_k
|
||||
@@ -317,6 +327,75 @@ def kv_cache_as_quant_view(
|
||||
return kv_cache.unsqueeze(-2)
|
||||
|
||||
|
||||
class PrefillRowShard:
|
||||
"""One TP rank's contiguous slice of the replicated prefill rows.
|
||||
|
||||
Rows are independent, so each rank scores only its slice and the group
|
||||
exchanges final int32 top-k indices instead of the logits."""
|
||||
|
||||
def __init__(self, num_decode_tokens: int, sizes: list[int], rank: int):
|
||||
self.num_decode_tokens = num_decode_tokens
|
||||
self.sizes = sizes
|
||||
self.start = num_decode_tokens + sum(sizes[:rank])
|
||||
self.stop = self.start + sizes[rank]
|
||||
|
||||
@classmethod
|
||||
def from_metadata(
|
||||
cls, prefill_metadata, num_decode_tokens: int
|
||||
) -> "PrefillRowShard | None":
|
||||
sizes = getattr(prefill_metadata, "row_shard_sizes", None)
|
||||
if sizes is None:
|
||||
return None
|
||||
return cls(num_decode_tokens, sizes, get_tensor_model_parallel_rank())
|
||||
|
||||
def narrow(self, chunk) -> "tuple[int, int, torch.Tensor, torch.Tensor] | None":
|
||||
"""Row window and narrowed causal bounds of ``chunk`` for this rank."""
|
||||
start = max(chunk.token_start, self.start)
|
||||
end = min(chunk.token_end, self.stop)
|
||||
if start >= end:
|
||||
return None
|
||||
lo, hi = start - chunk.token_start, end - chunk.token_start
|
||||
return start, end, chunk.cu_seqlen_ks[lo:hi], chunk.cu_seqlen_ke[lo:hi]
|
||||
|
||||
def exchange_topk(self, topk_buffer: torch.Tensor, width: int) -> None:
|
||||
"""Gather per-rank final indices back into global row order."""
|
||||
# The local input must outlive the async PyNCCL gather's enqueued copy.
|
||||
local = topk_buffer[self.start : self.stop, :width].contiguous()
|
||||
end = self.num_decode_tokens + sum(self.sizes)
|
||||
topk_buffer[self.num_decode_tokens : end, :width] = get_tp_group().all_gatherv(
|
||||
local, dim=0, sizes=self.sizes
|
||||
)
|
||||
|
||||
|
||||
def _reserve_row_shard_exchange_transient(
|
||||
topk_indices_buffer: torch.Tensor, device: torch.device
|
||||
) -> None:
|
||||
"""Profiling/dummy runs return before ``exchange_topk`` executes; reserve
|
||||
its worst-case transient so the memory profile accounts for it.
|
||||
|
||||
The exchange stages this rank's contiguous slice and the all_gatherv
|
||||
wrapper allocates the full gathered tensor; each is bounded by the
|
||||
buffer's row capacity times its (tile-padded) width.
|
||||
"""
|
||||
cfg = get_current_vllm_config_or_none()
|
||||
if cfg is None:
|
||||
return
|
||||
parallel = cfg.parallel_config
|
||||
if not tp_prefill_row_sharding_supported(
|
||||
cfg,
|
||||
parallel.decode_context_parallel_size,
|
||||
parallel.prefill_context_parallel_size > 1,
|
||||
get_tp_group().world_size,
|
||||
):
|
||||
return
|
||||
rows, width = topk_indices_buffer.shape[:2]
|
||||
_ = torch.empty(
|
||||
2 * rows * width * topk_indices_buffer.element_size(),
|
||||
dtype=torch.uint8,
|
||||
device=device,
|
||||
)
|
||||
|
||||
|
||||
@eager_break_during_capture
|
||||
def sparse_attn_indexer(
|
||||
hidden_states: torch.Tensor,
|
||||
@@ -388,6 +467,10 @@ def sparse_attn_indexer(
|
||||
max_logits_elems, dtype=torch.uint8, device=hidden_states.device
|
||||
)
|
||||
|
||||
# The dummy run returns before exchange_topk executes, so reserve its
|
||||
# worst-case transient for the memory profile.
|
||||
_reserve_row_shard_exchange_transient(topk_indices_buffer, hidden_states.device)
|
||||
|
||||
return sparse_attn_indexer_fake(
|
||||
hidden_states,
|
||||
k_cache_prefix,
|
||||
@@ -515,6 +598,12 @@ def sparse_attn_indexer(
|
||||
scales_spec,
|
||||
*gather_specs,
|
||||
)
|
||||
# One row window per rank; DSV4.1 candidate rows stay replicated.
|
||||
row_shard = (
|
||||
None
|
||||
if candidate_blocks is not None
|
||||
else PrefillRowShard.from_metadata(prefill_metadata, num_decode_tokens)
|
||||
)
|
||||
for chunk in prefill_metadata.chunks:
|
||||
cu_seqlen_ks = chunk.cu_seqlen_ks
|
||||
cu_seqlen_ke = chunk.cu_seqlen_ke
|
||||
@@ -530,6 +619,15 @@ def sparse_attn_indexer(
|
||||
chunk.local_cu_seq_lens,
|
||||
)
|
||||
|
||||
# Narrow scoring to this rank; KV gathers remain unconditional for
|
||||
# continuation chunks that reuse the workspace.
|
||||
row_start, row_end = chunk.token_start, chunk.token_end
|
||||
if row_shard is not None:
|
||||
narrowed = row_shard.narrow(chunk)
|
||||
if narrowed is None:
|
||||
continue
|
||||
row_start, row_end, cu_seqlen_ks, cu_seqlen_ke = narrowed
|
||||
|
||||
# PCP + DCP KV all-gather.
|
||||
deinterleave_idx = chunk.pcp_deinterleave_idx
|
||||
if deinterleave_idx is not None:
|
||||
@@ -547,15 +645,9 @@ def sparse_attn_indexer(
|
||||
out_scales,
|
||||
)
|
||||
|
||||
q_slice = q_quant[chunk.token_start : chunk.token_end]
|
||||
q_scale_slice = (
|
||||
q_scale[chunk.token_start : chunk.token_end]
|
||||
if q_scale is not None
|
||||
else None
|
||||
)
|
||||
topk_indices = topk_indices_buffer[
|
||||
chunk.token_start : chunk.token_end, :topk_tokens
|
||||
]
|
||||
q_slice = q_quant[row_start:row_end]
|
||||
q_scale_slice = q_scale[row_start:row_end] if q_scale is not None else None
|
||||
topk_indices = topk_indices_buffer[row_start:row_end, :topk_tokens]
|
||||
|
||||
if chunk.local_total_seq_lens == 0 or q_slice.shape[0] == 0:
|
||||
logits = q_slice.new_empty((q_slice.shape[0], 0), dtype=torch.float32)
|
||||
@@ -578,7 +670,7 @@ def sparse_attn_indexer(
|
||||
q_slice_cast,
|
||||
k_quant_cast,
|
||||
k_scale_cast,
|
||||
weights[chunk.token_start : chunk.token_end],
|
||||
weights[row_start:row_end],
|
||||
cu_seqlen_ks,
|
||||
cu_seqlen_ke,
|
||||
)
|
||||
@@ -586,7 +678,7 @@ def sparse_attn_indexer(
|
||||
logits = fp8_fp4_mqa_logits(
|
||||
(q_slice_cast, q_scale_slice),
|
||||
(k_quant_cast, k_scale_cast),
|
||||
weights[chunk.token_start : chunk.token_end],
|
||||
weights[row_start:row_end],
|
||||
cu_seqlen_ks,
|
||||
cu_seqlen_ke,
|
||||
clean_logits=False,
|
||||
@@ -637,9 +729,13 @@ def sparse_attn_indexer(
|
||||
dcp_rank,
|
||||
dcp_world_size,
|
||||
cp_kv_cache_interleave_size,
|
||||
row_starts=chunk.cu_seqlen_ks,
|
||||
row_starts=cu_seqlen_ks,
|
||||
)
|
||||
|
||||
if row_shard is not None:
|
||||
# Rows are already fully ranked, so this is a layout gather.
|
||||
row_shard.exchange_topk(topk_indices_buffer, topk_tokens)
|
||||
|
||||
if has_decode:
|
||||
decode_metadata = attn_metadata_narrowed.decode
|
||||
assert decode_metadata is not None
|
||||
|
||||
Reference in New Issue
Block a user