[Perf][GLM-5.3-Flash] Shard long-context indexer prefill rows across TP (#54951)

Signed-off-by: Zheng Cai <8370601+zigzagcai@users.noreply.github.com>
Signed-off-by: Zheng Cai <caizheng1993@gmail.com>
Signed-off-by: Codex <noreply@openai.com>
Co-authored-by: root <root@rjob-fb29567b88fec0a9-88c9a4c797eb2ec1-0.caizheng.ailab-sys.svc.pjlab.local>
Co-authored-by: OpenAI Codex <noreply@openai.com>
Co-authored-by: zigzagcai <zigzagcai@users.noreply.github.com>
Co-authored-by: Claude Code <noreply@anthropic.com>
Co-authored-by: Wentao Ye <44945378+yewentao256@users.noreply.github.com>
This commit is contained in:
Zheng Cai
2026-10-11 10:43:37 -04:00
committed by GitHub
co-authored by root OpenAI Codex zigzagcai Claude Code Wentao Ye
parent bd5bbce6e4
commit 3e8e1364ff
4 changed files with 436 additions and 38 deletions
+110 -14
View File
@@ -9,8 +9,17 @@ import vllm.envs as envs
from vllm import _custom_ops as ops
from vllm._aiter_ops import rocm_aiter_ops
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
from vllm.config import CUDAGraphMode, get_current_vllm_config
from vllm.distributed import get_dcp_group, get_pcp_group
from vllm.config import (
CUDAGraphMode,
get_current_vllm_config,
get_current_vllm_config_or_none,
)
from vllm.distributed import (
get_dcp_group,
get_pcp_group,
get_tensor_model_parallel_rank,
get_tp_group,
)
from vllm.forward_context import get_forward_context
from vllm.logger import init_logger
from vllm.model_executor.custom_op import CustomOp
@@ -43,6 +52,7 @@ from vllm.utils.torch_utils import (
)
from vllm.v1.attention.backends.mla.indexer import (
DeepseekV32IndexerMetadata,
tp_prefill_row_sharding_supported,
)
from vllm.v1.attention.ops.common import pack_seq_triton, unpack_seq_triton
from vllm.v1.attention.ops.pcp import maybe_gather_indexer_k
@@ -317,6 +327,75 @@ def kv_cache_as_quant_view(
return kv_cache.unsqueeze(-2)
class PrefillRowShard:
"""One TP rank's contiguous slice of the replicated prefill rows.
Rows are independent, so each rank scores only its slice and the group
exchanges final int32 top-k indices instead of the logits."""
def __init__(self, num_decode_tokens: int, sizes: list[int], rank: int):
self.num_decode_tokens = num_decode_tokens
self.sizes = sizes
self.start = num_decode_tokens + sum(sizes[:rank])
self.stop = self.start + sizes[rank]
@classmethod
def from_metadata(
cls, prefill_metadata, num_decode_tokens: int
) -> "PrefillRowShard | None":
sizes = getattr(prefill_metadata, "row_shard_sizes", None)
if sizes is None:
return None
return cls(num_decode_tokens, sizes, get_tensor_model_parallel_rank())
def narrow(self, chunk) -> "tuple[int, int, torch.Tensor, torch.Tensor] | None":
"""Row window and narrowed causal bounds of ``chunk`` for this rank."""
start = max(chunk.token_start, self.start)
end = min(chunk.token_end, self.stop)
if start >= end:
return None
lo, hi = start - chunk.token_start, end - chunk.token_start
return start, end, chunk.cu_seqlen_ks[lo:hi], chunk.cu_seqlen_ke[lo:hi]
def exchange_topk(self, topk_buffer: torch.Tensor, width: int) -> None:
"""Gather per-rank final indices back into global row order."""
# The local input must outlive the async PyNCCL gather's enqueued copy.
local = topk_buffer[self.start : self.stop, :width].contiguous()
end = self.num_decode_tokens + sum(self.sizes)
topk_buffer[self.num_decode_tokens : end, :width] = get_tp_group().all_gatherv(
local, dim=0, sizes=self.sizes
)
def _reserve_row_shard_exchange_transient(
topk_indices_buffer: torch.Tensor, device: torch.device
) -> None:
"""Profiling/dummy runs return before ``exchange_topk`` executes; reserve
its worst-case transient so the memory profile accounts for it.
The exchange stages this rank's contiguous slice and the all_gatherv
wrapper allocates the full gathered tensor; each is bounded by the
buffer's row capacity times its (tile-padded) width.
"""
cfg = get_current_vllm_config_or_none()
if cfg is None:
return
parallel = cfg.parallel_config
if not tp_prefill_row_sharding_supported(
cfg,
parallel.decode_context_parallel_size,
parallel.prefill_context_parallel_size > 1,
get_tp_group().world_size,
):
return
rows, width = topk_indices_buffer.shape[:2]
_ = torch.empty(
2 * rows * width * topk_indices_buffer.element_size(),
dtype=torch.uint8,
device=device,
)
@eager_break_during_capture
def sparse_attn_indexer(
hidden_states: torch.Tensor,
@@ -388,6 +467,10 @@ def sparse_attn_indexer(
max_logits_elems, dtype=torch.uint8, device=hidden_states.device
)
# The dummy run returns before exchange_topk executes, so reserve its
# worst-case transient for the memory profile.
_reserve_row_shard_exchange_transient(topk_indices_buffer, hidden_states.device)
return sparse_attn_indexer_fake(
hidden_states,
k_cache_prefix,
@@ -515,6 +598,12 @@ def sparse_attn_indexer(
scales_spec,
*gather_specs,
)
# One row window per rank; DSV4.1 candidate rows stay replicated.
row_shard = (
None
if candidate_blocks is not None
else PrefillRowShard.from_metadata(prefill_metadata, num_decode_tokens)
)
for chunk in prefill_metadata.chunks:
cu_seqlen_ks = chunk.cu_seqlen_ks
cu_seqlen_ke = chunk.cu_seqlen_ke
@@ -530,6 +619,15 @@ def sparse_attn_indexer(
chunk.local_cu_seq_lens,
)
# Narrow scoring to this rank; KV gathers remain unconditional for
# continuation chunks that reuse the workspace.
row_start, row_end = chunk.token_start, chunk.token_end
if row_shard is not None:
narrowed = row_shard.narrow(chunk)
if narrowed is None:
continue
row_start, row_end, cu_seqlen_ks, cu_seqlen_ke = narrowed
# PCP + DCP KV all-gather.
deinterleave_idx = chunk.pcp_deinterleave_idx
if deinterleave_idx is not None:
@@ -547,15 +645,9 @@ def sparse_attn_indexer(
out_scales,
)
q_slice = q_quant[chunk.token_start : chunk.token_end]
q_scale_slice = (
q_scale[chunk.token_start : chunk.token_end]
if q_scale is not None
else None
)
topk_indices = topk_indices_buffer[
chunk.token_start : chunk.token_end, :topk_tokens
]
q_slice = q_quant[row_start:row_end]
q_scale_slice = q_scale[row_start:row_end] if q_scale is not None else None
topk_indices = topk_indices_buffer[row_start:row_end, :topk_tokens]
if chunk.local_total_seq_lens == 0 or q_slice.shape[0] == 0:
logits = q_slice.new_empty((q_slice.shape[0], 0), dtype=torch.float32)
@@ -578,7 +670,7 @@ def sparse_attn_indexer(
q_slice_cast,
k_quant_cast,
k_scale_cast,
weights[chunk.token_start : chunk.token_end],
weights[row_start:row_end],
cu_seqlen_ks,
cu_seqlen_ke,
)
@@ -586,7 +678,7 @@ def sparse_attn_indexer(
logits = fp8_fp4_mqa_logits(
(q_slice_cast, q_scale_slice),
(k_quant_cast, k_scale_cast),
weights[chunk.token_start : chunk.token_end],
weights[row_start:row_end],
cu_seqlen_ks,
cu_seqlen_ke,
clean_logits=False,
@@ -637,9 +729,13 @@ def sparse_attn_indexer(
dcp_rank,
dcp_world_size,
cp_kv_cache_interleave_size,
row_starts=chunk.cu_seqlen_ks,
row_starts=cu_seqlen_ks,
)
if row_shard is not None:
# Rows are already fully ranked, so this is a layout gather.
row_shard.exchange_topk(topk_indices_buffer, topk_tokens)
if has_decode:
decode_metadata = attn_metadata_narrowed.decode
assert decode_metadata is not None