[Perf][GLM-5.3-Flash] Shard long-context indexer prefill rows across TP (#54951)

Signed-off-by: Zheng Cai <8370601+zigzagcai@users.noreply.github.com>
Signed-off-by: Zheng Cai <caizheng1993@gmail.com>
Signed-off-by: Codex <noreply@openai.com>
Co-authored-by: root <root@rjob-fb29567b88fec0a9-88c9a4c797eb2ec1-0.caizheng.ailab-sys.svc.pjlab.local>
Co-authored-by: OpenAI Codex <noreply@openai.com>
Co-authored-by: zigzagcai <zigzagcai@users.noreply.github.com>
Co-authored-by: Claude Code <noreply@anthropic.com>
Co-authored-by: Wentao Ye <44945378+yewentao256@users.noreply.github.com>
This commit is contained in:
Zheng Cai
2026-10-11 10:43:37 -04:00
committed by GitHub
co-authored by root OpenAI Codex zigzagcai Claude Code Wentao Ye
parent bd5bbce6e4
commit 3e8e1364ff
4 changed files with 436 additions and 38 deletions
+40 -23
View File
@@ -12,7 +12,11 @@ from vllm.forward_context import get_forward_context
from vllm.logger import init_logger
from vllm.model_executor.custom_op import CustomOp
from vllm.model_executor.layers.indexer_topk import get_indexer_topk
from vllm.model_executor.layers.sparse_attn_indexer import _merge_dcp_topk_global
from vllm.model_executor.layers.sparse_attn_indexer import (
PrefillRowShard,
_merge_dcp_topk_global,
_reserve_row_shard_exchange_transient,
)
from vllm.models.glm5next.common.sparse_indexer import (
RADIX_TOPK_WORKSPACE_SIZE,
_build_decode_scatter_indices,
@@ -169,6 +173,10 @@ def sparse_attn_indexer_kpool(
max_logits_elems, dtype=torch.uint8, device=hidden_states.device
)
# The dummy run returns before exchange_topk executes, so reserve its
# worst-case transient for the memory profile.
_reserve_row_shard_exchange_transient(topk_indices_buffer, hidden_states.device)
return topk_indices_buffer
attn_metadata_narrowed = attn_metadata[k_cache_prefix]
assert isinstance(attn_metadata_narrowed, DeepseekV32IndexerMetadata)
@@ -282,6 +290,12 @@ def sparse_attn_indexer_kpool(
_pos = positions[num_decode_tokens:num_tokens].to(torch.int32)
_buf = topk_indices_buffer[num_decode_tokens:num_tokens]
_fill_causal_indices(_buf, _pos)
# Short prefills score nothing locally; suppress the shard there.
row_shard = None
if not short_prefill:
row_shard = PrefillRowShard.from_metadata(
prefill_metadata, num_decode_tokens
)
# Get the full shared workspace buffers once (will allocate on first use).
# Layout switches between FP8 (head_dim bytes + 4-byte fp32 scale) and
@@ -310,12 +324,16 @@ def sparse_attn_indexer_kpool(
chunk.local_cu_seq_lens,
)
q_slice = q_quant[chunk.token_start : chunk.token_end]
q_scale_slice = (
q_scale[chunk.token_start : chunk.token_end]
if q_scale is not None
else None
)
row_start, row_end = chunk.token_start, chunk.token_end
cu_seqlen_ks, cu_seqlen_ke = chunk.cu_seqlen_ks, chunk.cu_seqlen_ke
if row_shard is not None:
narrowed = row_shard.narrow(chunk)
if narrowed is None:
continue
row_start, row_end, cu_seqlen_ks, cu_seqlen_ke = narrowed
q_slice = q_quant[row_start:row_end]
q_scale_slice = q_scale[row_start:row_end] if q_scale is not None else None
# DeepGEMM scalar-type tags (zero-copy): MXFP4 values → int8
# (kPackedFP4), scales → int32 squeezed to 1-D kv_sf / 2-D q_sf.
if use_fp4_cache:
@@ -335,9 +353,9 @@ def sparse_attn_indexer_kpool(
logits = fp8_fp4_mqa_logits(
(q_slice_cast, q_scale_slice),
(k_quant_cast, k_scale_cast),
weights[chunk.token_start : chunk.token_end],
chunk.cu_seqlen_ks,
chunk.cu_seqlen_ke,
weights[row_start:row_end],
cu_seqlen_ks,
cu_seqlen_ke,
clean_logits=False,
)
@@ -351,17 +369,15 @@ def sparse_attn_indexer_kpool(
)
topk_dst = pool_topk
else:
topk_dst = topk_indices_buffer[
chunk.token_start : chunk.token_end, :topk_tokens
]
topk_dst = topk_indices_buffer[row_start:row_end, :topk_tokens]
if logits.shape[1] == 0:
topk_dst.fill_(-1)
else:
torch.ops._C.top_k_per_row_prefill(
logits,
chunk.cu_seqlen_ks,
chunk.cu_seqlen_ke,
cu_seqlen_ks,
cu_seqlen_ke,
topk_dst,
num_rows,
logits.stride(0),
@@ -376,7 +392,7 @@ def sparse_attn_indexer_kpool(
dcp_rank,
dcp_world_size,
pool_cp_interleave,
row_starts=chunk.cu_seqlen_ks,
row_starts=cu_seqlen_ks,
)
if index_kpool > 1:
@@ -385,10 +401,7 @@ def sparse_attn_indexer_kpool(
# Fused expand-pools + append-tail into one Triton kernel
# (replaces ~25 elementwise ops). seq_len is token-granular
# (pos+1); the kernel derives pool_len internally.
q_seq = (
positions[chunk.token_start : chunk.token_end].to(torch.int32)
+ 1
)
q_seq = positions[row_start:row_end].to(torch.int32) + 1
expanded = kpool_ops.expand_pools_and_append_tail(
pool_ids, q_seq, index_kpool
)
@@ -397,9 +410,13 @@ def sparse_attn_indexer_kpool(
expanded = kpool_ops.expand_pools_to_tokens(
pool_ids, valid, topk_tokens, index_kpool
)
topk_indices_buffer[
chunk.token_start : chunk.token_end, : expanded.shape[-1]
] = expanded
topk_indices_buffer[row_start:row_end, : expanded.shape[-1]] = expanded
if row_shard is not None:
# The k-pool expansion appends the request's incomplete tail
# (index_kpool - 1 entries) after the logical top-k history; those
# columns are part of the attention index and are exchanged too.
row_shard.exchange_topk(topk_indices_buffer, topk_tokens + index_kpool - 1)
if has_decode:
decode_metadata = attn_metadata_narrowed.decode