[Perf][GLM-5.3-Flash] Shard long-context indexer prefill rows across TP (#54951)

Signed-off-by: Zheng Cai <8370601+zigzagcai@users.noreply.github.com>
Signed-off-by: Zheng Cai <caizheng1993@gmail.com>
Signed-off-by: Codex <noreply@openai.com>
Co-authored-by: root <root@rjob-fb29567b88fec0a9-88c9a4c797eb2ec1-0.caizheng.ailab-sys.svc.pjlab.local>
Co-authored-by: OpenAI Codex <noreply@openai.com>
Co-authored-by: zigzagcai <zigzagcai@users.noreply.github.com>
Co-authored-by: Claude Code <noreply@anthropic.com>
Co-authored-by: Wentao Ye <44945378+yewentao256@users.noreply.github.com>
This commit is contained in:
Zheng Cai
2026-10-11 10:43:37 -04:00
committed by GitHub
co-authored by root OpenAI Codex zigzagcai Claude Code Wentao Ye
parent bd5bbce6e4
commit 3e8e1364ff
4 changed files with 436 additions and 38 deletions
@@ -0,0 +1,213 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""TP query-row sharding of the DSA indexer prefill (see PrefillRowShard):
each rank scores a contiguous slice; the group exchanges int32s per row."""
from types import SimpleNamespace
import pytest
import torch
import vllm.model_executor.layers.sparse_attn_indexer as sparse_indexer
import vllm.v1.attention.backends.mla.indexer as indexer
from vllm.config import CUDAGraphMode
from vllm.model_executor.layers.sparse_attn_indexer import PrefillRowShard
def _chunk(start: int, end: int) -> SimpleNamespace:
"""Chunk over rows [start, end); bounds derive from the global row."""
n = end - start
return SimpleNamespace(
token_start=start,
token_end=end,
cu_seqlen_ks=torch.zeros(n, dtype=torch.int32),
cu_seqlen_ke=((torch.arange(n) * 7 + start) % 65).int(),
)
@pytest.mark.parametrize("world", [2, 3, 8])
def test_shard_narrowing_covers_every_row_once(world: int) -> None:
"""Narrowed windows tile the prefill rows exactly once, carry the causal
bounds of those global rows, and skip chunks foreign to this rank."""
decode, rows = 5, 997
edges = [decode, decode + 330, decode + 337, decode + rows]
chunks = [_chunk(lo, hi) for lo, hi in zip(edges, edges[1:])]
sizes = [rows // world + int(r < rows % world) for r in range(world)]
covered = []
for rank in range(world):
shard = PrefillRowShard(decode, sizes, rank)
for chunk in chunks:
window = shard.narrow(chunk)
if window is None:
continue
start, end, ks, ke = window
lo, hi = start - chunk.token_start, end - chunk.token_start
torch.testing.assert_close(ks, chunk.cu_seqlen_ks[lo:hi])
torch.testing.assert_close(ke, chunk.cu_seqlen_ke[lo:hi])
covered.append((start, end))
covered.sort()
# Windows must be disjoint, contiguous and exactly cover the prefill rows.
assert [s for s, _ in covered] == [decode, *[e for _, e in covered][:-1]]
assert covered[-1][1] == decode + rows
def test_shard_exchange_reassembles_rows_only(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""Gathered indices land only on real prefill rows: leading decode rows
and trailing CUDA-graph padding stay untouched, and the full k-pool
width (tail columns included) is exchanged."""
decode, pad, topk, kpool = 5, 3, 8, 4
width = topk + kpool - 1
sizes = [17, 9, 21]
buffer = torch.full((decode + sum(sizes) + pad, width), -7, dtype=torch.int32)
gathered = torch.arange(sum(sizes) * width, dtype=torch.int32).reshape(-1, width)
calls = []
def fake_group():
def all_gatherv(local, dim, sizes):
calls.append((local, sizes))
return gathered
return SimpleNamespace(all_gatherv=all_gatherv)
monkeypatch.setattr(sparse_indexer, "get_tp_group", fake_group)
for rank in range(len(sizes)):
PrefillRowShard(decode, sizes, rank).exchange_topk(buffer, width)
assert calls[0][1] == sizes and calls[0][0].is_contiguous()
torch.testing.assert_close(buffer[decode : decode + sum(sizes)], gathered)
assert torch.all(buffer[:decode] == -7), "decode rows stay outside"
assert torch.all(buffer[decode + sum(sizes) :] == -7), "padding untouched"
@pytest.mark.parametrize("tp_size", [2, 4])
def test_balanced_row_shard_partitions_and_declines(tp_size: int) -> None:
"""The split covers every row once with >= 1 row per rank and declines
below the activation floor (or on a single rank)."""
n = indexer.MIN_TP_SHARD_ROWS_PER_RANK * tp_size
shapes = [
([4 * n], [4 * n]), # fresh prompt: the full causal cost ramp
# uneven requests at different context depths
([2 * n, 7, 4 * n, n // 3 + 5], [2 * n, 9007, 4 * n + 500, n // 3 + 60005]),
]
ratio = 4
for query_lens, seq_lens in shapes:
sizes = indexer.balanced_prefill_row_shard(
torch.tensor(seq_lens, dtype=torch.int32),
torch.tensor(query_lens, dtype=torch.int32),
ratio,
tp_size,
)
assert sizes is not None and len(sizes) == tp_size
assert sum(sizes) == sum(query_lens) and min(sizes) >= 1
lens = torch.tensor([n - 1], dtype=torch.int32)
assert indexer.balanced_prefill_row_shard(lens, lens, ratio, tp_size) is None
assert indexer.balanced_prefill_row_shard(lens, lens, ratio, 1) is None
def test_row_sharding_gate_rejects_unsupported_configurations(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""The gate is the whole safety envelope; nothing else guards the exchange."""
monkeypatch.setattr(indexer.current_platform, "is_cuda", lambda: True)
def supported(**kwargs):
mode = kwargs.pop("cudagraph_mode", CUDAGraphMode.PIECEWISE)
cfg = SimpleNamespace(compilation_config=SimpleNamespace(cudagraph_mode=mode))
args = {"dcp_world_size": 1, "use_pcp": False, "tp_size": 4, **kwargs}
return indexer.tp_prefill_row_sharding_supported(cfg, **args)
assert supported()
for bad in ({"tp_size": 1}, {"dcp_world_size": 2}, {"use_pcp": True}):
assert not supported(**bad)
for name in (
"VLLM_DISABLE_PYNCCL",
"VLLM_USE_NCCL_SYMM_MEM",
"VLLM_BATCH_INVARIANT",
):
with monkeypatch.context() as m:
m.setattr(indexer.envs, name, True)
assert not supported()
# FULL captures mixed batches whole, so the exchange must stay out.
assert not supported(cudagraph_mode=CUDAGraphMode.FULL)
assert supported(cudagraph_mode=CUDAGraphMode.FULL_AND_PIECEWISE)
def test_profiling_run_reserves_exchange_transient(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""The dummy run returns before the exchange executes, so its all_gatherv
output plus staging copy must be reserved up front when row sharding is
active -- and only then."""
rows, topk = 128, 8
buffer = torch.empty((rows, topk), dtype=torch.int32)
peak = 2 * rows * topk * buffer.element_size()
specs = (((1,), torch.uint8), ((1,), torch.uint8))
numels: list[int] = []
real_empty = torch.empty
def recording_empty(*args, **kwargs):
out = real_empty(*args, **kwargs)
numels.append(out.numel())
return out
def run_dummy(tp_world: int) -> None:
numels.clear()
with monkeypatch.context() as m:
m.setattr(torch, "empty", recording_empty)
m.setattr(
sparse_indexer,
"get_forward_context",
lambda: SimpleNamespace(attn_metadata=None),
)
m.setattr(sparse_indexer, "_gather_workspace_shapes", lambda *a: specs)
m.setattr(
sparse_indexer,
"current_workspace_manager",
lambda: SimpleNamespace(get_simultaneous=lambda *s: None),
)
m.setattr(
sparse_indexer,
"get_tp_group",
lambda: SimpleNamespace(world_size=tp_world),
)
m.setattr(
sparse_indexer,
"get_current_vllm_config_or_none",
lambda: SimpleNamespace(
compilation_config=SimpleNamespace(
cudagraph_mode=CUDAGraphMode.PIECEWISE
),
parallel_config=SimpleNamespace(
decode_context_parallel_size=1,
prefill_context_parallel_size=1,
),
),
)
m.setattr(sparse_indexer.envs, "VLLM_SPARSE_INDEXER_MAX_LOGITS_MB", 1)
m.setattr(indexer.current_platform, "is_cuda", lambda: True)
sparse_indexer.sparse_attn_indexer(
hidden_states=torch.empty(4, 4),
k_cache_prefix="k",
kv_cache=torch.empty(4, 4),
q_quant=torch.empty(4, 4),
q_scale=None,
k=None,
weights=torch.empty(4),
quant_block_size=128,
scale_fmt=None,
topk_tokens=topk,
head_dim=64,
max_model_len=1024,
total_seq_lens=512,
topk_indices_buffer=buffer,
skip_k_cache_insert=True,
use_pcp=False,
pcp_shard_decode_requests=False,
dense_mha_metadata_layer_name="",
)
run_dummy(tp_world=2)
assert peak in numels
run_dummy(tp_world=1)
assert peak not in numels
+110 -14
View File
@@ -9,8 +9,17 @@ import vllm.envs as envs
from vllm import _custom_ops as ops
from vllm._aiter_ops import rocm_aiter_ops
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
from vllm.config import CUDAGraphMode, get_current_vllm_config
from vllm.distributed import get_dcp_group, get_pcp_group
from vllm.config import (
CUDAGraphMode,
get_current_vllm_config,
get_current_vllm_config_or_none,
)
from vllm.distributed import (
get_dcp_group,
get_pcp_group,
get_tensor_model_parallel_rank,
get_tp_group,
)
from vllm.forward_context import get_forward_context
from vllm.logger import init_logger
from vllm.model_executor.custom_op import CustomOp
@@ -43,6 +52,7 @@ from vllm.utils.torch_utils import (
)
from vllm.v1.attention.backends.mla.indexer import (
DeepseekV32IndexerMetadata,
tp_prefill_row_sharding_supported,
)
from vllm.v1.attention.ops.common import pack_seq_triton, unpack_seq_triton
from vllm.v1.attention.ops.pcp import maybe_gather_indexer_k
@@ -317,6 +327,75 @@ def kv_cache_as_quant_view(
return kv_cache.unsqueeze(-2)
class PrefillRowShard:
"""One TP rank's contiguous slice of the replicated prefill rows.
Rows are independent, so each rank scores only its slice and the group
exchanges final int32 top-k indices instead of the logits."""
def __init__(self, num_decode_tokens: int, sizes: list[int], rank: int):
self.num_decode_tokens = num_decode_tokens
self.sizes = sizes
self.start = num_decode_tokens + sum(sizes[:rank])
self.stop = self.start + sizes[rank]
@classmethod
def from_metadata(
cls, prefill_metadata, num_decode_tokens: int
) -> "PrefillRowShard | None":
sizes = getattr(prefill_metadata, "row_shard_sizes", None)
if sizes is None:
return None
return cls(num_decode_tokens, sizes, get_tensor_model_parallel_rank())
def narrow(self, chunk) -> "tuple[int, int, torch.Tensor, torch.Tensor] | None":
"""Row window and narrowed causal bounds of ``chunk`` for this rank."""
start = max(chunk.token_start, self.start)
end = min(chunk.token_end, self.stop)
if start >= end:
return None
lo, hi = start - chunk.token_start, end - chunk.token_start
return start, end, chunk.cu_seqlen_ks[lo:hi], chunk.cu_seqlen_ke[lo:hi]
def exchange_topk(self, topk_buffer: torch.Tensor, width: int) -> None:
"""Gather per-rank final indices back into global row order."""
# The local input must outlive the async PyNCCL gather's enqueued copy.
local = topk_buffer[self.start : self.stop, :width].contiguous()
end = self.num_decode_tokens + sum(self.sizes)
topk_buffer[self.num_decode_tokens : end, :width] = get_tp_group().all_gatherv(
local, dim=0, sizes=self.sizes
)
def _reserve_row_shard_exchange_transient(
topk_indices_buffer: torch.Tensor, device: torch.device
) -> None:
"""Profiling/dummy runs return before ``exchange_topk`` executes; reserve
its worst-case transient so the memory profile accounts for it.
The exchange stages this rank's contiguous slice and the all_gatherv
wrapper allocates the full gathered tensor; each is bounded by the
buffer's row capacity times its (tile-padded) width.
"""
cfg = get_current_vllm_config_or_none()
if cfg is None:
return
parallel = cfg.parallel_config
if not tp_prefill_row_sharding_supported(
cfg,
parallel.decode_context_parallel_size,
parallel.prefill_context_parallel_size > 1,
get_tp_group().world_size,
):
return
rows, width = topk_indices_buffer.shape[:2]
_ = torch.empty(
2 * rows * width * topk_indices_buffer.element_size(),
dtype=torch.uint8,
device=device,
)
@eager_break_during_capture
def sparse_attn_indexer(
hidden_states: torch.Tensor,
@@ -388,6 +467,10 @@ def sparse_attn_indexer(
max_logits_elems, dtype=torch.uint8, device=hidden_states.device
)
# The dummy run returns before exchange_topk executes, so reserve its
# worst-case transient for the memory profile.
_reserve_row_shard_exchange_transient(topk_indices_buffer, hidden_states.device)
return sparse_attn_indexer_fake(
hidden_states,
k_cache_prefix,
@@ -515,6 +598,12 @@ def sparse_attn_indexer(
scales_spec,
*gather_specs,
)
# One row window per rank; DSV4.1 candidate rows stay replicated.
row_shard = (
None
if candidate_blocks is not None
else PrefillRowShard.from_metadata(prefill_metadata, num_decode_tokens)
)
for chunk in prefill_metadata.chunks:
cu_seqlen_ks = chunk.cu_seqlen_ks
cu_seqlen_ke = chunk.cu_seqlen_ke
@@ -530,6 +619,15 @@ def sparse_attn_indexer(
chunk.local_cu_seq_lens,
)
# Narrow scoring to this rank; KV gathers remain unconditional for
# continuation chunks that reuse the workspace.
row_start, row_end = chunk.token_start, chunk.token_end
if row_shard is not None:
narrowed = row_shard.narrow(chunk)
if narrowed is None:
continue
row_start, row_end, cu_seqlen_ks, cu_seqlen_ke = narrowed
# PCP + DCP KV all-gather.
deinterleave_idx = chunk.pcp_deinterleave_idx
if deinterleave_idx is not None:
@@ -547,15 +645,9 @@ def sparse_attn_indexer(
out_scales,
)
q_slice = q_quant[chunk.token_start : chunk.token_end]
q_scale_slice = (
q_scale[chunk.token_start : chunk.token_end]
if q_scale is not None
else None
)
topk_indices = topk_indices_buffer[
chunk.token_start : chunk.token_end, :topk_tokens
]
q_slice = q_quant[row_start:row_end]
q_scale_slice = q_scale[row_start:row_end] if q_scale is not None else None
topk_indices = topk_indices_buffer[row_start:row_end, :topk_tokens]
if chunk.local_total_seq_lens == 0 or q_slice.shape[0] == 0:
logits = q_slice.new_empty((q_slice.shape[0], 0), dtype=torch.float32)
@@ -578,7 +670,7 @@ def sparse_attn_indexer(
q_slice_cast,
k_quant_cast,
k_scale_cast,
weights[chunk.token_start : chunk.token_end],
weights[row_start:row_end],
cu_seqlen_ks,
cu_seqlen_ke,
)
@@ -586,7 +678,7 @@ def sparse_attn_indexer(
logits = fp8_fp4_mqa_logits(
(q_slice_cast, q_scale_slice),
(k_quant_cast, k_scale_cast),
weights[chunk.token_start : chunk.token_end],
weights[row_start:row_end],
cu_seqlen_ks,
cu_seqlen_ke,
clean_logits=False,
@@ -637,9 +729,13 @@ def sparse_attn_indexer(
dcp_rank,
dcp_world_size,
cp_kv_cache_interleave_size,
row_starts=chunk.cu_seqlen_ks,
row_starts=cu_seqlen_ks,
)
if row_shard is not None:
# Rows are already fully ranked, so this is a layout gather.
row_shard.exchange_topk(topk_indices_buffer, topk_tokens)
if has_decode:
decode_metadata = attn_metadata_narrowed.decode
assert decode_metadata is not None
+40 -23
View File
@@ -12,7 +12,11 @@ from vllm.forward_context import get_forward_context
from vllm.logger import init_logger
from vllm.model_executor.custom_op import CustomOp
from vllm.model_executor.layers.indexer_topk import get_indexer_topk
from vllm.model_executor.layers.sparse_attn_indexer import _merge_dcp_topk_global
from vllm.model_executor.layers.sparse_attn_indexer import (
PrefillRowShard,
_merge_dcp_topk_global,
_reserve_row_shard_exchange_transient,
)
from vllm.models.glm5next.common.sparse_indexer import (
RADIX_TOPK_WORKSPACE_SIZE,
_build_decode_scatter_indices,
@@ -169,6 +173,10 @@ def sparse_attn_indexer_kpool(
max_logits_elems, dtype=torch.uint8, device=hidden_states.device
)
# The dummy run returns before exchange_topk executes, so reserve its
# worst-case transient for the memory profile.
_reserve_row_shard_exchange_transient(topk_indices_buffer, hidden_states.device)
return topk_indices_buffer
attn_metadata_narrowed = attn_metadata[k_cache_prefix]
assert isinstance(attn_metadata_narrowed, DeepseekV32IndexerMetadata)
@@ -282,6 +290,12 @@ def sparse_attn_indexer_kpool(
_pos = positions[num_decode_tokens:num_tokens].to(torch.int32)
_buf = topk_indices_buffer[num_decode_tokens:num_tokens]
_fill_causal_indices(_buf, _pos)
# Short prefills score nothing locally; suppress the shard there.
row_shard = None
if not short_prefill:
row_shard = PrefillRowShard.from_metadata(
prefill_metadata, num_decode_tokens
)
# Get the full shared workspace buffers once (will allocate on first use).
# Layout switches between FP8 (head_dim bytes + 4-byte fp32 scale) and
@@ -310,12 +324,16 @@ def sparse_attn_indexer_kpool(
chunk.local_cu_seq_lens,
)
q_slice = q_quant[chunk.token_start : chunk.token_end]
q_scale_slice = (
q_scale[chunk.token_start : chunk.token_end]
if q_scale is not None
else None
)
row_start, row_end = chunk.token_start, chunk.token_end
cu_seqlen_ks, cu_seqlen_ke = chunk.cu_seqlen_ks, chunk.cu_seqlen_ke
if row_shard is not None:
narrowed = row_shard.narrow(chunk)
if narrowed is None:
continue
row_start, row_end, cu_seqlen_ks, cu_seqlen_ke = narrowed
q_slice = q_quant[row_start:row_end]
q_scale_slice = q_scale[row_start:row_end] if q_scale is not None else None
# DeepGEMM scalar-type tags (zero-copy): MXFP4 values → int8
# (kPackedFP4), scales → int32 squeezed to 1-D kv_sf / 2-D q_sf.
if use_fp4_cache:
@@ -335,9 +353,9 @@ def sparse_attn_indexer_kpool(
logits = fp8_fp4_mqa_logits(
(q_slice_cast, q_scale_slice),
(k_quant_cast, k_scale_cast),
weights[chunk.token_start : chunk.token_end],
chunk.cu_seqlen_ks,
chunk.cu_seqlen_ke,
weights[row_start:row_end],
cu_seqlen_ks,
cu_seqlen_ke,
clean_logits=False,
)
@@ -351,17 +369,15 @@ def sparse_attn_indexer_kpool(
)
topk_dst = pool_topk
else:
topk_dst = topk_indices_buffer[
chunk.token_start : chunk.token_end, :topk_tokens
]
topk_dst = topk_indices_buffer[row_start:row_end, :topk_tokens]
if logits.shape[1] == 0:
topk_dst.fill_(-1)
else:
torch.ops._C.top_k_per_row_prefill(
logits,
chunk.cu_seqlen_ks,
chunk.cu_seqlen_ke,
cu_seqlen_ks,
cu_seqlen_ke,
topk_dst,
num_rows,
logits.stride(0),
@@ -376,7 +392,7 @@ def sparse_attn_indexer_kpool(
dcp_rank,
dcp_world_size,
pool_cp_interleave,
row_starts=chunk.cu_seqlen_ks,
row_starts=cu_seqlen_ks,
)
if index_kpool > 1:
@@ -385,10 +401,7 @@ def sparse_attn_indexer_kpool(
# Fused expand-pools + append-tail into one Triton kernel
# (replaces ~25 elementwise ops). seq_len is token-granular
# (pos+1); the kernel derives pool_len internally.
q_seq = (
positions[chunk.token_start : chunk.token_end].to(torch.int32)
+ 1
)
q_seq = positions[row_start:row_end].to(torch.int32) + 1
expanded = kpool_ops.expand_pools_and_append_tail(
pool_ids, q_seq, index_kpool
)
@@ -397,9 +410,13 @@ def sparse_attn_indexer_kpool(
expanded = kpool_ops.expand_pools_to_tokens(
pool_ids, valid, topk_tokens, index_kpool
)
topk_indices_buffer[
chunk.token_start : chunk.token_end, : expanded.shape[-1]
] = expanded
topk_indices_buffer[row_start:row_end, : expanded.shape[-1]] = expanded
if row_shard is not None:
# The k-pool expansion appends the request's incomplete tail
# (index_kpool - 1 entries) after the logical top-k history; those
# columns are part of the attention index and are exchanged too.
row_shard.exchange_topk(topk_indices_buffer, topk_tokens + index_kpool - 1)
if has_decode:
decode_metadata = attn_metadata_narrowed.decode
+73 -1
View File
@@ -7,7 +7,7 @@ import numpy as np
import torch
import vllm.envs as envs
from vllm.config import VllmConfig, get_current_vllm_config
from vllm.config import CUDAGraphMode, VllmConfig, get_current_vllm_config
from vllm.distributed import get_dcp_group, get_pcp_group
from vllm.logger import init_logger
from vllm.model_executor.warmup.jit_warmup import kernel_launcher, zip_inputs
@@ -204,6 +204,64 @@ class PrepareUniformDecodeKernel(
)
# Below this many rows per rank, stay replicated (TP4 engages at 64K rows).
MIN_TP_SHARD_ROWS_PER_RANK = 16_384
def balanced_prefill_row_shard(
seq_lens_cpu: torch.Tensor,
query_lens_cpu: torch.Tensor,
compress_ratio: int,
tp_size: int,
) -> list[int] | None:
"""Return contiguous TP row counts balanced by indexer MQA work."""
num_rows = int(query_lens_cpu.sum())
if tp_size < 2 or num_rows < MIN_TP_SHARD_ROWS_PER_RANK * tp_size:
return None
query_lens = query_lens_cpu.to(torch.int64)
first_key = torch.repeat_interleave(
seq_lens_cpu.to(torch.int64) - query_lens + 1, query_lens
)
row_in_request = torch.arange(num_rows) - torch.repeat_interleave(
torch.cumsum(query_lens, 0) - query_lens, query_lens
)
cost = torch.cumsum((first_key + row_in_request) // compress_ratio, 0)
total = int(cost[-1]) if cost.numel() else 0
if total <= 0:
return None
targets = torch.arange(1, tp_size) * total // tp_size
bounds = [0, *torch.searchsorted(cost, targets).tolist(), num_rows]
# Keep >= 1 row per rank (the floor above leaves ample room); this also
# handles repeated costs from pool compression.
for i in range(1, tp_size):
bounds[i] = max(bounds[i], bounds[i - 1] + 1)
for i in range(tp_size - 1, 0, -1):
bounds[i] = min(bounds[i], bounds[i + 1] - 1)
return [bounds[i + 1] - bounds[i] for i in range(tp_size)]
def tp_prefill_row_sharding_supported(
vllm_config: VllmConfig,
dcp_world_size: int,
use_pcp: bool,
tp_size: int,
) -> bool:
"""Whether replicated prefill rows may be sharded across TP ranks."""
cudagraph_mode = vllm_config.compilation_config.cudagraph_mode or CUDAGraphMode.NONE
return (
current_platform.is_cuda()
and dcp_world_size == 1
and not use_pcp
and tp_size > 1
and not envs.VLLM_DISABLE_PYNCCL
and not envs.VLLM_USE_NCCL_SYMM_MEM
and not envs.VLLM_BATCH_INVARIANT
and cudagraph_mode.mixed_mode() != CUDAGraphMode.FULL
)
class DeepseekV32IndexerBackend(AttentionBackend):
@classmethod
def supports_pcp(cls) -> bool:
@@ -654,6 +712,8 @@ class BuildPrefillChunkMetadataKernel(
class DeepseekV32IndexerPrefillMetadata:
chunks: list[DeepseekV32IndexerPrefillChunkMetadata]
max_prefill_seq_len: int = -1
# Per-TP-rank contiguous row counts, or None for the replicated path.
row_shard_sizes: list[int] | None = None
@dataclass
@@ -941,6 +1001,7 @@ class DeepseekV32IndexerMetadataBuilder(AttentionMetadataBuilder):
scheduler_config = self.vllm_config.scheduler_config
parallel_config = self.vllm_config.parallel_config
self.dcp_world_size = parallel_config.decode_context_parallel_size
self.tp_size = parallel_config.tensor_parallel_size
self.dcp_rank = get_dcp_group().rank_in_group if self.dcp_world_size > 1 else 0
self.pcp_world_size = parallel_config.prefill_context_parallel_size
self.use_pcp = self.pcp_world_size > 1
@@ -1512,6 +1573,16 @@ class DeepseekV32IndexerMetadataBuilder(AttentionMetadataBuilder):
# Skip when total_seq_lens is 0 (i.e., no compressed token).
if metadata is not None:
chunks.append(metadata)
row_shard_sizes = None
if tp_prefill_row_sharding_supported(
self.vllm_config, self.dcp_world_size, self.use_pcp, self.tp_size
):
row_shard_sizes = balanced_prefill_row_shard(
seq_lens_cpu[num_decodes:],
prefill_query_lens_cpu,
self.compress_ratio,
self.tp_size,
)
prefill_metadata = DeepseekV32IndexerPrefillMetadata(
chunks,
max_prefill_seq_len=(
@@ -1519,6 +1590,7 @@ class DeepseekV32IndexerMetadataBuilder(AttentionMetadataBuilder):
if num_prefills > 0
else 0
),
row_shard_sizes=row_shard_sizes,
)
decode_metadata = None