diff --git a/tests/v1/attention/test_indexer_tp_row_shard.py b/tests/v1/attention/test_indexer_tp_row_shard.py new file mode 100644 index 0000000000..e65e4789f1 --- /dev/null +++ b/tests/v1/attention/test_indexer_tp_row_shard.py @@ -0,0 +1,213 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""TP query-row sharding of the DSA indexer prefill (see PrefillRowShard): +each rank scores a contiguous slice; the group exchanges int32s per row.""" + +from types import SimpleNamespace + +import pytest +import torch + +import vllm.model_executor.layers.sparse_attn_indexer as sparse_indexer +import vllm.v1.attention.backends.mla.indexer as indexer +from vllm.config import CUDAGraphMode +from vllm.model_executor.layers.sparse_attn_indexer import PrefillRowShard + + +def _chunk(start: int, end: int) -> SimpleNamespace: + """Chunk over rows [start, end); bounds derive from the global row.""" + n = end - start + return SimpleNamespace( + token_start=start, + token_end=end, + cu_seqlen_ks=torch.zeros(n, dtype=torch.int32), + cu_seqlen_ke=((torch.arange(n) * 7 + start) % 65).int(), + ) + + +@pytest.mark.parametrize("world", [2, 3, 8]) +def test_shard_narrowing_covers_every_row_once(world: int) -> None: + """Narrowed windows tile the prefill rows exactly once, carry the causal + bounds of those global rows, and skip chunks foreign to this rank.""" + decode, rows = 5, 997 + edges = [decode, decode + 330, decode + 337, decode + rows] + chunks = [_chunk(lo, hi) for lo, hi in zip(edges, edges[1:])] + sizes = [rows // world + int(r < rows % world) for r in range(world)] + covered = [] + for rank in range(world): + shard = PrefillRowShard(decode, sizes, rank) + for chunk in chunks: + window = shard.narrow(chunk) + if window is None: + continue + start, end, ks, ke = window + lo, hi = start - chunk.token_start, end - chunk.token_start + torch.testing.assert_close(ks, chunk.cu_seqlen_ks[lo:hi]) + torch.testing.assert_close(ke, chunk.cu_seqlen_ke[lo:hi]) + covered.append((start, end)) + covered.sort() + # Windows must be disjoint, contiguous and exactly cover the prefill rows. + assert [s for s, _ in covered] == [decode, *[e for _, e in covered][:-1]] + assert covered[-1][1] == decode + rows + + +def test_shard_exchange_reassembles_rows_only( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Gathered indices land only on real prefill rows: leading decode rows + and trailing CUDA-graph padding stay untouched, and the full k-pool + width (tail columns included) is exchanged.""" + decode, pad, topk, kpool = 5, 3, 8, 4 + width = topk + kpool - 1 + sizes = [17, 9, 21] + buffer = torch.full((decode + sum(sizes) + pad, width), -7, dtype=torch.int32) + gathered = torch.arange(sum(sizes) * width, dtype=torch.int32).reshape(-1, width) + calls = [] + + def fake_group(): + def all_gatherv(local, dim, sizes): + calls.append((local, sizes)) + return gathered + + return SimpleNamespace(all_gatherv=all_gatherv) + + monkeypatch.setattr(sparse_indexer, "get_tp_group", fake_group) + for rank in range(len(sizes)): + PrefillRowShard(decode, sizes, rank).exchange_topk(buffer, width) + assert calls[0][1] == sizes and calls[0][0].is_contiguous() + torch.testing.assert_close(buffer[decode : decode + sum(sizes)], gathered) + assert torch.all(buffer[:decode] == -7), "decode rows stay outside" + assert torch.all(buffer[decode + sum(sizes) :] == -7), "padding untouched" + + +@pytest.mark.parametrize("tp_size", [2, 4]) +def test_balanced_row_shard_partitions_and_declines(tp_size: int) -> None: + """The split covers every row once with >= 1 row per rank and declines + below the activation floor (or on a single rank).""" + n = indexer.MIN_TP_SHARD_ROWS_PER_RANK * tp_size + shapes = [ + ([4 * n], [4 * n]), # fresh prompt: the full causal cost ramp + # uneven requests at different context depths + ([2 * n, 7, 4 * n, n // 3 + 5], [2 * n, 9007, 4 * n + 500, n // 3 + 60005]), + ] + ratio = 4 + for query_lens, seq_lens in shapes: + sizes = indexer.balanced_prefill_row_shard( + torch.tensor(seq_lens, dtype=torch.int32), + torch.tensor(query_lens, dtype=torch.int32), + ratio, + tp_size, + ) + assert sizes is not None and len(sizes) == tp_size + assert sum(sizes) == sum(query_lens) and min(sizes) >= 1 + lens = torch.tensor([n - 1], dtype=torch.int32) + assert indexer.balanced_prefill_row_shard(lens, lens, ratio, tp_size) is None + assert indexer.balanced_prefill_row_shard(lens, lens, ratio, 1) is None + + +def test_row_sharding_gate_rejects_unsupported_configurations( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The gate is the whole safety envelope; nothing else guards the exchange.""" + monkeypatch.setattr(indexer.current_platform, "is_cuda", lambda: True) + + def supported(**kwargs): + mode = kwargs.pop("cudagraph_mode", CUDAGraphMode.PIECEWISE) + cfg = SimpleNamespace(compilation_config=SimpleNamespace(cudagraph_mode=mode)) + args = {"dcp_world_size": 1, "use_pcp": False, "tp_size": 4, **kwargs} + return indexer.tp_prefill_row_sharding_supported(cfg, **args) + + assert supported() + for bad in ({"tp_size": 1}, {"dcp_world_size": 2}, {"use_pcp": True}): + assert not supported(**bad) + for name in ( + "VLLM_DISABLE_PYNCCL", + "VLLM_USE_NCCL_SYMM_MEM", + "VLLM_BATCH_INVARIANT", + ): + with monkeypatch.context() as m: + m.setattr(indexer.envs, name, True) + assert not supported() + # FULL captures mixed batches whole, so the exchange must stay out. + assert not supported(cudagraph_mode=CUDAGraphMode.FULL) + assert supported(cudagraph_mode=CUDAGraphMode.FULL_AND_PIECEWISE) + + +def test_profiling_run_reserves_exchange_transient( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The dummy run returns before the exchange executes, so its all_gatherv + output plus staging copy must be reserved up front when row sharding is + active -- and only then.""" + rows, topk = 128, 8 + buffer = torch.empty((rows, topk), dtype=torch.int32) + peak = 2 * rows * topk * buffer.element_size() + specs = (((1,), torch.uint8), ((1,), torch.uint8)) + numels: list[int] = [] + real_empty = torch.empty + + def recording_empty(*args, **kwargs): + out = real_empty(*args, **kwargs) + numels.append(out.numel()) + return out + + def run_dummy(tp_world: int) -> None: + numels.clear() + with monkeypatch.context() as m: + m.setattr(torch, "empty", recording_empty) + m.setattr( + sparse_indexer, + "get_forward_context", + lambda: SimpleNamespace(attn_metadata=None), + ) + m.setattr(sparse_indexer, "_gather_workspace_shapes", lambda *a: specs) + m.setattr( + sparse_indexer, + "current_workspace_manager", + lambda: SimpleNamespace(get_simultaneous=lambda *s: None), + ) + m.setattr( + sparse_indexer, + "get_tp_group", + lambda: SimpleNamespace(world_size=tp_world), + ) + m.setattr( + sparse_indexer, + "get_current_vllm_config_or_none", + lambda: SimpleNamespace( + compilation_config=SimpleNamespace( + cudagraph_mode=CUDAGraphMode.PIECEWISE + ), + parallel_config=SimpleNamespace( + decode_context_parallel_size=1, + prefill_context_parallel_size=1, + ), + ), + ) + m.setattr(sparse_indexer.envs, "VLLM_SPARSE_INDEXER_MAX_LOGITS_MB", 1) + m.setattr(indexer.current_platform, "is_cuda", lambda: True) + sparse_indexer.sparse_attn_indexer( + hidden_states=torch.empty(4, 4), + k_cache_prefix="k", + kv_cache=torch.empty(4, 4), + q_quant=torch.empty(4, 4), + q_scale=None, + k=None, + weights=torch.empty(4), + quant_block_size=128, + scale_fmt=None, + topk_tokens=topk, + head_dim=64, + max_model_len=1024, + total_seq_lens=512, + topk_indices_buffer=buffer, + skip_k_cache_insert=True, + use_pcp=False, + pcp_shard_decode_requests=False, + dense_mha_metadata_layer_name="", + ) + + run_dummy(tp_world=2) + assert peak in numels + run_dummy(tp_world=1) + assert peak not in numels diff --git a/vllm/model_executor/layers/sparse_attn_indexer.py b/vllm/model_executor/layers/sparse_attn_indexer.py index f8fedf046a..935e4c0d2d 100644 --- a/vllm/model_executor/layers/sparse_attn_indexer.py +++ b/vllm/model_executor/layers/sparse_attn_indexer.py @@ -9,8 +9,17 @@ import vllm.envs as envs from vllm import _custom_ops as ops from vllm._aiter_ops import rocm_aiter_ops from vllm.compilation.breakable_cudagraph import eager_break_during_capture -from vllm.config import CUDAGraphMode, get_current_vllm_config -from vllm.distributed import get_dcp_group, get_pcp_group +from vllm.config import ( + CUDAGraphMode, + get_current_vllm_config, + get_current_vllm_config_or_none, +) +from vllm.distributed import ( + get_dcp_group, + get_pcp_group, + get_tensor_model_parallel_rank, + get_tp_group, +) from vllm.forward_context import get_forward_context from vllm.logger import init_logger from vllm.model_executor.custom_op import CustomOp @@ -43,6 +52,7 @@ from vllm.utils.torch_utils import ( ) from vllm.v1.attention.backends.mla.indexer import ( DeepseekV32IndexerMetadata, + tp_prefill_row_sharding_supported, ) from vllm.v1.attention.ops.common import pack_seq_triton, unpack_seq_triton from vllm.v1.attention.ops.pcp import maybe_gather_indexer_k @@ -317,6 +327,75 @@ def kv_cache_as_quant_view( return kv_cache.unsqueeze(-2) +class PrefillRowShard: + """One TP rank's contiguous slice of the replicated prefill rows. + + Rows are independent, so each rank scores only its slice and the group + exchanges final int32 top-k indices instead of the logits.""" + + def __init__(self, num_decode_tokens: int, sizes: list[int], rank: int): + self.num_decode_tokens = num_decode_tokens + self.sizes = sizes + self.start = num_decode_tokens + sum(sizes[:rank]) + self.stop = self.start + sizes[rank] + + @classmethod + def from_metadata( + cls, prefill_metadata, num_decode_tokens: int + ) -> "PrefillRowShard | None": + sizes = getattr(prefill_metadata, "row_shard_sizes", None) + if sizes is None: + return None + return cls(num_decode_tokens, sizes, get_tensor_model_parallel_rank()) + + def narrow(self, chunk) -> "tuple[int, int, torch.Tensor, torch.Tensor] | None": + """Row window and narrowed causal bounds of ``chunk`` for this rank.""" + start = max(chunk.token_start, self.start) + end = min(chunk.token_end, self.stop) + if start >= end: + return None + lo, hi = start - chunk.token_start, end - chunk.token_start + return start, end, chunk.cu_seqlen_ks[lo:hi], chunk.cu_seqlen_ke[lo:hi] + + def exchange_topk(self, topk_buffer: torch.Tensor, width: int) -> None: + """Gather per-rank final indices back into global row order.""" + # The local input must outlive the async PyNCCL gather's enqueued copy. + local = topk_buffer[self.start : self.stop, :width].contiguous() + end = self.num_decode_tokens + sum(self.sizes) + topk_buffer[self.num_decode_tokens : end, :width] = get_tp_group().all_gatherv( + local, dim=0, sizes=self.sizes + ) + + +def _reserve_row_shard_exchange_transient( + topk_indices_buffer: torch.Tensor, device: torch.device +) -> None: + """Profiling/dummy runs return before ``exchange_topk`` executes; reserve + its worst-case transient so the memory profile accounts for it. + + The exchange stages this rank's contiguous slice and the all_gatherv + wrapper allocates the full gathered tensor; each is bounded by the + buffer's row capacity times its (tile-padded) width. + """ + cfg = get_current_vllm_config_or_none() + if cfg is None: + return + parallel = cfg.parallel_config + if not tp_prefill_row_sharding_supported( + cfg, + parallel.decode_context_parallel_size, + parallel.prefill_context_parallel_size > 1, + get_tp_group().world_size, + ): + return + rows, width = topk_indices_buffer.shape[:2] + _ = torch.empty( + 2 * rows * width * topk_indices_buffer.element_size(), + dtype=torch.uint8, + device=device, + ) + + @eager_break_during_capture def sparse_attn_indexer( hidden_states: torch.Tensor, @@ -388,6 +467,10 @@ def sparse_attn_indexer( max_logits_elems, dtype=torch.uint8, device=hidden_states.device ) + # The dummy run returns before exchange_topk executes, so reserve its + # worst-case transient for the memory profile. + _reserve_row_shard_exchange_transient(topk_indices_buffer, hidden_states.device) + return sparse_attn_indexer_fake( hidden_states, k_cache_prefix, @@ -515,6 +598,12 @@ def sparse_attn_indexer( scales_spec, *gather_specs, ) + # One row window per rank; DSV4.1 candidate rows stay replicated. + row_shard = ( + None + if candidate_blocks is not None + else PrefillRowShard.from_metadata(prefill_metadata, num_decode_tokens) + ) for chunk in prefill_metadata.chunks: cu_seqlen_ks = chunk.cu_seqlen_ks cu_seqlen_ke = chunk.cu_seqlen_ke @@ -530,6 +619,15 @@ def sparse_attn_indexer( chunk.local_cu_seq_lens, ) + # Narrow scoring to this rank; KV gathers remain unconditional for + # continuation chunks that reuse the workspace. + row_start, row_end = chunk.token_start, chunk.token_end + if row_shard is not None: + narrowed = row_shard.narrow(chunk) + if narrowed is None: + continue + row_start, row_end, cu_seqlen_ks, cu_seqlen_ke = narrowed + # PCP + DCP KV all-gather. deinterleave_idx = chunk.pcp_deinterleave_idx if deinterleave_idx is not None: @@ -547,15 +645,9 @@ def sparse_attn_indexer( out_scales, ) - q_slice = q_quant[chunk.token_start : chunk.token_end] - q_scale_slice = ( - q_scale[chunk.token_start : chunk.token_end] - if q_scale is not None - else None - ) - topk_indices = topk_indices_buffer[ - chunk.token_start : chunk.token_end, :topk_tokens - ] + q_slice = q_quant[row_start:row_end] + q_scale_slice = q_scale[row_start:row_end] if q_scale is not None else None + topk_indices = topk_indices_buffer[row_start:row_end, :topk_tokens] if chunk.local_total_seq_lens == 0 or q_slice.shape[0] == 0: logits = q_slice.new_empty((q_slice.shape[0], 0), dtype=torch.float32) @@ -578,7 +670,7 @@ def sparse_attn_indexer( q_slice_cast, k_quant_cast, k_scale_cast, - weights[chunk.token_start : chunk.token_end], + weights[row_start:row_end], cu_seqlen_ks, cu_seqlen_ke, ) @@ -586,7 +678,7 @@ def sparse_attn_indexer( logits = fp8_fp4_mqa_logits( (q_slice_cast, q_scale_slice), (k_quant_cast, k_scale_cast), - weights[chunk.token_start : chunk.token_end], + weights[row_start:row_end], cu_seqlen_ks, cu_seqlen_ke, clean_logits=False, @@ -637,9 +729,13 @@ def sparse_attn_indexer( dcp_rank, dcp_world_size, cp_kv_cache_interleave_size, - row_starts=chunk.cu_seqlen_ks, + row_starts=cu_seqlen_ks, ) + if row_shard is not None: + # Rows are already fully ranked, so this is a layout gather. + row_shard.exchange_topk(topk_indices_buffer, topk_tokens) + if has_decode: decode_metadata = attn_metadata_narrowed.decode assert decode_metadata is not None diff --git a/vllm/models/glm5next/nvidia/sparse_indexer.py b/vllm/models/glm5next/nvidia/sparse_indexer.py index 77fa4c4091..154fdcda90 100644 --- a/vllm/models/glm5next/nvidia/sparse_indexer.py +++ b/vllm/models/glm5next/nvidia/sparse_indexer.py @@ -12,7 +12,11 @@ from vllm.forward_context import get_forward_context from vllm.logger import init_logger from vllm.model_executor.custom_op import CustomOp from vllm.model_executor.layers.indexer_topk import get_indexer_topk -from vllm.model_executor.layers.sparse_attn_indexer import _merge_dcp_topk_global +from vllm.model_executor.layers.sparse_attn_indexer import ( + PrefillRowShard, + _merge_dcp_topk_global, + _reserve_row_shard_exchange_transient, +) from vllm.models.glm5next.common.sparse_indexer import ( RADIX_TOPK_WORKSPACE_SIZE, _build_decode_scatter_indices, @@ -169,6 +173,10 @@ def sparse_attn_indexer_kpool( max_logits_elems, dtype=torch.uint8, device=hidden_states.device ) + # The dummy run returns before exchange_topk executes, so reserve its + # worst-case transient for the memory profile. + _reserve_row_shard_exchange_transient(topk_indices_buffer, hidden_states.device) + return topk_indices_buffer attn_metadata_narrowed = attn_metadata[k_cache_prefix] assert isinstance(attn_metadata_narrowed, DeepseekV32IndexerMetadata) @@ -282,6 +290,12 @@ def sparse_attn_indexer_kpool( _pos = positions[num_decode_tokens:num_tokens].to(torch.int32) _buf = topk_indices_buffer[num_decode_tokens:num_tokens] _fill_causal_indices(_buf, _pos) + # Short prefills score nothing locally; suppress the shard there. + row_shard = None + if not short_prefill: + row_shard = PrefillRowShard.from_metadata( + prefill_metadata, num_decode_tokens + ) # Get the full shared workspace buffers once (will allocate on first use). # Layout switches between FP8 (head_dim bytes + 4-byte fp32 scale) and @@ -310,12 +324,16 @@ def sparse_attn_indexer_kpool( chunk.local_cu_seq_lens, ) - q_slice = q_quant[chunk.token_start : chunk.token_end] - q_scale_slice = ( - q_scale[chunk.token_start : chunk.token_end] - if q_scale is not None - else None - ) + row_start, row_end = chunk.token_start, chunk.token_end + cu_seqlen_ks, cu_seqlen_ke = chunk.cu_seqlen_ks, chunk.cu_seqlen_ke + if row_shard is not None: + narrowed = row_shard.narrow(chunk) + if narrowed is None: + continue + row_start, row_end, cu_seqlen_ks, cu_seqlen_ke = narrowed + + q_slice = q_quant[row_start:row_end] + q_scale_slice = q_scale[row_start:row_end] if q_scale is not None else None # DeepGEMM scalar-type tags (zero-copy): MXFP4 values → int8 # (kPackedFP4), scales → int32 squeezed to 1-D kv_sf / 2-D q_sf. if use_fp4_cache: @@ -335,9 +353,9 @@ def sparse_attn_indexer_kpool( logits = fp8_fp4_mqa_logits( (q_slice_cast, q_scale_slice), (k_quant_cast, k_scale_cast), - weights[chunk.token_start : chunk.token_end], - chunk.cu_seqlen_ks, - chunk.cu_seqlen_ke, + weights[row_start:row_end], + cu_seqlen_ks, + cu_seqlen_ke, clean_logits=False, ) @@ -351,17 +369,15 @@ def sparse_attn_indexer_kpool( ) topk_dst = pool_topk else: - topk_dst = topk_indices_buffer[ - chunk.token_start : chunk.token_end, :topk_tokens - ] + topk_dst = topk_indices_buffer[row_start:row_end, :topk_tokens] if logits.shape[1] == 0: topk_dst.fill_(-1) else: torch.ops._C.top_k_per_row_prefill( logits, - chunk.cu_seqlen_ks, - chunk.cu_seqlen_ke, + cu_seqlen_ks, + cu_seqlen_ke, topk_dst, num_rows, logits.stride(0), @@ -376,7 +392,7 @@ def sparse_attn_indexer_kpool( dcp_rank, dcp_world_size, pool_cp_interleave, - row_starts=chunk.cu_seqlen_ks, + row_starts=cu_seqlen_ks, ) if index_kpool > 1: @@ -385,10 +401,7 @@ def sparse_attn_indexer_kpool( # Fused expand-pools + append-tail into one Triton kernel # (replaces ~25 elementwise ops). seq_len is token-granular # (pos+1); the kernel derives pool_len internally. - q_seq = ( - positions[chunk.token_start : chunk.token_end].to(torch.int32) - + 1 - ) + q_seq = positions[row_start:row_end].to(torch.int32) + 1 expanded = kpool_ops.expand_pools_and_append_tail( pool_ids, q_seq, index_kpool ) @@ -397,9 +410,13 @@ def sparse_attn_indexer_kpool( expanded = kpool_ops.expand_pools_to_tokens( pool_ids, valid, topk_tokens, index_kpool ) - topk_indices_buffer[ - chunk.token_start : chunk.token_end, : expanded.shape[-1] - ] = expanded + topk_indices_buffer[row_start:row_end, : expanded.shape[-1]] = expanded + + if row_shard is not None: + # The k-pool expansion appends the request's incomplete tail + # (index_kpool - 1 entries) after the logical top-k history; those + # columns are part of the attention index and are exchanged too. + row_shard.exchange_topk(topk_indices_buffer, topk_tokens + index_kpool - 1) if has_decode: decode_metadata = attn_metadata_narrowed.decode diff --git a/vllm/v1/attention/backends/mla/indexer.py b/vllm/v1/attention/backends/mla/indexer.py index e776057b05..97de9ffa9b 100644 --- a/vllm/v1/attention/backends/mla/indexer.py +++ b/vllm/v1/attention/backends/mla/indexer.py @@ -7,7 +7,7 @@ import numpy as np import torch import vllm.envs as envs -from vllm.config import VllmConfig, get_current_vllm_config +from vllm.config import CUDAGraphMode, VllmConfig, get_current_vllm_config from vllm.distributed import get_dcp_group, get_pcp_group from vllm.logger import init_logger from vllm.model_executor.warmup.jit_warmup import kernel_launcher, zip_inputs @@ -204,6 +204,64 @@ class PrepareUniformDecodeKernel( ) +# Below this many rows per rank, stay replicated (TP4 engages at 64K rows). +MIN_TP_SHARD_ROWS_PER_RANK = 16_384 + + +def balanced_prefill_row_shard( + seq_lens_cpu: torch.Tensor, + query_lens_cpu: torch.Tensor, + compress_ratio: int, + tp_size: int, +) -> list[int] | None: + """Return contiguous TP row counts balanced by indexer MQA work.""" + num_rows = int(query_lens_cpu.sum()) + if tp_size < 2 or num_rows < MIN_TP_SHARD_ROWS_PER_RANK * tp_size: + return None + + query_lens = query_lens_cpu.to(torch.int64) + first_key = torch.repeat_interleave( + seq_lens_cpu.to(torch.int64) - query_lens + 1, query_lens + ) + row_in_request = torch.arange(num_rows) - torch.repeat_interleave( + torch.cumsum(query_lens, 0) - query_lens, query_lens + ) + cost = torch.cumsum((first_key + row_in_request) // compress_ratio, 0) + total = int(cost[-1]) if cost.numel() else 0 + if total <= 0: + return None + + targets = torch.arange(1, tp_size) * total // tp_size + bounds = [0, *torch.searchsorted(cost, targets).tolist(), num_rows] + # Keep >= 1 row per rank (the floor above leaves ample room); this also + # handles repeated costs from pool compression. + for i in range(1, tp_size): + bounds[i] = max(bounds[i], bounds[i - 1] + 1) + for i in range(tp_size - 1, 0, -1): + bounds[i] = min(bounds[i], bounds[i + 1] - 1) + return [bounds[i + 1] - bounds[i] for i in range(tp_size)] + + +def tp_prefill_row_sharding_supported( + vllm_config: VllmConfig, + dcp_world_size: int, + use_pcp: bool, + tp_size: int, +) -> bool: + """Whether replicated prefill rows may be sharded across TP ranks.""" + cudagraph_mode = vllm_config.compilation_config.cudagraph_mode or CUDAGraphMode.NONE + return ( + current_platform.is_cuda() + and dcp_world_size == 1 + and not use_pcp + and tp_size > 1 + and not envs.VLLM_DISABLE_PYNCCL + and not envs.VLLM_USE_NCCL_SYMM_MEM + and not envs.VLLM_BATCH_INVARIANT + and cudagraph_mode.mixed_mode() != CUDAGraphMode.FULL + ) + + class DeepseekV32IndexerBackend(AttentionBackend): @classmethod def supports_pcp(cls) -> bool: @@ -654,6 +712,8 @@ class BuildPrefillChunkMetadataKernel( class DeepseekV32IndexerPrefillMetadata: chunks: list[DeepseekV32IndexerPrefillChunkMetadata] max_prefill_seq_len: int = -1 + # Per-TP-rank contiguous row counts, or None for the replicated path. + row_shard_sizes: list[int] | None = None @dataclass @@ -941,6 +1001,7 @@ class DeepseekV32IndexerMetadataBuilder(AttentionMetadataBuilder): scheduler_config = self.vllm_config.scheduler_config parallel_config = self.vllm_config.parallel_config self.dcp_world_size = parallel_config.decode_context_parallel_size + self.tp_size = parallel_config.tensor_parallel_size self.dcp_rank = get_dcp_group().rank_in_group if self.dcp_world_size > 1 else 0 self.pcp_world_size = parallel_config.prefill_context_parallel_size self.use_pcp = self.pcp_world_size > 1 @@ -1512,6 +1573,16 @@ class DeepseekV32IndexerMetadataBuilder(AttentionMetadataBuilder): # Skip when total_seq_lens is 0 (i.e., no compressed token). if metadata is not None: chunks.append(metadata) + row_shard_sizes = None + if tp_prefill_row_sharding_supported( + self.vllm_config, self.dcp_world_size, self.use_pcp, self.tp_size + ): + row_shard_sizes = balanced_prefill_row_shard( + seq_lens_cpu[num_decodes:], + prefill_query_lens_cpu, + self.compress_ratio, + self.tp_size, + ) prefill_metadata = DeepseekV32IndexerPrefillMetadata( chunks, max_prefill_seq_len=( @@ -1519,6 +1590,7 @@ class DeepseekV32IndexerMetadataBuilder(AttentionMetadataBuilder): if num_prefills > 0 else 0 ), + row_shard_sizes=row_shard_sizes, ) decode_metadata = None