[Bugfix] Engage indexer TP row sharding on total rows, not per-rank rows (#61647)

Signed-off-by: Zheng Cai <caizheng1993@gmail.com>
Signed-off-by: zjy0516 <riverclouds.zhu@qq.com>
Co-authored-by: Claude Opus 4.7 <noreply@anthropic.com>
Co-authored-by: zjy0516 <riverclouds.zhu@qq.com>
This commit is contained in:
Zheng Cai
2026-10-15 13:54:14 +00:00
committed by GitHub
co-authored by Claude Opus 4.7 zjy0516
parent 69425846f8
commit 24f632ca7e
3 changed files with 40 additions and 10 deletions
@@ -10,9 +10,11 @@ import torch
import vllm.model_executor.layers.sparse_attn_indexer as sparse_indexer import vllm.model_executor.layers.sparse_attn_indexer as sparse_indexer
import vllm.v1.attention.backends.mla.indexer as indexer import vllm.v1.attention.backends.mla.indexer as indexer
from vllm.config import CUDAGraphMode from vllm.config import AttentionConfig, CUDAGraphMode
from vllm.model_executor.layers.sparse_attn_indexer import PrefillRowShard from vllm.model_executor.layers.sparse_attn_indexer import PrefillRowShard
MIN_ROWS = AttentionConfig().indexer_tp_shard_min_rows
def _chunk(start: int, end: int) -> SimpleNamespace: def _chunk(start: int, end: int) -> SimpleNamespace:
"""Chunk over rows [start, end); bounds derive from the global row.""" """Chunk over rows [start, end); bounds derive from the global row."""
@@ -80,11 +82,11 @@ def test_shard_exchange_reassembles_rows_only(
assert torch.all(buffer[decode + sum(sizes) :] == -7), "padding untouched" assert torch.all(buffer[decode + sum(sizes) :] == -7), "padding untouched"
@pytest.mark.parametrize("tp_size", [2, 4]) @pytest.mark.parametrize("tp_size", [2, 4, 8])
def test_balanced_row_shard_partitions_and_declines(tp_size: int) -> None: def test_balanced_row_shard_partitions_and_declines(tp_size: int) -> None:
"""The split covers every row once with >= 1 row per rank and declines """The split covers every row once with >= 1 row per rank and declines
below the activation floor (or on a single rank).""" below the activation floor (or on a single rank)."""
n = indexer.MIN_TP_SHARD_ROWS_PER_RANK * tp_size n = MIN_ROWS
shapes = [ shapes = [
([4 * n], [4 * n]), # fresh prompt: the full causal cost ramp ([4 * n], [4 * n]), # fresh prompt: the full causal cost ramp
# uneven requests at different context depths # uneven requests at different context depths
@@ -97,12 +99,38 @@ def test_balanced_row_shard_partitions_and_declines(tp_size: int) -> None:
torch.tensor(query_lens, dtype=torch.int32), torch.tensor(query_lens, dtype=torch.int32),
ratio, ratio,
tp_size, tp_size,
n,
) )
assert sizes is not None and len(sizes) == tp_size assert sizes is not None and len(sizes) == tp_size
assert sum(sizes) == sum(query_lens) and min(sizes) >= 1 assert sum(sizes) == sum(query_lens) and min(sizes) >= 1
lens = torch.tensor([n - 1], dtype=torch.int32) lens = torch.tensor([n - 1], dtype=torch.int32)
assert indexer.balanced_prefill_row_shard(lens, lens, ratio, tp_size) is None assert indexer.balanced_prefill_row_shard(lens, lens, ratio, tp_size, n) is None
assert indexer.balanced_prefill_row_shard(lens, lens, ratio, 1) is None assert indexer.balanced_prefill_row_shard(lens, lens, ratio, 1, n) is None
def test_row_shard_threshold_is_total_not_per_rank() -> None:
"""The activation floor counts TOTAL rows regardless of tp_size: a
per-rank floor would need 128K rows at TP8, i.e. an unschedulable
--max-num-batched-tokens of 128K on 80 GB Hopper nodes."""
lens = torch.tensor([MIN_ROWS], dtype=torch.int32)
for tp_size in (2, 4, 8):
sizes = indexer.balanced_prefill_row_shard(lens, lens, 4, tp_size, MIN_ROWS)
assert sizes is not None
def test_tp2_old_per_rank_window_stays_replicated() -> None:
"""32K-64K total rows engaged on TP2 under the old per-rank floor
(16K rows/rank); the total floor now keeps them replicated — the
all_gatherv latency term is largest at small tp."""
window = torch.tensor([3 * MIN_ROWS // 4], dtype=torch.int32)
assert indexer.balanced_prefill_row_shard(window, window, 4, 2, MIN_ROWS) is None
def test_row_shard_floor_is_at_least_tp_size() -> None:
"""A floor below tp_size still leaves every rank at least one row."""
lens = torch.tensor([8], dtype=torch.int32)
assert indexer.balanced_prefill_row_shard(lens, lens, 4, 8, 1) is not None
assert indexer.balanced_prefill_row_shard(lens - 1, lens - 1, 4, 8, 1) is None
def test_row_sharding_gate_rejects_unsupported_configurations( def test_row_sharding_gate_rejects_unsupported_configurations(
+4
View File
@@ -103,6 +103,10 @@ class AttentionConfig:
candidate pool instead, with fp32 logits, and keeps the masked dense walk candidate pool instead, with fp32 logits, and keeps the masked dense walk
for steps whose contexts are too short for the pool to pay.""" for steps whose contexts are too short for the pool to pay."""
indexer_tp_shard_min_rows: int = Field(default=65536, ge=1)
"""Minimum total prefill rows for the sparse-attention indexer to split
them across TP ranks instead of scoring every row on each rank."""
hisparse_config: HiSparseConfig | None = None hisparse_config: HiSparseConfig | None = None
"""HiSparse host-resident KV configuration. Setting this enables experimental """HiSparse host-resident KV configuration. Setting this enables experimental
Model Runner V2-only HiSparse sparse-MLA decode hot-buffering. It is inferred Model Runner V2-only HiSparse sparse-MLA decode hot-buffering. It is inferred
+3 -5
View File
@@ -205,19 +205,16 @@ class PrepareUniformDecodeKernel(
) )
# Below this many rows per rank, stay replicated (TP4 engages at 64K rows).
MIN_TP_SHARD_ROWS_PER_RANK = 16_384
def balanced_prefill_row_shard( def balanced_prefill_row_shard(
seq_lens_cpu: torch.Tensor, seq_lens_cpu: torch.Tensor,
query_lens_cpu: torch.Tensor, query_lens_cpu: torch.Tensor,
compress_ratio: int, compress_ratio: int,
tp_size: int, tp_size: int,
min_rows: int,
) -> list[int] | None: ) -> list[int] | None:
"""Return contiguous TP row counts balanced by indexer MQA work.""" """Return contiguous TP row counts balanced by indexer MQA work."""
num_rows = int(query_lens_cpu.sum()) num_rows = int(query_lens_cpu.sum())
if tp_size < 2 or num_rows < MIN_TP_SHARD_ROWS_PER_RANK * tp_size: if tp_size < 2 or num_rows < max(min_rows, tp_size):
return None return None
query_lens = query_lens_cpu.to(torch.int64) query_lens = query_lens_cpu.to(torch.int64)
@@ -1584,6 +1581,7 @@ class DeepseekV32IndexerMetadataBuilder(AttentionMetadataBuilder):
prefill_query_lens_cpu, prefill_query_lens_cpu,
self.compress_ratio, self.compress_ratio,
self.tp_size, self.tp_size,
self.vllm_config.attention_config.indexer_tp_shard_min_rows,
) )
prefill_metadata = DeepseekV32IndexerPrefillMetadata( prefill_metadata = DeepseekV32IndexerPrefillMetadata(
chunks, chunks,