[Bugfix] Engage indexer TP row sharding on total rows, not per-rank rows (#61647)

Signed-off-by: Zheng Cai <caizheng1993@gmail.com>
Signed-off-by: zjy0516 <riverclouds.zhu@qq.com>
Co-authored-by: Claude Opus 4.7 <noreply@anthropic.com>
Co-authored-by: zjy0516 <riverclouds.zhu@qq.com>
This commit is contained in:
Zheng Cai
2026-10-15 13:54:14 +00:00
committed by GitHub
co-authored by Claude Opus 4.7 zjy0516
parent 69425846f8
commit 24f632ca7e
3 changed files with 40 additions and 10 deletions
@@ -10,9 +10,11 @@ import torch
import vllm.model_executor.layers.sparse_attn_indexer as sparse_indexer
import vllm.v1.attention.backends.mla.indexer as indexer
from vllm.config import CUDAGraphMode
from vllm.config import AttentionConfig, CUDAGraphMode
from vllm.model_executor.layers.sparse_attn_indexer import PrefillRowShard
MIN_ROWS = AttentionConfig().indexer_tp_shard_min_rows
def _chunk(start: int, end: int) -> SimpleNamespace:
"""Chunk over rows [start, end); bounds derive from the global row."""
@@ -80,11 +82,11 @@ def test_shard_exchange_reassembles_rows_only(
assert torch.all(buffer[decode + sum(sizes) :] == -7), "padding untouched"
@pytest.mark.parametrize("tp_size", [2, 4])
@pytest.mark.parametrize("tp_size", [2, 4, 8])
def test_balanced_row_shard_partitions_and_declines(tp_size: int) -> None:
"""The split covers every row once with >= 1 row per rank and declines
below the activation floor (or on a single rank)."""
n = indexer.MIN_TP_SHARD_ROWS_PER_RANK * tp_size
n = MIN_ROWS
shapes = [
([4 * n], [4 * n]), # fresh prompt: the full causal cost ramp
# uneven requests at different context depths
@@ -97,12 +99,38 @@ def test_balanced_row_shard_partitions_and_declines(tp_size: int) -> None:
torch.tensor(query_lens, dtype=torch.int32),
ratio,
tp_size,
n,
)
assert sizes is not None and len(sizes) == tp_size
assert sum(sizes) == sum(query_lens) and min(sizes) >= 1
lens = torch.tensor([n - 1], dtype=torch.int32)
assert indexer.balanced_prefill_row_shard(lens, lens, ratio, tp_size) is None
assert indexer.balanced_prefill_row_shard(lens, lens, ratio, 1) is None
assert indexer.balanced_prefill_row_shard(lens, lens, ratio, tp_size, n) is None
assert indexer.balanced_prefill_row_shard(lens, lens, ratio, 1, n) is None
def test_row_shard_threshold_is_total_not_per_rank() -> None:
"""The activation floor counts TOTAL rows regardless of tp_size: a
per-rank floor would need 128K rows at TP8, i.e. an unschedulable
--max-num-batched-tokens of 128K on 80 GB Hopper nodes."""
lens = torch.tensor([MIN_ROWS], dtype=torch.int32)
for tp_size in (2, 4, 8):
sizes = indexer.balanced_prefill_row_shard(lens, lens, 4, tp_size, MIN_ROWS)
assert sizes is not None
def test_tp2_old_per_rank_window_stays_replicated() -> None:
"""32K-64K total rows engaged on TP2 under the old per-rank floor
(16K rows/rank); the total floor now keeps them replicated — the
all_gatherv latency term is largest at small tp."""
window = torch.tensor([3 * MIN_ROWS // 4], dtype=torch.int32)
assert indexer.balanced_prefill_row_shard(window, window, 4, 2, MIN_ROWS) is None
def test_row_shard_floor_is_at_least_tp_size() -> None:
"""A floor below tp_size still leaves every rank at least one row."""
lens = torch.tensor([8], dtype=torch.int32)
assert indexer.balanced_prefill_row_shard(lens, lens, 4, 8, 1) is not None
assert indexer.balanced_prefill_row_shard(lens - 1, lens - 1, 4, 8, 1) is None
def test_row_sharding_gate_rejects_unsupported_configurations(