[Bugfix] Engage indexer TP row sharding on total rows, not per-rank rows (#61647)

Signed-off-by: Zheng Cai <caizheng1993@gmail.com>
Signed-off-by: zjy0516 <riverclouds.zhu@qq.com>
Co-authored-by: Claude Opus 4.7 <noreply@anthropic.com>
Co-authored-by: zjy0516 <riverclouds.zhu@qq.com>
This commit is contained in:
Zheng Cai
2026-10-15 13:54:14 +00:00
committed by GitHub
co-authored by Claude Opus 4.7 zjy0516
parent 69425846f8
commit 24f632ca7e
3 changed files with 40 additions and 10 deletions
+4
View File
@@ -103,6 +103,10 @@ class AttentionConfig:
candidate pool instead, with fp32 logits, and keeps the masked dense walk
for steps whose contexts are too short for the pool to pay."""
indexer_tp_shard_min_rows: int = Field(default=65536, ge=1)
"""Minimum total prefill rows for the sparse-attention indexer to split
them across TP ranks instead of scoring every row on each rank."""
hisparse_config: HiSparseConfig | None = None
"""HiSparse host-resident KV configuration. Setting this enables experimental
Model Runner V2-only HiSparse sparse-MLA decode hot-buffering. It is inferred
+3 -5
View File
@@ -205,19 +205,16 @@ class PrepareUniformDecodeKernel(
)
# Below this many rows per rank, stay replicated (TP4 engages at 64K rows).
MIN_TP_SHARD_ROWS_PER_RANK = 16_384
def balanced_prefill_row_shard(
seq_lens_cpu: torch.Tensor,
query_lens_cpu: torch.Tensor,
compress_ratio: int,
tp_size: int,
min_rows: int,
) -> list[int] | None:
"""Return contiguous TP row counts balanced by indexer MQA work."""
num_rows = int(query_lens_cpu.sum())
if tp_size < 2 or num_rows < MIN_TP_SHARD_ROWS_PER_RANK * tp_size:
if tp_size < 2 or num_rows < max(min_rows, tp_size):
return None
query_lens = query_lens_cpu.to(torch.int64)
@@ -1584,6 +1581,7 @@ class DeepseekV32IndexerMetadataBuilder(AttentionMetadataBuilder):
prefill_query_lens_cpu,
self.compress_ratio,
self.tp_size,
self.vllm_config.attention_config.indexer_tp_shard_min_rows,
)
prefill_metadata = DeepseekV32IndexerPrefillMetadata(
chunks,