[Bugfix] Engage indexer TP row sharding on total rows, not per-rank rows (#61647)
Signed-off-by: Zheng Cai <caizheng1993@gmail.com> Signed-off-by: zjy0516 <riverclouds.zhu@qq.com> Co-authored-by: Claude Opus 4.7 <noreply@anthropic.com> Co-authored-by: zjy0516 <riverclouds.zhu@qq.com>
This commit is contained in:
co-authored by
Claude Opus 4.7
zjy0516
parent
69425846f8
commit
24f632ca7e
@@ -103,6 +103,10 @@ class AttentionConfig:
|
||||
candidate pool instead, with fp32 logits, and keeps the masked dense walk
|
||||
for steps whose contexts are too short for the pool to pay."""
|
||||
|
||||
indexer_tp_shard_min_rows: int = Field(default=65536, ge=1)
|
||||
"""Minimum total prefill rows for the sparse-attention indexer to split
|
||||
them across TP ranks instead of scoring every row on each rank."""
|
||||
|
||||
hisparse_config: HiSparseConfig | None = None
|
||||
"""HiSparse host-resident KV configuration. Setting this enables experimental
|
||||
Model Runner V2-only HiSparse sparse-MLA decode hot-buffering. It is inferred
|
||||
|
||||
@@ -205,19 +205,16 @@ class PrepareUniformDecodeKernel(
|
||||
)
|
||||
|
||||
|
||||
# Below this many rows per rank, stay replicated (TP4 engages at 64K rows).
|
||||
MIN_TP_SHARD_ROWS_PER_RANK = 16_384
|
||||
|
||||
|
||||
def balanced_prefill_row_shard(
|
||||
seq_lens_cpu: torch.Tensor,
|
||||
query_lens_cpu: torch.Tensor,
|
||||
compress_ratio: int,
|
||||
tp_size: int,
|
||||
min_rows: int,
|
||||
) -> list[int] | None:
|
||||
"""Return contiguous TP row counts balanced by indexer MQA work."""
|
||||
num_rows = int(query_lens_cpu.sum())
|
||||
if tp_size < 2 or num_rows < MIN_TP_SHARD_ROWS_PER_RANK * tp_size:
|
||||
if tp_size < 2 or num_rows < max(min_rows, tp_size):
|
||||
return None
|
||||
|
||||
query_lens = query_lens_cpu.to(torch.int64)
|
||||
@@ -1584,6 +1581,7 @@ class DeepseekV32IndexerMetadataBuilder(AttentionMetadataBuilder):
|
||||
prefill_query_lens_cpu,
|
||||
self.compress_ratio,
|
||||
self.tp_size,
|
||||
self.vllm_config.attention_config.indexer_tp_shard_min_rows,
|
||||
)
|
||||
prefill_metadata = DeepseekV32IndexerPrefillMetadata(
|
||||
chunks,
|
||||
|
||||
Reference in New Issue
Block a user