[Perf][GLM-5.3-Flash] Shard long-context indexer prefill rows across TP (#54951)
Signed-off-by: Zheng Cai <8370601+zigzagcai@users.noreply.github.com> Signed-off-by: Zheng Cai <caizheng1993@gmail.com> Signed-off-by: Codex <noreply@openai.com> Co-authored-by: root <root@rjob-fb29567b88fec0a9-88c9a4c797eb2ec1-0.caizheng.ailab-sys.svc.pjlab.local> Co-authored-by: OpenAI Codex <noreply@openai.com> Co-authored-by: zigzagcai <zigzagcai@users.noreply.github.com> Co-authored-by: Claude Code <noreply@anthropic.com> Co-authored-by: Wentao Ye <44945378+yewentao256@users.noreply.github.com>
This commit is contained in:
co-authored by
root
OpenAI Codex
zigzagcai
Claude Code
Wentao Ye
parent
bd5bbce6e4
commit
3e8e1364ff
@@ -0,0 +1,213 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""TP query-row sharding of the DSA indexer prefill (see PrefillRowShard):
|
||||
each rank scores a contiguous slice; the group exchanges int32s per row."""
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
import vllm.model_executor.layers.sparse_attn_indexer as sparse_indexer
|
||||
import vllm.v1.attention.backends.mla.indexer as indexer
|
||||
from vllm.config import CUDAGraphMode
|
||||
from vllm.model_executor.layers.sparse_attn_indexer import PrefillRowShard
|
||||
|
||||
|
||||
def _chunk(start: int, end: int) -> SimpleNamespace:
|
||||
"""Chunk over rows [start, end); bounds derive from the global row."""
|
||||
n = end - start
|
||||
return SimpleNamespace(
|
||||
token_start=start,
|
||||
token_end=end,
|
||||
cu_seqlen_ks=torch.zeros(n, dtype=torch.int32),
|
||||
cu_seqlen_ke=((torch.arange(n) * 7 + start) % 65).int(),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("world", [2, 3, 8])
|
||||
def test_shard_narrowing_covers_every_row_once(world: int) -> None:
|
||||
"""Narrowed windows tile the prefill rows exactly once, carry the causal
|
||||
bounds of those global rows, and skip chunks foreign to this rank."""
|
||||
decode, rows = 5, 997
|
||||
edges = [decode, decode + 330, decode + 337, decode + rows]
|
||||
chunks = [_chunk(lo, hi) for lo, hi in zip(edges, edges[1:])]
|
||||
sizes = [rows // world + int(r < rows % world) for r in range(world)]
|
||||
covered = []
|
||||
for rank in range(world):
|
||||
shard = PrefillRowShard(decode, sizes, rank)
|
||||
for chunk in chunks:
|
||||
window = shard.narrow(chunk)
|
||||
if window is None:
|
||||
continue
|
||||
start, end, ks, ke = window
|
||||
lo, hi = start - chunk.token_start, end - chunk.token_start
|
||||
torch.testing.assert_close(ks, chunk.cu_seqlen_ks[lo:hi])
|
||||
torch.testing.assert_close(ke, chunk.cu_seqlen_ke[lo:hi])
|
||||
covered.append((start, end))
|
||||
covered.sort()
|
||||
# Windows must be disjoint, contiguous and exactly cover the prefill rows.
|
||||
assert [s for s, _ in covered] == [decode, *[e for _, e in covered][:-1]]
|
||||
assert covered[-1][1] == decode + rows
|
||||
|
||||
|
||||
def test_shard_exchange_reassembles_rows_only(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""Gathered indices land only on real prefill rows: leading decode rows
|
||||
and trailing CUDA-graph padding stay untouched, and the full k-pool
|
||||
width (tail columns included) is exchanged."""
|
||||
decode, pad, topk, kpool = 5, 3, 8, 4
|
||||
width = topk + kpool - 1
|
||||
sizes = [17, 9, 21]
|
||||
buffer = torch.full((decode + sum(sizes) + pad, width), -7, dtype=torch.int32)
|
||||
gathered = torch.arange(sum(sizes) * width, dtype=torch.int32).reshape(-1, width)
|
||||
calls = []
|
||||
|
||||
def fake_group():
|
||||
def all_gatherv(local, dim, sizes):
|
||||
calls.append((local, sizes))
|
||||
return gathered
|
||||
|
||||
return SimpleNamespace(all_gatherv=all_gatherv)
|
||||
|
||||
monkeypatch.setattr(sparse_indexer, "get_tp_group", fake_group)
|
||||
for rank in range(len(sizes)):
|
||||
PrefillRowShard(decode, sizes, rank).exchange_topk(buffer, width)
|
||||
assert calls[0][1] == sizes and calls[0][0].is_contiguous()
|
||||
torch.testing.assert_close(buffer[decode : decode + sum(sizes)], gathered)
|
||||
assert torch.all(buffer[:decode] == -7), "decode rows stay outside"
|
||||
assert torch.all(buffer[decode + sum(sizes) :] == -7), "padding untouched"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("tp_size", [2, 4])
|
||||
def test_balanced_row_shard_partitions_and_declines(tp_size: int) -> None:
|
||||
"""The split covers every row once with >= 1 row per rank and declines
|
||||
below the activation floor (or on a single rank)."""
|
||||
n = indexer.MIN_TP_SHARD_ROWS_PER_RANK * tp_size
|
||||
shapes = [
|
||||
([4 * n], [4 * n]), # fresh prompt: the full causal cost ramp
|
||||
# uneven requests at different context depths
|
||||
([2 * n, 7, 4 * n, n // 3 + 5], [2 * n, 9007, 4 * n + 500, n // 3 + 60005]),
|
||||
]
|
||||
ratio = 4
|
||||
for query_lens, seq_lens in shapes:
|
||||
sizes = indexer.balanced_prefill_row_shard(
|
||||
torch.tensor(seq_lens, dtype=torch.int32),
|
||||
torch.tensor(query_lens, dtype=torch.int32),
|
||||
ratio,
|
||||
tp_size,
|
||||
)
|
||||
assert sizes is not None and len(sizes) == tp_size
|
||||
assert sum(sizes) == sum(query_lens) and min(sizes) >= 1
|
||||
lens = torch.tensor([n - 1], dtype=torch.int32)
|
||||
assert indexer.balanced_prefill_row_shard(lens, lens, ratio, tp_size) is None
|
||||
assert indexer.balanced_prefill_row_shard(lens, lens, ratio, 1) is None
|
||||
|
||||
|
||||
def test_row_sharding_gate_rejects_unsupported_configurations(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""The gate is the whole safety envelope; nothing else guards the exchange."""
|
||||
monkeypatch.setattr(indexer.current_platform, "is_cuda", lambda: True)
|
||||
|
||||
def supported(**kwargs):
|
||||
mode = kwargs.pop("cudagraph_mode", CUDAGraphMode.PIECEWISE)
|
||||
cfg = SimpleNamespace(compilation_config=SimpleNamespace(cudagraph_mode=mode))
|
||||
args = {"dcp_world_size": 1, "use_pcp": False, "tp_size": 4, **kwargs}
|
||||
return indexer.tp_prefill_row_sharding_supported(cfg, **args)
|
||||
|
||||
assert supported()
|
||||
for bad in ({"tp_size": 1}, {"dcp_world_size": 2}, {"use_pcp": True}):
|
||||
assert not supported(**bad)
|
||||
for name in (
|
||||
"VLLM_DISABLE_PYNCCL",
|
||||
"VLLM_USE_NCCL_SYMM_MEM",
|
||||
"VLLM_BATCH_INVARIANT",
|
||||
):
|
||||
with monkeypatch.context() as m:
|
||||
m.setattr(indexer.envs, name, True)
|
||||
assert not supported()
|
||||
# FULL captures mixed batches whole, so the exchange must stay out.
|
||||
assert not supported(cudagraph_mode=CUDAGraphMode.FULL)
|
||||
assert supported(cudagraph_mode=CUDAGraphMode.FULL_AND_PIECEWISE)
|
||||
|
||||
|
||||
def test_profiling_run_reserves_exchange_transient(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""The dummy run returns before the exchange executes, so its all_gatherv
|
||||
output plus staging copy must be reserved up front when row sharding is
|
||||
active -- and only then."""
|
||||
rows, topk = 128, 8
|
||||
buffer = torch.empty((rows, topk), dtype=torch.int32)
|
||||
peak = 2 * rows * topk * buffer.element_size()
|
||||
specs = (((1,), torch.uint8), ((1,), torch.uint8))
|
||||
numels: list[int] = []
|
||||
real_empty = torch.empty
|
||||
|
||||
def recording_empty(*args, **kwargs):
|
||||
out = real_empty(*args, **kwargs)
|
||||
numels.append(out.numel())
|
||||
return out
|
||||
|
||||
def run_dummy(tp_world: int) -> None:
|
||||
numels.clear()
|
||||
with monkeypatch.context() as m:
|
||||
m.setattr(torch, "empty", recording_empty)
|
||||
m.setattr(
|
||||
sparse_indexer,
|
||||
"get_forward_context",
|
||||
lambda: SimpleNamespace(attn_metadata=None),
|
||||
)
|
||||
m.setattr(sparse_indexer, "_gather_workspace_shapes", lambda *a: specs)
|
||||
m.setattr(
|
||||
sparse_indexer,
|
||||
"current_workspace_manager",
|
||||
lambda: SimpleNamespace(get_simultaneous=lambda *s: None),
|
||||
)
|
||||
m.setattr(
|
||||
sparse_indexer,
|
||||
"get_tp_group",
|
||||
lambda: SimpleNamespace(world_size=tp_world),
|
||||
)
|
||||
m.setattr(
|
||||
sparse_indexer,
|
||||
"get_current_vllm_config_or_none",
|
||||
lambda: SimpleNamespace(
|
||||
compilation_config=SimpleNamespace(
|
||||
cudagraph_mode=CUDAGraphMode.PIECEWISE
|
||||
),
|
||||
parallel_config=SimpleNamespace(
|
||||
decode_context_parallel_size=1,
|
||||
prefill_context_parallel_size=1,
|
||||
),
|
||||
),
|
||||
)
|
||||
m.setattr(sparse_indexer.envs, "VLLM_SPARSE_INDEXER_MAX_LOGITS_MB", 1)
|
||||
m.setattr(indexer.current_platform, "is_cuda", lambda: True)
|
||||
sparse_indexer.sparse_attn_indexer(
|
||||
hidden_states=torch.empty(4, 4),
|
||||
k_cache_prefix="k",
|
||||
kv_cache=torch.empty(4, 4),
|
||||
q_quant=torch.empty(4, 4),
|
||||
q_scale=None,
|
||||
k=None,
|
||||
weights=torch.empty(4),
|
||||
quant_block_size=128,
|
||||
scale_fmt=None,
|
||||
topk_tokens=topk,
|
||||
head_dim=64,
|
||||
max_model_len=1024,
|
||||
total_seq_lens=512,
|
||||
topk_indices_buffer=buffer,
|
||||
skip_k_cache_insert=True,
|
||||
use_pcp=False,
|
||||
pcp_shard_decode_requests=False,
|
||||
dense_mha_metadata_layer_name="",
|
||||
)
|
||||
|
||||
run_dummy(tp_world=2)
|
||||
assert peak in numels
|
||||
run_dummy(tp_world=1)
|
||||
assert peak not in numels
|
||||
@@ -9,8 +9,17 @@ import vllm.envs as envs
|
||||
from vllm import _custom_ops as ops
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
|
||||
from vllm.config import CUDAGraphMode, get_current_vllm_config
|
||||
from vllm.distributed import get_dcp_group, get_pcp_group
|
||||
from vllm.config import (
|
||||
CUDAGraphMode,
|
||||
get_current_vllm_config,
|
||||
get_current_vllm_config_or_none,
|
||||
)
|
||||
from vllm.distributed import (
|
||||
get_dcp_group,
|
||||
get_pcp_group,
|
||||
get_tensor_model_parallel_rank,
|
||||
get_tp_group,
|
||||
)
|
||||
from vllm.forward_context import get_forward_context
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.custom_op import CustomOp
|
||||
@@ -43,6 +52,7 @@ from vllm.utils.torch_utils import (
|
||||
)
|
||||
from vllm.v1.attention.backends.mla.indexer import (
|
||||
DeepseekV32IndexerMetadata,
|
||||
tp_prefill_row_sharding_supported,
|
||||
)
|
||||
from vllm.v1.attention.ops.common import pack_seq_triton, unpack_seq_triton
|
||||
from vllm.v1.attention.ops.pcp import maybe_gather_indexer_k
|
||||
@@ -317,6 +327,75 @@ def kv_cache_as_quant_view(
|
||||
return kv_cache.unsqueeze(-2)
|
||||
|
||||
|
||||
class PrefillRowShard:
|
||||
"""One TP rank's contiguous slice of the replicated prefill rows.
|
||||
|
||||
Rows are independent, so each rank scores only its slice and the group
|
||||
exchanges final int32 top-k indices instead of the logits."""
|
||||
|
||||
def __init__(self, num_decode_tokens: int, sizes: list[int], rank: int):
|
||||
self.num_decode_tokens = num_decode_tokens
|
||||
self.sizes = sizes
|
||||
self.start = num_decode_tokens + sum(sizes[:rank])
|
||||
self.stop = self.start + sizes[rank]
|
||||
|
||||
@classmethod
|
||||
def from_metadata(
|
||||
cls, prefill_metadata, num_decode_tokens: int
|
||||
) -> "PrefillRowShard | None":
|
||||
sizes = getattr(prefill_metadata, "row_shard_sizes", None)
|
||||
if sizes is None:
|
||||
return None
|
||||
return cls(num_decode_tokens, sizes, get_tensor_model_parallel_rank())
|
||||
|
||||
def narrow(self, chunk) -> "tuple[int, int, torch.Tensor, torch.Tensor] | None":
|
||||
"""Row window and narrowed causal bounds of ``chunk`` for this rank."""
|
||||
start = max(chunk.token_start, self.start)
|
||||
end = min(chunk.token_end, self.stop)
|
||||
if start >= end:
|
||||
return None
|
||||
lo, hi = start - chunk.token_start, end - chunk.token_start
|
||||
return start, end, chunk.cu_seqlen_ks[lo:hi], chunk.cu_seqlen_ke[lo:hi]
|
||||
|
||||
def exchange_topk(self, topk_buffer: torch.Tensor, width: int) -> None:
|
||||
"""Gather per-rank final indices back into global row order."""
|
||||
# The local input must outlive the async PyNCCL gather's enqueued copy.
|
||||
local = topk_buffer[self.start : self.stop, :width].contiguous()
|
||||
end = self.num_decode_tokens + sum(self.sizes)
|
||||
topk_buffer[self.num_decode_tokens : end, :width] = get_tp_group().all_gatherv(
|
||||
local, dim=0, sizes=self.sizes
|
||||
)
|
||||
|
||||
|
||||
def _reserve_row_shard_exchange_transient(
|
||||
topk_indices_buffer: torch.Tensor, device: torch.device
|
||||
) -> None:
|
||||
"""Profiling/dummy runs return before ``exchange_topk`` executes; reserve
|
||||
its worst-case transient so the memory profile accounts for it.
|
||||
|
||||
The exchange stages this rank's contiguous slice and the all_gatherv
|
||||
wrapper allocates the full gathered tensor; each is bounded by the
|
||||
buffer's row capacity times its (tile-padded) width.
|
||||
"""
|
||||
cfg = get_current_vllm_config_or_none()
|
||||
if cfg is None:
|
||||
return
|
||||
parallel = cfg.parallel_config
|
||||
if not tp_prefill_row_sharding_supported(
|
||||
cfg,
|
||||
parallel.decode_context_parallel_size,
|
||||
parallel.prefill_context_parallel_size > 1,
|
||||
get_tp_group().world_size,
|
||||
):
|
||||
return
|
||||
rows, width = topk_indices_buffer.shape[:2]
|
||||
_ = torch.empty(
|
||||
2 * rows * width * topk_indices_buffer.element_size(),
|
||||
dtype=torch.uint8,
|
||||
device=device,
|
||||
)
|
||||
|
||||
|
||||
@eager_break_during_capture
|
||||
def sparse_attn_indexer(
|
||||
hidden_states: torch.Tensor,
|
||||
@@ -388,6 +467,10 @@ def sparse_attn_indexer(
|
||||
max_logits_elems, dtype=torch.uint8, device=hidden_states.device
|
||||
)
|
||||
|
||||
# The dummy run returns before exchange_topk executes, so reserve its
|
||||
# worst-case transient for the memory profile.
|
||||
_reserve_row_shard_exchange_transient(topk_indices_buffer, hidden_states.device)
|
||||
|
||||
return sparse_attn_indexer_fake(
|
||||
hidden_states,
|
||||
k_cache_prefix,
|
||||
@@ -515,6 +598,12 @@ def sparse_attn_indexer(
|
||||
scales_spec,
|
||||
*gather_specs,
|
||||
)
|
||||
# One row window per rank; DSV4.1 candidate rows stay replicated.
|
||||
row_shard = (
|
||||
None
|
||||
if candidate_blocks is not None
|
||||
else PrefillRowShard.from_metadata(prefill_metadata, num_decode_tokens)
|
||||
)
|
||||
for chunk in prefill_metadata.chunks:
|
||||
cu_seqlen_ks = chunk.cu_seqlen_ks
|
||||
cu_seqlen_ke = chunk.cu_seqlen_ke
|
||||
@@ -530,6 +619,15 @@ def sparse_attn_indexer(
|
||||
chunk.local_cu_seq_lens,
|
||||
)
|
||||
|
||||
# Narrow scoring to this rank; KV gathers remain unconditional for
|
||||
# continuation chunks that reuse the workspace.
|
||||
row_start, row_end = chunk.token_start, chunk.token_end
|
||||
if row_shard is not None:
|
||||
narrowed = row_shard.narrow(chunk)
|
||||
if narrowed is None:
|
||||
continue
|
||||
row_start, row_end, cu_seqlen_ks, cu_seqlen_ke = narrowed
|
||||
|
||||
# PCP + DCP KV all-gather.
|
||||
deinterleave_idx = chunk.pcp_deinterleave_idx
|
||||
if deinterleave_idx is not None:
|
||||
@@ -547,15 +645,9 @@ def sparse_attn_indexer(
|
||||
out_scales,
|
||||
)
|
||||
|
||||
q_slice = q_quant[chunk.token_start : chunk.token_end]
|
||||
q_scale_slice = (
|
||||
q_scale[chunk.token_start : chunk.token_end]
|
||||
if q_scale is not None
|
||||
else None
|
||||
)
|
||||
topk_indices = topk_indices_buffer[
|
||||
chunk.token_start : chunk.token_end, :topk_tokens
|
||||
]
|
||||
q_slice = q_quant[row_start:row_end]
|
||||
q_scale_slice = q_scale[row_start:row_end] if q_scale is not None else None
|
||||
topk_indices = topk_indices_buffer[row_start:row_end, :topk_tokens]
|
||||
|
||||
if chunk.local_total_seq_lens == 0 or q_slice.shape[0] == 0:
|
||||
logits = q_slice.new_empty((q_slice.shape[0], 0), dtype=torch.float32)
|
||||
@@ -578,7 +670,7 @@ def sparse_attn_indexer(
|
||||
q_slice_cast,
|
||||
k_quant_cast,
|
||||
k_scale_cast,
|
||||
weights[chunk.token_start : chunk.token_end],
|
||||
weights[row_start:row_end],
|
||||
cu_seqlen_ks,
|
||||
cu_seqlen_ke,
|
||||
)
|
||||
@@ -586,7 +678,7 @@ def sparse_attn_indexer(
|
||||
logits = fp8_fp4_mqa_logits(
|
||||
(q_slice_cast, q_scale_slice),
|
||||
(k_quant_cast, k_scale_cast),
|
||||
weights[chunk.token_start : chunk.token_end],
|
||||
weights[row_start:row_end],
|
||||
cu_seqlen_ks,
|
||||
cu_seqlen_ke,
|
||||
clean_logits=False,
|
||||
@@ -637,9 +729,13 @@ def sparse_attn_indexer(
|
||||
dcp_rank,
|
||||
dcp_world_size,
|
||||
cp_kv_cache_interleave_size,
|
||||
row_starts=chunk.cu_seqlen_ks,
|
||||
row_starts=cu_seqlen_ks,
|
||||
)
|
||||
|
||||
if row_shard is not None:
|
||||
# Rows are already fully ranked, so this is a layout gather.
|
||||
row_shard.exchange_topk(topk_indices_buffer, topk_tokens)
|
||||
|
||||
if has_decode:
|
||||
decode_metadata = attn_metadata_narrowed.decode
|
||||
assert decode_metadata is not None
|
||||
|
||||
@@ -12,7 +12,11 @@ from vllm.forward_context import get_forward_context
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.custom_op import CustomOp
|
||||
from vllm.model_executor.layers.indexer_topk import get_indexer_topk
|
||||
from vllm.model_executor.layers.sparse_attn_indexer import _merge_dcp_topk_global
|
||||
from vllm.model_executor.layers.sparse_attn_indexer import (
|
||||
PrefillRowShard,
|
||||
_merge_dcp_topk_global,
|
||||
_reserve_row_shard_exchange_transient,
|
||||
)
|
||||
from vllm.models.glm5next.common.sparse_indexer import (
|
||||
RADIX_TOPK_WORKSPACE_SIZE,
|
||||
_build_decode_scatter_indices,
|
||||
@@ -169,6 +173,10 @@ def sparse_attn_indexer_kpool(
|
||||
max_logits_elems, dtype=torch.uint8, device=hidden_states.device
|
||||
)
|
||||
|
||||
# The dummy run returns before exchange_topk executes, so reserve its
|
||||
# worst-case transient for the memory profile.
|
||||
_reserve_row_shard_exchange_transient(topk_indices_buffer, hidden_states.device)
|
||||
|
||||
return topk_indices_buffer
|
||||
attn_metadata_narrowed = attn_metadata[k_cache_prefix]
|
||||
assert isinstance(attn_metadata_narrowed, DeepseekV32IndexerMetadata)
|
||||
@@ -282,6 +290,12 @@ def sparse_attn_indexer_kpool(
|
||||
_pos = positions[num_decode_tokens:num_tokens].to(torch.int32)
|
||||
_buf = topk_indices_buffer[num_decode_tokens:num_tokens]
|
||||
_fill_causal_indices(_buf, _pos)
|
||||
# Short prefills score nothing locally; suppress the shard there.
|
||||
row_shard = None
|
||||
if not short_prefill:
|
||||
row_shard = PrefillRowShard.from_metadata(
|
||||
prefill_metadata, num_decode_tokens
|
||||
)
|
||||
|
||||
# Get the full shared workspace buffers once (will allocate on first use).
|
||||
# Layout switches between FP8 (head_dim bytes + 4-byte fp32 scale) and
|
||||
@@ -310,12 +324,16 @@ def sparse_attn_indexer_kpool(
|
||||
chunk.local_cu_seq_lens,
|
||||
)
|
||||
|
||||
q_slice = q_quant[chunk.token_start : chunk.token_end]
|
||||
q_scale_slice = (
|
||||
q_scale[chunk.token_start : chunk.token_end]
|
||||
if q_scale is not None
|
||||
else None
|
||||
)
|
||||
row_start, row_end = chunk.token_start, chunk.token_end
|
||||
cu_seqlen_ks, cu_seqlen_ke = chunk.cu_seqlen_ks, chunk.cu_seqlen_ke
|
||||
if row_shard is not None:
|
||||
narrowed = row_shard.narrow(chunk)
|
||||
if narrowed is None:
|
||||
continue
|
||||
row_start, row_end, cu_seqlen_ks, cu_seqlen_ke = narrowed
|
||||
|
||||
q_slice = q_quant[row_start:row_end]
|
||||
q_scale_slice = q_scale[row_start:row_end] if q_scale is not None else None
|
||||
# DeepGEMM scalar-type tags (zero-copy): MXFP4 values → int8
|
||||
# (kPackedFP4), scales → int32 squeezed to 1-D kv_sf / 2-D q_sf.
|
||||
if use_fp4_cache:
|
||||
@@ -335,9 +353,9 @@ def sparse_attn_indexer_kpool(
|
||||
logits = fp8_fp4_mqa_logits(
|
||||
(q_slice_cast, q_scale_slice),
|
||||
(k_quant_cast, k_scale_cast),
|
||||
weights[chunk.token_start : chunk.token_end],
|
||||
chunk.cu_seqlen_ks,
|
||||
chunk.cu_seqlen_ke,
|
||||
weights[row_start:row_end],
|
||||
cu_seqlen_ks,
|
||||
cu_seqlen_ke,
|
||||
clean_logits=False,
|
||||
)
|
||||
|
||||
@@ -351,17 +369,15 @@ def sparse_attn_indexer_kpool(
|
||||
)
|
||||
topk_dst = pool_topk
|
||||
else:
|
||||
topk_dst = topk_indices_buffer[
|
||||
chunk.token_start : chunk.token_end, :topk_tokens
|
||||
]
|
||||
topk_dst = topk_indices_buffer[row_start:row_end, :topk_tokens]
|
||||
|
||||
if logits.shape[1] == 0:
|
||||
topk_dst.fill_(-1)
|
||||
else:
|
||||
torch.ops._C.top_k_per_row_prefill(
|
||||
logits,
|
||||
chunk.cu_seqlen_ks,
|
||||
chunk.cu_seqlen_ke,
|
||||
cu_seqlen_ks,
|
||||
cu_seqlen_ke,
|
||||
topk_dst,
|
||||
num_rows,
|
||||
logits.stride(0),
|
||||
@@ -376,7 +392,7 @@ def sparse_attn_indexer_kpool(
|
||||
dcp_rank,
|
||||
dcp_world_size,
|
||||
pool_cp_interleave,
|
||||
row_starts=chunk.cu_seqlen_ks,
|
||||
row_starts=cu_seqlen_ks,
|
||||
)
|
||||
|
||||
if index_kpool > 1:
|
||||
@@ -385,10 +401,7 @@ def sparse_attn_indexer_kpool(
|
||||
# Fused expand-pools + append-tail into one Triton kernel
|
||||
# (replaces ~25 elementwise ops). seq_len is token-granular
|
||||
# (pos+1); the kernel derives pool_len internally.
|
||||
q_seq = (
|
||||
positions[chunk.token_start : chunk.token_end].to(torch.int32)
|
||||
+ 1
|
||||
)
|
||||
q_seq = positions[row_start:row_end].to(torch.int32) + 1
|
||||
expanded = kpool_ops.expand_pools_and_append_tail(
|
||||
pool_ids, q_seq, index_kpool
|
||||
)
|
||||
@@ -397,9 +410,13 @@ def sparse_attn_indexer_kpool(
|
||||
expanded = kpool_ops.expand_pools_to_tokens(
|
||||
pool_ids, valid, topk_tokens, index_kpool
|
||||
)
|
||||
topk_indices_buffer[
|
||||
chunk.token_start : chunk.token_end, : expanded.shape[-1]
|
||||
] = expanded
|
||||
topk_indices_buffer[row_start:row_end, : expanded.shape[-1]] = expanded
|
||||
|
||||
if row_shard is not None:
|
||||
# The k-pool expansion appends the request's incomplete tail
|
||||
# (index_kpool - 1 entries) after the logical top-k history; those
|
||||
# columns are part of the attention index and are exchanged too.
|
||||
row_shard.exchange_topk(topk_indices_buffer, topk_tokens + index_kpool - 1)
|
||||
|
||||
if has_decode:
|
||||
decode_metadata = attn_metadata_narrowed.decode
|
||||
|
||||
@@ -7,7 +7,7 @@ import numpy as np
|
||||
import torch
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.config import VllmConfig, get_current_vllm_config
|
||||
from vllm.config import CUDAGraphMode, VllmConfig, get_current_vllm_config
|
||||
from vllm.distributed import get_dcp_group, get_pcp_group
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.warmup.jit_warmup import kernel_launcher, zip_inputs
|
||||
@@ -204,6 +204,64 @@ class PrepareUniformDecodeKernel(
|
||||
)
|
||||
|
||||
|
||||
# Below this many rows per rank, stay replicated (TP4 engages at 64K rows).
|
||||
MIN_TP_SHARD_ROWS_PER_RANK = 16_384
|
||||
|
||||
|
||||
def balanced_prefill_row_shard(
|
||||
seq_lens_cpu: torch.Tensor,
|
||||
query_lens_cpu: torch.Tensor,
|
||||
compress_ratio: int,
|
||||
tp_size: int,
|
||||
) -> list[int] | None:
|
||||
"""Return contiguous TP row counts balanced by indexer MQA work."""
|
||||
num_rows = int(query_lens_cpu.sum())
|
||||
if tp_size < 2 or num_rows < MIN_TP_SHARD_ROWS_PER_RANK * tp_size:
|
||||
return None
|
||||
|
||||
query_lens = query_lens_cpu.to(torch.int64)
|
||||
first_key = torch.repeat_interleave(
|
||||
seq_lens_cpu.to(torch.int64) - query_lens + 1, query_lens
|
||||
)
|
||||
row_in_request = torch.arange(num_rows) - torch.repeat_interleave(
|
||||
torch.cumsum(query_lens, 0) - query_lens, query_lens
|
||||
)
|
||||
cost = torch.cumsum((first_key + row_in_request) // compress_ratio, 0)
|
||||
total = int(cost[-1]) if cost.numel() else 0
|
||||
if total <= 0:
|
||||
return None
|
||||
|
||||
targets = torch.arange(1, tp_size) * total // tp_size
|
||||
bounds = [0, *torch.searchsorted(cost, targets).tolist(), num_rows]
|
||||
# Keep >= 1 row per rank (the floor above leaves ample room); this also
|
||||
# handles repeated costs from pool compression.
|
||||
for i in range(1, tp_size):
|
||||
bounds[i] = max(bounds[i], bounds[i - 1] + 1)
|
||||
for i in range(tp_size - 1, 0, -1):
|
||||
bounds[i] = min(bounds[i], bounds[i + 1] - 1)
|
||||
return [bounds[i + 1] - bounds[i] for i in range(tp_size)]
|
||||
|
||||
|
||||
def tp_prefill_row_sharding_supported(
|
||||
vllm_config: VllmConfig,
|
||||
dcp_world_size: int,
|
||||
use_pcp: bool,
|
||||
tp_size: int,
|
||||
) -> bool:
|
||||
"""Whether replicated prefill rows may be sharded across TP ranks."""
|
||||
cudagraph_mode = vllm_config.compilation_config.cudagraph_mode or CUDAGraphMode.NONE
|
||||
return (
|
||||
current_platform.is_cuda()
|
||||
and dcp_world_size == 1
|
||||
and not use_pcp
|
||||
and tp_size > 1
|
||||
and not envs.VLLM_DISABLE_PYNCCL
|
||||
and not envs.VLLM_USE_NCCL_SYMM_MEM
|
||||
and not envs.VLLM_BATCH_INVARIANT
|
||||
and cudagraph_mode.mixed_mode() != CUDAGraphMode.FULL
|
||||
)
|
||||
|
||||
|
||||
class DeepseekV32IndexerBackend(AttentionBackend):
|
||||
@classmethod
|
||||
def supports_pcp(cls) -> bool:
|
||||
@@ -654,6 +712,8 @@ class BuildPrefillChunkMetadataKernel(
|
||||
class DeepseekV32IndexerPrefillMetadata:
|
||||
chunks: list[DeepseekV32IndexerPrefillChunkMetadata]
|
||||
max_prefill_seq_len: int = -1
|
||||
# Per-TP-rank contiguous row counts, or None for the replicated path.
|
||||
row_shard_sizes: list[int] | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -941,6 +1001,7 @@ class DeepseekV32IndexerMetadataBuilder(AttentionMetadataBuilder):
|
||||
scheduler_config = self.vllm_config.scheduler_config
|
||||
parallel_config = self.vllm_config.parallel_config
|
||||
self.dcp_world_size = parallel_config.decode_context_parallel_size
|
||||
self.tp_size = parallel_config.tensor_parallel_size
|
||||
self.dcp_rank = get_dcp_group().rank_in_group if self.dcp_world_size > 1 else 0
|
||||
self.pcp_world_size = parallel_config.prefill_context_parallel_size
|
||||
self.use_pcp = self.pcp_world_size > 1
|
||||
@@ -1512,6 +1573,16 @@ class DeepseekV32IndexerMetadataBuilder(AttentionMetadataBuilder):
|
||||
# Skip when total_seq_lens is 0 (i.e., no compressed token).
|
||||
if metadata is not None:
|
||||
chunks.append(metadata)
|
||||
row_shard_sizes = None
|
||||
if tp_prefill_row_sharding_supported(
|
||||
self.vllm_config, self.dcp_world_size, self.use_pcp, self.tp_size
|
||||
):
|
||||
row_shard_sizes = balanced_prefill_row_shard(
|
||||
seq_lens_cpu[num_decodes:],
|
||||
prefill_query_lens_cpu,
|
||||
self.compress_ratio,
|
||||
self.tp_size,
|
||||
)
|
||||
prefill_metadata = DeepseekV32IndexerPrefillMetadata(
|
||||
chunks,
|
||||
max_prefill_seq_len=(
|
||||
@@ -1519,6 +1590,7 @@ class DeepseekV32IndexerMetadataBuilder(AttentionMetadataBuilder):
|
||||
if num_prefills > 0
|
||||
else 0
|
||||
),
|
||||
row_shard_sizes=row_shard_sizes,
|
||||
)
|
||||
|
||||
decode_metadata = None
|
||||
|
||||
Reference in New Issue
Block a user