[Model][GLM-5.3-Flash] Support DCP for the kpool sparse indexer (#59211)

Signed-off-by: NickLucche <nicolo.lucchesi@mistral.ai>
This commit is contained in:
Nicolò Lucchesi
2026-10-05 16:44:14 +02:00
committed by GitHub
parent f9c9e8ac24
commit 30d4032363
9 changed files with 243 additions and 38 deletions
@@ -315,6 +315,9 @@ def test_compressed_slot_mapping_warmup_includes_index_kpool():
config = SimpleNamespace(
cache_config=SimpleNamespace(block_size=256),
model_config=SimpleNamespace(hf_text_config=SimpleNamespace(index_kpool=32)),
parallel_config=SimpleNamespace(
decode_context_parallel_size=1, cp_kv_cache_interleave_size=1
),
)
keys = CompressedSlotMappingKernel().get_warmup_keys(config)
@@ -343,6 +346,41 @@ def test_compressed_slot_mapping_inherits_padded_token_slots():
assert compressed.tolist() == [-1, -1, -1, -1, -1, 3 * 4 + 2, -1, 3 * 4 + 3]
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
def test_compressed_slot_mapping_dcp_uses_rank_local_state_positions():
"""Under DCP each rank writes only the states it owns, at the state's
rank-local position."""
device = torch.device("cuda")
dcp, interleave, ratio, block_size = 2, 4, 2, 4
num_tokens = 32
query_start_loc = torch.tensor([0, num_tokens], dtype=torch.int32, device=device)
seq_lens = torch.tensor([num_tokens], dtype=torch.int32, device=device)
block_table = torch.tensor([[5, 7]], dtype=torch.int32, device=device)
pos = torch.arange(num_tokens)
for rank in range(dcp):
owned = (pos // interleave) % dcp == rank
slot_mapping = torch.where(owned, 0, -1).to(torch.int64).to(device)
compressed = get_compressed_slot_mapping(
num_tokens,
slot_mapping,
query_start_loc,
seq_lens,
block_table,
block_size=block_size,
compress_ratio=ratio,
dcp_world_size=dcp,
cp_interleave=interleave,
)
local_pos = pos // (interleave * dcp) * interleave + pos % interleave
state = local_pos // ratio
expected = torch.where(
owned & ((pos + 1) % ratio == 0),
block_table[0].cpu()[state // block_size] * block_size + state % block_size,
-1,
)
assert compressed.tolist() == expected.tolist()
def test_index_conversion_warmup_uses_physical_block_stride():
config = SimpleNamespace(
cache_config=SimpleNamespace(block_size=64),
@@ -370,6 +408,8 @@ def test_zero_token_pcp_rank_participates_in_compressed_mapping_gather(monkeypat
builder.use_pcp = True
builder.pcp_world_size = 4
builder.pcp_rank = 0
builder.dcp_world_size = 1
builder.cp_kv_cache_interleave_size = 1
builder.kernel_block_size = None
builder.kv_cache_spec = SimpleNamespace(block_size=64, num_states=64)
builder.compressed_slot_mapping_buffer = torch.zeros(8, dtype=torch.int64)
@@ -383,8 +423,11 @@ def test_zero_token_pcp_rank_participates_in_compressed_mapping_gather(monkeypat
block_size,
compress_ratio,
out,
dcp_world_size,
cp_interleave,
):
assert num_tokens == 0
assert dcp_world_size == 1
assert block_size == 64
assert compress_ratio == 4
out.fill_(-1)
+21
View File
@@ -1873,6 +1873,27 @@ def test_dcp_target_allocates_replicated_draft_independently(
assert [len(group) for group in cached_blocks.blocks] == [1, 4]
def test_dcp_replicated_kpool_tail_keeps_block_interior_layout(monkeypatch):
"""A replicated kpool tail ring is per-request state, not a draft group."""
from vllm.v1.attention.backends.utils import resolve_kv_cache_layout
monkeypatch.delenv("VLLM_KV_CACHE_LAYOUT", raising=False)
config = VllmConfig(model_config=ModelConfig(max_model_len=1024))
config.parallel_config.decode_context_parallel_size = 4
config.cache_config.kv_cache_layout = None
tail = KpoolTailSpec(
block_size=4,
num_kv_heads=2,
head_size=128,
head_size_v=0,
dtype=torch.bfloat16,
sliding_window=4,
dcp_sharded=False,
)
layout = resolve_kv_cache_layout(config, [["LBHNC"]], [new_mla_spec(), tail])
assert layout == KVCacheLayout.LBHNC
@pytest.mark.parametrize("use_mla", [False, True])
def test_full_attention_merge_preserves_replicated_cache_geometry(use_mla):
config = VllmConfig(model_config=ModelConfig(max_model_len=1024))
@@ -25,6 +25,22 @@ from vllm.model_executor.warmup.jit_warmup_triton_helper import (
from vllm.triton_utils import tl, triton
def _dcp_merge_topk_and_interleave(vllm_config: Any) -> tuple[int, int]:
"""(topk, cp_interleave) the DCP merge runs with.
Kpool indexers (GLM-5.x) merge pool ids: topk and the interleave are both
in units of ``index_kpool`` tokens.
"""
hf_text_config = vllm_config.model_config.hf_text_config
topk = hf_text_config.index_topk
cp_interleave = vllm_config.parallel_config.cp_kv_cache_interleave_size
kpool = getattr(hf_text_config, "index_kpool", None) or 1
if kpool > 1:
topk //= kpool
cp_interleave //= kpool
return topk, cp_interleave
def stable_topk_from_gathered_candidates_cutedsl(
gathered: torch.Tensor,
topk: int,
@@ -171,8 +187,7 @@ class PackDCPTopkCandidatesKernel(
dcp_world_size = vllm_config.parallel_config.decode_context_parallel_size
if dcp_world_size <= 1:
return []
cp_interleave = vllm_config.parallel_config.cp_kv_cache_interleave_size
topk = vllm_config.model_config.hf_text_config.index_topk
topk, cp_interleave = _dcp_merge_topk_and_interleave(vllm_config)
if topk <= 0:
return []
@@ -545,7 +560,7 @@ class StableTopKFromGatheredCandidatesKernel(
dcp_world_size = vllm_config.parallel_config.decode_context_parallel_size
if dcp_world_size <= 1:
return []
topk = vllm_config.model_config.hf_text_config.index_topk
topk, _ = _dcp_merge_topk_and_interleave(vllm_config)
if topk <= 0:
return []
return self._trace_dispatch(self.dispatch)(
@@ -733,6 +733,10 @@ class SparseAttnIndexerKpool(CustomOp):
self.topk_backend = (
cfg.kernel_config.sparse_indexer_topk_backend if cfg is not None else "auto"
)
if cfg is not None and cfg.parallel_config.decode_context_parallel_size > 1:
raise NotImplementedError(
"SparseAttnIndexerKpool on ROCm does not support DCP."
)
def forward_hip(
self,
+3
View File
@@ -209,6 +209,9 @@ class Glm5NextTailCache(DeepseekV32IndexerCache):
head_size_v=0,
dtype=torch.bfloat16,
sliding_window=ring,
# Tiny per-request ring; every DCP rank keeps the full tail so the
# rank owning a pool can compress it.
dcp_sharded=False,
)
def get_attn_backend(self):
+86 -27
View File
@@ -7,10 +7,12 @@ import torch
import vllm.envs as envs
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
from vllm.config import get_current_vllm_config_or_none
from vllm.distributed import get_dcp_group
from vllm.forward_context import get_forward_context
from vllm.logger import init_logger
from vllm.model_executor.custom_op import CustomOp
from vllm.model_executor.layers.indexer_topk import get_indexer_topk
from vllm.model_executor.layers.sparse_attn_indexer import _merge_dcp_topk_global
from vllm.models.glm5next.common.sparse_indexer import (
RADIX_TOPK_WORKSPACE_SIZE,
_build_decode_scatter_indices,
@@ -121,6 +123,9 @@ def sparse_attn_indexer_kpool(
tail_kv_cache: torch.Tensor | None = None,
tail_prefix: str | None = None,
topk_backend: str = "auto",
dcp_rank: int = 0,
dcp_world_size: int = 1,
cp_kv_cache_interleave_size: int = 1,
) -> torch.Tensor:
# careful! this will be None in dummy run
attn_metadata = get_forward_context().attn_metadata
@@ -170,6 +175,12 @@ def sparse_attn_indexer_kpool(
has_decode = attn_metadata_narrowed.num_decodes > 0
has_prefill = attn_metadata_narrowed.num_prefills > 0
num_decode_tokens = attn_metadata_narrowed.num_decode_tokens
# Pools are the DCP sharding unit of the index K cache.
pool_cp_interleave = (
cp_kv_cache_interleave_size // max(index_kpool, 1)
if dcp_world_size > 1
else cp_kv_cache_interleave_size
)
# q_scale is required iff the FP4 cache path is enabled; the FP8 path
# folds the Q scale into `weights` inside fused_indexer_q_rope_quant.
@@ -284,16 +295,18 @@ def sparse_attn_indexer_kpool(
scales_spec,
)
for chunk in prefill_metadata.chunks if not short_prefill else ():
k_quant = k_quant_full[: chunk.total_seq_lens]
k_scale = k_scale_full[: chunk.total_seq_lens]
# Under DCP this rank only gathers the pools it owns.
assert chunk.local_cu_seq_lens is not None
k_quant = k_quant_full[: chunk.max_local_total_seq_lens]
k_scale = k_scale_full[: chunk.max_local_total_seq_lens]
if not chunk.skip_kv_gather:
if not chunk.skip_kv_gather and chunk.local_total_seq_lens > 0:
ops.cp_gather_indexer_k_quant_cache(
kv_cache,
k_quant,
k_scale,
chunk.block_table,
chunk.cu_seq_lens,
chunk.local_cu_seq_lens,
)
q_slice = q_quant[chunk.token_start : chunk.token_end]
@@ -314,15 +327,18 @@ def sparse_attn_indexer_kpool(
k_scale_cast = k_scale.view(torch.float32).squeeze(-1)
from vllm.utils.deep_gemm import fp8_fp4_mqa_logits
logits = fp8_fp4_mqa_logits(
(q_slice_cast, q_scale_slice),
(k_quant_cast, k_scale_cast),
weights[chunk.token_start : chunk.token_end],
chunk.cu_seqlen_ks,
chunk.cu_seqlen_ke,
clean_logits=False,
)
num_rows = logits.shape[0]
num_rows = q_slice.shape[0]
if chunk.local_total_seq_lens == 0:
logits = q_slice.new_empty((num_rows, 0), dtype=torch.float32)
else:
logits = fp8_fp4_mqa_logits(
(q_slice_cast, q_scale_slice),
(k_quant_cast, k_scale_cast),
weights[chunk.token_start : chunk.token_end],
chunk.cu_seqlen_ks,
chunk.cu_seqlen_ke,
clean_logits=False,
)
# kpool: logits are pool-granular (compress_ratio == index_kpool),
# so topk selects pools. We pick topk_tokens // kpool pools then
@@ -338,15 +354,28 @@ def sparse_attn_indexer_kpool(
chunk.token_start : chunk.token_end, :topk_tokens
]
torch.ops._C.top_k_per_row_prefill(
if logits.shape[1] == 0:
topk_dst.fill_(-1)
else:
torch.ops._C.top_k_per_row_prefill(
logits,
chunk.cu_seqlen_ks,
chunk.cu_seqlen_ke,
topk_dst,
num_rows,
logits.stride(0),
logits.stride(1),
select_k,
)
# Local pool ids -> request-relative global pool ids.
_merge_dcp_topk_global(
logits,
chunk.cu_seqlen_ks,
chunk.cu_seqlen_ke,
topk_dst,
num_rows,
logits.stride(0),
logits.stride(1),
select_k,
dcp_rank,
dcp_world_size,
pool_cp_interleave,
row_starts=chunk.cu_seqlen_ks,
)
if index_kpool > 1:
@@ -584,6 +613,14 @@ def sparse_attn_indexer_kpool(
select_k,
attn_metadata_narrowed.max_seq_len,
)
_merge_dcp_topk_global(
logits,
topk_dst,
select_k,
dcp_rank,
dcp_world_size,
pool_cp_interleave,
)
# Resolve to token-level indices in the output buffer.
if index_kpool > 1:
@@ -669,14 +706,33 @@ class SparseAttnIndexerKpool(CustomOp):
else "auto"
)
_parallel = _cfg.parallel_config if _cfg is not None else None
if (
_parallel is not None
and _parallel.prefill_context_parallel_size > 1
and _parallel.decode_context_parallel_size > 1
):
raise NotImplementedError(
"SparseAttnIndexerKpool does not support PCP+DCP."
)
self._parallel_config = _parallel
self._cp_kv_cache_interleave_size: int | None = None
self.dcp_world_size = 1
self.dcp_rank = 0
if _parallel is not None and _parallel.decode_context_parallel_size > 1:
if _parallel.prefill_context_parallel_size > 1:
raise NotImplementedError(
"SparseAttnIndexerKpool does not support PCP+DCP."
)
self.dcp_world_size = _parallel.decode_context_parallel_size
self.dcp_rank = get_dcp_group().rank_in_group
@property
def cp_kv_cache_interleave_size(self) -> int:
"""Resolved lazily like ``SparseAttnIndexer``: NIXL P/D can adjust it
after the model is built, and the indexer metadata builder sees the
adjusted value.
"""
if self.dcp_world_size == 1:
return 1
if self._cp_kv_cache_interleave_size is None:
assert self._parallel_config is not None
value = self._parallel_config.cp_kv_cache_interleave_size
if isinstance(get_forward_context().attn_metadata, dict):
self._cp_kv_cache_interleave_size = value
return value
return self._cp_kv_cache_interleave_size
def forward_native(
self,
@@ -743,4 +799,7 @@ class SparseAttnIndexerKpool(CustomOp):
self.tail_cache.kv_cache if self.tail_cache is not None else None,
self.tail_cache.prefix if self.tail_cache is not None else None,
self.topk_backend,
self.dcp_rank,
self.dcp_world_size,
self.cp_kv_cache_interleave_size,
)
@@ -37,6 +37,8 @@ class CompressedSlotMappingKernel(
compress_ratio: int
triton_block_size: int
block_size: int
dcp_world_size: int
cp_interleave: int
@staticmethod
@triton.jit(do_not_specialize=["block_table_stride"])
@@ -53,6 +55,8 @@ class CompressedSlotMappingKernel(
block_table_ptr,
block_table_stride,
block_size,
dcp_world_size,
cp_interleave,
COMPRESS_RATIO: tl.constexpr,
PAD_ID: tl.constexpr,
TRITON_BLOCK_SIZE: tl.constexpr,
@@ -77,7 +81,14 @@ class CompressedSlotMappingKernel(
slot_mapping_ptr + query_start + offset, mask=mask, other=PAD_ID
)
is_valid = ((pos + 1) % COMPRESS_RATIO == 0) & (slot != PAD_ID)
pos_after_compress = pos // COMPRESS_RATIO
# Under DCP a PAD slot also marks tokens owned by other ranks; the
# interleave is a multiple of COMPRESS_RATIO, so a state lives
# entirely on the rank owning its last token.
local_pos = (
pos // (cp_interleave * dcp_world_size) * cp_interleave
+ pos % cp_interleave
)
pos_after_compress = local_pos // COMPRESS_RATIO
block_ids = pos_after_compress // block_size
block_numbers = tl.load(
@@ -97,11 +108,15 @@ class CompressedSlotMappingKernel(
*,
compress_ratio: int,
block_size: int,
dcp_world_size: int,
cp_interleave: int,
) -> CompileKey:
return self.CompileKey(
compress_ratio=compress_ratio,
triton_block_size=self.TRITON_BLOCK_SIZE,
block_size=triton_scalar_specialization_rep(block_size),
dcp_world_size=triton_scalar_specialization_rep(dcp_world_size),
cp_interleave=triton_scalar_specialization_rep(cp_interleave),
)
def get_warmup_keys(self, vllm_config: Any) -> list[CompileKey]:
@@ -115,12 +130,15 @@ class CompressedSlotMappingKernel(
)
if not compress_ratios:
return []
parallel_config = vllm_config.parallel_config
return self._trace_dispatch(self.dispatch)(
zip_inputs(
*(
dict(
compress_ratio=ratio,
block_size=vllm_config.cache_config.block_size // ratio,
dcp_world_size=parallel_config.decode_context_parallel_size,
cp_interleave=parallel_config.cp_kv_cache_interleave_size,
)
for ratio in compress_ratios
)
@@ -137,6 +155,8 @@ class CompressedSlotMappingKernel(
block_table=int32_ptr,
block_size=compile_key.block_size,
compress_ratio=compile_key.compress_ratio,
dcp_world_size=compile_key.dcp_world_size,
cp_interleave=compile_key.cp_interleave,
)
@kernel_launcher
@@ -149,6 +169,8 @@ class CompressedSlotMappingKernel(
block_table: torch.Tensor,
block_size: int,
compress_ratio: int,
dcp_world_size: int,
cp_interleave: int,
) -> LaunchSpec:
return (block_table.shape[0],), dict(
block_table_stride=block_table.stride(0),
@@ -167,6 +189,8 @@ def get_compressed_slot_mapping(
block_size: int,
compress_ratio: int,
out: torch.Tensor | None = None,
dcp_world_size: int = 1,
cp_interleave: int = 1,
) -> torch.Tensor:
"""Slot mapping for writing the compressed states of ``num_tokens`` tokens.
@@ -196,6 +220,8 @@ def get_compressed_slot_mapping(
block_table,
block_size,
compress_ratio,
dcp_world_size,
cp_interleave,
)
return compressed_slot_mapping
+36 -6
View File
@@ -554,7 +554,15 @@ class BuildPrefillChunkMetadataKernel(
# Cover Triton's divisible, exact-one, and generic i32 classes.
query_slice_start=(0, 1, 2),
query_slice_stop=(1, 2 * max_tokens - 1, 2 * max_tokens),
DCP_INTERLEAVE=dcp_interleave,
# Compressed indexers localize in units of compressed states.
DCP_INTERLEAVE=list(
dict.fromkeys(
dcp_interleave // ratio
for ratio in compress_ratios
if dcp_interleave % ratio == 0
)
)
or [dcp_interleave],
BLOCK_SIZE=self.BLOCK_SIZE,
COMPRESS_RATIO=list(compress_ratios),
# PCP's global cumulative lengths are the second row of one packed
@@ -1010,11 +1018,31 @@ class DeepseekV32IndexerMetadataBuilder(AttentionMetadataBuilder):
(self.num_sms + 1, 2), dtype=torch.int32, device=self.device
)
if self.dcp_world_size > 1 and self.compress_ratio > 1:
raise NotImplementedError(
"DCP is not supported with sparse indexer KV compression "
f"(compress_ratio={self.compress_ratio})."
# Each compressed state must live on a single rank, so the DCP
# interleave has to cover whole states.
if (
self.dcp_world_size > 1
and self.compress_ratio > 1
and (
(
isinstance(self.kv_cache_spec, MLAAttentionSpec)
and self.kv_cache_spec.model_version == "deepseek_v4"
)
or self.cp_kv_cache_interleave_size % self.compress_ratio != 0
)
):
raise NotImplementedError(
"DCP with sparse indexer KV compression "
f"(compress_ratio={self.compress_ratio}) requires "
"--cp-kv-cache-interleave-size to be a multiple of it; "
"DeepSeek-V4 is not supported."
)
# DCP interleave in units of compressed states.
self.compressed_cp_interleave_size = (
self.cp_kv_cache_interleave_size // self.compress_ratio
if self.dcp_world_size > 1 and self.compress_ratio > 1
else self.cp_kv_cache_interleave_size
)
# Pre-allocate buffers for CUDA graph compatibility when
if self.compress_ratio > 1:
@@ -1348,6 +1376,8 @@ class DeepseekV32IndexerMetadataBuilder(AttentionMetadataBuilder):
self.kv_cache_spec.num_states,
self.compress_ratio,
out=self.compressed_slot_mapping_buffer,
dcp_world_size=self.dcp_world_size,
cp_interleave=self.cp_kv_cache_interleave_size,
)
if self.pcp_world_size > 1:
compressed_slot_mapping = get_pcp_group().all_gather(
@@ -1467,7 +1497,7 @@ class DeepseekV32IndexerMetadataBuilder(AttentionMetadataBuilder):
skip_kv_gather=query_slice.start > 0,
dcp_rank=self.dcp_rank,
dcp_world_size=self.dcp_world_size,
cp_kv_cache_interleave_size=self.cp_kv_cache_interleave_size,
cp_kv_cache_interleave_size=self.compressed_cp_interleave_size,
pcp_plan=pcp_plan,
)
# Skip when total_seq_lens is 0 (i.e., no compressed token).
+5 -1
View File
@@ -307,8 +307,12 @@ def resolve_kv_cache_layout(
f"none is in every supported set: {supported_layouts}."
)
# Self-addressed per-request rings (e.g. the kpool tail) are replicated
# state like Mamba and don't need a separate draft group.
dcp_sharding = {
spec.dcp_sharded for spec in kv_cache_specs if isinstance(spec, AttentionSpec)
spec.dcp_sharded
for spec in kv_cache_specs
if isinstance(spec, AttentionSpec) and spec.uses_slot_mapping
}
page_sizes = {spec.page_size_bytes for spec in kv_cache_specs}
if len(dcp_sharding) > 1 and len(page_sizes) > 1: