From 30d4032363d4a1f6412d9a235049673b41309f98 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Lucchesi?= Date: Mon, 5 Oct 2026 16:44:14 +0200 Subject: [PATCH] [Model][GLM-5.3-Flash] Support DCP for the kpool sparse indexer (#59211) Signed-off-by: NickLucche --- .../test_indexer_deepseek_v4_slot_mapping.py | 43 +++++++ tests/v1/core/test_kv_cache_utils.py | 21 ++++ .../attention/dsa/dcp_indexer_cutedsl.py | 21 +++- vllm/models/glm5next/amd/sparse_indexer.py | 4 + vllm/models/glm5next/common/attention.py | 3 + vllm/models/glm5next/nvidia/sparse_indexer.py | 113 +++++++++++++----- .../backends/mla/compressor_utils.py | 28 ++++- vllm/v1/attention/backends/mla/indexer.py | 42 ++++++- vllm/v1/attention/backends/utils.py | 6 +- 9 files changed, 243 insertions(+), 38 deletions(-) diff --git a/tests/v1/attention/test_indexer_deepseek_v4_slot_mapping.py b/tests/v1/attention/test_indexer_deepseek_v4_slot_mapping.py index 8acfc0a334..052088a621 100644 --- a/tests/v1/attention/test_indexer_deepseek_v4_slot_mapping.py +++ b/tests/v1/attention/test_indexer_deepseek_v4_slot_mapping.py @@ -315,6 +315,9 @@ def test_compressed_slot_mapping_warmup_includes_index_kpool(): config = SimpleNamespace( cache_config=SimpleNamespace(block_size=256), model_config=SimpleNamespace(hf_text_config=SimpleNamespace(index_kpool=32)), + parallel_config=SimpleNamespace( + decode_context_parallel_size=1, cp_kv_cache_interleave_size=1 + ), ) keys = CompressedSlotMappingKernel().get_warmup_keys(config) @@ -343,6 +346,41 @@ def test_compressed_slot_mapping_inherits_padded_token_slots(): assert compressed.tolist() == [-1, -1, -1, -1, -1, 3 * 4 + 2, -1, 3 * 4 + 3] +@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA") +def test_compressed_slot_mapping_dcp_uses_rank_local_state_positions(): + """Under DCP each rank writes only the states it owns, at the state's + rank-local position.""" + device = torch.device("cuda") + dcp, interleave, ratio, block_size = 2, 4, 2, 4 + num_tokens = 32 + query_start_loc = torch.tensor([0, num_tokens], dtype=torch.int32, device=device) + seq_lens = torch.tensor([num_tokens], dtype=torch.int32, device=device) + block_table = torch.tensor([[5, 7]], dtype=torch.int32, device=device) + pos = torch.arange(num_tokens) + for rank in range(dcp): + owned = (pos // interleave) % dcp == rank + slot_mapping = torch.where(owned, 0, -1).to(torch.int64).to(device) + compressed = get_compressed_slot_mapping( + num_tokens, + slot_mapping, + query_start_loc, + seq_lens, + block_table, + block_size=block_size, + compress_ratio=ratio, + dcp_world_size=dcp, + cp_interleave=interleave, + ) + local_pos = pos // (interleave * dcp) * interleave + pos % interleave + state = local_pos // ratio + expected = torch.where( + owned & ((pos + 1) % ratio == 0), + block_table[0].cpu()[state // block_size] * block_size + state % block_size, + -1, + ) + assert compressed.tolist() == expected.tolist() + + def test_index_conversion_warmup_uses_physical_block_stride(): config = SimpleNamespace( cache_config=SimpleNamespace(block_size=64), @@ -370,6 +408,8 @@ def test_zero_token_pcp_rank_participates_in_compressed_mapping_gather(monkeypat builder.use_pcp = True builder.pcp_world_size = 4 builder.pcp_rank = 0 + builder.dcp_world_size = 1 + builder.cp_kv_cache_interleave_size = 1 builder.kernel_block_size = None builder.kv_cache_spec = SimpleNamespace(block_size=64, num_states=64) builder.compressed_slot_mapping_buffer = torch.zeros(8, dtype=torch.int64) @@ -383,8 +423,11 @@ def test_zero_token_pcp_rank_participates_in_compressed_mapping_gather(monkeypat block_size, compress_ratio, out, + dcp_world_size, + cp_interleave, ): assert num_tokens == 0 + assert dcp_world_size == 1 assert block_size == 64 assert compress_ratio == 4 out.fill_(-1) diff --git a/tests/v1/core/test_kv_cache_utils.py b/tests/v1/core/test_kv_cache_utils.py index 7923e34d0f..08db9a286b 100644 --- a/tests/v1/core/test_kv_cache_utils.py +++ b/tests/v1/core/test_kv_cache_utils.py @@ -1873,6 +1873,27 @@ def test_dcp_target_allocates_replicated_draft_independently( assert [len(group) for group in cached_blocks.blocks] == [1, 4] +def test_dcp_replicated_kpool_tail_keeps_block_interior_layout(monkeypatch): + """A replicated kpool tail ring is per-request state, not a draft group.""" + from vllm.v1.attention.backends.utils import resolve_kv_cache_layout + + monkeypatch.delenv("VLLM_KV_CACHE_LAYOUT", raising=False) + config = VllmConfig(model_config=ModelConfig(max_model_len=1024)) + config.parallel_config.decode_context_parallel_size = 4 + config.cache_config.kv_cache_layout = None + tail = KpoolTailSpec( + block_size=4, + num_kv_heads=2, + head_size=128, + head_size_v=0, + dtype=torch.bfloat16, + sliding_window=4, + dcp_sharded=False, + ) + layout = resolve_kv_cache_layout(config, [["LBHNC"]], [new_mla_spec(), tail]) + assert layout == KVCacheLayout.LBHNC + + @pytest.mark.parametrize("use_mla", [False, True]) def test_full_attention_merge_preserves_replicated_cache_geometry(use_mla): config = VllmConfig(model_config=ModelConfig(max_model_len=1024)) diff --git a/vllm/model_executor/kernels/attention/dsa/dcp_indexer_cutedsl.py b/vllm/model_executor/kernels/attention/dsa/dcp_indexer_cutedsl.py index 34ce32856a..8b5d6ab1a5 100644 --- a/vllm/model_executor/kernels/attention/dsa/dcp_indexer_cutedsl.py +++ b/vllm/model_executor/kernels/attention/dsa/dcp_indexer_cutedsl.py @@ -25,6 +25,22 @@ from vllm.model_executor.warmup.jit_warmup_triton_helper import ( from vllm.triton_utils import tl, triton +def _dcp_merge_topk_and_interleave(vllm_config: Any) -> tuple[int, int]: + """(topk, cp_interleave) the DCP merge runs with. + + Kpool indexers (GLM-5.x) merge pool ids: topk and the interleave are both + in units of ``index_kpool`` tokens. + """ + hf_text_config = vllm_config.model_config.hf_text_config + topk = hf_text_config.index_topk + cp_interleave = vllm_config.parallel_config.cp_kv_cache_interleave_size + kpool = getattr(hf_text_config, "index_kpool", None) or 1 + if kpool > 1: + topk //= kpool + cp_interleave //= kpool + return topk, cp_interleave + + def stable_topk_from_gathered_candidates_cutedsl( gathered: torch.Tensor, topk: int, @@ -171,8 +187,7 @@ class PackDCPTopkCandidatesKernel( dcp_world_size = vllm_config.parallel_config.decode_context_parallel_size if dcp_world_size <= 1: return [] - cp_interleave = vllm_config.parallel_config.cp_kv_cache_interleave_size - topk = vllm_config.model_config.hf_text_config.index_topk + topk, cp_interleave = _dcp_merge_topk_and_interleave(vllm_config) if topk <= 0: return [] @@ -545,7 +560,7 @@ class StableTopKFromGatheredCandidatesKernel( dcp_world_size = vllm_config.parallel_config.decode_context_parallel_size if dcp_world_size <= 1: return [] - topk = vllm_config.model_config.hf_text_config.index_topk + topk, _ = _dcp_merge_topk_and_interleave(vllm_config) if topk <= 0: return [] return self._trace_dispatch(self.dispatch)( diff --git a/vllm/models/glm5next/amd/sparse_indexer.py b/vllm/models/glm5next/amd/sparse_indexer.py index c357608c47..13e67de04b 100644 --- a/vllm/models/glm5next/amd/sparse_indexer.py +++ b/vllm/models/glm5next/amd/sparse_indexer.py @@ -733,6 +733,10 @@ class SparseAttnIndexerKpool(CustomOp): self.topk_backend = ( cfg.kernel_config.sparse_indexer_topk_backend if cfg is not None else "auto" ) + if cfg is not None and cfg.parallel_config.decode_context_parallel_size > 1: + raise NotImplementedError( + "SparseAttnIndexerKpool on ROCm does not support DCP." + ) def forward_hip( self, diff --git a/vllm/models/glm5next/common/attention.py b/vllm/models/glm5next/common/attention.py index d7bf70af07..a264717f0d 100644 --- a/vllm/models/glm5next/common/attention.py +++ b/vllm/models/glm5next/common/attention.py @@ -209,6 +209,9 @@ class Glm5NextTailCache(DeepseekV32IndexerCache): head_size_v=0, dtype=torch.bfloat16, sliding_window=ring, + # Tiny per-request ring; every DCP rank keeps the full tail so the + # rank owning a pool can compress it. + dcp_sharded=False, ) def get_attn_backend(self): diff --git a/vllm/models/glm5next/nvidia/sparse_indexer.py b/vllm/models/glm5next/nvidia/sparse_indexer.py index 00f51cc6a7..806b633c58 100644 --- a/vllm/models/glm5next/nvidia/sparse_indexer.py +++ b/vllm/models/glm5next/nvidia/sparse_indexer.py @@ -7,10 +7,12 @@ import torch import vllm.envs as envs from vllm.compilation.breakable_cudagraph import eager_break_during_capture from vllm.config import get_current_vllm_config_or_none +from vllm.distributed import get_dcp_group from vllm.forward_context import get_forward_context from vllm.logger import init_logger from vllm.model_executor.custom_op import CustomOp from vllm.model_executor.layers.indexer_topk import get_indexer_topk +from vllm.model_executor.layers.sparse_attn_indexer import _merge_dcp_topk_global from vllm.models.glm5next.common.sparse_indexer import ( RADIX_TOPK_WORKSPACE_SIZE, _build_decode_scatter_indices, @@ -121,6 +123,9 @@ def sparse_attn_indexer_kpool( tail_kv_cache: torch.Tensor | None = None, tail_prefix: str | None = None, topk_backend: str = "auto", + dcp_rank: int = 0, + dcp_world_size: int = 1, + cp_kv_cache_interleave_size: int = 1, ) -> torch.Tensor: # careful! this will be None in dummy run attn_metadata = get_forward_context().attn_metadata @@ -170,6 +175,12 @@ def sparse_attn_indexer_kpool( has_decode = attn_metadata_narrowed.num_decodes > 0 has_prefill = attn_metadata_narrowed.num_prefills > 0 num_decode_tokens = attn_metadata_narrowed.num_decode_tokens + # Pools are the DCP sharding unit of the index K cache. + pool_cp_interleave = ( + cp_kv_cache_interleave_size // max(index_kpool, 1) + if dcp_world_size > 1 + else cp_kv_cache_interleave_size + ) # q_scale is required iff the FP4 cache path is enabled; the FP8 path # folds the Q scale into `weights` inside fused_indexer_q_rope_quant. @@ -284,16 +295,18 @@ def sparse_attn_indexer_kpool( scales_spec, ) for chunk in prefill_metadata.chunks if not short_prefill else (): - k_quant = k_quant_full[: chunk.total_seq_lens] - k_scale = k_scale_full[: chunk.total_seq_lens] + # Under DCP this rank only gathers the pools it owns. + assert chunk.local_cu_seq_lens is not None + k_quant = k_quant_full[: chunk.max_local_total_seq_lens] + k_scale = k_scale_full[: chunk.max_local_total_seq_lens] - if not chunk.skip_kv_gather: + if not chunk.skip_kv_gather and chunk.local_total_seq_lens > 0: ops.cp_gather_indexer_k_quant_cache( kv_cache, k_quant, k_scale, chunk.block_table, - chunk.cu_seq_lens, + chunk.local_cu_seq_lens, ) q_slice = q_quant[chunk.token_start : chunk.token_end] @@ -314,15 +327,18 @@ def sparse_attn_indexer_kpool( k_scale_cast = k_scale.view(torch.float32).squeeze(-1) from vllm.utils.deep_gemm import fp8_fp4_mqa_logits - logits = fp8_fp4_mqa_logits( - (q_slice_cast, q_scale_slice), - (k_quant_cast, k_scale_cast), - weights[chunk.token_start : chunk.token_end], - chunk.cu_seqlen_ks, - chunk.cu_seqlen_ke, - clean_logits=False, - ) - num_rows = logits.shape[0] + num_rows = q_slice.shape[0] + if chunk.local_total_seq_lens == 0: + logits = q_slice.new_empty((num_rows, 0), dtype=torch.float32) + else: + logits = fp8_fp4_mqa_logits( + (q_slice_cast, q_scale_slice), + (k_quant_cast, k_scale_cast), + weights[chunk.token_start : chunk.token_end], + chunk.cu_seqlen_ks, + chunk.cu_seqlen_ke, + clean_logits=False, + ) # kpool: logits are pool-granular (compress_ratio == index_kpool), # so topk selects pools. We pick topk_tokens // kpool pools then @@ -338,15 +354,28 @@ def sparse_attn_indexer_kpool( chunk.token_start : chunk.token_end, :topk_tokens ] - torch.ops._C.top_k_per_row_prefill( + if logits.shape[1] == 0: + topk_dst.fill_(-1) + else: + torch.ops._C.top_k_per_row_prefill( + logits, + chunk.cu_seqlen_ks, + chunk.cu_seqlen_ke, + topk_dst, + num_rows, + logits.stride(0), + logits.stride(1), + select_k, + ) + # Local pool ids -> request-relative global pool ids. + _merge_dcp_topk_global( logits, - chunk.cu_seqlen_ks, - chunk.cu_seqlen_ke, topk_dst, - num_rows, - logits.stride(0), - logits.stride(1), select_k, + dcp_rank, + dcp_world_size, + pool_cp_interleave, + row_starts=chunk.cu_seqlen_ks, ) if index_kpool > 1: @@ -584,6 +613,14 @@ def sparse_attn_indexer_kpool( select_k, attn_metadata_narrowed.max_seq_len, ) + _merge_dcp_topk_global( + logits, + topk_dst, + select_k, + dcp_rank, + dcp_world_size, + pool_cp_interleave, + ) # Resolve to token-level indices in the output buffer. if index_kpool > 1: @@ -669,14 +706,33 @@ class SparseAttnIndexerKpool(CustomOp): else "auto" ) _parallel = _cfg.parallel_config if _cfg is not None else None - if ( - _parallel is not None - and _parallel.prefill_context_parallel_size > 1 - and _parallel.decode_context_parallel_size > 1 - ): - raise NotImplementedError( - "SparseAttnIndexerKpool does not support PCP+DCP." - ) + self._parallel_config = _parallel + self._cp_kv_cache_interleave_size: int | None = None + self.dcp_world_size = 1 + self.dcp_rank = 0 + if _parallel is not None and _parallel.decode_context_parallel_size > 1: + if _parallel.prefill_context_parallel_size > 1: + raise NotImplementedError( + "SparseAttnIndexerKpool does not support PCP+DCP." + ) + self.dcp_world_size = _parallel.decode_context_parallel_size + self.dcp_rank = get_dcp_group().rank_in_group + + @property + def cp_kv_cache_interleave_size(self) -> int: + """Resolved lazily like ``SparseAttnIndexer``: NIXL P/D can adjust it + after the model is built, and the indexer metadata builder sees the + adjusted value. + """ + if self.dcp_world_size == 1: + return 1 + if self._cp_kv_cache_interleave_size is None: + assert self._parallel_config is not None + value = self._parallel_config.cp_kv_cache_interleave_size + if isinstance(get_forward_context().attn_metadata, dict): + self._cp_kv_cache_interleave_size = value + return value + return self._cp_kv_cache_interleave_size def forward_native( self, @@ -743,4 +799,7 @@ class SparseAttnIndexerKpool(CustomOp): self.tail_cache.kv_cache if self.tail_cache is not None else None, self.tail_cache.prefix if self.tail_cache is not None else None, self.topk_backend, + self.dcp_rank, + self.dcp_world_size, + self.cp_kv_cache_interleave_size, ) diff --git a/vllm/v1/attention/backends/mla/compressor_utils.py b/vllm/v1/attention/backends/mla/compressor_utils.py index 25e24f347c..41d973f296 100644 --- a/vllm/v1/attention/backends/mla/compressor_utils.py +++ b/vllm/v1/attention/backends/mla/compressor_utils.py @@ -37,6 +37,8 @@ class CompressedSlotMappingKernel( compress_ratio: int triton_block_size: int block_size: int + dcp_world_size: int + cp_interleave: int @staticmethod @triton.jit(do_not_specialize=["block_table_stride"]) @@ -53,6 +55,8 @@ class CompressedSlotMappingKernel( block_table_ptr, block_table_stride, block_size, + dcp_world_size, + cp_interleave, COMPRESS_RATIO: tl.constexpr, PAD_ID: tl.constexpr, TRITON_BLOCK_SIZE: tl.constexpr, @@ -77,7 +81,14 @@ class CompressedSlotMappingKernel( slot_mapping_ptr + query_start + offset, mask=mask, other=PAD_ID ) is_valid = ((pos + 1) % COMPRESS_RATIO == 0) & (slot != PAD_ID) - pos_after_compress = pos // COMPRESS_RATIO + # Under DCP a PAD slot also marks tokens owned by other ranks; the + # interleave is a multiple of COMPRESS_RATIO, so a state lives + # entirely on the rank owning its last token. + local_pos = ( + pos // (cp_interleave * dcp_world_size) * cp_interleave + + pos % cp_interleave + ) + pos_after_compress = local_pos // COMPRESS_RATIO block_ids = pos_after_compress // block_size block_numbers = tl.load( @@ -97,11 +108,15 @@ class CompressedSlotMappingKernel( *, compress_ratio: int, block_size: int, + dcp_world_size: int, + cp_interleave: int, ) -> CompileKey: return self.CompileKey( compress_ratio=compress_ratio, triton_block_size=self.TRITON_BLOCK_SIZE, block_size=triton_scalar_specialization_rep(block_size), + dcp_world_size=triton_scalar_specialization_rep(dcp_world_size), + cp_interleave=triton_scalar_specialization_rep(cp_interleave), ) def get_warmup_keys(self, vllm_config: Any) -> list[CompileKey]: @@ -115,12 +130,15 @@ class CompressedSlotMappingKernel( ) if not compress_ratios: return [] + parallel_config = vllm_config.parallel_config return self._trace_dispatch(self.dispatch)( zip_inputs( *( dict( compress_ratio=ratio, block_size=vllm_config.cache_config.block_size // ratio, + dcp_world_size=parallel_config.decode_context_parallel_size, + cp_interleave=parallel_config.cp_kv_cache_interleave_size, ) for ratio in compress_ratios ) @@ -137,6 +155,8 @@ class CompressedSlotMappingKernel( block_table=int32_ptr, block_size=compile_key.block_size, compress_ratio=compile_key.compress_ratio, + dcp_world_size=compile_key.dcp_world_size, + cp_interleave=compile_key.cp_interleave, ) @kernel_launcher @@ -149,6 +169,8 @@ class CompressedSlotMappingKernel( block_table: torch.Tensor, block_size: int, compress_ratio: int, + dcp_world_size: int, + cp_interleave: int, ) -> LaunchSpec: return (block_table.shape[0],), dict( block_table_stride=block_table.stride(0), @@ -167,6 +189,8 @@ def get_compressed_slot_mapping( block_size: int, compress_ratio: int, out: torch.Tensor | None = None, + dcp_world_size: int = 1, + cp_interleave: int = 1, ) -> torch.Tensor: """Slot mapping for writing the compressed states of ``num_tokens`` tokens. @@ -196,6 +220,8 @@ def get_compressed_slot_mapping( block_table, block_size, compress_ratio, + dcp_world_size, + cp_interleave, ) return compressed_slot_mapping diff --git a/vllm/v1/attention/backends/mla/indexer.py b/vllm/v1/attention/backends/mla/indexer.py index 5cfaf6ca5f..9d3823b880 100644 --- a/vllm/v1/attention/backends/mla/indexer.py +++ b/vllm/v1/attention/backends/mla/indexer.py @@ -554,7 +554,15 @@ class BuildPrefillChunkMetadataKernel( # Cover Triton's divisible, exact-one, and generic i32 classes. query_slice_start=(0, 1, 2), query_slice_stop=(1, 2 * max_tokens - 1, 2 * max_tokens), - DCP_INTERLEAVE=dcp_interleave, + # Compressed indexers localize in units of compressed states. + DCP_INTERLEAVE=list( + dict.fromkeys( + dcp_interleave // ratio + for ratio in compress_ratios + if dcp_interleave % ratio == 0 + ) + ) + or [dcp_interleave], BLOCK_SIZE=self.BLOCK_SIZE, COMPRESS_RATIO=list(compress_ratios), # PCP's global cumulative lengths are the second row of one packed @@ -1010,11 +1018,31 @@ class DeepseekV32IndexerMetadataBuilder(AttentionMetadataBuilder): (self.num_sms + 1, 2), dtype=torch.int32, device=self.device ) - if self.dcp_world_size > 1 and self.compress_ratio > 1: - raise NotImplementedError( - "DCP is not supported with sparse indexer KV compression " - f"(compress_ratio={self.compress_ratio})." + # Each compressed state must live on a single rank, so the DCP + # interleave has to cover whole states. + if ( + self.dcp_world_size > 1 + and self.compress_ratio > 1 + and ( + ( + isinstance(self.kv_cache_spec, MLAAttentionSpec) + and self.kv_cache_spec.model_version == "deepseek_v4" + ) + or self.cp_kv_cache_interleave_size % self.compress_ratio != 0 ) + ): + raise NotImplementedError( + "DCP with sparse indexer KV compression " + f"(compress_ratio={self.compress_ratio}) requires " + "--cp-kv-cache-interleave-size to be a multiple of it; " + "DeepSeek-V4 is not supported." + ) + # DCP interleave in units of compressed states. + self.compressed_cp_interleave_size = ( + self.cp_kv_cache_interleave_size // self.compress_ratio + if self.dcp_world_size > 1 and self.compress_ratio > 1 + else self.cp_kv_cache_interleave_size + ) # Pre-allocate buffers for CUDA graph compatibility when if self.compress_ratio > 1: @@ -1348,6 +1376,8 @@ class DeepseekV32IndexerMetadataBuilder(AttentionMetadataBuilder): self.kv_cache_spec.num_states, self.compress_ratio, out=self.compressed_slot_mapping_buffer, + dcp_world_size=self.dcp_world_size, + cp_interleave=self.cp_kv_cache_interleave_size, ) if self.pcp_world_size > 1: compressed_slot_mapping = get_pcp_group().all_gather( @@ -1467,7 +1497,7 @@ class DeepseekV32IndexerMetadataBuilder(AttentionMetadataBuilder): skip_kv_gather=query_slice.start > 0, dcp_rank=self.dcp_rank, dcp_world_size=self.dcp_world_size, - cp_kv_cache_interleave_size=self.cp_kv_cache_interleave_size, + cp_kv_cache_interleave_size=self.compressed_cp_interleave_size, pcp_plan=pcp_plan, ) # Skip when total_seq_lens is 0 (i.e., no compressed token). diff --git a/vllm/v1/attention/backends/utils.py b/vllm/v1/attention/backends/utils.py index 529d816c06..f93156d53f 100644 --- a/vllm/v1/attention/backends/utils.py +++ b/vllm/v1/attention/backends/utils.py @@ -307,8 +307,12 @@ def resolve_kv_cache_layout( f"none is in every supported set: {supported_layouts}." ) + # Self-addressed per-request rings (e.g. the kpool tail) are replicated + # state like Mamba and don't need a separate draft group. dcp_sharding = { - spec.dcp_sharded for spec in kv_cache_specs if isinstance(spec, AttentionSpec) + spec.dcp_sharded + for spec in kv_cache_specs + if isinstance(spec, AttentionSpec) and spec.uses_slot_mapping } page_sizes = {spec.page_size_bytes for spec in kv_cache_specs} if len(dcp_sharding) > 1 and len(page_sizes) > 1: