[Model][GLM-5.3-Flash] Support DCP for the kpool sparse indexer (#59211)
Signed-off-by: NickLucche <nicolo.lucchesi@mistral.ai>
This commit is contained in:
@@ -315,6 +315,9 @@ def test_compressed_slot_mapping_warmup_includes_index_kpool():
|
||||
config = SimpleNamespace(
|
||||
cache_config=SimpleNamespace(block_size=256),
|
||||
model_config=SimpleNamespace(hf_text_config=SimpleNamespace(index_kpool=32)),
|
||||
parallel_config=SimpleNamespace(
|
||||
decode_context_parallel_size=1, cp_kv_cache_interleave_size=1
|
||||
),
|
||||
)
|
||||
|
||||
keys = CompressedSlotMappingKernel().get_warmup_keys(config)
|
||||
@@ -343,6 +346,41 @@ def test_compressed_slot_mapping_inherits_padded_token_slots():
|
||||
assert compressed.tolist() == [-1, -1, -1, -1, -1, 3 * 4 + 2, -1, 3 * 4 + 3]
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
|
||||
def test_compressed_slot_mapping_dcp_uses_rank_local_state_positions():
|
||||
"""Under DCP each rank writes only the states it owns, at the state's
|
||||
rank-local position."""
|
||||
device = torch.device("cuda")
|
||||
dcp, interleave, ratio, block_size = 2, 4, 2, 4
|
||||
num_tokens = 32
|
||||
query_start_loc = torch.tensor([0, num_tokens], dtype=torch.int32, device=device)
|
||||
seq_lens = torch.tensor([num_tokens], dtype=torch.int32, device=device)
|
||||
block_table = torch.tensor([[5, 7]], dtype=torch.int32, device=device)
|
||||
pos = torch.arange(num_tokens)
|
||||
for rank in range(dcp):
|
||||
owned = (pos // interleave) % dcp == rank
|
||||
slot_mapping = torch.where(owned, 0, -1).to(torch.int64).to(device)
|
||||
compressed = get_compressed_slot_mapping(
|
||||
num_tokens,
|
||||
slot_mapping,
|
||||
query_start_loc,
|
||||
seq_lens,
|
||||
block_table,
|
||||
block_size=block_size,
|
||||
compress_ratio=ratio,
|
||||
dcp_world_size=dcp,
|
||||
cp_interleave=interleave,
|
||||
)
|
||||
local_pos = pos // (interleave * dcp) * interleave + pos % interleave
|
||||
state = local_pos // ratio
|
||||
expected = torch.where(
|
||||
owned & ((pos + 1) % ratio == 0),
|
||||
block_table[0].cpu()[state // block_size] * block_size + state % block_size,
|
||||
-1,
|
||||
)
|
||||
assert compressed.tolist() == expected.tolist()
|
||||
|
||||
|
||||
def test_index_conversion_warmup_uses_physical_block_stride():
|
||||
config = SimpleNamespace(
|
||||
cache_config=SimpleNamespace(block_size=64),
|
||||
@@ -370,6 +408,8 @@ def test_zero_token_pcp_rank_participates_in_compressed_mapping_gather(monkeypat
|
||||
builder.use_pcp = True
|
||||
builder.pcp_world_size = 4
|
||||
builder.pcp_rank = 0
|
||||
builder.dcp_world_size = 1
|
||||
builder.cp_kv_cache_interleave_size = 1
|
||||
builder.kernel_block_size = None
|
||||
builder.kv_cache_spec = SimpleNamespace(block_size=64, num_states=64)
|
||||
builder.compressed_slot_mapping_buffer = torch.zeros(8, dtype=torch.int64)
|
||||
@@ -383,8 +423,11 @@ def test_zero_token_pcp_rank_participates_in_compressed_mapping_gather(monkeypat
|
||||
block_size,
|
||||
compress_ratio,
|
||||
out,
|
||||
dcp_world_size,
|
||||
cp_interleave,
|
||||
):
|
||||
assert num_tokens == 0
|
||||
assert dcp_world_size == 1
|
||||
assert block_size == 64
|
||||
assert compress_ratio == 4
|
||||
out.fill_(-1)
|
||||
|
||||
@@ -1873,6 +1873,27 @@ def test_dcp_target_allocates_replicated_draft_independently(
|
||||
assert [len(group) for group in cached_blocks.blocks] == [1, 4]
|
||||
|
||||
|
||||
def test_dcp_replicated_kpool_tail_keeps_block_interior_layout(monkeypatch):
|
||||
"""A replicated kpool tail ring is per-request state, not a draft group."""
|
||||
from vllm.v1.attention.backends.utils import resolve_kv_cache_layout
|
||||
|
||||
monkeypatch.delenv("VLLM_KV_CACHE_LAYOUT", raising=False)
|
||||
config = VllmConfig(model_config=ModelConfig(max_model_len=1024))
|
||||
config.parallel_config.decode_context_parallel_size = 4
|
||||
config.cache_config.kv_cache_layout = None
|
||||
tail = KpoolTailSpec(
|
||||
block_size=4,
|
||||
num_kv_heads=2,
|
||||
head_size=128,
|
||||
head_size_v=0,
|
||||
dtype=torch.bfloat16,
|
||||
sliding_window=4,
|
||||
dcp_sharded=False,
|
||||
)
|
||||
layout = resolve_kv_cache_layout(config, [["LBHNC"]], [new_mla_spec(), tail])
|
||||
assert layout == KVCacheLayout.LBHNC
|
||||
|
||||
|
||||
@pytest.mark.parametrize("use_mla", [False, True])
|
||||
def test_full_attention_merge_preserves_replicated_cache_geometry(use_mla):
|
||||
config = VllmConfig(model_config=ModelConfig(max_model_len=1024))
|
||||
|
||||
@@ -25,6 +25,22 @@ from vllm.model_executor.warmup.jit_warmup_triton_helper import (
|
||||
from vllm.triton_utils import tl, triton
|
||||
|
||||
|
||||
def _dcp_merge_topk_and_interleave(vllm_config: Any) -> tuple[int, int]:
|
||||
"""(topk, cp_interleave) the DCP merge runs with.
|
||||
|
||||
Kpool indexers (GLM-5.x) merge pool ids: topk and the interleave are both
|
||||
in units of ``index_kpool`` tokens.
|
||||
"""
|
||||
hf_text_config = vllm_config.model_config.hf_text_config
|
||||
topk = hf_text_config.index_topk
|
||||
cp_interleave = vllm_config.parallel_config.cp_kv_cache_interleave_size
|
||||
kpool = getattr(hf_text_config, "index_kpool", None) or 1
|
||||
if kpool > 1:
|
||||
topk //= kpool
|
||||
cp_interleave //= kpool
|
||||
return topk, cp_interleave
|
||||
|
||||
|
||||
def stable_topk_from_gathered_candidates_cutedsl(
|
||||
gathered: torch.Tensor,
|
||||
topk: int,
|
||||
@@ -171,8 +187,7 @@ class PackDCPTopkCandidatesKernel(
|
||||
dcp_world_size = vllm_config.parallel_config.decode_context_parallel_size
|
||||
if dcp_world_size <= 1:
|
||||
return []
|
||||
cp_interleave = vllm_config.parallel_config.cp_kv_cache_interleave_size
|
||||
topk = vllm_config.model_config.hf_text_config.index_topk
|
||||
topk, cp_interleave = _dcp_merge_topk_and_interleave(vllm_config)
|
||||
if topk <= 0:
|
||||
return []
|
||||
|
||||
@@ -545,7 +560,7 @@ class StableTopKFromGatheredCandidatesKernel(
|
||||
dcp_world_size = vllm_config.parallel_config.decode_context_parallel_size
|
||||
if dcp_world_size <= 1:
|
||||
return []
|
||||
topk = vllm_config.model_config.hf_text_config.index_topk
|
||||
topk, _ = _dcp_merge_topk_and_interleave(vllm_config)
|
||||
if topk <= 0:
|
||||
return []
|
||||
return self._trace_dispatch(self.dispatch)(
|
||||
|
||||
@@ -733,6 +733,10 @@ class SparseAttnIndexerKpool(CustomOp):
|
||||
self.topk_backend = (
|
||||
cfg.kernel_config.sparse_indexer_topk_backend if cfg is not None else "auto"
|
||||
)
|
||||
if cfg is not None and cfg.parallel_config.decode_context_parallel_size > 1:
|
||||
raise NotImplementedError(
|
||||
"SparseAttnIndexerKpool on ROCm does not support DCP."
|
||||
)
|
||||
|
||||
def forward_hip(
|
||||
self,
|
||||
|
||||
@@ -209,6 +209,9 @@ class Glm5NextTailCache(DeepseekV32IndexerCache):
|
||||
head_size_v=0,
|
||||
dtype=torch.bfloat16,
|
||||
sliding_window=ring,
|
||||
# Tiny per-request ring; every DCP rank keeps the full tail so the
|
||||
# rank owning a pool can compress it.
|
||||
dcp_sharded=False,
|
||||
)
|
||||
|
||||
def get_attn_backend(self):
|
||||
|
||||
@@ -7,10 +7,12 @@ import torch
|
||||
import vllm.envs as envs
|
||||
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
|
||||
from vllm.config import get_current_vllm_config_or_none
|
||||
from vllm.distributed import get_dcp_group
|
||||
from vllm.forward_context import get_forward_context
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.custom_op import CustomOp
|
||||
from vllm.model_executor.layers.indexer_topk import get_indexer_topk
|
||||
from vllm.model_executor.layers.sparse_attn_indexer import _merge_dcp_topk_global
|
||||
from vllm.models.glm5next.common.sparse_indexer import (
|
||||
RADIX_TOPK_WORKSPACE_SIZE,
|
||||
_build_decode_scatter_indices,
|
||||
@@ -121,6 +123,9 @@ def sparse_attn_indexer_kpool(
|
||||
tail_kv_cache: torch.Tensor | None = None,
|
||||
tail_prefix: str | None = None,
|
||||
topk_backend: str = "auto",
|
||||
dcp_rank: int = 0,
|
||||
dcp_world_size: int = 1,
|
||||
cp_kv_cache_interleave_size: int = 1,
|
||||
) -> torch.Tensor:
|
||||
# careful! this will be None in dummy run
|
||||
attn_metadata = get_forward_context().attn_metadata
|
||||
@@ -170,6 +175,12 @@ def sparse_attn_indexer_kpool(
|
||||
has_decode = attn_metadata_narrowed.num_decodes > 0
|
||||
has_prefill = attn_metadata_narrowed.num_prefills > 0
|
||||
num_decode_tokens = attn_metadata_narrowed.num_decode_tokens
|
||||
# Pools are the DCP sharding unit of the index K cache.
|
||||
pool_cp_interleave = (
|
||||
cp_kv_cache_interleave_size // max(index_kpool, 1)
|
||||
if dcp_world_size > 1
|
||||
else cp_kv_cache_interleave_size
|
||||
)
|
||||
|
||||
# q_scale is required iff the FP4 cache path is enabled; the FP8 path
|
||||
# folds the Q scale into `weights` inside fused_indexer_q_rope_quant.
|
||||
@@ -284,16 +295,18 @@ def sparse_attn_indexer_kpool(
|
||||
scales_spec,
|
||||
)
|
||||
for chunk in prefill_metadata.chunks if not short_prefill else ():
|
||||
k_quant = k_quant_full[: chunk.total_seq_lens]
|
||||
k_scale = k_scale_full[: chunk.total_seq_lens]
|
||||
# Under DCP this rank only gathers the pools it owns.
|
||||
assert chunk.local_cu_seq_lens is not None
|
||||
k_quant = k_quant_full[: chunk.max_local_total_seq_lens]
|
||||
k_scale = k_scale_full[: chunk.max_local_total_seq_lens]
|
||||
|
||||
if not chunk.skip_kv_gather:
|
||||
if not chunk.skip_kv_gather and chunk.local_total_seq_lens > 0:
|
||||
ops.cp_gather_indexer_k_quant_cache(
|
||||
kv_cache,
|
||||
k_quant,
|
||||
k_scale,
|
||||
chunk.block_table,
|
||||
chunk.cu_seq_lens,
|
||||
chunk.local_cu_seq_lens,
|
||||
)
|
||||
|
||||
q_slice = q_quant[chunk.token_start : chunk.token_end]
|
||||
@@ -314,15 +327,18 @@ def sparse_attn_indexer_kpool(
|
||||
k_scale_cast = k_scale.view(torch.float32).squeeze(-1)
|
||||
from vllm.utils.deep_gemm import fp8_fp4_mqa_logits
|
||||
|
||||
logits = fp8_fp4_mqa_logits(
|
||||
(q_slice_cast, q_scale_slice),
|
||||
(k_quant_cast, k_scale_cast),
|
||||
weights[chunk.token_start : chunk.token_end],
|
||||
chunk.cu_seqlen_ks,
|
||||
chunk.cu_seqlen_ke,
|
||||
clean_logits=False,
|
||||
)
|
||||
num_rows = logits.shape[0]
|
||||
num_rows = q_slice.shape[0]
|
||||
if chunk.local_total_seq_lens == 0:
|
||||
logits = q_slice.new_empty((num_rows, 0), dtype=torch.float32)
|
||||
else:
|
||||
logits = fp8_fp4_mqa_logits(
|
||||
(q_slice_cast, q_scale_slice),
|
||||
(k_quant_cast, k_scale_cast),
|
||||
weights[chunk.token_start : chunk.token_end],
|
||||
chunk.cu_seqlen_ks,
|
||||
chunk.cu_seqlen_ke,
|
||||
clean_logits=False,
|
||||
)
|
||||
|
||||
# kpool: logits are pool-granular (compress_ratio == index_kpool),
|
||||
# so topk selects pools. We pick topk_tokens // kpool pools then
|
||||
@@ -338,15 +354,28 @@ def sparse_attn_indexer_kpool(
|
||||
chunk.token_start : chunk.token_end, :topk_tokens
|
||||
]
|
||||
|
||||
torch.ops._C.top_k_per_row_prefill(
|
||||
if logits.shape[1] == 0:
|
||||
topk_dst.fill_(-1)
|
||||
else:
|
||||
torch.ops._C.top_k_per_row_prefill(
|
||||
logits,
|
||||
chunk.cu_seqlen_ks,
|
||||
chunk.cu_seqlen_ke,
|
||||
topk_dst,
|
||||
num_rows,
|
||||
logits.stride(0),
|
||||
logits.stride(1),
|
||||
select_k,
|
||||
)
|
||||
# Local pool ids -> request-relative global pool ids.
|
||||
_merge_dcp_topk_global(
|
||||
logits,
|
||||
chunk.cu_seqlen_ks,
|
||||
chunk.cu_seqlen_ke,
|
||||
topk_dst,
|
||||
num_rows,
|
||||
logits.stride(0),
|
||||
logits.stride(1),
|
||||
select_k,
|
||||
dcp_rank,
|
||||
dcp_world_size,
|
||||
pool_cp_interleave,
|
||||
row_starts=chunk.cu_seqlen_ks,
|
||||
)
|
||||
|
||||
if index_kpool > 1:
|
||||
@@ -584,6 +613,14 @@ def sparse_attn_indexer_kpool(
|
||||
select_k,
|
||||
attn_metadata_narrowed.max_seq_len,
|
||||
)
|
||||
_merge_dcp_topk_global(
|
||||
logits,
|
||||
topk_dst,
|
||||
select_k,
|
||||
dcp_rank,
|
||||
dcp_world_size,
|
||||
pool_cp_interleave,
|
||||
)
|
||||
|
||||
# Resolve to token-level indices in the output buffer.
|
||||
if index_kpool > 1:
|
||||
@@ -669,14 +706,33 @@ class SparseAttnIndexerKpool(CustomOp):
|
||||
else "auto"
|
||||
)
|
||||
_parallel = _cfg.parallel_config if _cfg is not None else None
|
||||
if (
|
||||
_parallel is not None
|
||||
and _parallel.prefill_context_parallel_size > 1
|
||||
and _parallel.decode_context_parallel_size > 1
|
||||
):
|
||||
raise NotImplementedError(
|
||||
"SparseAttnIndexerKpool does not support PCP+DCP."
|
||||
)
|
||||
self._parallel_config = _parallel
|
||||
self._cp_kv_cache_interleave_size: int | None = None
|
||||
self.dcp_world_size = 1
|
||||
self.dcp_rank = 0
|
||||
if _parallel is not None and _parallel.decode_context_parallel_size > 1:
|
||||
if _parallel.prefill_context_parallel_size > 1:
|
||||
raise NotImplementedError(
|
||||
"SparseAttnIndexerKpool does not support PCP+DCP."
|
||||
)
|
||||
self.dcp_world_size = _parallel.decode_context_parallel_size
|
||||
self.dcp_rank = get_dcp_group().rank_in_group
|
||||
|
||||
@property
|
||||
def cp_kv_cache_interleave_size(self) -> int:
|
||||
"""Resolved lazily like ``SparseAttnIndexer``: NIXL P/D can adjust it
|
||||
after the model is built, and the indexer metadata builder sees the
|
||||
adjusted value.
|
||||
"""
|
||||
if self.dcp_world_size == 1:
|
||||
return 1
|
||||
if self._cp_kv_cache_interleave_size is None:
|
||||
assert self._parallel_config is not None
|
||||
value = self._parallel_config.cp_kv_cache_interleave_size
|
||||
if isinstance(get_forward_context().attn_metadata, dict):
|
||||
self._cp_kv_cache_interleave_size = value
|
||||
return value
|
||||
return self._cp_kv_cache_interleave_size
|
||||
|
||||
def forward_native(
|
||||
self,
|
||||
@@ -743,4 +799,7 @@ class SparseAttnIndexerKpool(CustomOp):
|
||||
self.tail_cache.kv_cache if self.tail_cache is not None else None,
|
||||
self.tail_cache.prefix if self.tail_cache is not None else None,
|
||||
self.topk_backend,
|
||||
self.dcp_rank,
|
||||
self.dcp_world_size,
|
||||
self.cp_kv_cache_interleave_size,
|
||||
)
|
||||
|
||||
@@ -37,6 +37,8 @@ class CompressedSlotMappingKernel(
|
||||
compress_ratio: int
|
||||
triton_block_size: int
|
||||
block_size: int
|
||||
dcp_world_size: int
|
||||
cp_interleave: int
|
||||
|
||||
@staticmethod
|
||||
@triton.jit(do_not_specialize=["block_table_stride"])
|
||||
@@ -53,6 +55,8 @@ class CompressedSlotMappingKernel(
|
||||
block_table_ptr,
|
||||
block_table_stride,
|
||||
block_size,
|
||||
dcp_world_size,
|
||||
cp_interleave,
|
||||
COMPRESS_RATIO: tl.constexpr,
|
||||
PAD_ID: tl.constexpr,
|
||||
TRITON_BLOCK_SIZE: tl.constexpr,
|
||||
@@ -77,7 +81,14 @@ class CompressedSlotMappingKernel(
|
||||
slot_mapping_ptr + query_start + offset, mask=mask, other=PAD_ID
|
||||
)
|
||||
is_valid = ((pos + 1) % COMPRESS_RATIO == 0) & (slot != PAD_ID)
|
||||
pos_after_compress = pos // COMPRESS_RATIO
|
||||
# Under DCP a PAD slot also marks tokens owned by other ranks; the
|
||||
# interleave is a multiple of COMPRESS_RATIO, so a state lives
|
||||
# entirely on the rank owning its last token.
|
||||
local_pos = (
|
||||
pos // (cp_interleave * dcp_world_size) * cp_interleave
|
||||
+ pos % cp_interleave
|
||||
)
|
||||
pos_after_compress = local_pos // COMPRESS_RATIO
|
||||
|
||||
block_ids = pos_after_compress // block_size
|
||||
block_numbers = tl.load(
|
||||
@@ -97,11 +108,15 @@ class CompressedSlotMappingKernel(
|
||||
*,
|
||||
compress_ratio: int,
|
||||
block_size: int,
|
||||
dcp_world_size: int,
|
||||
cp_interleave: int,
|
||||
) -> CompileKey:
|
||||
return self.CompileKey(
|
||||
compress_ratio=compress_ratio,
|
||||
triton_block_size=self.TRITON_BLOCK_SIZE,
|
||||
block_size=triton_scalar_specialization_rep(block_size),
|
||||
dcp_world_size=triton_scalar_specialization_rep(dcp_world_size),
|
||||
cp_interleave=triton_scalar_specialization_rep(cp_interleave),
|
||||
)
|
||||
|
||||
def get_warmup_keys(self, vllm_config: Any) -> list[CompileKey]:
|
||||
@@ -115,12 +130,15 @@ class CompressedSlotMappingKernel(
|
||||
)
|
||||
if not compress_ratios:
|
||||
return []
|
||||
parallel_config = vllm_config.parallel_config
|
||||
return self._trace_dispatch(self.dispatch)(
|
||||
zip_inputs(
|
||||
*(
|
||||
dict(
|
||||
compress_ratio=ratio,
|
||||
block_size=vllm_config.cache_config.block_size // ratio,
|
||||
dcp_world_size=parallel_config.decode_context_parallel_size,
|
||||
cp_interleave=parallel_config.cp_kv_cache_interleave_size,
|
||||
)
|
||||
for ratio in compress_ratios
|
||||
)
|
||||
@@ -137,6 +155,8 @@ class CompressedSlotMappingKernel(
|
||||
block_table=int32_ptr,
|
||||
block_size=compile_key.block_size,
|
||||
compress_ratio=compile_key.compress_ratio,
|
||||
dcp_world_size=compile_key.dcp_world_size,
|
||||
cp_interleave=compile_key.cp_interleave,
|
||||
)
|
||||
|
||||
@kernel_launcher
|
||||
@@ -149,6 +169,8 @@ class CompressedSlotMappingKernel(
|
||||
block_table: torch.Tensor,
|
||||
block_size: int,
|
||||
compress_ratio: int,
|
||||
dcp_world_size: int,
|
||||
cp_interleave: int,
|
||||
) -> LaunchSpec:
|
||||
return (block_table.shape[0],), dict(
|
||||
block_table_stride=block_table.stride(0),
|
||||
@@ -167,6 +189,8 @@ def get_compressed_slot_mapping(
|
||||
block_size: int,
|
||||
compress_ratio: int,
|
||||
out: torch.Tensor | None = None,
|
||||
dcp_world_size: int = 1,
|
||||
cp_interleave: int = 1,
|
||||
) -> torch.Tensor:
|
||||
"""Slot mapping for writing the compressed states of ``num_tokens`` tokens.
|
||||
|
||||
@@ -196,6 +220,8 @@ def get_compressed_slot_mapping(
|
||||
block_table,
|
||||
block_size,
|
||||
compress_ratio,
|
||||
dcp_world_size,
|
||||
cp_interleave,
|
||||
)
|
||||
return compressed_slot_mapping
|
||||
|
||||
|
||||
@@ -554,7 +554,15 @@ class BuildPrefillChunkMetadataKernel(
|
||||
# Cover Triton's divisible, exact-one, and generic i32 classes.
|
||||
query_slice_start=(0, 1, 2),
|
||||
query_slice_stop=(1, 2 * max_tokens - 1, 2 * max_tokens),
|
||||
DCP_INTERLEAVE=dcp_interleave,
|
||||
# Compressed indexers localize in units of compressed states.
|
||||
DCP_INTERLEAVE=list(
|
||||
dict.fromkeys(
|
||||
dcp_interleave // ratio
|
||||
for ratio in compress_ratios
|
||||
if dcp_interleave % ratio == 0
|
||||
)
|
||||
)
|
||||
or [dcp_interleave],
|
||||
BLOCK_SIZE=self.BLOCK_SIZE,
|
||||
COMPRESS_RATIO=list(compress_ratios),
|
||||
# PCP's global cumulative lengths are the second row of one packed
|
||||
@@ -1010,11 +1018,31 @@ class DeepseekV32IndexerMetadataBuilder(AttentionMetadataBuilder):
|
||||
(self.num_sms + 1, 2), dtype=torch.int32, device=self.device
|
||||
)
|
||||
|
||||
if self.dcp_world_size > 1 and self.compress_ratio > 1:
|
||||
raise NotImplementedError(
|
||||
"DCP is not supported with sparse indexer KV compression "
|
||||
f"(compress_ratio={self.compress_ratio})."
|
||||
# Each compressed state must live on a single rank, so the DCP
|
||||
# interleave has to cover whole states.
|
||||
if (
|
||||
self.dcp_world_size > 1
|
||||
and self.compress_ratio > 1
|
||||
and (
|
||||
(
|
||||
isinstance(self.kv_cache_spec, MLAAttentionSpec)
|
||||
and self.kv_cache_spec.model_version == "deepseek_v4"
|
||||
)
|
||||
or self.cp_kv_cache_interleave_size % self.compress_ratio != 0
|
||||
)
|
||||
):
|
||||
raise NotImplementedError(
|
||||
"DCP with sparse indexer KV compression "
|
||||
f"(compress_ratio={self.compress_ratio}) requires "
|
||||
"--cp-kv-cache-interleave-size to be a multiple of it; "
|
||||
"DeepSeek-V4 is not supported."
|
||||
)
|
||||
# DCP interleave in units of compressed states.
|
||||
self.compressed_cp_interleave_size = (
|
||||
self.cp_kv_cache_interleave_size // self.compress_ratio
|
||||
if self.dcp_world_size > 1 and self.compress_ratio > 1
|
||||
else self.cp_kv_cache_interleave_size
|
||||
)
|
||||
|
||||
# Pre-allocate buffers for CUDA graph compatibility when
|
||||
if self.compress_ratio > 1:
|
||||
@@ -1348,6 +1376,8 @@ class DeepseekV32IndexerMetadataBuilder(AttentionMetadataBuilder):
|
||||
self.kv_cache_spec.num_states,
|
||||
self.compress_ratio,
|
||||
out=self.compressed_slot_mapping_buffer,
|
||||
dcp_world_size=self.dcp_world_size,
|
||||
cp_interleave=self.cp_kv_cache_interleave_size,
|
||||
)
|
||||
if self.pcp_world_size > 1:
|
||||
compressed_slot_mapping = get_pcp_group().all_gather(
|
||||
@@ -1467,7 +1497,7 @@ class DeepseekV32IndexerMetadataBuilder(AttentionMetadataBuilder):
|
||||
skip_kv_gather=query_slice.start > 0,
|
||||
dcp_rank=self.dcp_rank,
|
||||
dcp_world_size=self.dcp_world_size,
|
||||
cp_kv_cache_interleave_size=self.cp_kv_cache_interleave_size,
|
||||
cp_kv_cache_interleave_size=self.compressed_cp_interleave_size,
|
||||
pcp_plan=pcp_plan,
|
||||
)
|
||||
# Skip when total_seq_lens is 0 (i.e., no compressed token).
|
||||
|
||||
@@ -307,8 +307,12 @@ def resolve_kv_cache_layout(
|
||||
f"none is in every supported set: {supported_layouts}."
|
||||
)
|
||||
|
||||
# Self-addressed per-request rings (e.g. the kpool tail) are replicated
|
||||
# state like Mamba and don't need a separate draft group.
|
||||
dcp_sharding = {
|
||||
spec.dcp_sharded for spec in kv_cache_specs if isinstance(spec, AttentionSpec)
|
||||
spec.dcp_sharded
|
||||
for spec in kv_cache_specs
|
||||
if isinstance(spec, AttentionSpec) and spec.uses_slot_mapping
|
||||
}
|
||||
page_sizes = {spec.page_size_bytes for spec in kv_cache_specs}
|
||||
if len(dcp_sharding) > 1 and len(page_sizes) > 1:
|
||||
|
||||
Reference in New Issue
Block a user