[Kimi-K3] support DCP partial prefix cache hit (#50493)

Signed-off-by: Summer Yang <girasoleyang@gmail.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai>
This commit is contained in:
Summer Yang
2026-08-17 17:05:51 -07:00
committed by GitHub
co-authored by mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Yifan Qiao
parent 0e8989b416
commit 0db502c8d8
8 changed files with 523 additions and 37 deletions
+6 -10
View File
@@ -60,7 +60,6 @@ from vllm.multimodal.encoder_budget import (
)
from vllm.sequence import IntermediateTensors
from vllm.tasks import SupportedTask
from vllm.utils.math_utils import cdiv
from vllm.utils.mem_utils import DeviceMemoryProfiler, format_gib
from vllm.utils.torch_utils import STR_DTYPE_TO_TORCH_DTYPE
from vllm.v1.core.sched.output import GrammarOutput, SchedulerOutput
@@ -517,17 +516,14 @@ class GPUModelRunner(LoRAModelRunnerMixin):
for kv_cache_group in kv_cache_config.kv_cache_groups:
spec = kv_cache_group.kv_cache_spec
block_sizes.append(spec.block_size)
# When using DCP, each request's KV cache is sharded among different ranks.
# As a result, one block on the current rank covers `block_size * cp_size`
# tokens in the full, global (unsharded) sequence.
max_num_blocks = cdiv(
block_table_max_model_len, spec.block_size * self.dcp_size
# Let each cache type account for CP. Attention KV is DCP-sharded,
# while Mamba/GDN recurrent state is replicated across DCP ranks.
max_num_blocks = spec.max_num_blocks_per_req(
self.vllm_config, block_table_max_model_len
)
# For Mamba/Hybrid Model, KVCaches need extra blocks for speculative tokens
# Preserve each cache type's alignment requirements after applying
# its topology-aware block-table width.
if isinstance(spec, MambaSpec):
max_num_blocks = (
max_num_blocks if self.cache_config.enable_prefix_caching else 1
) + spec.num_speculative_blocks
max_num_blocks = get_block_table_width(
max_num_blocks, spec.block_size, token_alignment=None
)