[Kimi-K3] support DCP partial prefix cache hit (#50493)
Signed-off-by: Summer Yang <girasoleyang@gmail.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Yifan Qiao
parent
0e8989b416
commit
0db502c8d8
@@ -60,7 +60,6 @@ from vllm.multimodal.encoder_budget import (
|
||||
)
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.tasks import SupportedTask
|
||||
from vllm.utils.math_utils import cdiv
|
||||
from vllm.utils.mem_utils import DeviceMemoryProfiler, format_gib
|
||||
from vllm.utils.torch_utils import STR_DTYPE_TO_TORCH_DTYPE
|
||||
from vllm.v1.core.sched.output import GrammarOutput, SchedulerOutput
|
||||
@@ -517,17 +516,14 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
for kv_cache_group in kv_cache_config.kv_cache_groups:
|
||||
spec = kv_cache_group.kv_cache_spec
|
||||
block_sizes.append(spec.block_size)
|
||||
# When using DCP, each request's KV cache is sharded among different ranks.
|
||||
# As a result, one block on the current rank covers `block_size * cp_size`
|
||||
# tokens in the full, global (unsharded) sequence.
|
||||
max_num_blocks = cdiv(
|
||||
block_table_max_model_len, spec.block_size * self.dcp_size
|
||||
# Let each cache type account for CP. Attention KV is DCP-sharded,
|
||||
# while Mamba/GDN recurrent state is replicated across DCP ranks.
|
||||
max_num_blocks = spec.max_num_blocks_per_req(
|
||||
self.vllm_config, block_table_max_model_len
|
||||
)
|
||||
# For Mamba/Hybrid Model, KVCaches need extra blocks for speculative tokens
|
||||
# Preserve each cache type's alignment requirements after applying
|
||||
# its topology-aware block-table width.
|
||||
if isinstance(spec, MambaSpec):
|
||||
max_num_blocks = (
|
||||
max_num_blocks if self.cache_config.enable_prefix_caching else 1
|
||||
) + spec.num_speculative_blocks
|
||||
max_num_blocks = get_block_table_width(
|
||||
max_num_blocks, spec.block_size, token_alignment=None
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user