[Bugfix][Prefix caching] Fix inconsistent hybrid Mamba prefix cache boundaries (#60533)
Signed-off-by: Wei Zhao <weizha@aws-cmh-slurm-1-login-01.cm.cluster> Signed-off-by: zjy0516 <riverclouds.zhu@qq.com> Co-authored-by: Wei Zhao <weizha@aws-cmh-slurm-1-login-01.cm.cluster> Co-authored-by: Codex <noreply@openai.com> Co-authored-by: zjy0516 <riverclouds.zhu@qq.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
Wei Zhao
Codex
zjy0516
Claude Opus 5.5
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
1664c4f444
commit
d8fa463e2b
@@ -53,6 +53,7 @@ from vllm.v1.core.kv_cache_utils import (
|
||||
get_kv_cache_configs,
|
||||
get_request_block_hasher,
|
||||
init_none_hash,
|
||||
resolve_cache_hit_alignment_tokens,
|
||||
resolve_kv_cache_block_sizes,
|
||||
update_kv_cache_capacity,
|
||||
)
|
||||
@@ -352,6 +353,19 @@ class EngineCore:
|
||||
|
||||
vllm_config.validate_block_size()
|
||||
|
||||
scheduler_block_size, hash_block_size = resolve_kv_cache_block_sizes(
|
||||
scheduler_kv_cache_config, vllm_config
|
||||
)
|
||||
cache_hit_alignment_tokens = resolve_cache_hit_alignment_tokens(
|
||||
scheduler_kv_cache_config,
|
||||
vllm_config,
|
||||
scheduler_block_size,
|
||||
hash_block_size,
|
||||
)
|
||||
for kv_cache_config in kv_cache_configs:
|
||||
kv_cache_config.hash_block_size = hash_block_size
|
||||
kv_cache_config.cache_hit_alignment_tokens = cache_hit_alignment_tokens
|
||||
|
||||
self.model_executor.initialize_from_config(kv_cache_configs)
|
||||
if not envs.VLLM_ELASTIC_EP_SCALE_UP_LAUNCH:
|
||||
self.model_executor.compile_or_warm_up_model()
|
||||
|
||||
Reference in New Issue
Block a user