[Feature][Spec] Support disabling trailing prefix-cache block dropping (#53388)

Signed-off-by: Ziming Huang <zelda.huanghuang@gmail.com>
Signed-off-by: Ziming Huang <48115868+ZeldaHuang@users.noreply.github.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
Ziming Huang
2026-09-01 19:12:36 +08:00
committed by GitHub
co-authored by OpenAI Codex
parent 9e905f7450
commit 481839ad9e
13 changed files with 112 additions and 21 deletions
+1 -1
View File
@@ -1841,7 +1841,7 @@ def _annotate_eagle_groups(
use_deepseek_v4_fallback: Enable rule 2 for a DeepseekV4 packed group.
"""
spec_config = vllm_config.speculative_config
if spec_config is None or not spec_config.use_eagle():
if spec_config is None or not spec_config.use_eagle_block_drop():
return
for group in kv_cache_groups:
+10 -2
View File
@@ -255,6 +255,7 @@ class Scheduler(SchedulerInterface):
)
speculative_config = vllm_config.speculative_config
self.use_eagle = False
self.use_eagle_block_drop = False
self.num_spec_tokens = vllm_config.num_speculative_tokens
self.num_lookahead_tokens = vllm_config.num_lookahead_tokens
# Positions past the computed tokens that the drafter reads mid-prefill.
@@ -279,6 +280,13 @@ class Scheduler(SchedulerInterface):
if speculative_config.use_multi_module_mtp()
else 1
)
self.use_eagle_block_drop = speculative_config.use_eagle_block_drop()
if self.use_eagle and not self.use_eagle_block_drop:
logger.warning(
"EAGLE trailing prefix-cache block dropping is disabled. "
"This is experimental and may affect speculative-token "
"acceptance rates."
)
# Create the KV cache manager.
if hash_block_size is None:
@@ -289,7 +297,7 @@ class Scheduler(SchedulerInterface):
max_model_len=self.max_model_len,
max_in_flight_tokens=vllm_config.max_in_flight_tokens,
enable_caching=self.cache_config.enable_prefix_caching,
use_eagle=self.use_eagle,
use_eagle=self.use_eagle_block_drop,
num_prefill_lookahead=self.num_prefill_lookahead,
log_stats=self.log_stats,
enable_kv_cache_events=self.enable_kv_cache_events,
@@ -414,7 +422,7 @@ class Scheduler(SchedulerInterface):
# Eagle, FullAttn prunes the last matching block, so back off one
# block to avoid a Mamba cache miss.
last_cache_position = request.num_tokens - request.num_tokens % block_size
if self.use_eagle:
if self.use_eagle_block_drop:
last_cache_position = max(last_cache_position - block_size, 0)
end = start + num_new_tokens
+3 -4
View File
@@ -925,10 +925,9 @@ class SlidingWindowManager(SingleTypeKVCacheManager):
)
assert dcp_world_size == 1, "DCP not support sliding window attn now."
assert pcp_world_size == 1, "PCP not support sliding window attn now."
# Fine-grained partial hits are not supported for sliding window now
assert alignment_tokens % kv_cache_spec.block_size == 0, (
"SlidingWindowManager does not support fine-grained (partial) cache hits"
)
# Sliding-window cache hits must stay at the group's physical block
# granularity. resolve_block_hashes() converts finer-grained hashes to
# that view when the hybrid-cache alignment is smaller than block_size.
block_hashes = resolve_block_hashes(
block_hashes,
block_pool.hash_block_size,