[Feature][Spec] Support disabling trailing prefix-cache block dropping (#53388)
Signed-off-by: Ziming Huang <zelda.huanghuang@gmail.com> Signed-off-by: Ziming Huang <48115868+ZeldaHuang@users.noreply.github.com> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
OpenAI Codex
parent
9e905f7450
commit
481839ad9e
@@ -1841,7 +1841,7 @@ def _annotate_eagle_groups(
|
||||
use_deepseek_v4_fallback: Enable rule 2 for a DeepseekV4 packed group.
|
||||
"""
|
||||
spec_config = vllm_config.speculative_config
|
||||
if spec_config is None or not spec_config.use_eagle():
|
||||
if spec_config is None or not spec_config.use_eagle_block_drop():
|
||||
return
|
||||
|
||||
for group in kv_cache_groups:
|
||||
|
||||
@@ -255,6 +255,7 @@ class Scheduler(SchedulerInterface):
|
||||
)
|
||||
speculative_config = vllm_config.speculative_config
|
||||
self.use_eagle = False
|
||||
self.use_eagle_block_drop = False
|
||||
self.num_spec_tokens = vllm_config.num_speculative_tokens
|
||||
self.num_lookahead_tokens = vllm_config.num_lookahead_tokens
|
||||
# Positions past the computed tokens that the drafter reads mid-prefill.
|
||||
@@ -279,6 +280,13 @@ class Scheduler(SchedulerInterface):
|
||||
if speculative_config.use_multi_module_mtp()
|
||||
else 1
|
||||
)
|
||||
self.use_eagle_block_drop = speculative_config.use_eagle_block_drop()
|
||||
if self.use_eagle and not self.use_eagle_block_drop:
|
||||
logger.warning(
|
||||
"EAGLE trailing prefix-cache block dropping is disabled. "
|
||||
"This is experimental and may affect speculative-token "
|
||||
"acceptance rates."
|
||||
)
|
||||
|
||||
# Create the KV cache manager.
|
||||
if hash_block_size is None:
|
||||
@@ -289,7 +297,7 @@ class Scheduler(SchedulerInterface):
|
||||
max_model_len=self.max_model_len,
|
||||
max_in_flight_tokens=vllm_config.max_in_flight_tokens,
|
||||
enable_caching=self.cache_config.enable_prefix_caching,
|
||||
use_eagle=self.use_eagle,
|
||||
use_eagle=self.use_eagle_block_drop,
|
||||
num_prefill_lookahead=self.num_prefill_lookahead,
|
||||
log_stats=self.log_stats,
|
||||
enable_kv_cache_events=self.enable_kv_cache_events,
|
||||
@@ -414,7 +422,7 @@ class Scheduler(SchedulerInterface):
|
||||
# Eagle, FullAttn prunes the last matching block, so back off one
|
||||
# block to avoid a Mamba cache miss.
|
||||
last_cache_position = request.num_tokens - request.num_tokens % block_size
|
||||
if self.use_eagle:
|
||||
if self.use_eagle_block_drop:
|
||||
last_cache_position = max(last_cache_position - block_size, 0)
|
||||
|
||||
end = start + num_new_tokens
|
||||
|
||||
@@ -925,10 +925,9 @@ class SlidingWindowManager(SingleTypeKVCacheManager):
|
||||
)
|
||||
assert dcp_world_size == 1, "DCP not support sliding window attn now."
|
||||
assert pcp_world_size == 1, "PCP not support sliding window attn now."
|
||||
# Fine-grained partial hits are not supported for sliding window now
|
||||
assert alignment_tokens % kv_cache_spec.block_size == 0, (
|
||||
"SlidingWindowManager does not support fine-grained (partial) cache hits"
|
||||
)
|
||||
# Sliding-window cache hits must stay at the group's physical block
|
||||
# granularity. resolve_block_hashes() converts finer-grained hashes to
|
||||
# that view when the hybrid-cache alignment is smaller than block_size.
|
||||
block_hashes = resolve_block_hashes(
|
||||
block_hashes,
|
||||
block_pool.hash_block_size,
|
||||
|
||||
Reference in New Issue
Block a user