[Feature][Spec] Support disabling trailing prefix-cache block dropping (#53388)
Signed-off-by: Ziming Huang <zelda.huanghuang@gmail.com> Signed-off-by: Ziming Huang <48115868+ZeldaHuang@users.noreply.github.com> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
OpenAI Codex
parent
9e905f7450
commit
481839ad9e
@@ -2152,6 +2152,23 @@ def test_draft_sample_method_probabilistic_is_accepted():
|
|||||||
assert speculative_config.draft_sample_method == "probabilistic"
|
assert speculative_config.draft_sample_method == "probabilistic"
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("disable_eagle_block_drop", [False, True])
|
||||||
|
def test_eagle_block_drop_can_be_disabled_without_disabling_eagle(
|
||||||
|
disable_eagle_block_drop: bool,
|
||||||
|
):
|
||||||
|
# Start from an ngram config to avoid loading model metadata: these predicates
|
||||||
|
# depend only on the speculative method and the new switch.
|
||||||
|
speculative_config = SpeculativeConfig(
|
||||||
|
method="ngram",
|
||||||
|
num_speculative_tokens=3,
|
||||||
|
disable_eagle_block_drop=disable_eagle_block_drop,
|
||||||
|
)
|
||||||
|
speculative_config.method = "eagle3"
|
||||||
|
|
||||||
|
assert speculative_config.use_eagle()
|
||||||
|
assert speculative_config.use_eagle_block_drop() is not disable_eagle_block_drop
|
||||||
|
|
||||||
|
|
||||||
def test_draft_sample_method_gumbel_is_rejected():
|
def test_draft_sample_method_gumbel_is_rejected():
|
||||||
with pytest.raises(ValidationError):
|
with pytest.raises(ValidationError):
|
||||||
SpeculativeConfig(
|
SpeculativeConfig(
|
||||||
|
|||||||
@@ -170,7 +170,7 @@ def test_mamba_align_split_partial_tail_schedule(dcp_world_size: int):
|
|||||||
cache_config=SimpleNamespace(block_size=block_size),
|
cache_config=SimpleNamespace(block_size=block_size),
|
||||||
max_num_scheduled_tokens=8192,
|
max_num_scheduled_tokens=8192,
|
||||||
scheduler_config=SimpleNamespace(long_prefill_token_threshold=0),
|
scheduler_config=SimpleNamespace(long_prefill_token_threshold=0),
|
||||||
use_eagle=False,
|
use_eagle_block_drop=False,
|
||||||
hash_block_size=hash_block_size,
|
hash_block_size=hash_block_size,
|
||||||
dcp_world_size=dcp_world_size,
|
dcp_world_size=dcp_world_size,
|
||||||
scheduler_block_size=scheduler_block_size,
|
scheduler_block_size=scheduler_block_size,
|
||||||
@@ -218,7 +218,7 @@ def test_mamba_align_split_when_block_exceeds_scheduling_budget():
|
|||||||
cache_config=SimpleNamespace(block_size=block_size),
|
cache_config=SimpleNamespace(block_size=block_size),
|
||||||
max_num_scheduled_tokens=token_budget,
|
max_num_scheduled_tokens=token_budget,
|
||||||
scheduler_config=SimpleNamespace(long_prefill_token_threshold=0),
|
scheduler_config=SimpleNamespace(long_prefill_token_threshold=0),
|
||||||
use_eagle=False,
|
use_eagle_block_drop=False,
|
||||||
hash_block_size=32,
|
hash_block_size=32,
|
||||||
mamba_partial_cache_hit=False,
|
mamba_partial_cache_hit=False,
|
||||||
mamba_has_prefill_checkpoint_blocks=False,
|
mamba_has_prefill_checkpoint_blocks=False,
|
||||||
@@ -257,7 +257,7 @@ def test_mamba_align_split_when_block_exceeds_long_prefill_threshold():
|
|||||||
scheduler_config=SimpleNamespace(
|
scheduler_config=SimpleNamespace(
|
||||||
long_prefill_token_threshold=long_prefill_threshold
|
long_prefill_token_threshold=long_prefill_threshold
|
||||||
),
|
),
|
||||||
use_eagle=False,
|
use_eagle_block_drop=False,
|
||||||
hash_block_size=32,
|
hash_block_size=32,
|
||||||
mamba_partial_cache_hit=False,
|
mamba_partial_cache_hit=False,
|
||||||
mamba_has_prefill_checkpoint_blocks=False,
|
mamba_has_prefill_checkpoint_blocks=False,
|
||||||
|
|||||||
@@ -3296,6 +3296,7 @@ def _spec_decode_grouping_config(method="dspark", model_type=None):
|
|||||||
speculative_config=SimpleNamespace(
|
speculative_config=SimpleNamespace(
|
||||||
method=method,
|
method=method,
|
||||||
use_eagle=lambda: True,
|
use_eagle=lambda: True,
|
||||||
|
use_eagle_block_drop=lambda: True,
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|||||||
@@ -80,13 +80,16 @@ def _split(
|
|||||||
request: Request,
|
request: Request,
|
||||||
num_new_tokens: int,
|
num_new_tokens: int,
|
||||||
use_eagle: bool = True,
|
use_eagle: bool = True,
|
||||||
|
use_eagle_block_drop: bool | None = None,
|
||||||
partial_hit: bool = False,
|
partial_hit: bool = False,
|
||||||
num_prefill_checkpoint_blocks: int = 0,
|
num_prefill_checkpoint_blocks: int = 0,
|
||||||
) -> int:
|
) -> int:
|
||||||
"""Call the real `Scheduler._mamba_block_aligned_split` on a stub self."""
|
"""Call the real `Scheduler._mamba_block_aligned_split` on a stub self."""
|
||||||
|
if use_eagle_block_drop is None:
|
||||||
|
use_eagle_block_drop = use_eagle
|
||||||
stub = SimpleNamespace(
|
stub = SimpleNamespace(
|
||||||
cache_config=SimpleNamespace(block_size=MAMBA_BLOCK_SIZE),
|
cache_config=SimpleNamespace(block_size=MAMBA_BLOCK_SIZE),
|
||||||
use_eagle=use_eagle,
|
use_eagle_block_drop=use_eagle_block_drop,
|
||||||
max_num_scheduled_tokens=16384,
|
max_num_scheduled_tokens=16384,
|
||||||
scheduler_config=SimpleNamespace(long_prefill_token_threshold=0),
|
scheduler_config=SimpleNamespace(long_prefill_token_threshold=0),
|
||||||
# `prefix_match_unit` finer than the block size (#46384).
|
# `prefix_match_unit` finer than the block size (#46384).
|
||||||
@@ -128,6 +131,16 @@ def test_internal_checkpoint_split(
|
|||||||
assert all(not block.is_null for block in blocks) # checkpoint + running state
|
assert all(not block.is_null for block in blocks) # checkpoint + running state
|
||||||
|
|
||||||
|
|
||||||
|
def test_disabling_eagle_block_drop_keeps_the_trailing_cache_boundary() -> None:
|
||||||
|
(request,) = create_requests(1, num_tokens=3602, block_size=ATTN_BLOCK_SIZE)
|
||||||
|
|
||||||
|
with_drop = _split(request, request.num_tokens, use_eagle_block_drop=True)
|
||||||
|
without_drop = _split(request, request.num_tokens, use_eagle_block_drop=False)
|
||||||
|
|
||||||
|
assert with_drop == MAMBA_BLOCK_SIZE
|
||||||
|
assert without_drop == 2 * MAMBA_BLOCK_SIZE
|
||||||
|
|
||||||
|
|
||||||
def _run_chunked_prefill(
|
def _run_chunked_prefill(
|
||||||
manager: KVCacheManager, request: Request, budgets: list[int]
|
manager: KVCacheManager, request: Request, budgets: list[int]
|
||||||
) -> dict[int, int]:
|
) -> dict[int, int]:
|
||||||
|
|||||||
@@ -1163,7 +1163,7 @@ def test_hybrid_cache_mamba_align_shared_prefix_detection():
|
|||||||
cache_config=SimpleNamespace(block_size=block_size),
|
cache_config=SimpleNamespace(block_size=block_size),
|
||||||
max_num_scheduled_tokens=3 * block_size,
|
max_num_scheduled_tokens=3 * block_size,
|
||||||
scheduler_config=SimpleNamespace(long_prefill_token_threshold=0),
|
scheduler_config=SimpleNamespace(long_prefill_token_threshold=0),
|
||||||
use_eagle=False,
|
use_eagle_block_drop=False,
|
||||||
hash_block_size=block_size,
|
hash_block_size=block_size,
|
||||||
mamba_partial_cache_hit=False,
|
mamba_partial_cache_hit=False,
|
||||||
mamba_has_prefill_checkpoint_blocks=False,
|
mamba_has_prefill_checkpoint_blocks=False,
|
||||||
|
|||||||
@@ -399,6 +399,47 @@ def test_sliding_window_possible_cached_prefix():
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_sliding_window_cache_hit_with_finer_hash_alignment():
|
||||||
|
"""Sliding-window lookup uses full blocks with finer hybrid-cache hashes."""
|
||||||
|
hash_block_size = 2
|
||||||
|
block_size = 4
|
||||||
|
sliding_window_spec = SlidingWindowSpec(
|
||||||
|
block_size=block_size,
|
||||||
|
num_kv_heads=1,
|
||||||
|
head_size=1,
|
||||||
|
dtype=torch.float32,
|
||||||
|
sliding_window=8,
|
||||||
|
)
|
||||||
|
block_pool = BlockPool(
|
||||||
|
num_gpu_blocks=100,
|
||||||
|
enable_caching=True,
|
||||||
|
hash_block_size=hash_block_size,
|
||||||
|
)
|
||||||
|
manager = get_sliding_window_manager(sliding_window_spec, block_pool)
|
||||||
|
block_hashes = [BlockHash(str(i).encode()) for i in range(4)]
|
||||||
|
|
||||||
|
# Each physical block uses the last chained hash in its block-size view.
|
||||||
|
for block_hash, block in zip(
|
||||||
|
(block_hashes[1], block_hashes[3]), block_pool.blocks[10:12]
|
||||||
|
):
|
||||||
|
block_pool.cached_block_hash_to_block.insert(
|
||||||
|
make_block_hash_with_group_id(block_hash, 0), block
|
||||||
|
)
|
||||||
|
|
||||||
|
computed_blocks, hit_length = manager.find_longest_cache_hit(
|
||||||
|
block_hashes=block_hashes,
|
||||||
|
max_length=8,
|
||||||
|
kv_cache_group_ids=[0],
|
||||||
|
block_pool=block_pool,
|
||||||
|
kv_cache_spec=sliding_window_spec,
|
||||||
|
drop_eagle_block=False,
|
||||||
|
alignment_tokens=hash_block_size,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert hit_length == 8
|
||||||
|
assert computed_blocks[0] == block_pool.blocks[10:12]
|
||||||
|
|
||||||
|
|
||||||
def test_chunked_local_attention_remove_skipped_blocks():
|
def test_chunked_local_attention_remove_skipped_blocks():
|
||||||
attention_spec = ChunkedLocalAttentionSpec(
|
attention_spec = ChunkedLocalAttentionSpec(
|
||||||
block_size=2,
|
block_size=2,
|
||||||
|
|||||||
@@ -436,6 +436,11 @@ class SpeculativeConfig:
|
|||||||
speculative input batches can contain sequences of different lengths,
|
speculative input batches can contain sequences of different lengths,
|
||||||
which may only be supported by certain attention backends. This currently
|
which may only be supported by certain attention backends. This currently
|
||||||
only affects the EAGLE method of speculation."""
|
only affects the EAGLE method of speculation."""
|
||||||
|
disable_eagle_block_drop: bool = False
|
||||||
|
"""Disable dropping the trailing prefix-cache block for EAGLE-like
|
||||||
|
speculative methods. This is an experimental option for measuring the
|
||||||
|
acceptance-rate impact of reusing that block. It does not disable the
|
||||||
|
speculative drafter itself."""
|
||||||
use_local_argmax_reduction: bool = False
|
use_local_argmax_reduction: bool = False
|
||||||
"""Use vocab-parallel local argmax instead of all-gathering full logits
|
"""Use vocab-parallel local argmax instead of all-gathering full logits
|
||||||
for draft token generation. Reduces communication from O(vocab_size) to
|
for draft token generation. Reduces communication from O(vocab_size) to
|
||||||
@@ -1846,6 +1851,10 @@ class SpeculativeConfig:
|
|||||||
# TODO(ben): Refactor this so the naming is clearer
|
# TODO(ben): Refactor this so the naming is clearer
|
||||||
return self.method in ("eagle", "eagle3", "mtp", "dflash", "dspark")
|
return self.method in ("eagle", "eagle3", "mtp", "dflash", "dspark")
|
||||||
|
|
||||||
|
def use_eagle_block_drop(self) -> bool:
|
||||||
|
"""Whether volatile trailing cache blocks should be discarded."""
|
||||||
|
return self.use_eagle() and not self.disable_eagle_block_drop
|
||||||
|
|
||||||
def use_dflash(self) -> bool:
|
def use_dflash(self) -> bool:
|
||||||
return self.method == "dflash"
|
return self.method == "dflash"
|
||||||
|
|
||||||
|
|||||||
@@ -1593,16 +1593,17 @@ class MooncakeStoreWorker:
|
|||||||
for group in kv_cache_config.transfer_groups
|
for group in kv_cache_config.transfer_groups
|
||||||
]
|
]
|
||||||
spec_cfg = getattr(vllm_config, "speculative_config", None)
|
spec_cfg = getattr(vllm_config, "speculative_config", None)
|
||||||
use_eagle = bool(
|
use_eagle_block_drop = bool(
|
||||||
spec_cfg.use_eagle()
|
spec_cfg.use_eagle_block_drop()
|
||||||
if spec_cfg is not None and callable(getattr(spec_cfg, "use_eagle", None))
|
if spec_cfg is not None
|
||||||
|
and callable(getattr(spec_cfg, "use_eagle_block_drop", None))
|
||||||
else False
|
else False
|
||||||
)
|
)
|
||||||
self.coord = MooncakeStoreCoordinator(
|
self.coord = MooncakeStoreCoordinator(
|
||||||
self._kv_cache_groups,
|
self._kv_cache_groups,
|
||||||
scheduler_block_size=self.block_size,
|
scheduler_block_size=self.block_size,
|
||||||
hash_block_size=self.hash_block_size,
|
hash_block_size=self.hash_block_size,
|
||||||
use_eagle=use_eagle,
|
use_eagle=use_eagle_block_drop,
|
||||||
retention_interval=kv_cache_config.prefix_cache_retention_interval,
|
retention_interval=kv_cache_config.prefix_cache_retention_interval,
|
||||||
dcp_world_size=self.dcp_size,
|
dcp_world_size=self.dcp_size,
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -223,11 +223,11 @@ class SchedulerOffloadConfig(NamedTuple):
|
|||||||
if g.is_eagle_group
|
if g.is_eagle_group
|
||||||
}
|
}
|
||||||
|
|
||||||
use_eagle = (
|
use_eagle_block_drop = (
|
||||||
vllm_config.speculative_config is not None
|
vllm_config.speculative_config is not None
|
||||||
and vllm_config.speculative_config.use_eagle()
|
and vllm_config.speculative_config.use_eagle_block_drop()
|
||||||
)
|
)
|
||||||
if use_eagle and not eagle_groups:
|
if use_eagle_block_drop and not eagle_groups:
|
||||||
eagle_groups = set(range(len(kv_cache_config.kv_cache_groups)))
|
eagle_groups = set(range(len(kv_cache_config.kv_cache_groups)))
|
||||||
|
|
||||||
if eagle_groups:
|
if eagle_groups:
|
||||||
|
|||||||
@@ -1841,7 +1841,7 @@ def _annotate_eagle_groups(
|
|||||||
use_deepseek_v4_fallback: Enable rule 2 for a DeepseekV4 packed group.
|
use_deepseek_v4_fallback: Enable rule 2 for a DeepseekV4 packed group.
|
||||||
"""
|
"""
|
||||||
spec_config = vllm_config.speculative_config
|
spec_config = vllm_config.speculative_config
|
||||||
if spec_config is None or not spec_config.use_eagle():
|
if spec_config is None or not spec_config.use_eagle_block_drop():
|
||||||
return
|
return
|
||||||
|
|
||||||
for group in kv_cache_groups:
|
for group in kv_cache_groups:
|
||||||
|
|||||||
@@ -255,6 +255,7 @@ class Scheduler(SchedulerInterface):
|
|||||||
)
|
)
|
||||||
speculative_config = vllm_config.speculative_config
|
speculative_config = vllm_config.speculative_config
|
||||||
self.use_eagle = False
|
self.use_eagle = False
|
||||||
|
self.use_eagle_block_drop = False
|
||||||
self.num_spec_tokens = vllm_config.num_speculative_tokens
|
self.num_spec_tokens = vllm_config.num_speculative_tokens
|
||||||
self.num_lookahead_tokens = vllm_config.num_lookahead_tokens
|
self.num_lookahead_tokens = vllm_config.num_lookahead_tokens
|
||||||
# Positions past the computed tokens that the drafter reads mid-prefill.
|
# Positions past the computed tokens that the drafter reads mid-prefill.
|
||||||
@@ -279,6 +280,13 @@ class Scheduler(SchedulerInterface):
|
|||||||
if speculative_config.use_multi_module_mtp()
|
if speculative_config.use_multi_module_mtp()
|
||||||
else 1
|
else 1
|
||||||
)
|
)
|
||||||
|
self.use_eagle_block_drop = speculative_config.use_eagle_block_drop()
|
||||||
|
if self.use_eagle and not self.use_eagle_block_drop:
|
||||||
|
logger.warning(
|
||||||
|
"EAGLE trailing prefix-cache block dropping is disabled. "
|
||||||
|
"This is experimental and may affect speculative-token "
|
||||||
|
"acceptance rates."
|
||||||
|
)
|
||||||
|
|
||||||
# Create the KV cache manager.
|
# Create the KV cache manager.
|
||||||
if hash_block_size is None:
|
if hash_block_size is None:
|
||||||
@@ -289,7 +297,7 @@ class Scheduler(SchedulerInterface):
|
|||||||
max_model_len=self.max_model_len,
|
max_model_len=self.max_model_len,
|
||||||
max_in_flight_tokens=vllm_config.max_in_flight_tokens,
|
max_in_flight_tokens=vllm_config.max_in_flight_tokens,
|
||||||
enable_caching=self.cache_config.enable_prefix_caching,
|
enable_caching=self.cache_config.enable_prefix_caching,
|
||||||
use_eagle=self.use_eagle,
|
use_eagle=self.use_eagle_block_drop,
|
||||||
num_prefill_lookahead=self.num_prefill_lookahead,
|
num_prefill_lookahead=self.num_prefill_lookahead,
|
||||||
log_stats=self.log_stats,
|
log_stats=self.log_stats,
|
||||||
enable_kv_cache_events=self.enable_kv_cache_events,
|
enable_kv_cache_events=self.enable_kv_cache_events,
|
||||||
@@ -414,7 +422,7 @@ class Scheduler(SchedulerInterface):
|
|||||||
# Eagle, FullAttn prunes the last matching block, so back off one
|
# Eagle, FullAttn prunes the last matching block, so back off one
|
||||||
# block to avoid a Mamba cache miss.
|
# block to avoid a Mamba cache miss.
|
||||||
last_cache_position = request.num_tokens - request.num_tokens % block_size
|
last_cache_position = request.num_tokens - request.num_tokens % block_size
|
||||||
if self.use_eagle:
|
if self.use_eagle_block_drop:
|
||||||
last_cache_position = max(last_cache_position - block_size, 0)
|
last_cache_position = max(last_cache_position - block_size, 0)
|
||||||
|
|
||||||
end = start + num_new_tokens
|
end = start + num_new_tokens
|
||||||
|
|||||||
@@ -925,10 +925,9 @@ class SlidingWindowManager(SingleTypeKVCacheManager):
|
|||||||
)
|
)
|
||||||
assert dcp_world_size == 1, "DCP not support sliding window attn now."
|
assert dcp_world_size == 1, "DCP not support sliding window attn now."
|
||||||
assert pcp_world_size == 1, "PCP not support sliding window attn now."
|
assert pcp_world_size == 1, "PCP not support sliding window attn now."
|
||||||
# Fine-grained partial hits are not supported for sliding window now
|
# Sliding-window cache hits must stay at the group's physical block
|
||||||
assert alignment_tokens % kv_cache_spec.block_size == 0, (
|
# granularity. resolve_block_hashes() converts finer-grained hashes to
|
||||||
"SlidingWindowManager does not support fine-grained (partial) cache hits"
|
# that view when the hybrid-cache alignment is smaller than block_size.
|
||||||
)
|
|
||||||
block_hashes = resolve_block_hashes(
|
block_hashes = resolve_block_hashes(
|
||||||
block_hashes,
|
block_hashes,
|
||||||
block_pool.hash_block_size,
|
block_pool.hash_block_size,
|
||||||
|
|||||||
@@ -125,12 +125,14 @@ class SimpleCPUOffloadScheduler:
|
|||||||
)
|
)
|
||||||
|
|
||||||
spec_config = vllm_config.speculative_config
|
spec_config = vllm_config.speculative_config
|
||||||
use_eagle = spec_config is not None and spec_config.use_eagle()
|
use_eagle_block_drop = (
|
||||||
|
spec_config is not None and spec_config.use_eagle_block_drop()
|
||||||
|
)
|
||||||
self.cpu_coordinator: KVCacheCoordinator = get_kv_cache_coordinator(
|
self.cpu_coordinator: KVCacheCoordinator = get_kv_cache_coordinator(
|
||||||
kv_cache_config=self.cpu_kv_cache_config,
|
kv_cache_config=self.cpu_kv_cache_config,
|
||||||
max_model_len=vllm_config.model_config.max_model_len,
|
max_model_len=vllm_config.model_config.max_model_len,
|
||||||
max_in_flight_tokens=vllm_config.max_in_flight_tokens,
|
max_in_flight_tokens=vllm_config.max_in_flight_tokens,
|
||||||
use_eagle=use_eagle,
|
use_eagle=use_eagle_block_drop,
|
||||||
enable_caching=True,
|
enable_caching=True,
|
||||||
enable_kv_cache_events=self.enable_kv_cache_events,
|
enable_kv_cache_events=self.enable_kv_cache_events,
|
||||||
dcp_world_size=dcp_world_size,
|
dcp_world_size=dcp_world_size,
|
||||||
|
|||||||
Reference in New Issue
Block a user