[Feature][Spec] Support disabling trailing prefix-cache block dropping (#53388)

Signed-off-by: Ziming Huang <zelda.huanghuang@gmail.com>
Signed-off-by: Ziming Huang <48115868+ZeldaHuang@users.noreply.github.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
Ziming Huang
2026-09-01 19:12:36 +08:00
committed by GitHub
co-authored by OpenAI Codex
parent 9e905f7450
commit 481839ad9e
13 changed files with 112 additions and 21 deletions
+17
View File
@@ -2152,6 +2152,23 @@ def test_draft_sample_method_probabilistic_is_accepted():
assert speculative_config.draft_sample_method == "probabilistic"
@pytest.mark.parametrize("disable_eagle_block_drop", [False, True])
def test_eagle_block_drop_can_be_disabled_without_disabling_eagle(
disable_eagle_block_drop: bool,
):
# Start from an ngram config to avoid loading model metadata: these predicates
# depend only on the speculative method and the new switch.
speculative_config = SpeculativeConfig(
method="ngram",
num_speculative_tokens=3,
disable_eagle_block_drop=disable_eagle_block_drop,
)
speculative_config.method = "eagle3"
assert speculative_config.use_eagle()
assert speculative_config.use_eagle_block_drop() is not disable_eagle_block_drop
def test_draft_sample_method_gumbel_is_rejected():
with pytest.raises(ValidationError):
SpeculativeConfig(
@@ -170,7 +170,7 @@ def test_mamba_align_split_partial_tail_schedule(dcp_world_size: int):
cache_config=SimpleNamespace(block_size=block_size),
max_num_scheduled_tokens=8192,
scheduler_config=SimpleNamespace(long_prefill_token_threshold=0),
use_eagle=False,
use_eagle_block_drop=False,
hash_block_size=hash_block_size,
dcp_world_size=dcp_world_size,
scheduler_block_size=scheduler_block_size,
@@ -218,7 +218,7 @@ def test_mamba_align_split_when_block_exceeds_scheduling_budget():
cache_config=SimpleNamespace(block_size=block_size),
max_num_scheduled_tokens=token_budget,
scheduler_config=SimpleNamespace(long_prefill_token_threshold=0),
use_eagle=False,
use_eagle_block_drop=False,
hash_block_size=32,
mamba_partial_cache_hit=False,
mamba_has_prefill_checkpoint_blocks=False,
@@ -257,7 +257,7 @@ def test_mamba_align_split_when_block_exceeds_long_prefill_threshold():
scheduler_config=SimpleNamespace(
long_prefill_token_threshold=long_prefill_threshold
),
use_eagle=False,
use_eagle_block_drop=False,
hash_block_size=32,
mamba_partial_cache_hit=False,
mamba_has_prefill_checkpoint_blocks=False,
+1
View File
@@ -3296,6 +3296,7 @@ def _spec_decode_grouping_config(method="dspark", model_type=None):
speculative_config=SimpleNamespace(
method=method,
use_eagle=lambda: True,
use_eagle_block_drop=lambda: True,
),
)
+14 -1
View File
@@ -80,13 +80,16 @@ def _split(
request: Request,
num_new_tokens: int,
use_eagle: bool = True,
use_eagle_block_drop: bool | None = None,
partial_hit: bool = False,
num_prefill_checkpoint_blocks: int = 0,
) -> int:
"""Call the real `Scheduler._mamba_block_aligned_split` on a stub self."""
if use_eagle_block_drop is None:
use_eagle_block_drop = use_eagle
stub = SimpleNamespace(
cache_config=SimpleNamespace(block_size=MAMBA_BLOCK_SIZE),
use_eagle=use_eagle,
use_eagle_block_drop=use_eagle_block_drop,
max_num_scheduled_tokens=16384,
scheduler_config=SimpleNamespace(long_prefill_token_threshold=0),
# `prefix_match_unit` finer than the block size (#46384).
@@ -128,6 +131,16 @@ def test_internal_checkpoint_split(
assert all(not block.is_null for block in blocks) # checkpoint + running state
def test_disabling_eagle_block_drop_keeps_the_trailing_cache_boundary() -> None:
(request,) = create_requests(1, num_tokens=3602, block_size=ATTN_BLOCK_SIZE)
with_drop = _split(request, request.num_tokens, use_eagle_block_drop=True)
without_drop = _split(request, request.num_tokens, use_eagle_block_drop=False)
assert with_drop == MAMBA_BLOCK_SIZE
assert without_drop == 2 * MAMBA_BLOCK_SIZE
def _run_chunked_prefill(
manager: KVCacheManager, request: Request, budgets: list[int]
) -> dict[int, int]:
+1 -1
View File
@@ -1163,7 +1163,7 @@ def test_hybrid_cache_mamba_align_shared_prefix_detection():
cache_config=SimpleNamespace(block_size=block_size),
max_num_scheduled_tokens=3 * block_size,
scheduler_config=SimpleNamespace(long_prefill_token_threshold=0),
use_eagle=False,
use_eagle_block_drop=False,
hash_block_size=block_size,
mamba_partial_cache_hit=False,
mamba_has_prefill_checkpoint_blocks=False,
@@ -399,6 +399,47 @@ def test_sliding_window_possible_cached_prefix():
)
def test_sliding_window_cache_hit_with_finer_hash_alignment():
"""Sliding-window lookup uses full blocks with finer hybrid-cache hashes."""
hash_block_size = 2
block_size = 4
sliding_window_spec = SlidingWindowSpec(
block_size=block_size,
num_kv_heads=1,
head_size=1,
dtype=torch.float32,
sliding_window=8,
)
block_pool = BlockPool(
num_gpu_blocks=100,
enable_caching=True,
hash_block_size=hash_block_size,
)
manager = get_sliding_window_manager(sliding_window_spec, block_pool)
block_hashes = [BlockHash(str(i).encode()) for i in range(4)]
# Each physical block uses the last chained hash in its block-size view.
for block_hash, block in zip(
(block_hashes[1], block_hashes[3]), block_pool.blocks[10:12]
):
block_pool.cached_block_hash_to_block.insert(
make_block_hash_with_group_id(block_hash, 0), block
)
computed_blocks, hit_length = manager.find_longest_cache_hit(
block_hashes=block_hashes,
max_length=8,
kv_cache_group_ids=[0],
block_pool=block_pool,
kv_cache_spec=sliding_window_spec,
drop_eagle_block=False,
alignment_tokens=hash_block_size,
)
assert hit_length == 8
assert computed_blocks[0] == block_pool.blocks[10:12]
def test_chunked_local_attention_remove_skipped_blocks():
attention_spec = ChunkedLocalAttentionSpec(
block_size=2,