[Feature][Spec] Support disabling trailing prefix-cache block dropping (#53388)
Signed-off-by: Ziming Huang <zelda.huanghuang@gmail.com> Signed-off-by: Ziming Huang <48115868+ZeldaHuang@users.noreply.github.com> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
OpenAI Codex
parent
9e905f7450
commit
481839ad9e
@@ -2152,6 +2152,23 @@ def test_draft_sample_method_probabilistic_is_accepted():
|
||||
assert speculative_config.draft_sample_method == "probabilistic"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("disable_eagle_block_drop", [False, True])
|
||||
def test_eagle_block_drop_can_be_disabled_without_disabling_eagle(
|
||||
disable_eagle_block_drop: bool,
|
||||
):
|
||||
# Start from an ngram config to avoid loading model metadata: these predicates
|
||||
# depend only on the speculative method and the new switch.
|
||||
speculative_config = SpeculativeConfig(
|
||||
method="ngram",
|
||||
num_speculative_tokens=3,
|
||||
disable_eagle_block_drop=disable_eagle_block_drop,
|
||||
)
|
||||
speculative_config.method = "eagle3"
|
||||
|
||||
assert speculative_config.use_eagle()
|
||||
assert speculative_config.use_eagle_block_drop() is not disable_eagle_block_drop
|
||||
|
||||
|
||||
def test_draft_sample_method_gumbel_is_rejected():
|
||||
with pytest.raises(ValidationError):
|
||||
SpeculativeConfig(
|
||||
|
||||
@@ -170,7 +170,7 @@ def test_mamba_align_split_partial_tail_schedule(dcp_world_size: int):
|
||||
cache_config=SimpleNamespace(block_size=block_size),
|
||||
max_num_scheduled_tokens=8192,
|
||||
scheduler_config=SimpleNamespace(long_prefill_token_threshold=0),
|
||||
use_eagle=False,
|
||||
use_eagle_block_drop=False,
|
||||
hash_block_size=hash_block_size,
|
||||
dcp_world_size=dcp_world_size,
|
||||
scheduler_block_size=scheduler_block_size,
|
||||
@@ -218,7 +218,7 @@ def test_mamba_align_split_when_block_exceeds_scheduling_budget():
|
||||
cache_config=SimpleNamespace(block_size=block_size),
|
||||
max_num_scheduled_tokens=token_budget,
|
||||
scheduler_config=SimpleNamespace(long_prefill_token_threshold=0),
|
||||
use_eagle=False,
|
||||
use_eagle_block_drop=False,
|
||||
hash_block_size=32,
|
||||
mamba_partial_cache_hit=False,
|
||||
mamba_has_prefill_checkpoint_blocks=False,
|
||||
@@ -257,7 +257,7 @@ def test_mamba_align_split_when_block_exceeds_long_prefill_threshold():
|
||||
scheduler_config=SimpleNamespace(
|
||||
long_prefill_token_threshold=long_prefill_threshold
|
||||
),
|
||||
use_eagle=False,
|
||||
use_eagle_block_drop=False,
|
||||
hash_block_size=32,
|
||||
mamba_partial_cache_hit=False,
|
||||
mamba_has_prefill_checkpoint_blocks=False,
|
||||
|
||||
@@ -3296,6 +3296,7 @@ def _spec_decode_grouping_config(method="dspark", model_type=None):
|
||||
speculative_config=SimpleNamespace(
|
||||
method=method,
|
||||
use_eagle=lambda: True,
|
||||
use_eagle_block_drop=lambda: True,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@@ -80,13 +80,16 @@ def _split(
|
||||
request: Request,
|
||||
num_new_tokens: int,
|
||||
use_eagle: bool = True,
|
||||
use_eagle_block_drop: bool | None = None,
|
||||
partial_hit: bool = False,
|
||||
num_prefill_checkpoint_blocks: int = 0,
|
||||
) -> int:
|
||||
"""Call the real `Scheduler._mamba_block_aligned_split` on a stub self."""
|
||||
if use_eagle_block_drop is None:
|
||||
use_eagle_block_drop = use_eagle
|
||||
stub = SimpleNamespace(
|
||||
cache_config=SimpleNamespace(block_size=MAMBA_BLOCK_SIZE),
|
||||
use_eagle=use_eagle,
|
||||
use_eagle_block_drop=use_eagle_block_drop,
|
||||
max_num_scheduled_tokens=16384,
|
||||
scheduler_config=SimpleNamespace(long_prefill_token_threshold=0),
|
||||
# `prefix_match_unit` finer than the block size (#46384).
|
||||
@@ -128,6 +131,16 @@ def test_internal_checkpoint_split(
|
||||
assert all(not block.is_null for block in blocks) # checkpoint + running state
|
||||
|
||||
|
||||
def test_disabling_eagle_block_drop_keeps_the_trailing_cache_boundary() -> None:
|
||||
(request,) = create_requests(1, num_tokens=3602, block_size=ATTN_BLOCK_SIZE)
|
||||
|
||||
with_drop = _split(request, request.num_tokens, use_eagle_block_drop=True)
|
||||
without_drop = _split(request, request.num_tokens, use_eagle_block_drop=False)
|
||||
|
||||
assert with_drop == MAMBA_BLOCK_SIZE
|
||||
assert without_drop == 2 * MAMBA_BLOCK_SIZE
|
||||
|
||||
|
||||
def _run_chunked_prefill(
|
||||
manager: KVCacheManager, request: Request, budgets: list[int]
|
||||
) -> dict[int, int]:
|
||||
|
||||
@@ -1163,7 +1163,7 @@ def test_hybrid_cache_mamba_align_shared_prefix_detection():
|
||||
cache_config=SimpleNamespace(block_size=block_size),
|
||||
max_num_scheduled_tokens=3 * block_size,
|
||||
scheduler_config=SimpleNamespace(long_prefill_token_threshold=0),
|
||||
use_eagle=False,
|
||||
use_eagle_block_drop=False,
|
||||
hash_block_size=block_size,
|
||||
mamba_partial_cache_hit=False,
|
||||
mamba_has_prefill_checkpoint_blocks=False,
|
||||
|
||||
@@ -399,6 +399,47 @@ def test_sliding_window_possible_cached_prefix():
|
||||
)
|
||||
|
||||
|
||||
def test_sliding_window_cache_hit_with_finer_hash_alignment():
|
||||
"""Sliding-window lookup uses full blocks with finer hybrid-cache hashes."""
|
||||
hash_block_size = 2
|
||||
block_size = 4
|
||||
sliding_window_spec = SlidingWindowSpec(
|
||||
block_size=block_size,
|
||||
num_kv_heads=1,
|
||||
head_size=1,
|
||||
dtype=torch.float32,
|
||||
sliding_window=8,
|
||||
)
|
||||
block_pool = BlockPool(
|
||||
num_gpu_blocks=100,
|
||||
enable_caching=True,
|
||||
hash_block_size=hash_block_size,
|
||||
)
|
||||
manager = get_sliding_window_manager(sliding_window_spec, block_pool)
|
||||
block_hashes = [BlockHash(str(i).encode()) for i in range(4)]
|
||||
|
||||
# Each physical block uses the last chained hash in its block-size view.
|
||||
for block_hash, block in zip(
|
||||
(block_hashes[1], block_hashes[3]), block_pool.blocks[10:12]
|
||||
):
|
||||
block_pool.cached_block_hash_to_block.insert(
|
||||
make_block_hash_with_group_id(block_hash, 0), block
|
||||
)
|
||||
|
||||
computed_blocks, hit_length = manager.find_longest_cache_hit(
|
||||
block_hashes=block_hashes,
|
||||
max_length=8,
|
||||
kv_cache_group_ids=[0],
|
||||
block_pool=block_pool,
|
||||
kv_cache_spec=sliding_window_spec,
|
||||
drop_eagle_block=False,
|
||||
alignment_tokens=hash_block_size,
|
||||
)
|
||||
|
||||
assert hit_length == 8
|
||||
assert computed_blocks[0] == block_pool.blocks[10:12]
|
||||
|
||||
|
||||
def test_chunked_local_attention_remove_skipped_blocks():
|
||||
attention_spec = ChunkedLocalAttentionSpec(
|
||||
block_size=2,
|
||||
|
||||
Reference in New Issue
Block a user