Signed-off-by: Roi Koren <roik@nvidia.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
7d99c2c4fd
commit
49ee12d742
@@ -6,7 +6,7 @@ Full attention hits where it holds a key; EAGLE prunes one hash unit off that
|
||||
candidate and drops it. Mamba materializes state only on its own block grid, so
|
||||
the pruned position holds nothing and the hit floors back a whole Mamba block.
|
||||
|
||||
Gated by ``CacheConfig.enable_mamba_fine_grained_prefix_cache``. Every test here
|
||||
Gated by ``CacheConfig.enable_mamba_shared_prefix_checkpoint``. Every test here
|
||||
drives the real ``Scheduler._mamba_block_aligned_split`` and the real
|
||||
``KVCacheManager``; no chunk boundary is hard-coded.
|
||||
"""
|
||||
@@ -32,7 +32,7 @@ def _manager(
|
||||
block_size,
|
||||
hash_block_size,
|
||||
*,
|
||||
fine_grained=True,
|
||||
shared_prefix_checkpoint=True,
|
||||
num_blocks=8192,
|
||||
eagle_group=None,
|
||||
num_prefill_lookahead=0,
|
||||
@@ -52,7 +52,7 @@ def _manager(
|
||||
hash_block_size=hash_block_size,
|
||||
use_eagle=True,
|
||||
num_prefill_lookahead=num_prefill_lookahead,
|
||||
enable_mamba_fine_grained_prefix_cache=fine_grained,
|
||||
enable_mamba_shared_prefix_checkpoint=shared_prefix_checkpoint,
|
||||
)
|
||||
|
||||
|
||||
@@ -76,8 +76,8 @@ def _stub(manager, block_size, hash_block_size, *, block_drop=True):
|
||||
hash_block_size=hash_block_size,
|
||||
mamba_has_prefill_checkpoint_blocks=False, # forced False under eagle
|
||||
mamba_partial_cache_hit=partial_hit,
|
||||
mamba_fine_grained_prefix_cache=(
|
||||
partial_hit and manager.mamba_fine_grained_prefix_cache
|
||||
mamba_shared_prefix_checkpoint=(
|
||||
partial_hit and manager.mamba_shared_prefix_checkpoint
|
||||
),
|
||||
)
|
||||
|
||||
@@ -151,7 +151,7 @@ def test_sibling_resumes_from_the_observed_junction():
|
||||
block_size, hash_block_size = 512, 32
|
||||
manager = _manager(block_size, hash_block_size)
|
||||
stub = _stub(manager, block_size, hash_block_size)
|
||||
assert stub.mamba_fine_grained_prefix_cache, "the feature must be armed"
|
||||
assert stub.mamba_shared_prefix_checkpoint, "the feature must be armed"
|
||||
_orphaned_full_attention_tail(manager, stub, 2020)
|
||||
|
||||
consumer = make_request(
|
||||
@@ -325,11 +325,11 @@ def test_enabling_never_reduces_reuse(eagle_group, num_prefill_lookahead):
|
||||
"""
|
||||
block_size, hash_block_size = 512, 32
|
||||
|
||||
def sibling_hit(fine_grained):
|
||||
def sibling_hit(shared_prefix_checkpoint):
|
||||
manager = _manager(
|
||||
block_size,
|
||||
hash_block_size,
|
||||
fine_grained=fine_grained,
|
||||
shared_prefix_checkpoint=shared_prefix_checkpoint,
|
||||
eagle_group=eagle_group,
|
||||
num_prefill_lookahead=num_prefill_lookahead,
|
||||
)
|
||||
@@ -348,7 +348,7 @@ def test_enabling_never_reduces_reuse(eagle_group, num_prefill_lookahead):
|
||||
def test_disabled_by_default():
|
||||
"""Off, the junction is block-floored and no resume point is registered."""
|
||||
block_size, hash_block_size = 512, 32
|
||||
manager = _manager(block_size, hash_block_size, fine_grained=False)
|
||||
manager = _manager(block_size, hash_block_size, shared_prefix_checkpoint=False)
|
||||
stub = _stub(manager, block_size, hash_block_size)
|
||||
_orphaned_full_attention_tail(manager, stub, 2020)
|
||||
|
||||
|
||||
@@ -188,7 +188,7 @@ def test_mamba_align_split_partial_tail_schedule(dcp_world_size: int):
|
||||
dcp_world_size=dcp_world_size,
|
||||
scheduler_block_size=scheduler_block_size,
|
||||
mamba_partial_cache_hit=True,
|
||||
mamba_fine_grained_prefix_cache=False,
|
||||
mamba_shared_prefix_checkpoint=False,
|
||||
mamba_has_prefill_checkpoint_blocks=False,
|
||||
)
|
||||
split = Scheduler._mamba_block_aligned_split
|
||||
@@ -236,7 +236,7 @@ def test_mamba_align_split_when_block_exceeds_scheduling_budget():
|
||||
use_eagle_block_drop=False,
|
||||
hash_block_size=32,
|
||||
mamba_partial_cache_hit=False,
|
||||
mamba_fine_grained_prefix_cache=False,
|
||||
mamba_shared_prefix_checkpoint=False,
|
||||
mamba_has_prefill_checkpoint_blocks=False,
|
||||
)
|
||||
req = make_request("0", [0] * prompt_length, 32, sha256)
|
||||
@@ -277,7 +277,7 @@ def test_mamba_align_split_when_block_exceeds_long_prefill_threshold():
|
||||
use_eagle_block_drop=False,
|
||||
hash_block_size=32,
|
||||
mamba_partial_cache_hit=False,
|
||||
mamba_fine_grained_prefix_cache=False,
|
||||
mamba_shared_prefix_checkpoint=False,
|
||||
mamba_has_prefill_checkpoint_blocks=False,
|
||||
)
|
||||
req = make_request("0", [0] * prompt_length, 32, sha256)
|
||||
@@ -1826,7 +1826,7 @@ def test_mamba_align_split_stops_below_eagle_proof_boundary():
|
||||
use_eagle_block_drop=True,
|
||||
hash_block_size=hash_block_size,
|
||||
mamba_partial_cache_hit=True,
|
||||
mamba_fine_grained_prefix_cache=False,
|
||||
mamba_shared_prefix_checkpoint=False,
|
||||
mamba_has_prefill_checkpoint_blocks=False,
|
||||
)
|
||||
split = Scheduler._mamba_block_aligned_split
|
||||
|
||||
@@ -144,7 +144,7 @@ def _split(
|
||||
scheduler_config=SimpleNamespace(long_prefill_token_threshold=0),
|
||||
# `prefix_match_unit` finer than the block size (#46384).
|
||||
mamba_partial_cache_hit=partial_hit,
|
||||
mamba_fine_grained_prefix_cache=False,
|
||||
mamba_shared_prefix_checkpoint=False,
|
||||
hash_block_size=ATTN_BLOCK_SIZE,
|
||||
mamba_has_prefill_checkpoint_blocks=(num_prefill_checkpoint_blocks > 0),
|
||||
mamba_prefill_checkpoint_alignment=(
|
||||
|
||||
@@ -2269,7 +2269,7 @@ def test_hybrid_cache_mamba_align_shared_prefix_detection():
|
||||
use_eagle_block_drop=False,
|
||||
hash_block_size=block_size,
|
||||
mamba_partial_cache_hit=False,
|
||||
mamba_fine_grained_prefix_cache=False,
|
||||
mamba_shared_prefix_checkpoint=False,
|
||||
mamba_has_prefill_checkpoint_blocks=False,
|
||||
)
|
||||
req_2.shared_prefix_boundary = shared_prefix_boundary
|
||||
|
||||
Reference in New Issue
Block a user