[Bugfix][Core] Retire Mamba states across null gaps (#55450)
Signed-off-by: Luca Motz <luca.motz@icloud.com> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
This commit is contained in:
co-authored by
OpenAI Codex
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Cyrus Leung
parent
a9271c750f
commit
fadfe1c7d4
@@ -103,6 +103,82 @@ def test_mamba_speculative_block_relocation_requires_exclusive_ownership():
|
||||
manager._relocate_speculative_block([pinned_block], 0)
|
||||
|
||||
|
||||
def test_mamba_retirement_crosses_null_gaps():
|
||||
spec = MambaSpec(
|
||||
block_size=4,
|
||||
shapes=((1, 1),),
|
||||
dtypes=(torch.float32,),
|
||||
mamba_cache_mode="align",
|
||||
)
|
||||
pool = BlockPool(num_gpu_blocks=8, enable_caching=False, hash_block_size=4)
|
||||
manager = MambaManager(
|
||||
spec,
|
||||
block_pool=pool,
|
||||
enable_caching=False,
|
||||
kv_cache_group_id=0,
|
||||
scheduler_block_size=4,
|
||||
)
|
||||
old, committed, in_flight = pool.get_new_blocks(3)
|
||||
manager.req_to_blocks["r"] = [old, pool.null_block, committed, in_flight]
|
||||
|
||||
# The state at token 12 and the following in-flight state must survive.
|
||||
for _ in range(2):
|
||||
manager.remove_skipped_blocks("r", processed_computed_tokens=12)
|
||||
assert manager.req_to_blocks["r"] == [
|
||||
pool.null_block,
|
||||
pool.null_block,
|
||||
committed,
|
||||
in_flight,
|
||||
]
|
||||
assert old.ref_cnt == 0
|
||||
assert committed.ref_cnt == in_flight.ref_cnt == 1
|
||||
assert pool.get_num_free_blocks() == 5
|
||||
|
||||
manager.free("r")
|
||||
manager.req_to_blocks["r"] = pool.get_new_blocks(2)
|
||||
manager.remove_skipped_blocks("r", processed_computed_tokens=5)
|
||||
assert manager.req_to_blocks["r"][0].is_null
|
||||
assert manager.req_to_blocks["r"][1].ref_cnt == 1
|
||||
|
||||
|
||||
@pytest.mark.parametrize("block_size", [3584, 4608])
|
||||
@pytest.mark.parametrize("in_flight_chunks", [0, 1, 2])
|
||||
def test_mamba_retirement_bounds_prefill_states(block_size, in_flight_chunks):
|
||||
spec = MambaSpec(
|
||||
block_size=block_size,
|
||||
shapes=((1, 1),),
|
||||
dtypes=(torch.float32,),
|
||||
mamba_cache_mode="align",
|
||||
num_speculative_blocks=5,
|
||||
)
|
||||
pool = BlockPool(num_gpu_blocks=1000, enable_caching=True, hash_block_size=256)
|
||||
manager = MambaManager(
|
||||
spec,
|
||||
block_pool=pool,
|
||||
enable_caching=True,
|
||||
kv_cache_group_id=0,
|
||||
scheduler_block_size=block_size,
|
||||
)
|
||||
initial_free = pool.get_num_free_blocks()
|
||||
chunk = 8192 // block_size * block_size
|
||||
peak_held = 0
|
||||
# Replay full prefill chunks with an async committed-token lag.
|
||||
for target in range(chunk, 480031 + 1, chunk):
|
||||
computed = target - chunk
|
||||
committed = max(0, computed - in_flight_chunks * chunk)
|
||||
manager.remove_skipped_blocks("r", committed)
|
||||
manager.get_num_blocks_to_allocate(
|
||||
"r", target + 5, [], computed, computed, target
|
||||
)
|
||||
manager.allocate_new_blocks("r", target + 5, target)
|
||||
peak_held = max(peak_held, initial_free - pool.get_num_free_blocks())
|
||||
|
||||
# Five speculative blocks, current/previous states, and bounded in-flight states.
|
||||
assert peak_held == 7 + in_flight_chunks
|
||||
manager.free("r")
|
||||
assert pool.get_num_free_blocks() == initial_free
|
||||
|
||||
|
||||
def get_sliding_window_manager(
|
||||
sliding_window_spec,
|
||||
block_pool,
|
||||
|
||||
@@ -837,7 +837,7 @@ def get_mamba_prefix_cache_step_configs(
|
||||
StepAction(
|
||||
560 * 10,
|
||||
4,
|
||||
[0, 0, 0, 0, 1, 0, 0, 0, 1, 1, 1, 1, 1, 1]
|
||||
[0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1]
|
||||
if a
|
||||
else [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1],
|
||||
(9, 10),
|
||||
|
||||
Reference in New Issue
Block a user