[Perf] Support internal prefill checkpoints for Mamba prefix caching, 9%~25% TTFT improvement (#52789)
Signed-off-by: yewentao256 <zhyanwentao@126.com> Signed-off-by: Ziming Huang <zelda.huanghuang@gmail.com> Signed-off-by: Wentao Ye <44945378+yewentao256@users.noreply.github.com> Co-authored-by: Ziming Huang <zelda.huanghuang@gmail.com> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
Ziming Huang
OpenAI Codex
parent
020829f66b
commit
9eb9d9d395
@@ -77,6 +77,7 @@ def make_full_mamba_manager(
|
||||
mamba_block_size: int = 4,
|
||||
num_blocks: int = 32,
|
||||
use_eagle: bool = False,
|
||||
num_prefill_checkpoint_blocks: int = 0,
|
||||
):
|
||||
kv_cache_config = KVCacheConfig(
|
||||
num_blocks=num_blocks,
|
||||
@@ -98,6 +99,7 @@ def make_full_mamba_manager(
|
||||
shapes=(1, 1),
|
||||
dtypes=(torch.float32,),
|
||||
mamba_cache_mode="align",
|
||||
num_prefill_checkpoint_blocks=num_prefill_checkpoint_blocks,
|
||||
),
|
||||
),
|
||||
],
|
||||
@@ -135,6 +137,7 @@ def test_mamba_align_split_partial_tail_schedule(dcp_world_size: int):
|
||||
dcp_world_size=dcp_world_size,
|
||||
scheduler_block_size=scheduler_block_size,
|
||||
mamba_partial_cache_hit=True,
|
||||
mamba_has_prefill_checkpoint_blocks=False,
|
||||
)
|
||||
split = Scheduler._mamba_block_aligned_split
|
||||
|
||||
@@ -180,6 +183,7 @@ def test_mamba_align_split_when_block_exceeds_scheduling_budget():
|
||||
use_eagle=False,
|
||||
hash_block_size=32,
|
||||
mamba_partial_cache_hit=False,
|
||||
mamba_has_prefill_checkpoint_blocks=False,
|
||||
)
|
||||
req = make_request("0", [0] * prompt_length, 32, sha256)
|
||||
split = Scheduler._mamba_block_aligned_split
|
||||
@@ -218,6 +222,7 @@ def test_mamba_align_split_when_block_exceeds_long_prefill_threshold():
|
||||
use_eagle=False,
|
||||
hash_block_size=32,
|
||||
mamba_partial_cache_hit=False,
|
||||
mamba_has_prefill_checkpoint_blocks=False,
|
||||
)
|
||||
req = make_request("0", [0] * prompt_length, 32, sha256)
|
||||
split = Scheduler._mamba_block_aligned_split
|
||||
@@ -462,6 +467,85 @@ def test_hybrid_mamba_partial_tail_owner_uses_cow_on_continue():
|
||||
assert moved[0].block_hash_num_tokens == 6
|
||||
|
||||
|
||||
def test_partial_hit_then_internal_checkpoint_uses_distinct_mamba_blocks():
|
||||
hash_block_size = 2
|
||||
mamba_block_size = 4
|
||||
manager = make_full_mamba_manager(
|
||||
dcp_world_size=1,
|
||||
hash_block_size=hash_block_size,
|
||||
full_block_size=hash_block_size,
|
||||
mamba_block_size=mamba_block_size,
|
||||
num_prefill_checkpoint_blocks=1,
|
||||
)
|
||||
|
||||
owner = make_request("owner", [0, 0, 1, 1, 2, 2], hash_block_size, sha256)
|
||||
computed_blocks, num_computed, _ = manager.get_computed_blocks(owner)
|
||||
assert manager.allocate_slots(owner, 6, num_computed, computed_blocks) is not None
|
||||
manager.free(owner)
|
||||
manager.new_step_starts()
|
||||
|
||||
partial_hash = owner.block_hashes[2]
|
||||
partial_block = manager.block_pool.get_cached_block(partial_hash, [1])
|
||||
assert partial_block is not None
|
||||
partial_block_id = partial_block[0].block_id
|
||||
|
||||
replay = make_request(
|
||||
"replay",
|
||||
[0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6],
|
||||
hash_block_size,
|
||||
sha256,
|
||||
)
|
||||
computed_blocks, num_computed, _ = manager.get_computed_blocks(replay)
|
||||
assert num_computed == 6
|
||||
|
||||
realigned_blocks = manager.allocate_slots(replay, 2, num_computed, computed_blocks)
|
||||
assert realigned_blocks is not None
|
||||
mamba_cow_block_id = realigned_blocks.get_block_ids()[1][0]
|
||||
copies, retained = manager.take_kv_cache_block_copies()
|
||||
assert KVCacheBlockCopy(partial_block_id, mamba_cow_block_id) in copies
|
||||
manager.block_pool.free_blocks(retained)
|
||||
|
||||
replay.num_computed_tokens = 8
|
||||
manager.new_step_starts()
|
||||
final_blocks = manager.allocate_slots(replay, 6)
|
||||
assert final_blocks is not None
|
||||
|
||||
checkpoint_block_id, running_block_id = final_blocks.get_block_ids()[1]
|
||||
assert len({mamba_cow_block_id, checkpoint_block_id, running_block_id}) == 3
|
||||
mamba_blocks = manager.get_blocks(replay.request_id).blocks[1]
|
||||
assert mamba_blocks[2].block_id == checkpoint_block_id
|
||||
assert mamba_blocks[3].block_id == running_block_id
|
||||
|
||||
|
||||
def test_internal_checkpoint_requires_block_aligned_start():
|
||||
hash_block_size = 2
|
||||
mamba_block_size = 16
|
||||
manager = make_full_mamba_manager(
|
||||
dcp_world_size=1,
|
||||
hash_block_size=hash_block_size,
|
||||
full_block_size=hash_block_size,
|
||||
mamba_block_size=mamba_block_size,
|
||||
num_prefill_checkpoint_blocks=1,
|
||||
)
|
||||
request = make_request("producer", list(range(50)), hash_block_size, sha256)
|
||||
|
||||
# Compute token 0 first, so the next query starts at token 1, which is not
|
||||
# aligned to the Mamba block size.
|
||||
assert manager.allocate_slots(request, 1) is not None
|
||||
request.num_computed_tokens = 1
|
||||
manager.new_step_starts()
|
||||
|
||||
new_blocks = manager.allocate_slots(request, 49)
|
||||
|
||||
assert new_blocks is not None
|
||||
mamba_blocks = manager.get_blocks(request.request_id).blocks[1]
|
||||
checkpoint_block_idx = 48 // mamba_block_size - 1
|
||||
assert mamba_blocks[checkpoint_block_idx].is_null
|
||||
assert not mamba_blocks[-1].is_null
|
||||
checkpoint_hash = request.block_hashes[48 // hash_block_size - 1]
|
||||
assert manager.block_pool.get_cached_block(checkpoint_hash, [1]) is None
|
||||
|
||||
|
||||
def test_external_mamba_hit_same_block_uses_running_cow_on_continue():
|
||||
"""An external mid-block hit must become a running request even when its
|
||||
first continuation does not need another Mamba block."""
|
||||
@@ -560,6 +644,7 @@ def test_take_partial_tail_offloads_returns_cow_target():
|
||||
shapes=(1, 1),
|
||||
dtypes=(torch.float32,),
|
||||
mamba_cache_mode="align",
|
||||
num_prefill_checkpoint_blocks=1,
|
||||
),
|
||||
),
|
||||
],
|
||||
|
||||
Reference in New Issue
Block a user