[kv_offload]: Add DSv4 support (#43142)
Signed-off-by: Or Ozeri <oro@il.ibm.com>
This commit is contained in:
@@ -1041,3 +1041,124 @@ def test_reset_cache(request_runner, async_scheduling: bool):
|
||||
for req_status in runner.connector_scheduler._req_status.values():
|
||||
for group_state in req_status.group_states:
|
||||
assert group_state.next_stored_block_idx == 0
|
||||
|
||||
|
||||
@pytest.mark.parametrize("async_scheduling", [True, False])
|
||||
def test_swa_alignment_skip(request_runner, async_scheduling: bool):
|
||||
"""SWA blocks unreachable by the load path are skipped during store.
|
||||
|
||||
Simulates a DeepSeek V4-like hybrid architecture where SWA groups have
|
||||
much smaller block sizes than the full-attention (MLA) group, causing
|
||||
most SWA blocks to be unreachable by the alignment-based load path.
|
||||
|
||||
Setup:
|
||||
- Group 0: full attention (MLA-like), block_size=16
|
||||
- Group 1: SWA, block_size=4, sliding_window=8
|
||||
|
||||
alignment_block_count = 16 / 4 = 4 SWA blocks per alignment segment.
|
||||
sliding_window_size_in_blocks = ceil(8 / 4) = 2.
|
||||
Within each segment of 4 SWA blocks, only the trailing 2 are stored.
|
||||
|
||||
With 32 tokens (2 full-attn blocks, 8 SWA blocks):
|
||||
- Group 0 stores: blocks 0, 1 (all full-attn blocks)
|
||||
- Group 1 stores: blocks 2, 3, 6, 7 (skip 0,1,4,5)
|
||||
|
||||
For real DeepSeek V4 (100K tokens), this reduces SWA stores by ~78%.
|
||||
"""
|
||||
full_attn_block_size = 16
|
||||
swa_block_size = 4
|
||||
sliding_window = 8
|
||||
num_gpu_blocks = 200
|
||||
|
||||
kv_cache_groups = [
|
||||
KVCacheGroupSpec(
|
||||
["layer0"],
|
||||
FullAttentionSpec(
|
||||
block_size=full_attn_block_size,
|
||||
num_kv_heads=1,
|
||||
head_size=1,
|
||||
dtype=torch.float32,
|
||||
),
|
||||
),
|
||||
KVCacheGroupSpec(
|
||||
["layer1"],
|
||||
SlidingWindowSpec(
|
||||
block_size=swa_block_size,
|
||||
num_kv_heads=1,
|
||||
head_size=1,
|
||||
dtype=torch.float32,
|
||||
sliding_window=sliding_window,
|
||||
),
|
||||
),
|
||||
]
|
||||
|
||||
runner = request_runner(
|
||||
block_size=swa_block_size,
|
||||
num_gpu_blocks=num_gpu_blocks,
|
||||
async_scheduling=async_scheduling,
|
||||
kv_cache_groups=kv_cache_groups,
|
||||
)
|
||||
|
||||
# Verify config: alignment_block_count computed correctly
|
||||
kv_group_configs = runner.connector_scheduler.config.kv_group_configs
|
||||
assert len(kv_group_configs) == 2
|
||||
# Group 0: full attention -> no alignment skip
|
||||
assert kv_group_configs[0].alignment_block_count is None
|
||||
assert kv_group_configs[0].sliding_window_size_in_blocks is None
|
||||
assert kv_group_configs[0].offloaded_block_size == full_attn_block_size
|
||||
# Group 1: SWA -> alignment_block_count = 16/4 = 4, tail = 2
|
||||
assert kv_group_configs[1].alignment_block_count == 4
|
||||
assert kv_group_configs[1].sliding_window_size_in_blocks == 2
|
||||
assert kv_group_configs[1].offloaded_block_size == swa_block_size
|
||||
|
||||
# Send 32 tokens = 2 full-attn blocks (block_size=16) = 8 SWA blocks
|
||||
# (block_size=4). Decode 1 token to kick off processing (stores are
|
||||
# deferred to next step).
|
||||
num_tokens = 32
|
||||
runner.new_request(token_ids=[0] * num_tokens)
|
||||
runner.manager.prepare_store.side_effect = lambda keys, req_context: (
|
||||
generate_store_output(keys)
|
||||
)
|
||||
runner.run(decoded_tokens=[0])
|
||||
|
||||
# Decode 1 more token to complete the deferred stores from above.
|
||||
runner.manager.prepare_store.side_effect = lambda keys, req_context: (
|
||||
generate_store_output(keys)
|
||||
)
|
||||
runner.run(
|
||||
decoded_tokens=[EOS_TOKEN_ID],
|
||||
# Group 0 (full attn, block_size=16): 2 offloaded blocks
|
||||
# -> GPU blocks (0, 0) and (0, 1)
|
||||
# Group 1 (SWA, block_size=4): 8 offloaded blocks, skip first 2
|
||||
# per segment of 4:
|
||||
# Segment 0 (blocks 0-3): skip 0,1 -> store (1, 2), (1, 3)
|
||||
# Segment 1 (blocks 4-7): skip 4,5 -> store (1, 6), (1, 7)
|
||||
expected_stored=(
|
||||
(0, 0),
|
||||
(0, 1),
|
||||
(1, 2),
|
||||
(1, 3),
|
||||
(1, 6),
|
||||
(1, 7),
|
||||
),
|
||||
)
|
||||
|
||||
# Verify that loads still work correctly for the stored SWA blocks.
|
||||
runner.scheduler.reset_prefix_cache()
|
||||
runner.new_request(token_ids=[0] * num_tokens + [1])
|
||||
runner.manager.lookup.return_value = True
|
||||
runner.connector_scheduler._maximal_prefix_lookup = lambda key, req_context: 2
|
||||
runner.run(
|
||||
decoded_tokens=[EOS_TOKEN_ID],
|
||||
# Group 0: full prefix lookup hits 2 offloaded blocks
|
||||
# -> loads GPU blocks (0, 0), (0, 1)
|
||||
# Group 1: sliding window lookup finds trailing 2 from last segment
|
||||
# (blocks 6, 7 which were stored)
|
||||
# -> loads GPU blocks (1, 6), (1, 7)
|
||||
expected_loaded=(
|
||||
(0, 0),
|
||||
(0, 1),
|
||||
(1, 6),
|
||||
(1, 7),
|
||||
),
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user