|
|
|
@@ -1293,12 +1293,21 @@ def test_two_groups_different_block_sizes(request_runner, async_scheduling: bool
|
|
|
|
|
|
|
|
|
|
def _make_scheduler_with_lookup(
|
|
|
|
|
lookup_results: dict[int, LookupResult],
|
|
|
|
|
default: LookupResult = LookupResult.MISS,
|
|
|
|
|
) -> OffloadingConnectorScheduler:
|
|
|
|
|
"""Create an OffloadingConnectorScheduler with a mocked manager.lookup."""
|
|
|
|
|
"""Create an OffloadingConnectorScheduler with a mocked manager.lookup.
|
|
|
|
|
|
|
|
|
|
Keys are addressed by their integer hash; `default` answers anything else.
|
|
|
|
|
"""
|
|
|
|
|
manager = MagicMock(spec=OffloadingManager)
|
|
|
|
|
manager.lookup.side_effect = lambda key, req_context: lookup_results.get(
|
|
|
|
|
int(get_offload_block_hash(key).decode()), LookupResult.MISS
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
def lookup(key, req_context):
|
|
|
|
|
block_hash = get_offload_block_hash(key)
|
|
|
|
|
if not block_hash.isdigit():
|
|
|
|
|
return default
|
|
|
|
|
return lookup_results.get(int(block_hash.decode()), default)
|
|
|
|
|
|
|
|
|
|
manager.lookup.side_effect = lookup
|
|
|
|
|
|
|
|
|
|
scheduler = object.__new__(OffloadingConnectorScheduler)
|
|
|
|
|
scheduler.manager = manager
|
|
|
|
@@ -1322,6 +1331,32 @@ def _maximal_lookup(sched, keys, start_chunk_idx: int = 0):
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
# Lookups issued, and end index returned, by a window-1 scan over 3 keys all
|
|
|
|
|
# resolving to the same result. A result that keeps the streak alive ends the
|
|
|
|
|
# scan at the first key; one that resets it makes the scan walk every key,
|
|
|
|
|
# which is what widens the demanded set. A new member has to be added here.
|
|
|
|
|
_SCAN_BEHAVIOR = {
|
|
|
|
|
LookupResult.HIT: (1, 3),
|
|
|
|
|
LookupResult.HIT_PENDING: (1, None),
|
|
|
|
|
LookupResult.RETRY: (3, None),
|
|
|
|
|
LookupResult.MISS: (3, 0),
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
@pytest.mark.parametrize("result", list(LookupResult))
|
|
|
|
|
def test_scan_behavior_declared_for_every_lookup_result(result: LookupResult):
|
|
|
|
|
"""Whether a result keeps a sliding-window streak alive decides how wide
|
|
|
|
|
the demanded chunk set gets, so every member needs deliberate behavior."""
|
|
|
|
|
assert result in _SCAN_BEHAVIOR, f"{result} has no declared scan behavior"
|
|
|
|
|
expected_lookups, expected_end = _SCAN_BEHAVIOR[result]
|
|
|
|
|
|
|
|
|
|
keys = to_keys([1, 2, 3])
|
|
|
|
|
sched = _make_scheduler_with_lookup(dict.fromkeys([1, 2, 3], result))
|
|
|
|
|
|
|
|
|
|
assert sched._sliding_window_lookup(keys, 1, _EMPTY_REQ_CTX) == expected_end
|
|
|
|
|
assert len(sched.manager.lookup.call_args_list) == expected_lookups
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class TestMaximalPrefixLookup:
|
|
|
|
|
def test_all_hit(self):
|
|
|
|
|
sched = _make_scheduler_with_lookup({1: LookupResult.HIT, 2: LookupResult.HIT})
|
|
|
|
@@ -1556,6 +1591,62 @@ class TestSlidingWindowLookup:
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
# Tests for SWA store pruning vs. load demand
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
@pytest.mark.parametrize("alignment_chunk_count", [4, 8, 64])
|
|
|
|
|
@pytest.mark.parametrize("sliding_window_chunks", [1, 2, 3])
|
|
|
|
|
@pytest.mark.parametrize("is_eagle_group", [False, True])
|
|
|
|
|
@pytest.mark.parametrize("tail_chunks", [0, 1, 5])
|
|
|
|
|
def test_sliding_window_demand_is_store_reachable(
|
|
|
|
|
alignment_chunk_count: int,
|
|
|
|
|
sliding_window_chunks: int,
|
|
|
|
|
is_eagle_group: bool,
|
|
|
|
|
tail_chunks: int,
|
|
|
|
|
):
|
|
|
|
|
"""Every chunk the load path looks up must be one the store path keeps.
|
|
|
|
|
|
|
|
|
|
`is_store_reachable_swa_chunk` prunes chunks `_sliding_window_lookup` can
|
|
|
|
|
never query, but the two mirror each other by hand: this pins the relation,
|
|
|
|
|
where `test_is_store_reachable_swa_chunk` pins only the predicate's values.
|
|
|
|
|
A looked-up key is a demanded key, since a secondary-tier hit queues a
|
|
|
|
|
promotion immediately.
|
|
|
|
|
"""
|
|
|
|
|
storable_chunk_count = alignment_chunk_count + tail_chunks
|
|
|
|
|
# Both sides widen by one for an unverified EAGLE group.
|
|
|
|
|
required_window = sliding_window_chunks + int(is_eagle_group)
|
|
|
|
|
|
|
|
|
|
sched = _make_scheduler_with_lookup(
|
|
|
|
|
{i: LookupResult.HIT_PENDING for i in range(storable_chunk_count)}
|
|
|
|
|
)
|
|
|
|
|
sched._sliding_window_lookup(
|
|
|
|
|
to_keys(range(storable_chunk_count)), required_window, _EMPTY_REQ_CTX
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
demanded = {
|
|
|
|
|
int(get_offload_block_hash(lookup_call.args[0]).decode())
|
|
|
|
|
for lookup_call in sched.manager.lookup.call_args_list
|
|
|
|
|
}
|
|
|
|
|
assert demanded, "the scan must query something"
|
|
|
|
|
unsuppliable = sorted(
|
|
|
|
|
chunk_idx
|
|
|
|
|
for chunk_idx in demanded
|
|
|
|
|
if not is_store_reachable_swa_chunk(
|
|
|
|
|
chunk_idx,
|
|
|
|
|
storable_chunk_count,
|
|
|
|
|
alignment_chunk_count,
|
|
|
|
|
sliding_window_chunks,
|
|
|
|
|
is_eagle_group,
|
|
|
|
|
)
|
|
|
|
|
)
|
|
|
|
|
assert not unsuppliable, (
|
|
|
|
|
f"chunks {unsuppliable} are demanded by the load path but pruned by the "
|
|
|
|
|
f"store path, so a warm producer can never supply them"
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
@pytest.mark.parametrize("async_scheduling", [True, False])
|
|
|
|
|
def test_request_level_policy_stores_all_blocks(request_runner, async_scheduling: bool):
|
|
|
|
|
"""With REQUEST_LEVEL policy, all blocks are stored — including prefix hits."""
|
|
|
|
@@ -2308,6 +2399,146 @@ def test_swa_alignment_skip(request_runner, async_scheduling: bool):
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def _full_and_swa_groups(
|
|
|
|
|
full_attn_block_size: int, swa_block_size: int, sliding_window: int
|
|
|
|
|
) -> list[KVCacheGroupSpec]:
|
|
|
|
|
"""Group 0 full attention (MLA-like), group 1 sliding window."""
|
|
|
|
|
return [
|
|
|
|
|
KVCacheGroupSpec(
|
|
|
|
|
["layer0"],
|
|
|
|
|
FullAttentionSpec(
|
|
|
|
|
block_size=full_attn_block_size,
|
|
|
|
|
num_kv_heads=1,
|
|
|
|
|
head_size=1,
|
|
|
|
|
dtype=torch.float32,
|
|
|
|
|
),
|
|
|
|
|
),
|
|
|
|
|
KVCacheGroupSpec(
|
|
|
|
|
["layer1"],
|
|
|
|
|
SlidingWindowSpec(
|
|
|
|
|
block_size=swa_block_size,
|
|
|
|
|
num_kv_heads=1,
|
|
|
|
|
head_size=1,
|
|
|
|
|
dtype=torch.float32,
|
|
|
|
|
sliding_window=sliding_window,
|
|
|
|
|
),
|
|
|
|
|
),
|
|
|
|
|
]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def _demanded_keys(runner, offload_keys_per_group: list[list], num_tokens: int) -> set:
|
|
|
|
|
"""Keys a cold consumer's lookup scan would demand for `num_tokens`.
|
|
|
|
|
|
|
|
|
|
Drives the real scan functions with every key resolving as a promoting
|
|
|
|
|
secondary tier resolves it.
|
|
|
|
|
"""
|
|
|
|
|
scan = _make_scheduler_with_lookup({}, default=LookupResult.HIT_PENDING)
|
|
|
|
|
for group_config, offload_keys in zip(
|
|
|
|
|
runner.connector_scheduler.config.kv_group_configs, offload_keys_per_group
|
|
|
|
|
):
|
|
|
|
|
num_chunks = num_tokens // group_config.tokens_per_chunk
|
|
|
|
|
keys = offload_keys[:num_chunks]
|
|
|
|
|
window = group_config.sliding_window_size_in_chunks
|
|
|
|
|
if window is None:
|
|
|
|
|
_maximal_lookup(scan, keys)
|
|
|
|
|
else:
|
|
|
|
|
scan._sliding_window_lookup(keys, window, _EMPTY_REQ_CTX)
|
|
|
|
|
return {lookup_call.args[0] for lookup_call in scan.manager.lookup.call_args_list}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
@pytest.mark.parametrize("warmth", ["cold", "gpu_warm", "primary_warm"])
|
|
|
|
|
@pytest.mark.parametrize("with_swa_group", [False, True])
|
|
|
|
|
def test_request_level_supply_covers_consumer_demand(
|
|
|
|
|
request_runner, warmth: str, with_swa_group: bool
|
|
|
|
|
):
|
|
|
|
|
"""A REQUEST_LEVEL producer must offer every key a consumer will demand.
|
|
|
|
|
|
|
|
|
|
Same relation as `test_sliding_window_demand_is_store_reachable`, but at
|
|
|
|
|
the level where the key set is really built: `_build_store_jobs` prunes
|
|
|
|
|
chunks with a null GPU block and unreachable SWA chunks, while the demand
|
|
|
|
|
comes from an independent scan.
|
|
|
|
|
"""
|
|
|
|
|
full_attn_block_size = 16
|
|
|
|
|
swa_block_size = 4
|
|
|
|
|
num_tokens = 32
|
|
|
|
|
|
|
|
|
|
kv_cache_groups = (
|
|
|
|
|
_full_and_swa_groups(full_attn_block_size, swa_block_size, sliding_window=8)
|
|
|
|
|
if with_swa_group
|
|
|
|
|
else None
|
|
|
|
|
)
|
|
|
|
|
runner = request_runner(
|
|
|
|
|
block_size=swa_block_size if with_swa_group else full_attn_block_size,
|
|
|
|
|
num_gpu_blocks=200,
|
|
|
|
|
async_scheduling=False,
|
|
|
|
|
kv_cache_groups=kv_cache_groups,
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
def store_everything_offered():
|
|
|
|
|
runner.manager.prepare_store.side_effect = lambda keys, req_context: (
|
|
|
|
|
generate_store_output(keys)
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
# `_run` rather than `run`, which asserts on an expected GPU-block set this
|
|
|
|
|
# test deliberately does not hard-code.
|
|
|
|
|
if warmth != "cold":
|
|
|
|
|
runner.new_request(token_ids=[0] * num_tokens)
|
|
|
|
|
store_everything_offered()
|
|
|
|
|
runner._run(decoded_tokens=[0], complete_transfers=True)
|
|
|
|
|
runner._run(decoded_tokens=[EOS_TOKEN_ID], complete_transfers=True)
|
|
|
|
|
if warmth == "primary_warm":
|
|
|
|
|
# Warm only in the primary tier, so the producer must load first.
|
|
|
|
|
runner.scheduler.reset_prefix_cache()
|
|
|
|
|
|
|
|
|
|
# The producer leg: same prompt, REQUEST_LEVEL, so prefix hits stay in the
|
|
|
|
|
# store path instead of being skipped.
|
|
|
|
|
runner.manager.reset_mock()
|
|
|
|
|
store_everything_offered()
|
|
|
|
|
runner.manager.on_new_request.return_value = RequestOffloadingContext(
|
|
|
|
|
policy=OffloadPolicy.REQUEST_LEVEL
|
|
|
|
|
)
|
|
|
|
|
if warmth == "primary_warm":
|
|
|
|
|
# HIT is final; HIT_PENDING would defer the request on every step.
|
|
|
|
|
runner.manager.lookup.return_value = LookupResult.HIT
|
|
|
|
|
runner.new_request(token_ids=[0] * num_tokens)
|
|
|
|
|
|
|
|
|
|
offload_keys_per_group: list[list] = []
|
|
|
|
|
|
|
|
|
|
def capture_offload_keys() -> None:
|
|
|
|
|
for req_status in runner.connector_scheduler._req_status.values():
|
|
|
|
|
offload_keys_per_group.clear()
|
|
|
|
|
offload_keys_per_group.extend(
|
|
|
|
|
list(group_state.offload_keys)
|
|
|
|
|
for group_state in req_status.group_states
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
runner._run(
|
|
|
|
|
decoded_tokens=[0],
|
|
|
|
|
complete_transfers=True,
|
|
|
|
|
post_step_fn=capture_offload_keys,
|
|
|
|
|
)
|
|
|
|
|
runner._run(
|
|
|
|
|
decoded_tokens=[EOS_TOKEN_ID],
|
|
|
|
|
complete_transfers=True,
|
|
|
|
|
post_step_fn=capture_offload_keys,
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
assert offload_keys_per_group, "no request state was captured"
|
|
|
|
|
supplied = {
|
|
|
|
|
key
|
|
|
|
|
for store_call in runner.manager.prepare_store.call_args_list
|
|
|
|
|
for key in store_call.args[0]
|
|
|
|
|
}
|
|
|
|
|
demanded = _demanded_keys(runner, offload_keys_per_group, num_tokens)
|
|
|
|
|
|
|
|
|
|
assert demanded, "the scan must demand something"
|
|
|
|
|
missing = demanded - supplied
|
|
|
|
|
assert not missing, (
|
|
|
|
|
f"{len(missing)} of {len(demanded)} demanded keys are never offered to "
|
|
|
|
|
f"prepare_store, so a peer fetching them would wait out the load timeout"
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
@pytest.mark.parametrize("async_scheduling", [True, False])
|
|
|
|
|
def test_stale_sliding_window_block_after_prepare_store_failure(
|
|
|
|
|
request_runner, async_scheduling: bool
|
|
|
|
|