[Bugfix][SimpleCPUOffloadBackend] Dedup in-flight CPU offload stores across scheduler steps (#41289)
Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
a8c13d2837
commit
9ce74042d3
@@ -79,6 +79,7 @@ steps:
|
||||
- tests/v1/core
|
||||
- tests/v1/executor
|
||||
- tests/v1/kv_offload
|
||||
- tests/v1/simple_kv_offload
|
||||
- tests/v1/worker
|
||||
- tests/v1/kv_connector/unit
|
||||
- tests/v1/metrics
|
||||
@@ -90,6 +91,7 @@ steps:
|
||||
- pytest -v -s -m 'not cpu_test' v1/core
|
||||
- pytest -v -s v1/executor
|
||||
- pytest -v -s v1/kv_offload
|
||||
- pytest -v -s v1/simple_kv_offload
|
||||
- pytest -v -s v1/worker
|
||||
- pytest -v -s -m 'not cpu_test' v1/kv_connector/unit
|
||||
- pytest -v -s -m 'not cpu_test' v1/metrics
|
||||
|
||||
@@ -179,6 +179,7 @@ _req_counter = 0
|
||||
def make_request(
|
||||
num_blocks: int = 2,
|
||||
request_id: str | None = None,
|
||||
extra_tokens: int = 1,
|
||||
) -> Request:
|
||||
"""Create a Request with deterministic block hashes."""
|
||||
global _req_counter
|
||||
@@ -186,13 +187,7 @@ def make_request(
|
||||
if request_id is None:
|
||||
request_id = f"req-{_req_counter}"
|
||||
|
||||
# Add one extra token beyond the last full block so that
|
||||
# ``max_cache_hit_length = num_tokens - 1`` (see
|
||||
# KVCacheManager.get_computed_blocks) does not truncate the final
|
||||
# full block: ``find_longest_cache_hit`` uses
|
||||
# ``max_length // block_size`` and would otherwise drop one block
|
||||
# when the prompt is an exact multiple of block_size.
|
||||
num_tokens = num_blocks * BLOCK_SIZE + 1
|
||||
num_tokens = num_blocks * BLOCK_SIZE + extra_tokens
|
||||
start = _req_counter * 10000
|
||||
prompt_token_ids = list(range(start, start + num_tokens))
|
||||
sampling_params = SamplingParams(max_tokens=1)
|
||||
@@ -386,6 +381,8 @@ def test_eager_store_and_load_roundtrip() -> None:
|
||||
block_hasher=req._block_hasher,
|
||||
)
|
||||
hit_tokens, is_async = sched.get_num_new_matched_tokens(req2, num_computed_tokens=0)
|
||||
# make_request pads num_tokens by +1 beyond the last full block, so the
|
||||
# manager's max_hit_len = num_tokens - 1 cap leaves all full blocks intact.
|
||||
assert hit_tokens == num_blocks * BLOCK_SIZE
|
||||
assert is_async is True
|
||||
|
||||
@@ -405,7 +402,44 @@ def test_eager_store_and_load_roundtrip() -> None:
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Test 1b: Lazy store-and-load roundtrip
|
||||
# Test 1b: Boundary — max_hit_len cap drops the last full block when the
|
||||
# prompt is an exact multiple of BLOCK_SIZE.
|
||||
# ---------------------------------------------------------------------------
|
||||
def test_max_hit_len_cap_drops_last_full_block() -> None:
|
||||
"""When num_tokens is an exact multiple of BLOCK_SIZE, the manager's
|
||||
``max_hit_len = num_tokens - 1`` cap forces ``find_longest_cache_hit`` to
|
||||
drop the final block (since ``max_length // block_size`` rounds down).
|
||||
"""
|
||||
fix = make_scheduler(num_cpu_blocks=8, num_gpu_blocks=16, lazy=False)
|
||||
sched = fix.scheduler
|
||||
|
||||
num_blocks = 2
|
||||
req = make_request(num_blocks=num_blocks, extra_tokens=0)
|
||||
assert req.num_tokens == num_blocks * BLOCK_SIZE
|
||||
|
||||
kv_blocks = _alloc_and_register(fix, req, num_blocks)
|
||||
sched.update_state_after_alloc(req, kv_blocks, num_external_tokens=0)
|
||||
sched_out = make_scheduler_output(
|
||||
{req.request_id: num_blocks * BLOCK_SIZE},
|
||||
new_reqs={req.request_id: kv_blocks.get_block_ids()},
|
||||
)
|
||||
meta = sched.build_connector_meta(sched_out)
|
||||
simulate_store_completion(sched, meta.store_event)
|
||||
|
||||
req2 = Request(
|
||||
request_id="req-cap-boundary",
|
||||
prompt_token_ids=req.prompt_token_ids,
|
||||
sampling_params=req.sampling_params,
|
||||
pooling_params=None,
|
||||
mm_features=None,
|
||||
block_hasher=req._block_hasher,
|
||||
)
|
||||
hit_tokens, _ = sched.get_num_new_matched_tokens(req2, num_computed_tokens=0)
|
||||
assert hit_tokens == (num_blocks - 1) * BLOCK_SIZE
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Test 1c: Lazy store-and-load roundtrip
|
||||
# ---------------------------------------------------------------------------
|
||||
def _flush_old_blocks_to_lru_head(
|
||||
gpu_pool: BlockPool,
|
||||
@@ -469,8 +503,11 @@ def test_lazy_store_and_load_roundtrip() -> None:
|
||||
hit_tokens, is_async = sched.get_num_new_matched_tokens(
|
||||
req_old2, num_computed_tokens=0
|
||||
)
|
||||
assert hit_tokens == num_blocks * BLOCK_SIZE, (
|
||||
f"Expected {num_blocks * BLOCK_SIZE} hit tokens, got {hit_tokens}"
|
||||
# make_request pads num_tokens by +1 beyond the last full block, so the
|
||||
# manager's max_hit_len = num_tokens - 1 cap leaves all full blocks intact.
|
||||
expected_hit = num_blocks * BLOCK_SIZE
|
||||
assert hit_tokens == expected_hit, (
|
||||
f"Expected {expected_hit} hit tokens, got {hit_tokens}"
|
||||
)
|
||||
assert is_async is True
|
||||
|
||||
@@ -537,7 +574,72 @@ def test_eager_duplicate_store_skipped() -> None:
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Test 2b: Lazy duplicate store is skipped
|
||||
# Test 2b: Eager dedup of in-flight stores across consecutive steps
|
||||
# ---------------------------------------------------------------------------
|
||||
def test_eager_in_flight_store_dedup_across_steps() -> None:
|
||||
"""Eager: a second request sharing a prefix with an in-flight store
|
||||
must not re-offload the same GPU blocks before completion lands.
|
||||
|
||||
Simulates a GPU prefix-cache hit by reusing the first request's
|
||||
GPU block IDs in the second scheduler step, which is the path the
|
||||
real scheduler takes when two requests share a prefix.
|
||||
"""
|
||||
fix = make_scheduler(num_cpu_blocks=8, num_gpu_blocks=16, lazy=False)
|
||||
sched = fix.scheduler
|
||||
|
||||
num_blocks = 2
|
||||
req = make_request(num_blocks=num_blocks)
|
||||
|
||||
kv_blocks = _alloc_and_register(fix, req, num_blocks)
|
||||
sched.update_state_after_alloc(req, kv_blocks, num_external_tokens=0)
|
||||
block_ids = kv_blocks.get_block_ids()
|
||||
sched_out = make_scheduler_output(
|
||||
{req.request_id: num_blocks * BLOCK_SIZE},
|
||||
new_reqs={req.request_id: block_ids},
|
||||
)
|
||||
|
||||
meta1 = sched.build_connector_meta(sched_out)
|
||||
assert meta1.store_event >= 0
|
||||
assert len(meta1.store_cpu_blocks) == num_blocks
|
||||
# In-flight set tracks the scheduled GPU blocks until completion.
|
||||
assert sched._in_flight_store_gpu_blocks == set(meta1.store_gpu_blocks)
|
||||
cpu_free_after_first = get_cpu_free_blocks(sched)
|
||||
|
||||
# Second request shares the prefix and reuses the same GPU block IDs
|
||||
# (the real scheduler path: GPU prefix cache returns the same blocks).
|
||||
# Do NOT simulate completion — the first store is still in-flight.
|
||||
req2 = Request(
|
||||
request_id="req-dup-eager-inflight",
|
||||
prompt_token_ids=req.prompt_token_ids,
|
||||
sampling_params=req.sampling_params,
|
||||
pooling_params=None,
|
||||
mm_features=None,
|
||||
block_hasher=req._block_hasher,
|
||||
)
|
||||
req2.num_computed_tokens = num_blocks * BLOCK_SIZE
|
||||
sched.update_state_after_alloc(req2, kv_blocks, num_external_tokens=0)
|
||||
sched_out2 = make_scheduler_output(
|
||||
{req2.request_id: num_blocks * BLOCK_SIZE},
|
||||
new_reqs={req2.request_id: block_ids},
|
||||
)
|
||||
|
||||
meta2 = sched.build_connector_meta(sched_out2)
|
||||
if meta2.store_event >= 0:
|
||||
assert len(meta2.store_cpu_blocks) == 0, (
|
||||
"Expected no new CPU blocks for in-flight duplicate hashes"
|
||||
)
|
||||
assert get_cpu_free_blocks(sched) == cpu_free_after_first, (
|
||||
"Second request should not consume CPU blocks while the first "
|
||||
"store is still in-flight"
|
||||
)
|
||||
|
||||
# After completion, the in-flight set is cleared.
|
||||
simulate_store_completion(sched, meta1.store_event)
|
||||
assert sched._in_flight_store_gpu_blocks == set()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Test 2c: Lazy duplicate store is skipped
|
||||
# ---------------------------------------------------------------------------
|
||||
def test_lazy_duplicate_store_skipped() -> None:
|
||||
"""Lazy: blocks already offloaded to CPU should not be offloaded again.
|
||||
@@ -1092,11 +1194,9 @@ def test_partial_gpu_prefix_plus_cpu_load() -> None:
|
||||
hit_tokens, is_async = sched.get_num_new_matched_tokens(
|
||||
req2, num_computed_tokens=gpu_local_computed
|
||||
)
|
||||
# CPU should hit blocks 2,3 (not 4,5 — those are beyond the CPU range).
|
||||
num_cpu_hit_blocks = 2
|
||||
# Actually CPU has all 6 stored; it returns hits starting from position 2.
|
||||
# The number of CPU hit blocks = min(remaining request blocks, CPU cached).
|
||||
# Here remaining = 6 - 2 = 4 blocks are in CPU, so hit = 4 * BLOCK_SIZE.
|
||||
# CPU has all 6 blocks stored. make_request pads num_tokens by +1, so
|
||||
# the manager's num_tokens - 1 cap leaves all full blocks intact:
|
||||
# remaining hashable range = 6 - 2 = 4 blocks, all hit.
|
||||
num_cpu_hit_blocks = 4
|
||||
assert hit_tokens == num_cpu_hit_blocks * BLOCK_SIZE, (
|
||||
f"Expected {num_cpu_hit_blocks * BLOCK_SIZE} CPU hit tokens, got {hit_tokens}"
|
||||
|
||||
@@ -146,6 +146,7 @@ class SimpleCPUOffloadScheduler:
|
||||
# Eager mode only
|
||||
self._reqs_to_store: dict[str, StoreRequestState] = {}
|
||||
self._store_event_to_reqs: dict[int, list[str]] = {}
|
||||
self._in_flight_store_gpu_blocks: set[int] = set()
|
||||
|
||||
# Event counters
|
||||
self._load_event_counter: int = 0
|
||||
@@ -468,7 +469,8 @@ class SimpleCPUOffloadScheduler:
|
||||
num_free = cpu_block_pool.get_num_free_blocks()
|
||||
kv_cache_groups = self.cpu_kv_cache_config.kv_cache_groups
|
||||
num_groups = len(kv_cache_groups)
|
||||
gpu_blocks_this_step: set[int] = set()
|
||||
# Dedup against blocks already scheduled.
|
||||
in_flight = self._in_flight_store_gpu_blocks
|
||||
|
||||
for req_id, new_block_id_groups, preempted in yield_req_data(scheduler_output):
|
||||
state = self._reqs_to_store.get(req_id)
|
||||
@@ -523,10 +525,9 @@ class SimpleCPUOffloadScheduler:
|
||||
if bhash_with_group is None:
|
||||
break
|
||||
|
||||
# Check if this group's data is already scheduled for store
|
||||
# in this step or already cached in CPU.
|
||||
# Skip if already scheduled for store or already cached in CPU.
|
||||
if (
|
||||
gpu_block_id in gpu_blocks_this_step
|
||||
gpu_block_id in in_flight
|
||||
or cpu_block_pool.cached_block_hash_to_block.get_one_block(
|
||||
bhash_with_group
|
||||
)
|
||||
@@ -561,7 +562,7 @@ class SimpleCPUOffloadScheduler:
|
||||
req_ids.append(req_id)
|
||||
merged_gpu_block_ids.extend(gpu_block_ids)
|
||||
merged_cpu_block_ids.extend(cpu_block_ids)
|
||||
gpu_blocks_this_step.update(gpu_block_ids)
|
||||
in_flight.update(gpu_block_ids)
|
||||
|
||||
# Touch GPU blocks to prevent freeing during async copy
|
||||
gpu_block_pool.touch(
|
||||
@@ -608,6 +609,8 @@ class SimpleCPUOffloadScheduler:
|
||||
def _process_store_event(self, event_idx: int) -> None:
|
||||
"""Process a fully-completed store event."""
|
||||
transfer = self._store_event_to_blocks.pop(event_idx)
|
||||
if not self._lazy_mode:
|
||||
self._in_flight_store_gpu_blocks.difference_update(transfer.gpu_block_ids)
|
||||
self._process_store_completion(transfer.gpu_block_ids, transfer.cpu_block_ids)
|
||||
logger.debug(
|
||||
"Store event %d completed: cached %d blocks to CPU",
|
||||
|
||||
Reference in New Issue
Block a user