[Bugfix][KV Offloading] Fix eager SimpleCPUOffload cache registration and final flush (#53532)
Signed-off-by: maithilijoshi20 <maithilijoshi2003@gmail.com> Signed-off-by: maithilijoshi20 <97733343+maithilijoshi20@users.noreply.github.com> Co-authored-by: maithilijoshi20 <maithilijoshi2003@gmail.com> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai>
This commit is contained in:
co-authored by
maithilijoshi20
OpenAI Codex
Yifan Qiao
parent
dbf1a044ea
commit
12b9573c98
@@ -18,6 +18,9 @@ from vllm.config import (
|
||||
SchedulerConfig,
|
||||
VllmConfig,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.simple_cpu_offload_connector import (
|
||||
SimpleCPUOffloadConnector,
|
||||
)
|
||||
from vllm.utils.hashing import sha256
|
||||
from vllm.v1.core.block_pool import BlockPool
|
||||
from vllm.v1.core.kv_cache_manager import KVCacheBlocks
|
||||
@@ -39,12 +42,16 @@ from vllm.v1.kv_cache_interface import (
|
||||
KVCacheConfig,
|
||||
KVCacheGroupSpec,
|
||||
KVCacheTensor,
|
||||
SlidingWindowSpec,
|
||||
)
|
||||
from vllm.v1.outputs import KVConnectorOutput
|
||||
from vllm.v1.request import Request
|
||||
from vllm.v1.simple_kv_offload.manager import SimpleCPUOffloadScheduler
|
||||
from vllm.v1.simple_kv_offload.metadata import SimpleCPUOffloadWorkerMetadata
|
||||
|
||||
pytestmark = pytest.mark.skip_global_cleanup
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Constants
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -77,15 +84,26 @@ def _make_kv_cache_config(
|
||||
) # Ensure specs are registered for tests
|
||||
for g in range(num_groups):
|
||||
layer_names = [f"layer_{g}"]
|
||||
spec = (
|
||||
FullAttentionSpec(
|
||||
block_size=BLOCK_SIZE,
|
||||
num_kv_heads=NUM_KV_HEADS,
|
||||
head_size=HEAD_SIZE,
|
||||
dtype=DTYPE,
|
||||
)
|
||||
if g == 0
|
||||
else SlidingWindowSpec(
|
||||
block_size=BLOCK_SIZE,
|
||||
num_kv_heads=NUM_KV_HEADS,
|
||||
head_size=HEAD_SIZE,
|
||||
dtype=DTYPE,
|
||||
sliding_window=BLOCK_SIZE * 4,
|
||||
)
|
||||
)
|
||||
groups.append(
|
||||
KVCacheGroupSpec(
|
||||
layer_names,
|
||||
FullAttentionSpec(
|
||||
block_size=BLOCK_SIZE,
|
||||
num_kv_heads=NUM_KV_HEADS,
|
||||
head_size=HEAD_SIZE,
|
||||
dtype=DTYPE,
|
||||
),
|
||||
spec,
|
||||
)
|
||||
)
|
||||
tensors.append(
|
||||
@@ -412,6 +430,230 @@ def test_eager_store_and_load_roundtrip() -> None:
|
||||
assert len(meta2.load_cpu_blocks) == len(meta2.load_gpu_blocks)
|
||||
|
||||
|
||||
def test_eager_store_preserves_secondary_block_hashes() -> None:
|
||||
"""CPU copies retain fine-grained hashes owned by the GPU block."""
|
||||
fix = make_scheduler(num_cpu_blocks=8, num_gpu_blocks=16, lazy=False)
|
||||
sched = fix.scheduler
|
||||
req = make_request(num_blocks=1)
|
||||
kv_blocks = _alloc_and_register(fix, req, num_blocks=1)
|
||||
gpu_block = kv_blocks.blocks[0][0]
|
||||
primary_hash = gpu_block.block_hash
|
||||
primary_num_tokens = gpu_block.block_hash_num_tokens
|
||||
assert primary_hash is not None
|
||||
|
||||
fine_grained_req = Request(
|
||||
request_id="req-fine-grained-hash",
|
||||
prompt_token_ids=req.prompt_token_ids,
|
||||
sampling_params=req.sampling_params,
|
||||
pooling_params=None,
|
||||
mm_features=None,
|
||||
block_hasher=get_request_block_hasher(BLOCK_SIZE // 2, sha256),
|
||||
)
|
||||
secondary_hash = make_block_hash_with_group_id(fine_grained_req.block_hashes[0], 0)
|
||||
fix.gpu_block_pool._insert_block_hash(
|
||||
secondary_hash, gpu_block, num_tokens=BLOCK_SIZE // 2
|
||||
)
|
||||
assert (
|
||||
secondary_hash
|
||||
in fix.gpu_block_pool.cached_block_hashes_by_block[gpu_block.block_id]
|
||||
)
|
||||
|
||||
sched.update_state_after_alloc(req, kv_blocks, num_external_tokens=0)
|
||||
block_ids = kv_blocks.get_block_ids()
|
||||
meta = sched.build_connector_meta(
|
||||
make_scheduler_output(
|
||||
{req.request_id: BLOCK_SIZE},
|
||||
new_reqs={req.request_id: block_ids},
|
||||
)
|
||||
)
|
||||
simulate_store_completion(sched, meta.store_event)
|
||||
|
||||
cpu_block = sched.cpu_block_pool.cached_block_hash_to_block.get_one_block(
|
||||
secondary_hash
|
||||
)
|
||||
assert cpu_block is not None
|
||||
assert cpu_block.block_hash == primary_hash
|
||||
assert cpu_block.block_hash_num_tokens == primary_num_tokens
|
||||
assert (
|
||||
secondary_hash
|
||||
in sched.cpu_block_pool.cached_block_hashes_by_block[cpu_block.block_id]
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Test 1b: Finished eager stores
|
||||
# ---------------------------------------------------------------------------
|
||||
def test_finished_eager_stores_are_batched() -> None:
|
||||
"""Finished requests in one step retain, complete, and cache every store."""
|
||||
fix = make_scheduler(num_cpu_blocks=8, num_gpu_blocks=16, lazy=False)
|
||||
sched = fix.scheduler
|
||||
|
||||
requests = [make_request(num_blocks=2) for _ in range(2)]
|
||||
block_ids = []
|
||||
for request in requests:
|
||||
kv_blocks = _alloc_and_register(fix, request, num_blocks=2)
|
||||
sched.update_state_after_alloc(request, kv_blocks, num_external_tokens=0)
|
||||
block_ids.append(kv_blocks.get_block_ids())
|
||||
|
||||
for request, request_block_ids in zip(requests, block_ids):
|
||||
sched.request_finished_all_groups(request, request_block_ids)
|
||||
|
||||
meta = sched.build_connector_meta(make_scheduler_output({}))
|
||||
assert len(meta.store_gpu_blocks) == 4
|
||||
assert len(meta.store_cpu_blocks) == 4
|
||||
simulate_store_completion(sched, meta.store_event)
|
||||
|
||||
for request in requests:
|
||||
for block_hash in request.block_hashes[:2]:
|
||||
cached = sched.cpu_block_pool.cached_block_hash_to_block.get_one_block(
|
||||
make_block_hash_with_group_id(block_hash, 0)
|
||||
)
|
||||
assert cached is not None
|
||||
|
||||
request = requests[0]
|
||||
cached_request = Request(
|
||||
request_id="req-finished-eager-load",
|
||||
prompt_token_ids=request.prompt_token_ids,
|
||||
sampling_params=request.sampling_params,
|
||||
pooling_params=None,
|
||||
mm_features=None,
|
||||
block_hasher=request._block_hasher,
|
||||
)
|
||||
hit_tokens, is_async = sched.get_num_new_matched_tokens(
|
||||
cached_request, num_computed_tokens=0
|
||||
)
|
||||
assert hit_tokens == 2 * BLOCK_SIZE
|
||||
assert is_async is True
|
||||
|
||||
cached_gpu_blocks = fix.gpu_block_pool.get_new_blocks(2)
|
||||
sched.update_state_after_alloc(
|
||||
cached_request,
|
||||
KVCacheBlocks(blocks=(cached_gpu_blocks,)),
|
||||
num_external_tokens=hit_tokens,
|
||||
)
|
||||
load_meta = sched.build_connector_meta(
|
||||
make_scheduler_output(
|
||||
{cached_request.request_id: 1},
|
||||
new_reqs={
|
||||
cached_request.request_id: (
|
||||
[block.block_id for block in cached_gpu_blocks],
|
||||
)
|
||||
},
|
||||
)
|
||||
)
|
||||
assert load_meta.load_event >= 0
|
||||
assert len(load_meta.load_gpu_blocks) == 2
|
||||
assert len(load_meta.load_cpu_blocks) == 2
|
||||
|
||||
|
||||
def test_finished_eager_store_skips_existing_cpu_cache_entries() -> None:
|
||||
"""A final flush must not consume CPU capacity for an existing hash."""
|
||||
fix = make_scheduler(num_cpu_blocks=8, num_gpu_blocks=16, lazy=False)
|
||||
sched = fix.scheduler
|
||||
|
||||
first = make_request(num_blocks=2)
|
||||
first_blocks = _alloc_and_register(fix, first, num_blocks=2)
|
||||
sched.update_state_after_alloc(first, first_blocks, num_external_tokens=0)
|
||||
sched.request_finished_all_groups(first, first_blocks.get_block_ids())
|
||||
first_meta = sched.build_connector_meta(make_scheduler_output({}))
|
||||
simulate_store_completion(sched, first_meta.store_event)
|
||||
free_blocks = get_cpu_free_blocks(sched)
|
||||
|
||||
duplicate = Request(
|
||||
request_id="req-finished-eager-duplicate",
|
||||
prompt_token_ids=first.prompt_token_ids,
|
||||
sampling_params=first.sampling_params,
|
||||
pooling_params=None,
|
||||
mm_features=None,
|
||||
block_hasher=first._block_hasher,
|
||||
)
|
||||
duplicate_blocks = _alloc_and_register(fix, duplicate, num_blocks=2)
|
||||
sched.update_state_after_alloc(duplicate, duplicate_blocks, num_external_tokens=0)
|
||||
sched.request_finished_all_groups(duplicate, duplicate_blocks.get_block_ids())
|
||||
|
||||
duplicate_meta = sched.build_connector_meta(make_scheduler_output({}))
|
||||
assert duplicate_meta.store_event == -1
|
||||
assert get_cpu_free_blocks(sched) == free_blocks
|
||||
|
||||
|
||||
def test_finished_eager_store_is_reported_pending_before_metadata_build() -> None:
|
||||
"""Queued finished stores must keep the connector pending until submitted."""
|
||||
fix = make_scheduler(num_cpu_blocks=8, num_gpu_blocks=16, lazy=False)
|
||||
sched = fix.scheduler
|
||||
request = make_request(num_blocks=2)
|
||||
kv_blocks = _alloc_and_register(fix, request, num_blocks=2)
|
||||
connector = SimpleCPUOffloadConnector.__new__(SimpleCPUOffloadConnector)
|
||||
connector.scheduler_manager = sched
|
||||
sched.update_state_after_alloc(request, kv_blocks, num_external_tokens=0)
|
||||
|
||||
sched.request_finished_all_groups(request, kv_blocks.get_block_ids())
|
||||
|
||||
assert sched.has_pending_stores()
|
||||
assert connector.has_pending_push_work()
|
||||
meta = sched.build_connector_meta(make_scheduler_output({}))
|
||||
assert meta.store_event >= 0
|
||||
assert not sched._pending_finished_stores
|
||||
assert connector.has_pending_push_work()
|
||||
simulate_store_completion(sched, meta.store_event)
|
||||
assert not sched.has_pending_stores()
|
||||
assert not connector.has_pending_push_work()
|
||||
|
||||
|
||||
def test_finished_eager_store_caches_all_groups() -> None:
|
||||
"""The final flush retains completed blocks from every KV cache group."""
|
||||
fix = make_scheduler(num_cpu_blocks=8, num_gpu_blocks=16, num_groups=2, lazy=False)
|
||||
sched = fix.scheduler
|
||||
request = make_request(num_blocks=2)
|
||||
group_blocks = tuple(
|
||||
_allocate_gpu_blocks(fix.gpu_block_pool, request, 2, group_id=group_id)
|
||||
for group_id in range(2)
|
||||
)
|
||||
request.num_computed_tokens = 2 * BLOCK_SIZE
|
||||
kv_blocks = KVCacheBlocks(blocks=group_blocks)
|
||||
sched.update_state_after_alloc(request, kv_blocks, num_external_tokens=0)
|
||||
|
||||
sched.request_finished_all_groups(request, kv_blocks.get_block_ids())
|
||||
meta = sched.build_connector_meta(make_scheduler_output({}))
|
||||
assert len(meta.store_gpu_blocks) == 4
|
||||
simulate_store_completion(sched, meta.store_event)
|
||||
|
||||
for group_id in range(2):
|
||||
for block_hash in request.block_hashes[:2]:
|
||||
cached = sched.cpu_block_pool.cached_block_hash_to_block.get_one_block(
|
||||
make_block_hash_with_group_id(block_hash, group_id)
|
||||
)
|
||||
assert cached is not None
|
||||
|
||||
|
||||
def test_finished_eager_store_respects_cpu_capacity() -> None:
|
||||
"""A finished request skips excess blocks when the CPU cache is full."""
|
||||
fix = make_scheduler(num_cpu_blocks=3, num_gpu_blocks=16, lazy=False)
|
||||
sched = fix.scheduler
|
||||
request = make_request(num_blocks=3)
|
||||
kv_blocks = _alloc_and_register(fix, request, num_blocks=3)
|
||||
sched.update_state_after_alloc(request, kv_blocks, num_external_tokens=0)
|
||||
available = get_cpu_free_blocks(sched)
|
||||
|
||||
sched.request_finished_all_groups(request, kv_blocks.get_block_ids())
|
||||
meta = sched.build_connector_meta(make_scheduler_output({}))
|
||||
|
||||
assert len(meta.store_gpu_blocks) == available
|
||||
assert len(meta.store_cpu_blocks) == available
|
||||
|
||||
|
||||
def test_reset_releases_unsubmitted_finished_stores() -> None:
|
||||
"""Reset releases finished-store refs that were not sent to a worker."""
|
||||
fix = make_scheduler(num_cpu_blocks=8, num_gpu_blocks=16, lazy=False)
|
||||
sched = fix.scheduler
|
||||
request = make_request(num_blocks=2)
|
||||
kv_blocks = _alloc_and_register(fix, request, num_blocks=2)
|
||||
sched.update_state_after_alloc(request, kv_blocks, num_external_tokens=0)
|
||||
|
||||
sched.request_finished_all_groups(request, kv_blocks.get_block_ids())
|
||||
|
||||
assert sched.reset()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Test 1b: Boundary — max_hit_len cap drops the last full block when the
|
||||
# prompt is an exact multiple of BLOCK_SIZE.
|
||||
|
||||
@@ -291,8 +291,7 @@ class SimpleCPUOffloadConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
)
|
||||
return False, None
|
||||
|
||||
# NOTE: New API only for SimpleCPUOffloadConnector.
|
||||
def has_pending_transfers(self) -> bool:
|
||||
def has_pending_push_work(self) -> bool:
|
||||
if self.scheduler_manager is not None:
|
||||
return self.scheduler_manager.has_pending_stores()
|
||||
return False
|
||||
|
||||
@@ -174,6 +174,7 @@ class SimpleCPUOffloadScheduler:
|
||||
self._reqs_to_store: dict[str, StoreRequestState] = {}
|
||||
self._store_event_to_reqs: dict[int, list[str]] = {}
|
||||
self._in_flight_store_gpu_blocks: set[int] = set()
|
||||
self._pending_finished_stores: list[TransferMeta] = []
|
||||
self._abandoned_reqs_to_load: dict[str, LoadRequestState] = {}
|
||||
|
||||
# Event counters
|
||||
@@ -414,6 +415,11 @@ class SimpleCPUOffloadScheduler:
|
||||
# --- Stores ---
|
||||
store_event = -1
|
||||
store_gpu, store_cpu, store_req_ids = self.prepare_store_specs(scheduler_output)
|
||||
for transfer in self._pending_finished_stores:
|
||||
store_gpu.extend(transfer.gpu_block_ids)
|
||||
store_cpu.extend(transfer.cpu_block_ids)
|
||||
self._pending_finished_stores.clear()
|
||||
|
||||
if store_gpu:
|
||||
store_event = self._store_event_counter
|
||||
self._store_event_counter += 1
|
||||
@@ -492,7 +498,6 @@ class SimpleCPUOffloadScheduler:
|
||||
self._cursor = None
|
||||
|
||||
gpu_ids: list[int] = []
|
||||
block_hashes: list[bytes] = []
|
||||
last_visited = self._cursor
|
||||
|
||||
for covered, node in enumerate(free_queue.iter_blocks_after(self._cursor)):
|
||||
@@ -508,16 +513,13 @@ class SimpleCPUOffloadScheduler:
|
||||
and cpu_pool.cached_block_hash_to_block.get_one_block(bhash) is None
|
||||
):
|
||||
gpu_ids.append(node.block_id)
|
||||
block_hashes.append(bhash)
|
||||
|
||||
self._cursor = last_visited
|
||||
|
||||
# Batch-allocate CPU blocks and stamp hashes.
|
||||
# Batch-allocate CPU blocks.
|
||||
if gpu_ids:
|
||||
cpu_blocks = cpu_pool.get_new_blocks(len(gpu_ids))
|
||||
cpu_ids = [blk.block_id for blk in cpu_blocks]
|
||||
for cpu_blk, bhash in zip(cpu_blocks, block_hashes): # type: ignore[assignment]
|
||||
cpu_blk._block_hash = bhash # type: ignore[assignment]
|
||||
# Touch GPU blocks to prevent eviction during async copy.
|
||||
gpu_pool.touch([gpu_pool.blocks[bid] for bid in gpu_ids])
|
||||
else:
|
||||
@@ -531,9 +533,8 @@ class SimpleCPUOffloadScheduler:
|
||||
"""Identify newly computed blocks to offload from scheduler requests.
|
||||
|
||||
Only considers blocks whose KV data has been **confirmed computed** by
|
||||
the GPU. This means blocks from the current step are NOT stored until the
|
||||
next step. If a request finishes in the same step as its last full block,
|
||||
that block may be missed. (TODO: flush on finish.)
|
||||
the GPU. Blocks from the current step are stored on a later step, or by
|
||||
the finish-time flush if the request completes first.
|
||||
|
||||
Returns:
|
||||
(gpu_block_ids, cpu_block_ids, req_ids) for the store event.
|
||||
@@ -547,7 +548,6 @@ class SimpleCPUOffloadScheduler:
|
||||
if gpu_block_pool is None:
|
||||
return [], [], []
|
||||
cpu_block_pool = self.cpu_block_pool
|
||||
num_free = cpu_block_pool.get_num_free_blocks()
|
||||
kv_cache_groups = self.cpu_kv_cache_config.kv_cache_groups
|
||||
num_groups = len(kv_cache_groups)
|
||||
# Dedup against blocks already scheduled.
|
||||
@@ -575,73 +575,15 @@ class SimpleCPUOffloadScheduler:
|
||||
if not block_ids_by_group:
|
||||
continue
|
||||
|
||||
# --- Phase 1: Scan blocks, classify as cached vs to-store ---
|
||||
gpu_block_ids: list[int] = []
|
||||
block_hashes_to_store: list[bytes] = []
|
||||
advanced_per_group: list[int] = [0] * num_groups
|
||||
out_of_space = False
|
||||
# Confirmed tokens: KV data written and visible to all streams.
|
||||
req = state.request
|
||||
confirmed_tokens = req.num_computed_tokens - req.num_output_placeholders
|
||||
# Cap to blocks with confirmed KV data.
|
||||
aligned_tokens = confirmed_tokens // self.block_size * self.block_size
|
||||
gpu_block_ids, advanced_per_group = self._select_eager_blocks_to_store(
|
||||
state, block_ids_by_group
|
||||
)
|
||||
|
||||
for g in range(num_groups):
|
||||
# FIXME (yifan): handle CPU cache eviction, where
|
||||
# num_stored_blocks can be stale and omit evicted blocks in
|
||||
# the middle of the request.
|
||||
already_stored_g = state.num_stored_blocks[g]
|
||||
group_gpu_ids = block_ids_by_group[g]
|
||||
|
||||
g_block_size = (
|
||||
kv_cache_groups[g].kv_cache_spec.block_size * self.cp_world_size
|
||||
)
|
||||
ready_blocks_g = aligned_tokens // g_block_size
|
||||
scannable = group_gpu_ids[already_stored_g:ready_blocks_g]
|
||||
|
||||
for gpu_block_id in scannable:
|
||||
gpu_block = gpu_block_pool.blocks[gpu_block_id]
|
||||
if gpu_block.is_null:
|
||||
advanced_per_group[g] += 1
|
||||
continue
|
||||
|
||||
bhash_with_group = gpu_block.block_hash
|
||||
if bhash_with_group is None:
|
||||
# Masked-out SWA position the coordinator chose not to
|
||||
# hash; it can never serve a prefix-cache hit, so skip.
|
||||
advanced_per_group[g] += 1
|
||||
continue
|
||||
|
||||
# Skip if already scheduled for store or already cached in CPU.
|
||||
if (
|
||||
gpu_block_id in in_flight
|
||||
or cpu_block_pool.cached_block_hash_to_block.get_one_block(
|
||||
bhash_with_group
|
||||
)
|
||||
is not None
|
||||
):
|
||||
advanced_per_group[g] += 1
|
||||
continue
|
||||
|
||||
if num_free <= 0:
|
||||
out_of_space = True
|
||||
break
|
||||
num_free -= 1
|
||||
|
||||
gpu_block_ids.append(gpu_block_id)
|
||||
block_hashes_to_store.append(bhash_with_group)
|
||||
advanced_per_group[g] += 1
|
||||
|
||||
if out_of_space:
|
||||
break
|
||||
|
||||
# --- Phase 2: Batch allocate CPU blocks and stamp hashes ---
|
||||
# --- Phase 2: Batch allocate CPU blocks ---
|
||||
n_to_alloc = len(gpu_block_ids)
|
||||
if n_to_alloc > 0:
|
||||
cpu_blocks_alloc = cpu_block_pool.get_new_blocks(n_to_alloc)
|
||||
cpu_block_ids = [blk.block_id for blk in cpu_blocks_alloc]
|
||||
for cpu_blk, bhash in zip(cpu_blocks_alloc, block_hashes_to_store):
|
||||
cpu_blk._block_hash = bhash # type: ignore[assignment]
|
||||
else:
|
||||
cpu_block_ids = []
|
||||
|
||||
@@ -669,6 +611,53 @@ class SimpleCPUOffloadScheduler:
|
||||
|
||||
return merged_gpu_block_ids, merged_cpu_block_ids, req_ids
|
||||
|
||||
def _select_eager_blocks_to_store(
|
||||
self,
|
||||
state: StoreRequestState,
|
||||
block_ids_by_group: tuple[list[int], ...],
|
||||
) -> tuple[list[int], list[int]]:
|
||||
"""Return confirmed, uncached eager blocks and per-group cursor advances."""
|
||||
assert self._gpu_block_pool is not None
|
||||
gpu_block_ids: list[int] = []
|
||||
advanced_per_group = [0] * len(self.cpu_kv_cache_config.kv_cache_groups)
|
||||
confirmed_tokens = (
|
||||
state.request.num_computed_tokens - state.request.num_output_placeholders
|
||||
)
|
||||
aligned_tokens = confirmed_tokens // self.block_size * self.block_size
|
||||
num_free = self.cpu_block_pool.get_num_free_blocks()
|
||||
|
||||
for g, group_gpu_ids in enumerate(block_ids_by_group):
|
||||
if len(gpu_block_ids) >= num_free:
|
||||
break
|
||||
# FIXME (yifan): handle CPU cache eviction, where
|
||||
# num_stored_blocks can be stale and omit evicted blocks in
|
||||
# the middle of the request.
|
||||
group_size = (
|
||||
self.cpu_kv_cache_config.kv_cache_groups[g].kv_cache_spec.block_size
|
||||
* self.cp_world_size
|
||||
)
|
||||
ready = min(len(group_gpu_ids), aligned_tokens // group_size)
|
||||
for gpu_block_id in group_gpu_ids[state.num_stored_blocks[g] : ready]:
|
||||
gpu_block = self._gpu_block_pool.blocks[gpu_block_id]
|
||||
if gpu_block.is_null or gpu_block.block_hash is None:
|
||||
advanced_per_group[g] += 1
|
||||
continue
|
||||
if (
|
||||
gpu_block_id in self._in_flight_store_gpu_blocks
|
||||
or self.cpu_block_pool.cached_block_hash_to_block.get_one_block(
|
||||
gpu_block.block_hash
|
||||
)
|
||||
is not None
|
||||
):
|
||||
advanced_per_group[g] += 1
|
||||
continue
|
||||
if len(gpu_block_ids) >= num_free:
|
||||
break
|
||||
gpu_block_ids.append(gpu_block_id)
|
||||
advanced_per_group[g] += 1
|
||||
|
||||
return gpu_block_ids, advanced_per_group
|
||||
|
||||
def update_connector_output(self, connector_output: KVConnectorOutput) -> None:
|
||||
"""Handle async transfer completions from worker.
|
||||
|
||||
@@ -726,24 +715,31 @@ class SimpleCPUOffloadScheduler:
|
||||
def _process_store_completion(
|
||||
self, gpu_block_ids: list[int], cpu_block_ids: list[int]
|
||||
) -> None:
|
||||
"""Cache CPU blocks per-group and release GPU refs.
|
||||
|
||||
Block hashes were stamped on CPU blocks at allocation time (in
|
||||
``_prepare_*_store_specs``). Here we just register them in the
|
||||
cache map so they become discoverable by the load path.
|
||||
"""
|
||||
"""Register copied blocks in the CPU prefix cache and release refs."""
|
||||
assert len(cpu_block_ids) == len(gpu_block_ids)
|
||||
|
||||
cpu_blocks = [self.cpu_block_pool.blocks[bid] for bid in cpu_block_ids]
|
||||
|
||||
for cpu_block in cpu_blocks:
|
||||
bhash = cpu_block.block_hash
|
||||
assert bhash is not None
|
||||
self.cpu_block_pool.cached_block_hash_to_block.insert(bhash, cpu_block)
|
||||
assert self._gpu_block_pool is not None
|
||||
for gpu_block_id, cpu_block in zip(gpu_block_ids, cpu_blocks):
|
||||
gpu_block = self._gpu_block_pool.blocks[gpu_block_id]
|
||||
primary_hash = gpu_block.block_hash
|
||||
assert primary_hash is not None
|
||||
self.cpu_block_pool._insert_block_hash(
|
||||
primary_hash,
|
||||
cpu_block,
|
||||
num_tokens=gpu_block.block_hash_num_tokens,
|
||||
)
|
||||
secondary_hashes = self._gpu_block_pool.cached_block_hashes_by_block.get(
|
||||
gpu_block_id, ()
|
||||
)
|
||||
for block_hash in secondary_hashes:
|
||||
self.cpu_block_pool._insert_block_hash(
|
||||
block_hash, cpu_block, num_tokens=None
|
||||
)
|
||||
|
||||
# Free CPU and GPU blocks' ref counts to turn them into prefix cache
|
||||
self.cpu_block_pool.free_blocks(cpu_blocks)
|
||||
assert self._gpu_block_pool is not None
|
||||
self._gpu_block_pool.free_blocks(
|
||||
self._gpu_block_pool.blocks[bid] for bid in gpu_block_ids
|
||||
)
|
||||
@@ -751,8 +747,6 @@ class SimpleCPUOffloadScheduler:
|
||||
def _release_transfer_refs(self, transfer: TransferMeta) -> None:
|
||||
"""Release transfer refs without making copied data cacheable."""
|
||||
cpu_blocks = [self.cpu_block_pool.blocks[bid] for bid in transfer.cpu_block_ids]
|
||||
for cpu_block in cpu_blocks:
|
||||
cpu_block.reset_hash()
|
||||
self.cpu_block_pool.free_blocks(cpu_blocks)
|
||||
assert self._gpu_block_pool is not None
|
||||
self._gpu_block_pool.free_blocks(
|
||||
@@ -760,9 +754,11 @@ class SimpleCPUOffloadScheduler:
|
||||
)
|
||||
|
||||
def has_pending_stores(self) -> bool:
|
||||
"""Return True if there are in-flight store transfers."""
|
||||
"""Return True if a store transfer is queued or in flight."""
|
||||
return bool(
|
||||
self._store_event_to_blocks or self._abandoned_store_event_to_blocks
|
||||
self._pending_finished_stores
|
||||
or self._store_event_to_blocks
|
||||
or self._abandoned_store_event_to_blocks
|
||||
)
|
||||
|
||||
def request_finished(
|
||||
@@ -804,8 +800,27 @@ class SimpleCPUOffloadScheduler:
|
||||
request: "Request",
|
||||
block_ids: tuple[list[int], ...],
|
||||
) -> tuple[bool, dict[str, Any] | None]:
|
||||
self._queue_finished_eager_store(request, block_ids)
|
||||
return self.request_finished(request, block_ids=[])
|
||||
|
||||
def _queue_finished_eager_store(
|
||||
self, request: "Request", block_ids: tuple[list[int], ...]
|
||||
) -> None:
|
||||
"""Queue confirmed eager blocks omitted after a request enters decode."""
|
||||
state = self._reqs_to_store.get(request.request_id)
|
||||
gpu_pool = self._gpu_block_pool
|
||||
if state is None or gpu_pool is None:
|
||||
return
|
||||
gpu_ids, _ = self._select_eager_blocks_to_store(state, block_ids)
|
||||
if not gpu_ids:
|
||||
return
|
||||
cpu_blocks = self.cpu_block_pool.get_new_blocks(len(gpu_ids))
|
||||
self._pending_finished_stores.append(
|
||||
TransferMeta(gpu_ids, [b.block_id for b in cpu_blocks])
|
||||
)
|
||||
self._in_flight_store_gpu_blocks.update(gpu_ids)
|
||||
gpu_pool.touch([gpu_pool.blocks[bid] for bid in gpu_ids])
|
||||
|
||||
def _free_pending_cpu_hit(self, pending: tuple) -> None:
|
||||
"""Release the temporary CPU block pin taken in get_num_new_matched_tokens()."""
|
||||
cpu_hit_blocks, _ = pending
|
||||
@@ -879,6 +894,10 @@ class SimpleCPUOffloadScheduler:
|
||||
"""
|
||||
|
||||
self._abandoned_store_event_to_blocks.update(self._store_event_to_blocks)
|
||||
for transfer in self._pending_finished_stores:
|
||||
self._release_transfer_refs(transfer)
|
||||
self._pending_finished_stores.clear()
|
||||
|
||||
self._store_event_to_blocks.clear()
|
||||
self._in_flight_store_gpu_blocks.clear()
|
||||
|
||||
|
||||
Reference in New Issue
Block a user