[KV Offload] Per-job store completion for CPU offloading connector (#39186)

Signed-off-by: Itay Etelis <itay.etelis@ibm.com>
Signed-off-by: Itay Etelis <92247226+Etelis@users.noreply.github.com>
Co-authored-by: Itay Etelis <itay.etelis@ibm.com>
Co-authored-by: Or Ozeri <or@ozery.com>
Co-authored-by: Or Ozeri <oro@il.ibm.com>
This commit is contained in:
Itay Etelis
2026-04-29 08:52:55 +03:00
committed by GitHub
co-authored by Itay Etelis Or Ozeri Or Ozeri
parent a269744e9f
commit 8a8c9b564e
8 changed files with 428 additions and 143 deletions
@@ -1,15 +1,60 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from dataclasses import dataclass
from dataclasses import dataclass, field
from vllm.distributed.kv_transfer.kv_connector.v1.base import KVConnectorMetadata
from vllm.distributed.kv_transfer.kv_connector.v1.base import (
KVConnectorMetadata,
KVConnectorWorkerMetadata,
)
from vllm.v1.kv_offload.worker.worker import TransferSpec
ReqId = str
@dataclass
class TransferJob:
"""A transfer job bundling request context with transfer spec.
Used for both loads and stores, keyed by scheduler-assigned job ID.
The worker reports the job ID back when the transfer finishes,
and the scheduler processes the completion.
"""
req_id: ReqId
transfer_spec: TransferSpec
@dataclass
class OffloadingConnectorMetadata(KVConnectorMetadata):
reqs_to_load: dict[ReqId, TransferSpec]
reqs_to_store: dict[ReqId, TransferSpec]
reqs_to_flush: set[str] | None = None
# Keyed by scheduler-assigned job IDs.
load_jobs: dict[int, TransferJob]
store_jobs: dict[int, TransferJob]
jobs_to_flush: set[int] | None = None
@dataclass
class OffloadingWorkerMetadata(KVConnectorWorkerMetadata):
"""Worker -> Scheduler metadata for completed transfer jobs.
Each worker reports {job_id: 1} for newly completed transfer jobs
(load or store). aggregate() sums counts across workers within a step.
The scheduler accumulates across steps and processes
a transfer completion only when count reaches num_workers.
"""
completed_jobs: dict[int, int] = field(default_factory=dict)
def mark_completed(self, job_id: int) -> None:
"""Record a transfer job completion from this worker."""
self.completed_jobs[job_id] = 1
def aggregate(
self, other: "KVConnectorWorkerMetadata"
) -> "KVConnectorWorkerMetadata":
assert isinstance(other, OffloadingWorkerMetadata)
merged = dict(self.completed_jobs)
for job_id, v in other.completed_jobs.items():
merged[job_id] = merged.get(job_id, 0) + v
return OffloadingWorkerMetadata(completed_jobs=merged)
@@ -1,6 +1,5 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections import defaultdict
from collections.abc import Iterable, Sequence
from dataclasses import dataclass, field
from itertools import islice
@@ -11,7 +10,9 @@ from vllm.distributed.kv_transfer.kv_connector.utils import yield_req_data
from vllm.distributed.kv_transfer.kv_connector.v1.base import KVConnectorMetadata
from vllm.distributed.kv_transfer.kv_connector.v1.offloading.common import (
OffloadingConnectorMetadata,
OffloadingWorkerMetadata,
ReqId,
TransferJob,
)
from vllm.logger import init_logger
from vllm.utils.math_utils import cdiv
@@ -26,13 +27,27 @@ from vllm.v1.kv_offload.abstract import (
)
from vllm.v1.kv_offload.mediums import GPULoadStoreSpec
from vllm.v1.kv_offload.spec import OffloadingSpec
from vllm.v1.kv_offload.worker.worker import TransferSpec
from vllm.v1.outputs import KVConnectorOutput
from vllm.v1.request import Request
logger = init_logger(__name__)
@dataclass(slots=True)
class TransferJobStatus:
"""Tracks scheduler-side state for a single transfer job."""
req_id: ReqId
# Number of workers still pending. Starts at num_workers,
# decremented as each worker reports completion. Job is done at 0.
pending_count: int
# Offload keys this job covers; passed to manager.complete_*().
keys: set[OffloadKey]
is_store: bool
# GPU blocks the fence tracks. Store src blocks; None for loads.
gpu_block_ids: list[int] | None = None
class GroupOffloadConfig(NamedTuple):
group_idx: int
gpu_block_size: int
@@ -43,10 +58,12 @@ class GroupOffloadConfig(NamedTuple):
class SchedulerOffloadConfig(NamedTuple):
kv_group_configs: tuple[GroupOffloadConfig, ...]
block_size_factor: int
num_workers: int
@classmethod
def from_spec(cls, spec: OffloadingSpec) -> "SchedulerOffloadConfig":
return cls(
num_workers=spec.vllm_config.parallel_config.world_size,
kv_group_configs=tuple(
GroupOffloadConfig(
group_idx=idx,
@@ -79,6 +96,9 @@ class RequestOffloadState:
req_context: ReqContext = field(init=False)
# number of hits in the GPU cache
num_locally_computed_tokens: int = 0
# In-flight job IDs. Per the connector's invariant, at any given time
# this contains either a single load job, or one or more store jobs.
transfer_jobs: set[int] = field(default_factory=set)
def __post_init__(self) -> None:
self.group_states = tuple(
@@ -135,17 +155,26 @@ class OffloadingConnectorScheduler:
self.lookup_groups = attention_groups
self._req_status: dict[ReqId, RequestOffloadState] = {}
# requests to load for the current scheduler step
self._reqs_to_load: dict[ReqId, TransferSpec] = {}
self._current_batch_load_jobs: dict[int, TransferJob] = {}
self._current_batch_jobs_to_flush: set[int] = set()
# if GPU prefix caching is enabled,
# track loaded blocks to avoid redundant loads
self._blocks_being_loaded: set[OffloadKey] | None = (
set() if spec.vllm_config.cache_config.enable_prefix_caching else None
)
# request ID -> set(offload keys being stored/loaded)
self._reqs_being_stored = defaultdict[ReqId, set[OffloadKey]](set)
self._reqs_being_loaded = defaultdict[ReqId, set[OffloadKey]](set)
# Job ID counter shared by loads and stores.
self._job_counter: int = 0
self._jobs: dict[int, TransferJobStatus] = {}
# block_id -> pending store job_ids. Populated only for finished
# requests (running-request blocks are protected by their ref_cnt).
self._block_id_to_pending_jobs: dict[int, set[int]] = {}
def _generate_job_id(self) -> int:
job_id = self._job_counter
self._job_counter += 1
return job_id
def _maximal_prefix_lookup(
self, keys: Iterable[OffloadKey], req_context: ReqContext
@@ -369,23 +398,46 @@ class OffloadingConnectorScheduler:
# entire KV cache so a remote decode node can consume it.
group_state.next_stored_block_idx = num_blocks
# Fence dst blocks against finished-request pending stores.
if (
self._block_id_to_pending_jobs
and not self._block_id_to_pending_jobs.keys().isdisjoint(dst_block_ids)
):
self._current_batch_jobs_to_flush.update(
jid
for bid in dst_block_ids
for jid in self._block_id_to_pending_jobs.get(bid, ())
)
src_spec = self.manager.prepare_load(keys_to_load, req_status.req_context)
dst_spec = GPULoadStoreSpec(
dst_block_ids, group_sizes=group_sizes, block_indices=block_indices
)
self._reqs_to_load[request.request_id] = (src_spec, dst_spec)
req_blocks_being_loaded = self._reqs_being_loaded[request.request_id]
req_blocks_being_loaded.update(keys_to_load)
load_job_id = self._generate_job_id()
self._current_batch_load_jobs[load_job_id] = TransferJob(
req_id=request.request_id,
transfer_spec=(src_spec, dst_spec),
)
# a load can only be issued when no other jobs are pending.
assert not req_status.transfer_jobs
req_status.transfer_jobs.add(load_job_id)
self._jobs[load_job_id] = TransferJobStatus(
req_id=request.request_id,
pending_count=self.config.num_workers,
keys=set(keys_to_load),
is_store=False,
)
if self._blocks_being_loaded is not None:
self._blocks_being_loaded.update(req_blocks_being_loaded)
self._blocks_being_loaded.update(keys_to_load)
def _get_reqs_to_store(
self, scheduler_output: SchedulerOutput
) -> dict[ReqId, TransferSpec]:
def _build_store_jobs(
self,
scheduler_output: SchedulerOutput,
) -> dict[int, TransferJob]:
block_size_factor = self.config.block_size_factor
reqs_to_store: dict[ReqId, TransferSpec] = {}
store_jobs: dict[int, TransferJob] = {}
# iterate over both new and cached requests
for req_id, new_block_id_groups, preempted in yield_req_data(scheduler_output):
req_status = self._req_status[req_id]
@@ -398,6 +450,19 @@ class OffloadingConnectorScheduler:
if new_block_id_groups:
req_status.update_block_id_groups(new_block_id_groups)
# Fence new blocks against in-flight stores.
if self._block_id_to_pending_jobs:
new_blocks_flat = [
bid for new_blocks in new_block_id_groups for bid in new_blocks
]
if not self._block_id_to_pending_jobs.keys().isdisjoint(
new_blocks_flat
):
self._current_batch_jobs_to_flush.update(
jid
for bid in new_blocks_flat
for jid in self._block_id_to_pending_jobs.get(bid, ())
)
num_scheduled_tokens = scheduler_output.num_scheduled_tokens[req_id]
num_tokens_after_batch = req.num_computed_tokens + num_scheduled_tokens
@@ -491,36 +556,52 @@ class OffloadingConnectorScheduler:
)
dst_spec = store_output.store_spec
reqs_to_store[req_id] = (src_spec, dst_spec)
self._reqs_being_stored[req_id] |= keys_to_store
job_id = self._generate_job_id()
# a store can only be issued when no load is pending.
if req_status.transfer_jobs:
any_jid = next(iter(req_status.transfer_jobs))
assert self._jobs[any_jid].is_store
req_status.transfer_jobs.add(job_id)
self._jobs[job_id] = TransferJobStatus(
req_id=req_id,
pending_count=self.config.num_workers,
keys=set(keys_to_store),
is_store=True,
gpu_block_ids=src_block_ids,
)
store_jobs[job_id] = TransferJob(
req_id=req_id, transfer_spec=(src_spec, dst_spec)
)
logger.debug(
"Request %s offloading %s blocks upto %d tokens",
"Request %s offloading %s blocks upto %d tokens (job %d)",
req_id,
len(keys_to_store),
num_offloadable_tokens,
job_id,
)
return reqs_to_store
return store_jobs
def build_connector_meta(
self, scheduler_output: SchedulerOutput
) -> KVConnectorMetadata:
meta = OffloadingConnectorMetadata(
reqs_to_load=self._reqs_to_load,
reqs_to_store=self._get_reqs_to_store(scheduler_output),
reqs_to_flush=scheduler_output.preempted_req_ids,
)
self._reqs_to_load = {}
# NOTE (orozery): we should move this logic to update_connector_output
# once KVConnectorOutput allows us to report completed transfers
for req_id in scheduler_output.preempted_req_ids or ():
keys = self._reqs_being_stored.get(req_id)
if keys:
self.manager.complete_store(keys)
keys.clear()
req_status = self._req_status.get(req_id)
if req_status is None or not req_status.transfer_jobs:
continue
any_jid = next(iter(req_status.transfer_jobs))
assert self._jobs[any_jid].is_store
self._current_batch_jobs_to_flush.update(req_status.transfer_jobs)
meta = OffloadingConnectorMetadata(
load_jobs=self._current_batch_load_jobs,
store_jobs=self._build_store_jobs(scheduler_output),
jobs_to_flush=self._current_batch_jobs_to_flush,
)
self._current_batch_load_jobs = {}
self._current_batch_jobs_to_flush = set()
return meta
def update_connector_output(self, connector_output: KVConnectorOutput):
@@ -531,17 +612,37 @@ class OffloadingConnectorScheduler:
connector_output (KVConnectorOutput): the worker-side
connectors output.
"""
for req_id in connector_output.finished_sending or []:
keys = self._reqs_being_stored.pop(req_id, None)
if keys:
self.manager.complete_store(keys)
meta = connector_output.kv_connector_worker_meta
if not isinstance(meta, OffloadingWorkerMetadata):
assert meta is None
meta = OffloadingWorkerMetadata()
for job_id, count in meta.completed_jobs.items():
assert count > 0
job_status = self._jobs[job_id]
job_status.pending_count -= count
if job_status.pending_count > 0:
continue
assert job_status.pending_count == 0
for req_id in connector_output.finished_recving or []:
keys = self._reqs_being_loaded.pop(req_id, None)
if keys:
if job_status.is_store:
self.manager.complete_store(job_status.keys)
else:
self.manager.complete_load(job_status.keys)
if self._blocks_being_loaded:
self._blocks_being_loaded.difference_update(keys)
self.manager.complete_load(keys)
self._blocks_being_loaded.difference_update(job_status.keys)
req_status = self._req_status[job_status.req_id]
if self._block_id_to_pending_jobs and req_status.req.is_finished():
for bid in job_status.gpu_block_ids or ():
pending = self._block_id_to_pending_jobs[bid]
pending.remove(job_id)
if not pending:
del self._block_id_to_pending_jobs[bid]
del self._jobs[job_id]
req_status.transfer_jobs.remove(job_id)
if not req_status.transfer_jobs and req_status.req.is_finished():
del self._req_status[job_status.req_id]
def request_finished(
self,
@@ -558,14 +659,21 @@ class OffloadingConnectorScheduler:
Optional KVTransferParams to be included in the request outputs
returned by the engine.
"""
req_id = request.request_id
# TODO(orozery): possibly kickoff offload for last block
# which may have been deferred due to async scheduling
self._req_status.pop(req_id, None)
request_being_stored = req_id in self._reqs_being_stored
return request_being_stored, None
req_status = self._req_status.get(request.request_id)
if req_status is None:
return False, None
if not req_status.transfer_jobs:
del self._req_status[request.request_id]
return False, None
# Pending stores will outlive the request's block ownership.
# Register them so future block reuse triggers a flush.
for job_id in req_status.transfer_jobs:
job_status = self._jobs[job_id]
for bid in job_status.gpu_block_ids or ():
self._block_id_to_pending_jobs.setdefault(bid, set()).add(job_id)
return False, None
def take_events(self) -> Iterable[KVCacheEvent]:
"""Take the KV cache events from the connector.
@@ -11,6 +11,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.metrics import (
)
from vllm.distributed.kv_transfer.kv_connector.v1.offloading.common import (
OffloadingConnectorMetadata,
OffloadingWorkerMetadata,
ReqId,
)
from vllm.distributed.kv_transfer.kv_connector.v1.offloading.metrics import (
@@ -45,24 +46,11 @@ class OffloadingConnectorWorker:
self.spec = spec
self.worker = OffloadingWorker()
self._job_counter = 0
self.kv_connector_stats = OffloadingConnectorStats()
# req_id -> (job_id, store)
self._jobs: dict[int, tuple[ReqId, bool]] = {}
# req_id -> active job IDs
self._load_job: dict[ReqId, int] = {}
# req_id -> set(active job IDs)
self._store_jobs = defaultdict[ReqId, set[int]](set)
# list of store jobs pending submission (job_id, transfer_spec)
# job_id -> req_id for in-flight loads.
self._load_jobs: dict[int, ReqId] = {}
self._unsubmitted_store_jobs: list[tuple[int, TransferSpec]] = []
self._finished_reqs_waiting_for_store: set[ReqId] = set()
def _generate_job_id(self) -> int:
job_id = self._job_counter
self._job_counter = job_id + 1
return job_id
self._connector_worker_meta = OffloadingWorkerMetadata()
def _register_handlers(self, kv_caches: CanonicalKVCaches):
for src_cls, dst_cls, handler in self.spec.get_handlers(kv_caches):
@@ -301,10 +289,8 @@ class OffloadingConnectorWorker:
assert success
self._unsubmitted_store_jobs.clear()
for req_id in kv_connector_metadata.reqs_to_flush or ():
job_ids = self._store_jobs.get(req_id)
if job_ids:
self.worker.wait(job_ids)
if kv_connector_metadata.jobs_to_flush:
self.worker.wait(kv_connector_metadata.jobs_to_flush)
def start_kv_transfers(self, metadata: OffloadingConnectorMetadata):
for job_id, transfer_spec in self._unsubmitted_store_jobs:
@@ -312,41 +298,33 @@ class OffloadingConnectorWorker:
assert success
self._unsubmitted_store_jobs.clear()
for req_id, transfer_spec in metadata.reqs_to_load.items():
job_id = self._generate_job_id()
self._jobs[job_id] = (req_id, False)
assert req_id not in self._load_job
self._load_job[req_id] = job_id
success = self.worker.transfer_async(job_id, transfer_spec)
for job_id, entry in metadata.load_jobs.items():
self._load_jobs[job_id] = entry.req_id
success = self.worker.transfer_async(job_id, entry.transfer_spec)
assert success
def prepare_store_kv(self, metadata: OffloadingConnectorMetadata):
for req_id, transfer_spec in metadata.reqs_to_store.items():
job_id = self._generate_job_id()
self._jobs[job_id] = (req_id, True)
self._store_jobs[req_id].add(job_id)
# NOTE(orozery): defer the store to the beginning of the next engine step,
# so that offloading starts AFTER transfers related to token sampling,
# thereby avoiding delays to token generation due to offloading.
self._unsubmitted_store_jobs.append((job_id, transfer_spec))
for job_id, entry in metadata.store_jobs.items():
# NOTE(orozery): defer the store to the beginning of the next
# engine step, so that offloading starts AFTER transfers related
# to token sampling, thereby avoiding delays to token generation.
self._unsubmitted_store_jobs.append((job_id, entry.transfer_spec))
def get_finished(self, finished_req_ids: set[str]) -> tuple[set[str], set[str]]:
"""
Notifies worker-side connector ids of requests that have
finished generating tokens.
Returns a list of request IDs that finished loading or storing.
Returns:
ids of requests that have finished asynchronous transfer
tuple of (sending/saving ids, recving/loading ids).
tuple of (finished_sending, finished_recving). Stores never
emit finished_sending — the scheduler tracks store completion
via kv_connector_worker_meta.completed_jobs and fences any
block reuse via jobs_to_flush. Loads still emit
finished_recving so the base scheduler can resume requests
blocked on remote KV (and free aborted-during-load reqs).
"""
finished_sending = set()
finished_recving = set()
finished_recving: set[str] = set()
for transfer_result in self.worker.get_finished():
# we currently do not support job failures
job_id = transfer_result.job_id
assert transfer_result.success
req_id, store = self._jobs.pop(job_id)
if (
transfer_result.transfer_time
and transfer_result.transfer_size is not None
@@ -357,31 +335,21 @@ class OffloadingConnectorWorker:
time=transfer_result.transfer_time,
transfer_type=transfer_result.transfer_type,
)
if store:
req_jobs = self._store_jobs[req_id]
req_jobs.remove(job_id)
if req_jobs:
continue
if req_id in self._finished_reqs_waiting_for_store:
self._finished_reqs_waiting_for_store.remove(req_id)
finished_sending.add(req_id)
del self._store_jobs[req_id]
else:
req_job = self._load_job[req_id]
assert job_id == req_job
del self._load_job[req_id]
self._connector_worker_meta.mark_completed(job_id)
req_id = self._load_jobs.pop(job_id, None)
if req_id is not None:
finished_recving.add(req_id)
for req_id in finished_req_ids:
pending_req_jobs = self._store_jobs.get(req_id)
if pending_req_jobs:
self._finished_reqs_waiting_for_store.add(req_id)
elif pending_req_jobs is not None:
finished_sending.add(req_id)
del self._store_jobs[req_id]
return set(), finished_recving
return finished_sending, finished_recving
def build_connector_worker_meta(self) -> OffloadingWorkerMetadata | None:
"""Return completed transfer job IDs since the last call."""
if not self._connector_worker_meta.completed_jobs:
return None
meta = self._connector_worker_meta
self._connector_worker_meta = OffloadingWorkerMetadata()
return meta
def get_kv_connector_stats(self) -> KVConnectorStats | None:
"""
@@ -396,11 +364,7 @@ class OffloadingConnectorWorker:
return kv_connector_stats
def shutdown(self) -> None:
# Drop deferred store jobs: there is no point in submitting
# them during shutdown.
self._unsubmitted_store_jobs.clear()
self._jobs.clear()
self._load_job.clear()
self._store_jobs.clear()
self._finished_reqs_waiting_for_store.clear()
self._load_jobs.clear()
self._connector_worker_meta = OffloadingWorkerMetadata()
self.worker.shutdown()
@@ -20,6 +20,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.metrics import (
)
from vllm.distributed.kv_transfer.kv_connector.v1.offloading.common import (
OffloadingConnectorMetadata,
OffloadingWorkerMetadata,
)
from vllm.distributed.kv_transfer.kv_connector.v1.offloading.metrics import (
OffloadingConnectorStats,
@@ -111,6 +112,11 @@ class OffloadingConnector(KVConnectorBase_V1):
assert self.connector_worker is not None
return self.connector_worker.get_finished(finished_req_ids)
def build_connector_worker_meta(self) -> OffloadingWorkerMetadata | None:
if self.connector_worker is not None:
return self.connector_worker.build_connector_worker_meta()
return None
def get_num_new_matched_tokens(
self, request: "Request", num_computed_tokens: int
) -> tuple[int | None, bool]: