[KV Offload] Per-job store completion for CPU offloading connector (#39186)

Signed-off-by: Itay Etelis <itay.etelis@ibm.com>
Signed-off-by: Itay Etelis <92247226+Etelis@users.noreply.github.com>
Co-authored-by: Itay Etelis <itay.etelis@ibm.com>
Co-authored-by: Or Ozeri <or@ozery.com>
Co-authored-by: Or Ozeri <oro@il.ibm.com>
This commit is contained in:
Itay Etelis
2026-04-29 08:52:55 +03:00
committed by GitHub
co-authored by Itay Etelis Or Ozeri Or Ozeri
parent a269744e9f
commit 8a8c9b564e
8 changed files with 428 additions and 143 deletions
@@ -232,6 +232,9 @@ def test_request_preemption(request_runner, async_scheduling: bool):
expected_stored_gpu_block_indexes=(9, 10, 11),
)
# All stores completed before request_finished -> fence index empty.
assert runner.connector_scheduler._block_id_to_pending_jobs == {}
@pytest.mark.parametrize("async_scheduling", [True, False])
def test_concurrent_lookups_of_the_same_prefix(request_runner, async_scheduling: bool):
@@ -292,6 +295,9 @@ def test_concurrent_lookups_of_the_same_prefix(request_runner, async_scheduling:
# second request will use the GPU prefix cache
assert transfer_jobs == list(runner.offloading_spec.handler.transfer_specs)
# Fence index drained: stores completed before request_finished ran.
assert runner.connector_scheduler._block_id_to_pending_jobs == {}
@pytest.mark.parametrize("async_scheduling", [True, False])
def test_abort_loading_requests(request_runner, async_scheduling: bool):
@@ -534,3 +540,131 @@ def test_do_remote_decode_stores_all_blocks(request_runner, async_scheduling: bo
decoded_tokens=[EOS_TOKEN_ID],
expected_stored_gpu_block_indexes=(0, 1, 2, 3, 4, 5),
)
# All stores completed before request_finished -> fence index empty.
assert runner.connector_scheduler._block_id_to_pending_jobs == {}
# ---------------------------------------------------------------------------
# Tests for the per-job-store-completion design and fence invariants.
# ---------------------------------------------------------------------------
def test_loads_do_not_populate_fence_index(request_runner):
"""Loads don't populate _block_id_to_pending_jobs (protected by
delay_free_blocks while in flight)."""
runner = request_runner(
offloaded_block_size=12,
gpu_block_size=4,
num_gpu_blocks=100,
async_scheduling=False,
)
runner.new_request(token_ids=[0] * 12)
runner.connector_scheduler._maximal_prefix_lookup = lambda key, req_context: 1
runner.run(decoded_tokens=[], complete_transfers=False)
assert runner.connector_scheduler._block_id_to_pending_jobs == {}
def test_fence_at_update_state_after_alloc(request_runner):
"""A load reusing a finished request's pending-store block triggers
a flush via update_state_after_alloc's fence.
num_gpu_blocks=2 forces the BlockPool to give req2 the same block
req1 just freed.
"""
runner = request_runner(
offloaded_block_size=4,
gpu_block_size=4,
num_gpu_blocks=2,
async_scheduling=False,
)
runner.new_request(token_ids=[0] * 4)
runner.manager.prepare_store.side_effect = (
lambda keys, req_context: generate_store_output(keys)
)
runner.run(decoded_tokens=[EOS_TOKEN_ID], complete_transfers=False)
assert runner.connector_scheduler._block_id_to_pending_jobs
runner.scheduler.reset_prefix_cache()
runner.new_request(token_ids=[0] * 4)
runner.connector_scheduler._maximal_prefix_lookup = lambda key, req_context: 1
runner.manager.prepare_store.side_effect = (
lambda keys, req_context: generate_store_output([])
)
runner.run(
decoded_tokens=[],
complete_transfers=False,
expected_stored_gpu_block_indexes=(0,),
expected_flushed_gpu_block_indexes=(0,),
)
assert runner.connector_scheduler._block_id_to_pending_jobs == {}
def test_fence_at_build_store_jobs(request_runner):
"""A new prefill (no load -> update_state_after_alloc returns early)
reusing a finished request's pending-store block is flushed by
_build_store_jobs's fence."""
runner = request_runner(
offloaded_block_size=4,
gpu_block_size=4,
num_gpu_blocks=2,
async_scheduling=False,
)
runner.new_request(token_ids=[0] * 4)
runner.manager.prepare_store.side_effect = (
lambda keys, req_context: generate_store_output(keys)
)
runner.run(decoded_tokens=[EOS_TOKEN_ID], complete_transfers=False)
assert runner.connector_scheduler._block_id_to_pending_jobs
runner.scheduler.reset_prefix_cache()
runner.new_request(token_ids=[1] * 4)
runner.connector_scheduler._maximal_prefix_lookup = lambda key, req_context: 0
runner.manager.prepare_store.side_effect = (
lambda keys, req_context: generate_store_output([])
)
runner.run(
decoded_tokens=[EOS_TOKEN_ID],
expected_stored_gpu_block_indexes=(0,),
expected_flushed_gpu_block_indexes=(0,),
)
assert runner.connector_scheduler._block_id_to_pending_jobs == {}
@pytest.mark.parametrize("async_scheduling", [True, False])
def test_complete_store_called_per_job(request_runner, async_scheduling: bool):
"""complete_store fires per-job, not deferred to request finish.
Each call carries only that store's keys."""
offloaded_block_size = 12
runner = request_runner(
offloaded_block_size=offloaded_block_size,
gpu_block_size=4,
num_gpu_blocks=100,
async_scheduling=async_scheduling,
)
runner.new_request(token_ids=[0] * offloaded_block_size)
runner.manager.prepare_store.side_effect = (
lambda keys, req_context: generate_store_output(keys)
)
# First store: fires when block 0 is fully populated.
runner.run(decoded_tokens=[0, 0], expected_stored_gpu_block_indexes=(0, 1, 2))
assert runner.manager.complete_store.call_count == 1
first_call_keys = set(runner.manager.complete_store.call_args.args[0])
assert len(first_call_keys) == 1
runner.manager.complete_store.reset_mock()
# Second store: fires when block 1 is fully populated, with different keys.
runner.run(
decoded_tokens=[0] * (offloaded_block_size + 1),
expected_stored_gpu_block_indexes=(3, 4, 5),
)
assert runner.manager.complete_store.call_count == 1
second_call_keys = set(runner.manager.complete_store.call_args.args[0])
assert first_call_keys != second_call_keys
runner.manager.complete_store.reset_mock()
# Finish: no store pending -> no further call.
runner.run(decoded_tokens=[EOS_TOKEN_ID])
assert runner.manager.complete_store.call_count == 0
@@ -0,0 +1,32 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
from vllm.distributed.kv_transfer.kv_connector.v1.offloading.common import (
OffloadingWorkerMetadata,
)
pytestmark = pytest.mark.cpu_test
def test_aggregate_sums_counts():
meta1 = OffloadingWorkerMetadata(completed_jobs={42: 1, 7: 1})
meta2 = OffloadingWorkerMetadata(completed_jobs={42: 1, 7: 1})
result = meta1.aggregate(meta2)
assert result.completed_jobs == {42: 2, 7: 2}
def test_aggregate_disjoint_jobs():
meta1 = OffloadingWorkerMetadata(completed_jobs={42: 1, 7: 1})
meta2 = OffloadingWorkerMetadata(completed_jobs={43: 1, 8: 1})
result = meta1.aggregate(meta2)
assert result.completed_jobs == {42: 1, 7: 1, 43: 1, 8: 1}
def test_aggregate_multiple_workers():
meta1 = OffloadingWorkerMetadata(completed_jobs={42: 1, 43: 1, 7: 1})
meta2 = OffloadingWorkerMetadata(completed_jobs={42: 1, 7: 1, 8: 1})
meta3 = OffloadingWorkerMetadata(completed_jobs={42: 1, 43: 1, 8: 1})
result = meta1.aggregate(meta2).aggregate(meta3)
assert result.completed_jobs == {42: 3, 43: 2, 7: 2, 8: 2}
@@ -1,6 +1,5 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import copy
from collections.abc import Iterable, Iterator
from dataclasses import dataclass
from typing import Any
@@ -19,6 +18,7 @@ from vllm.config import KVTransferConfig, VllmConfig, set_current_vllm_config
from vllm.distributed.kv_transfer.kv_connector.v1 import KVConnectorRole
from vllm.distributed.kv_transfer.kv_connector.v1.offloading.common import (
OffloadingConnectorMetadata,
OffloadingWorkerMetadata,
)
from vllm.distributed.kv_transfer.kv_connector.v1.offloading_connector import (
OffloadingConnector,
@@ -51,7 +51,6 @@ from vllm.v1.kv_offload.worker.worker import (
TransferResult,
TransferSpec,
)
from vllm.v1.outputs import EMPTY_MODEL_RUNNER_OUTPUT, KVConnectorOutput
from vllm.v1.request import Request
from vllm.v1.structured_output import StructuredOutputManager
@@ -369,7 +368,12 @@ class RequestRunner:
prev_scheduler_output = None
prev_model_runner_output = None
while True:
assert self.scheduler.requests
# Strict-always-False frees the request immediately on EOS, but
# the worker may still have a deferred store queued. In production
# the next request's step drains it; in single-request tests we
# must keep stepping until the scheduler sees no in-flight jobs.
if not self.scheduler.requests and not self.connector_scheduler._jobs:
break
scheduler_output = self.scheduler.schedule()
self._update_gpu_block_idx()
@@ -392,6 +396,10 @@ class RequestRunner:
finished_sending, finished_recving = self.worker_connector.get_finished(
scheduler_output.finished_req_ids
)
worker_meta = (
self.worker_connector.build_connector_worker_meta()
or OffloadingWorkerMetadata()
)
self.worker_connector.clear_connector_metadata()
@@ -400,6 +408,7 @@ class RequestRunner:
finished_sending=finished_sending,
finished_recving=finished_recving,
token_id=token_id or 0,
kv_connector_worker_meta=worker_meta,
)
prev_token_id = token_id
@@ -420,7 +429,7 @@ class RequestRunner:
if (
prev_token_id == EOS_TOKEN_ID
and prev_token_id != token_id
and self.scheduler.requests
and (self.scheduler.requests or self.connector_scheduler._jobs)
):
# continue for one more step to allow offloading to kick off
continue
@@ -435,26 +444,9 @@ class RequestRunner:
self._parse_transfers()
# run one more step to update finished stored
if EOS_TOKEN_ID in decoded_tokens:
assert not self.scheduler.running
while self.scheduler.requests:
scheduler_output = self.scheduler.schedule()
finished_sending, finished_recving = self.worker_connector.get_finished(
scheduler_output.finished_req_ids
)
assert not finished_recving
model_runner_output = copy.deepcopy(EMPTY_MODEL_RUNNER_OUTPUT)
model_runner_output.kv_connector_output = KVConnectorOutput(
finished_sending=finished_sending
)
self.scheduler.update_from_output(scheduler_output, model_runner_output)
def run(
self,
decoded_tokens: list[int],
+4
View File
@@ -24,6 +24,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.base import (
KVConnectorBase_V1,
KVConnectorMetadata,
KVConnectorRole,
KVConnectorWorkerMetadata,
)
from vllm.distributed.kv_transfer.kv_connector.v1.example_connector import ( # noqa
ExampleConnector,
@@ -249,6 +250,7 @@ def create_model_runner_output(
invalid_block_ids: set[int] | None = None,
use_eos: bool = False,
token_id: int = 0,
kv_connector_worker_meta: KVConnectorWorkerMetadata | None = None,
) -> ModelRunnerOutput:
"""Make dummy model runner output for testing."""
@@ -266,11 +268,13 @@ def create_model_runner_output(
finished_sending is None
and finished_recving is None
and invalid_block_ids is None
and kv_connector_worker_meta is None
)
else KVConnectorOutput(
finished_sending=finished_sending,
finished_recving=finished_recving,
invalid_block_ids=invalid_block_ids or set(),
kv_connector_worker_meta=kv_connector_worker_meta,
)
)
@@ -1,15 +1,60 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from dataclasses import dataclass
from dataclasses import dataclass, field
from vllm.distributed.kv_transfer.kv_connector.v1.base import KVConnectorMetadata
from vllm.distributed.kv_transfer.kv_connector.v1.base import (
KVConnectorMetadata,
KVConnectorWorkerMetadata,
)
from vllm.v1.kv_offload.worker.worker import TransferSpec
ReqId = str
@dataclass
class TransferJob:
"""A transfer job bundling request context with transfer spec.
Used for both loads and stores, keyed by scheduler-assigned job ID.
The worker reports the job ID back when the transfer finishes,
and the scheduler processes the completion.
"""
req_id: ReqId
transfer_spec: TransferSpec
@dataclass
class OffloadingConnectorMetadata(KVConnectorMetadata):
reqs_to_load: dict[ReqId, TransferSpec]
reqs_to_store: dict[ReqId, TransferSpec]
reqs_to_flush: set[str] | None = None
# Keyed by scheduler-assigned job IDs.
load_jobs: dict[int, TransferJob]
store_jobs: dict[int, TransferJob]
jobs_to_flush: set[int] | None = None
@dataclass
class OffloadingWorkerMetadata(KVConnectorWorkerMetadata):
"""Worker -> Scheduler metadata for completed transfer jobs.
Each worker reports {job_id: 1} for newly completed transfer jobs
(load or store). aggregate() sums counts across workers within a step.
The scheduler accumulates across steps and processes
a transfer completion only when count reaches num_workers.
"""
completed_jobs: dict[int, int] = field(default_factory=dict)
def mark_completed(self, job_id: int) -> None:
"""Record a transfer job completion from this worker."""
self.completed_jobs[job_id] = 1
def aggregate(
self, other: "KVConnectorWorkerMetadata"
) -> "KVConnectorWorkerMetadata":
assert isinstance(other, OffloadingWorkerMetadata)
merged = dict(self.completed_jobs)
for job_id, v in other.completed_jobs.items():
merged[job_id] = merged.get(job_id, 0) + v
return OffloadingWorkerMetadata(completed_jobs=merged)
@@ -1,6 +1,5 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections import defaultdict
from collections.abc import Iterable, Sequence
from dataclasses import dataclass, field
from itertools import islice
@@ -11,7 +10,9 @@ from vllm.distributed.kv_transfer.kv_connector.utils import yield_req_data
from vllm.distributed.kv_transfer.kv_connector.v1.base import KVConnectorMetadata
from vllm.distributed.kv_transfer.kv_connector.v1.offloading.common import (
OffloadingConnectorMetadata,
OffloadingWorkerMetadata,
ReqId,
TransferJob,
)
from vllm.logger import init_logger
from vllm.utils.math_utils import cdiv
@@ -26,13 +27,27 @@ from vllm.v1.kv_offload.abstract import (
)
from vllm.v1.kv_offload.mediums import GPULoadStoreSpec
from vllm.v1.kv_offload.spec import OffloadingSpec
from vllm.v1.kv_offload.worker.worker import TransferSpec
from vllm.v1.outputs import KVConnectorOutput
from vllm.v1.request import Request
logger = init_logger(__name__)
@dataclass(slots=True)
class TransferJobStatus:
"""Tracks scheduler-side state for a single transfer job."""
req_id: ReqId
# Number of workers still pending. Starts at num_workers,
# decremented as each worker reports completion. Job is done at 0.
pending_count: int
# Offload keys this job covers; passed to manager.complete_*().
keys: set[OffloadKey]
is_store: bool
# GPU blocks the fence tracks. Store src blocks; None for loads.
gpu_block_ids: list[int] | None = None
class GroupOffloadConfig(NamedTuple):
group_idx: int
gpu_block_size: int
@@ -43,10 +58,12 @@ class GroupOffloadConfig(NamedTuple):
class SchedulerOffloadConfig(NamedTuple):
kv_group_configs: tuple[GroupOffloadConfig, ...]
block_size_factor: int
num_workers: int
@classmethod
def from_spec(cls, spec: OffloadingSpec) -> "SchedulerOffloadConfig":
return cls(
num_workers=spec.vllm_config.parallel_config.world_size,
kv_group_configs=tuple(
GroupOffloadConfig(
group_idx=idx,
@@ -79,6 +96,9 @@ class RequestOffloadState:
req_context: ReqContext = field(init=False)
# number of hits in the GPU cache
num_locally_computed_tokens: int = 0
# In-flight job IDs. Per the connector's invariant, at any given time
# this contains either a single load job, or one or more store jobs.
transfer_jobs: set[int] = field(default_factory=set)
def __post_init__(self) -> None:
self.group_states = tuple(
@@ -135,17 +155,26 @@ class OffloadingConnectorScheduler:
self.lookup_groups = attention_groups
self._req_status: dict[ReqId, RequestOffloadState] = {}
# requests to load for the current scheduler step
self._reqs_to_load: dict[ReqId, TransferSpec] = {}
self._current_batch_load_jobs: dict[int, TransferJob] = {}
self._current_batch_jobs_to_flush: set[int] = set()
# if GPU prefix caching is enabled,
# track loaded blocks to avoid redundant loads
self._blocks_being_loaded: set[OffloadKey] | None = (
set() if spec.vllm_config.cache_config.enable_prefix_caching else None
)
# request ID -> set(offload keys being stored/loaded)
self._reqs_being_stored = defaultdict[ReqId, set[OffloadKey]](set)
self._reqs_being_loaded = defaultdict[ReqId, set[OffloadKey]](set)
# Job ID counter shared by loads and stores.
self._job_counter: int = 0
self._jobs: dict[int, TransferJobStatus] = {}
# block_id -> pending store job_ids. Populated only for finished
# requests (running-request blocks are protected by their ref_cnt).
self._block_id_to_pending_jobs: dict[int, set[int]] = {}
def _generate_job_id(self) -> int:
job_id = self._job_counter
self._job_counter += 1
return job_id
def _maximal_prefix_lookup(
self, keys: Iterable[OffloadKey], req_context: ReqContext
@@ -369,23 +398,46 @@ class OffloadingConnectorScheduler:
# entire KV cache so a remote decode node can consume it.
group_state.next_stored_block_idx = num_blocks
# Fence dst blocks against finished-request pending stores.
if (
self._block_id_to_pending_jobs
and not self._block_id_to_pending_jobs.keys().isdisjoint(dst_block_ids)
):
self._current_batch_jobs_to_flush.update(
jid
for bid in dst_block_ids
for jid in self._block_id_to_pending_jobs.get(bid, ())
)
src_spec = self.manager.prepare_load(keys_to_load, req_status.req_context)
dst_spec = GPULoadStoreSpec(
dst_block_ids, group_sizes=group_sizes, block_indices=block_indices
)
self._reqs_to_load[request.request_id] = (src_spec, dst_spec)
req_blocks_being_loaded = self._reqs_being_loaded[request.request_id]
req_blocks_being_loaded.update(keys_to_load)
load_job_id = self._generate_job_id()
self._current_batch_load_jobs[load_job_id] = TransferJob(
req_id=request.request_id,
transfer_spec=(src_spec, dst_spec),
)
# a load can only be issued when no other jobs are pending.
assert not req_status.transfer_jobs
req_status.transfer_jobs.add(load_job_id)
self._jobs[load_job_id] = TransferJobStatus(
req_id=request.request_id,
pending_count=self.config.num_workers,
keys=set(keys_to_load),
is_store=False,
)
if self._blocks_being_loaded is not None:
self._blocks_being_loaded.update(req_blocks_being_loaded)
self._blocks_being_loaded.update(keys_to_load)
def _get_reqs_to_store(
self, scheduler_output: SchedulerOutput
) -> dict[ReqId, TransferSpec]:
def _build_store_jobs(
self,
scheduler_output: SchedulerOutput,
) -> dict[int, TransferJob]:
block_size_factor = self.config.block_size_factor
reqs_to_store: dict[ReqId, TransferSpec] = {}
store_jobs: dict[int, TransferJob] = {}
# iterate over both new and cached requests
for req_id, new_block_id_groups, preempted in yield_req_data(scheduler_output):
req_status = self._req_status[req_id]
@@ -398,6 +450,19 @@ class OffloadingConnectorScheduler:
if new_block_id_groups:
req_status.update_block_id_groups(new_block_id_groups)
# Fence new blocks against in-flight stores.
if self._block_id_to_pending_jobs:
new_blocks_flat = [
bid for new_blocks in new_block_id_groups for bid in new_blocks
]
if not self._block_id_to_pending_jobs.keys().isdisjoint(
new_blocks_flat
):
self._current_batch_jobs_to_flush.update(
jid
for bid in new_blocks_flat
for jid in self._block_id_to_pending_jobs.get(bid, ())
)
num_scheduled_tokens = scheduler_output.num_scheduled_tokens[req_id]
num_tokens_after_batch = req.num_computed_tokens + num_scheduled_tokens
@@ -491,36 +556,52 @@ class OffloadingConnectorScheduler:
)
dst_spec = store_output.store_spec
reqs_to_store[req_id] = (src_spec, dst_spec)
self._reqs_being_stored[req_id] |= keys_to_store
job_id = self._generate_job_id()
# a store can only be issued when no load is pending.
if req_status.transfer_jobs:
any_jid = next(iter(req_status.transfer_jobs))
assert self._jobs[any_jid].is_store
req_status.transfer_jobs.add(job_id)
self._jobs[job_id] = TransferJobStatus(
req_id=req_id,
pending_count=self.config.num_workers,
keys=set(keys_to_store),
is_store=True,
gpu_block_ids=src_block_ids,
)
store_jobs[job_id] = TransferJob(
req_id=req_id, transfer_spec=(src_spec, dst_spec)
)
logger.debug(
"Request %s offloading %s blocks upto %d tokens",
"Request %s offloading %s blocks upto %d tokens (job %d)",
req_id,
len(keys_to_store),
num_offloadable_tokens,
job_id,
)
return reqs_to_store
return store_jobs
def build_connector_meta(
self, scheduler_output: SchedulerOutput
) -> KVConnectorMetadata:
meta = OffloadingConnectorMetadata(
reqs_to_load=self._reqs_to_load,
reqs_to_store=self._get_reqs_to_store(scheduler_output),
reqs_to_flush=scheduler_output.preempted_req_ids,
)
self._reqs_to_load = {}
# NOTE (orozery): we should move this logic to update_connector_output
# once KVConnectorOutput allows us to report completed transfers
for req_id in scheduler_output.preempted_req_ids or ():
keys = self._reqs_being_stored.get(req_id)
if keys:
self.manager.complete_store(keys)
keys.clear()
req_status = self._req_status.get(req_id)
if req_status is None or not req_status.transfer_jobs:
continue
any_jid = next(iter(req_status.transfer_jobs))
assert self._jobs[any_jid].is_store
self._current_batch_jobs_to_flush.update(req_status.transfer_jobs)
meta = OffloadingConnectorMetadata(
load_jobs=self._current_batch_load_jobs,
store_jobs=self._build_store_jobs(scheduler_output),
jobs_to_flush=self._current_batch_jobs_to_flush,
)
self._current_batch_load_jobs = {}
self._current_batch_jobs_to_flush = set()
return meta
def update_connector_output(self, connector_output: KVConnectorOutput):
@@ -531,17 +612,37 @@ class OffloadingConnectorScheduler:
connector_output (KVConnectorOutput): the worker-side
connectors output.
"""
for req_id in connector_output.finished_sending or []:
keys = self._reqs_being_stored.pop(req_id, None)
if keys:
self.manager.complete_store(keys)
meta = connector_output.kv_connector_worker_meta
if not isinstance(meta, OffloadingWorkerMetadata):
assert meta is None
meta = OffloadingWorkerMetadata()
for job_id, count in meta.completed_jobs.items():
assert count > 0
job_status = self._jobs[job_id]
job_status.pending_count -= count
if job_status.pending_count > 0:
continue
assert job_status.pending_count == 0
for req_id in connector_output.finished_recving or []:
keys = self._reqs_being_loaded.pop(req_id, None)
if keys:
if job_status.is_store:
self.manager.complete_store(job_status.keys)
else:
self.manager.complete_load(job_status.keys)
if self._blocks_being_loaded:
self._blocks_being_loaded.difference_update(keys)
self.manager.complete_load(keys)
self._blocks_being_loaded.difference_update(job_status.keys)
req_status = self._req_status[job_status.req_id]
if self._block_id_to_pending_jobs and req_status.req.is_finished():
for bid in job_status.gpu_block_ids or ():
pending = self._block_id_to_pending_jobs[bid]
pending.remove(job_id)
if not pending:
del self._block_id_to_pending_jobs[bid]
del self._jobs[job_id]
req_status.transfer_jobs.remove(job_id)
if not req_status.transfer_jobs and req_status.req.is_finished():
del self._req_status[job_status.req_id]
def request_finished(
self,
@@ -558,14 +659,21 @@ class OffloadingConnectorScheduler:
Optional KVTransferParams to be included in the request outputs
returned by the engine.
"""
req_id = request.request_id
# TODO(orozery): possibly kickoff offload for last block
# which may have been deferred due to async scheduling
self._req_status.pop(req_id, None)
request_being_stored = req_id in self._reqs_being_stored
return request_being_stored, None
req_status = self._req_status.get(request.request_id)
if req_status is None:
return False, None
if not req_status.transfer_jobs:
del self._req_status[request.request_id]
return False, None
# Pending stores will outlive the request's block ownership.
# Register them so future block reuse triggers a flush.
for job_id in req_status.transfer_jobs:
job_status = self._jobs[job_id]
for bid in job_status.gpu_block_ids or ():
self._block_id_to_pending_jobs.setdefault(bid, set()).add(job_id)
return False, None
def take_events(self) -> Iterable[KVCacheEvent]:
"""Take the KV cache events from the connector.
@@ -11,6 +11,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.metrics import (
)
from vllm.distributed.kv_transfer.kv_connector.v1.offloading.common import (
OffloadingConnectorMetadata,
OffloadingWorkerMetadata,
ReqId,
)
from vllm.distributed.kv_transfer.kv_connector.v1.offloading.metrics import (
@@ -45,24 +46,11 @@ class OffloadingConnectorWorker:
self.spec = spec
self.worker = OffloadingWorker()
self._job_counter = 0
self.kv_connector_stats = OffloadingConnectorStats()
# req_id -> (job_id, store)
self._jobs: dict[int, tuple[ReqId, bool]] = {}
# req_id -> active job IDs
self._load_job: dict[ReqId, int] = {}
# req_id -> set(active job IDs)
self._store_jobs = defaultdict[ReqId, set[int]](set)
# list of store jobs pending submission (job_id, transfer_spec)
# job_id -> req_id for in-flight loads.
self._load_jobs: dict[int, ReqId] = {}
self._unsubmitted_store_jobs: list[tuple[int, TransferSpec]] = []
self._finished_reqs_waiting_for_store: set[ReqId] = set()
def _generate_job_id(self) -> int:
job_id = self._job_counter
self._job_counter = job_id + 1
return job_id
self._connector_worker_meta = OffloadingWorkerMetadata()
def _register_handlers(self, kv_caches: CanonicalKVCaches):
for src_cls, dst_cls, handler in self.spec.get_handlers(kv_caches):
@@ -301,10 +289,8 @@ class OffloadingConnectorWorker:
assert success
self._unsubmitted_store_jobs.clear()
for req_id in kv_connector_metadata.reqs_to_flush or ():
job_ids = self._store_jobs.get(req_id)
if job_ids:
self.worker.wait(job_ids)
if kv_connector_metadata.jobs_to_flush:
self.worker.wait(kv_connector_metadata.jobs_to_flush)
def start_kv_transfers(self, metadata: OffloadingConnectorMetadata):
for job_id, transfer_spec in self._unsubmitted_store_jobs:
@@ -312,41 +298,33 @@ class OffloadingConnectorWorker:
assert success
self._unsubmitted_store_jobs.clear()
for req_id, transfer_spec in metadata.reqs_to_load.items():
job_id = self._generate_job_id()
self._jobs[job_id] = (req_id, False)
assert req_id not in self._load_job
self._load_job[req_id] = job_id
success = self.worker.transfer_async(job_id, transfer_spec)
for job_id, entry in metadata.load_jobs.items():
self._load_jobs[job_id] = entry.req_id
success = self.worker.transfer_async(job_id, entry.transfer_spec)
assert success
def prepare_store_kv(self, metadata: OffloadingConnectorMetadata):
for req_id, transfer_spec in metadata.reqs_to_store.items():
job_id = self._generate_job_id()
self._jobs[job_id] = (req_id, True)
self._store_jobs[req_id].add(job_id)
# NOTE(orozery): defer the store to the beginning of the next engine step,
# so that offloading starts AFTER transfers related to token sampling,
# thereby avoiding delays to token generation due to offloading.
self._unsubmitted_store_jobs.append((job_id, transfer_spec))
for job_id, entry in metadata.store_jobs.items():
# NOTE(orozery): defer the store to the beginning of the next
# engine step, so that offloading starts AFTER transfers related
# to token sampling, thereby avoiding delays to token generation.
self._unsubmitted_store_jobs.append((job_id, entry.transfer_spec))
def get_finished(self, finished_req_ids: set[str]) -> tuple[set[str], set[str]]:
"""
Notifies worker-side connector ids of requests that have
finished generating tokens.
Returns a list of request IDs that finished loading or storing.
Returns:
ids of requests that have finished asynchronous transfer
tuple of (sending/saving ids, recving/loading ids).
tuple of (finished_sending, finished_recving). Stores never
emit finished_sending — the scheduler tracks store completion
via kv_connector_worker_meta.completed_jobs and fences any
block reuse via jobs_to_flush. Loads still emit
finished_recving so the base scheduler can resume requests
blocked on remote KV (and free aborted-during-load reqs).
"""
finished_sending = set()
finished_recving = set()
finished_recving: set[str] = set()
for transfer_result in self.worker.get_finished():
# we currently do not support job failures
job_id = transfer_result.job_id
assert transfer_result.success
req_id, store = self._jobs.pop(job_id)
if (
transfer_result.transfer_time
and transfer_result.transfer_size is not None
@@ -357,31 +335,21 @@ class OffloadingConnectorWorker:
time=transfer_result.transfer_time,
transfer_type=transfer_result.transfer_type,
)
if store:
req_jobs = self._store_jobs[req_id]
req_jobs.remove(job_id)
if req_jobs:
continue
if req_id in self._finished_reqs_waiting_for_store:
self._finished_reqs_waiting_for_store.remove(req_id)
finished_sending.add(req_id)
del self._store_jobs[req_id]
else:
req_job = self._load_job[req_id]
assert job_id == req_job
del self._load_job[req_id]
self._connector_worker_meta.mark_completed(job_id)
req_id = self._load_jobs.pop(job_id, None)
if req_id is not None:
finished_recving.add(req_id)
for req_id in finished_req_ids:
pending_req_jobs = self._store_jobs.get(req_id)
if pending_req_jobs:
self._finished_reqs_waiting_for_store.add(req_id)
elif pending_req_jobs is not None:
finished_sending.add(req_id)
del self._store_jobs[req_id]
return set(), finished_recving
return finished_sending, finished_recving
def build_connector_worker_meta(self) -> OffloadingWorkerMetadata | None:
"""Return completed transfer job IDs since the last call."""
if not self._connector_worker_meta.completed_jobs:
return None
meta = self._connector_worker_meta
self._connector_worker_meta = OffloadingWorkerMetadata()
return meta
def get_kv_connector_stats(self) -> KVConnectorStats | None:
"""
@@ -396,11 +364,7 @@ class OffloadingConnectorWorker:
return kv_connector_stats
def shutdown(self) -> None:
# Drop deferred store jobs: there is no point in submitting
# them during shutdown.
self._unsubmitted_store_jobs.clear()
self._jobs.clear()
self._load_job.clear()
self._store_jobs.clear()
self._finished_reqs_waiting_for_store.clear()
self._load_jobs.clear()
self._connector_worker_meta = OffloadingWorkerMetadata()
self.worker.shutdown()
@@ -20,6 +20,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.metrics import (
)
from vllm.distributed.kv_transfer.kv_connector.v1.offloading.common import (
OffloadingConnectorMetadata,
OffloadingWorkerMetadata,
)
from vllm.distributed.kv_transfer.kv_connector.v1.offloading.metrics import (
OffloadingConnectorStats,
@@ -111,6 +112,11 @@ class OffloadingConnector(KVConnectorBase_V1):
assert self.connector_worker is not None
return self.connector_worker.get_finished(finished_req_ids)
def build_connector_worker_meta(self) -> OffloadingWorkerMetadata | None:
if self.connector_worker is not None:
return self.connector_worker.build_connector_worker_meta()
return None
def get_num_new_matched_tokens(
self, request: "Request", num_computed_tokens: int
) -> tuple[int | None, bool]: