[KV Offload] Per-job store completion for CPU offloading connector (#39186)
Signed-off-by: Itay Etelis <itay.etelis@ibm.com> Signed-off-by: Itay Etelis <92247226+Etelis@users.noreply.github.com> Co-authored-by: Itay Etelis <itay.etelis@ibm.com> Co-authored-by: Or Ozeri <or@ozery.com> Co-authored-by: Or Ozeri <oro@il.ibm.com>
This commit is contained in:
co-authored by
Itay Etelis
Or Ozeri
Or Ozeri
parent
a269744e9f
commit
8a8c9b564e
@@ -1,15 +1,60 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from dataclasses import dataclass
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.base import KVConnectorMetadata
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.base import (
|
||||
KVConnectorMetadata,
|
||||
KVConnectorWorkerMetadata,
|
||||
)
|
||||
from vllm.v1.kv_offload.worker.worker import TransferSpec
|
||||
|
||||
ReqId = str
|
||||
|
||||
|
||||
@dataclass
|
||||
class TransferJob:
|
||||
"""A transfer job bundling request context with transfer spec.
|
||||
|
||||
Used for both loads and stores, keyed by scheduler-assigned job ID.
|
||||
The worker reports the job ID back when the transfer finishes,
|
||||
and the scheduler processes the completion.
|
||||
"""
|
||||
|
||||
req_id: ReqId
|
||||
transfer_spec: TransferSpec
|
||||
|
||||
|
||||
@dataclass
|
||||
class OffloadingConnectorMetadata(KVConnectorMetadata):
|
||||
reqs_to_load: dict[ReqId, TransferSpec]
|
||||
reqs_to_store: dict[ReqId, TransferSpec]
|
||||
reqs_to_flush: set[str] | None = None
|
||||
# Keyed by scheduler-assigned job IDs.
|
||||
load_jobs: dict[int, TransferJob]
|
||||
store_jobs: dict[int, TransferJob]
|
||||
jobs_to_flush: set[int] | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class OffloadingWorkerMetadata(KVConnectorWorkerMetadata):
|
||||
"""Worker -> Scheduler metadata for completed transfer jobs.
|
||||
|
||||
Each worker reports {job_id: 1} for newly completed transfer jobs
|
||||
(load or store). aggregate() sums counts across workers within a step.
|
||||
The scheduler accumulates across steps and processes
|
||||
a transfer completion only when count reaches num_workers.
|
||||
"""
|
||||
|
||||
completed_jobs: dict[int, int] = field(default_factory=dict)
|
||||
|
||||
def mark_completed(self, job_id: int) -> None:
|
||||
"""Record a transfer job completion from this worker."""
|
||||
self.completed_jobs[job_id] = 1
|
||||
|
||||
def aggregate(
|
||||
self, other: "KVConnectorWorkerMetadata"
|
||||
) -> "KVConnectorWorkerMetadata":
|
||||
assert isinstance(other, OffloadingWorkerMetadata)
|
||||
|
||||
merged = dict(self.completed_jobs)
|
||||
for job_id, v in other.completed_jobs.items():
|
||||
merged[job_id] = merged.get(job_id, 0) + v
|
||||
|
||||
return OffloadingWorkerMetadata(completed_jobs=merged)
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from collections import defaultdict
|
||||
from collections.abc import Iterable, Sequence
|
||||
from dataclasses import dataclass, field
|
||||
from itertools import islice
|
||||
@@ -11,7 +10,9 @@ from vllm.distributed.kv_transfer.kv_connector.utils import yield_req_data
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.base import KVConnectorMetadata
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.offloading.common import (
|
||||
OffloadingConnectorMetadata,
|
||||
OffloadingWorkerMetadata,
|
||||
ReqId,
|
||||
TransferJob,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils.math_utils import cdiv
|
||||
@@ -26,13 +27,27 @@ from vllm.v1.kv_offload.abstract import (
|
||||
)
|
||||
from vllm.v1.kv_offload.mediums import GPULoadStoreSpec
|
||||
from vllm.v1.kv_offload.spec import OffloadingSpec
|
||||
from vllm.v1.kv_offload.worker.worker import TransferSpec
|
||||
from vllm.v1.outputs import KVConnectorOutput
|
||||
from vllm.v1.request import Request
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class TransferJobStatus:
|
||||
"""Tracks scheduler-side state for a single transfer job."""
|
||||
|
||||
req_id: ReqId
|
||||
# Number of workers still pending. Starts at num_workers,
|
||||
# decremented as each worker reports completion. Job is done at 0.
|
||||
pending_count: int
|
||||
# Offload keys this job covers; passed to manager.complete_*().
|
||||
keys: set[OffloadKey]
|
||||
is_store: bool
|
||||
# GPU blocks the fence tracks. Store src blocks; None for loads.
|
||||
gpu_block_ids: list[int] | None = None
|
||||
|
||||
|
||||
class GroupOffloadConfig(NamedTuple):
|
||||
group_idx: int
|
||||
gpu_block_size: int
|
||||
@@ -43,10 +58,12 @@ class GroupOffloadConfig(NamedTuple):
|
||||
class SchedulerOffloadConfig(NamedTuple):
|
||||
kv_group_configs: tuple[GroupOffloadConfig, ...]
|
||||
block_size_factor: int
|
||||
num_workers: int
|
||||
|
||||
@classmethod
|
||||
def from_spec(cls, spec: OffloadingSpec) -> "SchedulerOffloadConfig":
|
||||
return cls(
|
||||
num_workers=spec.vllm_config.parallel_config.world_size,
|
||||
kv_group_configs=tuple(
|
||||
GroupOffloadConfig(
|
||||
group_idx=idx,
|
||||
@@ -79,6 +96,9 @@ class RequestOffloadState:
|
||||
req_context: ReqContext = field(init=False)
|
||||
# number of hits in the GPU cache
|
||||
num_locally_computed_tokens: int = 0
|
||||
# In-flight job IDs. Per the connector's invariant, at any given time
|
||||
# this contains either a single load job, or one or more store jobs.
|
||||
transfer_jobs: set[int] = field(default_factory=set)
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
self.group_states = tuple(
|
||||
@@ -135,17 +155,26 @@ class OffloadingConnectorScheduler:
|
||||
self.lookup_groups = attention_groups
|
||||
|
||||
self._req_status: dict[ReqId, RequestOffloadState] = {}
|
||||
# requests to load for the current scheduler step
|
||||
self._reqs_to_load: dict[ReqId, TransferSpec] = {}
|
||||
self._current_batch_load_jobs: dict[int, TransferJob] = {}
|
||||
self._current_batch_jobs_to_flush: set[int] = set()
|
||||
# if GPU prefix caching is enabled,
|
||||
# track loaded blocks to avoid redundant loads
|
||||
self._blocks_being_loaded: set[OffloadKey] | None = (
|
||||
set() if spec.vllm_config.cache_config.enable_prefix_caching else None
|
||||
)
|
||||
|
||||
# request ID -> set(offload keys being stored/loaded)
|
||||
self._reqs_being_stored = defaultdict[ReqId, set[OffloadKey]](set)
|
||||
self._reqs_being_loaded = defaultdict[ReqId, set[OffloadKey]](set)
|
||||
# Job ID counter shared by loads and stores.
|
||||
self._job_counter: int = 0
|
||||
self._jobs: dict[int, TransferJobStatus] = {}
|
||||
|
||||
# block_id -> pending store job_ids. Populated only for finished
|
||||
# requests (running-request blocks are protected by their ref_cnt).
|
||||
self._block_id_to_pending_jobs: dict[int, set[int]] = {}
|
||||
|
||||
def _generate_job_id(self) -> int:
|
||||
job_id = self._job_counter
|
||||
self._job_counter += 1
|
||||
return job_id
|
||||
|
||||
def _maximal_prefix_lookup(
|
||||
self, keys: Iterable[OffloadKey], req_context: ReqContext
|
||||
@@ -369,23 +398,46 @@ class OffloadingConnectorScheduler:
|
||||
# entire KV cache so a remote decode node can consume it.
|
||||
group_state.next_stored_block_idx = num_blocks
|
||||
|
||||
# Fence dst blocks against finished-request pending stores.
|
||||
if (
|
||||
self._block_id_to_pending_jobs
|
||||
and not self._block_id_to_pending_jobs.keys().isdisjoint(dst_block_ids)
|
||||
):
|
||||
self._current_batch_jobs_to_flush.update(
|
||||
jid
|
||||
for bid in dst_block_ids
|
||||
for jid in self._block_id_to_pending_jobs.get(bid, ())
|
||||
)
|
||||
|
||||
src_spec = self.manager.prepare_load(keys_to_load, req_status.req_context)
|
||||
dst_spec = GPULoadStoreSpec(
|
||||
dst_block_ids, group_sizes=group_sizes, block_indices=block_indices
|
||||
)
|
||||
|
||||
self._reqs_to_load[request.request_id] = (src_spec, dst_spec)
|
||||
req_blocks_being_loaded = self._reqs_being_loaded[request.request_id]
|
||||
req_blocks_being_loaded.update(keys_to_load)
|
||||
load_job_id = self._generate_job_id()
|
||||
self._current_batch_load_jobs[load_job_id] = TransferJob(
|
||||
req_id=request.request_id,
|
||||
transfer_spec=(src_spec, dst_spec),
|
||||
)
|
||||
# a load can only be issued when no other jobs are pending.
|
||||
assert not req_status.transfer_jobs
|
||||
req_status.transfer_jobs.add(load_job_id)
|
||||
self._jobs[load_job_id] = TransferJobStatus(
|
||||
req_id=request.request_id,
|
||||
pending_count=self.config.num_workers,
|
||||
keys=set(keys_to_load),
|
||||
is_store=False,
|
||||
)
|
||||
|
||||
if self._blocks_being_loaded is not None:
|
||||
self._blocks_being_loaded.update(req_blocks_being_loaded)
|
||||
self._blocks_being_loaded.update(keys_to_load)
|
||||
|
||||
def _get_reqs_to_store(
|
||||
self, scheduler_output: SchedulerOutput
|
||||
) -> dict[ReqId, TransferSpec]:
|
||||
def _build_store_jobs(
|
||||
self,
|
||||
scheduler_output: SchedulerOutput,
|
||||
) -> dict[int, TransferJob]:
|
||||
block_size_factor = self.config.block_size_factor
|
||||
reqs_to_store: dict[ReqId, TransferSpec] = {}
|
||||
store_jobs: dict[int, TransferJob] = {}
|
||||
# iterate over both new and cached requests
|
||||
for req_id, new_block_id_groups, preempted in yield_req_data(scheduler_output):
|
||||
req_status = self._req_status[req_id]
|
||||
@@ -398,6 +450,19 @@ class OffloadingConnectorScheduler:
|
||||
|
||||
if new_block_id_groups:
|
||||
req_status.update_block_id_groups(new_block_id_groups)
|
||||
# Fence new blocks against in-flight stores.
|
||||
if self._block_id_to_pending_jobs:
|
||||
new_blocks_flat = [
|
||||
bid for new_blocks in new_block_id_groups for bid in new_blocks
|
||||
]
|
||||
if not self._block_id_to_pending_jobs.keys().isdisjoint(
|
||||
new_blocks_flat
|
||||
):
|
||||
self._current_batch_jobs_to_flush.update(
|
||||
jid
|
||||
for bid in new_blocks_flat
|
||||
for jid in self._block_id_to_pending_jobs.get(bid, ())
|
||||
)
|
||||
|
||||
num_scheduled_tokens = scheduler_output.num_scheduled_tokens[req_id]
|
||||
num_tokens_after_batch = req.num_computed_tokens + num_scheduled_tokens
|
||||
@@ -491,36 +556,52 @@ class OffloadingConnectorScheduler:
|
||||
)
|
||||
dst_spec = store_output.store_spec
|
||||
|
||||
reqs_to_store[req_id] = (src_spec, dst_spec)
|
||||
self._reqs_being_stored[req_id] |= keys_to_store
|
||||
job_id = self._generate_job_id()
|
||||
# a store can only be issued when no load is pending.
|
||||
if req_status.transfer_jobs:
|
||||
any_jid = next(iter(req_status.transfer_jobs))
|
||||
assert self._jobs[any_jid].is_store
|
||||
req_status.transfer_jobs.add(job_id)
|
||||
self._jobs[job_id] = TransferJobStatus(
|
||||
req_id=req_id,
|
||||
pending_count=self.config.num_workers,
|
||||
keys=set(keys_to_store),
|
||||
is_store=True,
|
||||
gpu_block_ids=src_block_ids,
|
||||
)
|
||||
|
||||
store_jobs[job_id] = TransferJob(
|
||||
req_id=req_id, transfer_spec=(src_spec, dst_spec)
|
||||
)
|
||||
|
||||
logger.debug(
|
||||
"Request %s offloading %s blocks upto %d tokens",
|
||||
"Request %s offloading %s blocks upto %d tokens (job %d)",
|
||||
req_id,
|
||||
len(keys_to_store),
|
||||
num_offloadable_tokens,
|
||||
job_id,
|
||||
)
|
||||
|
||||
return reqs_to_store
|
||||
return store_jobs
|
||||
|
||||
def build_connector_meta(
|
||||
self, scheduler_output: SchedulerOutput
|
||||
) -> KVConnectorMetadata:
|
||||
meta = OffloadingConnectorMetadata(
|
||||
reqs_to_load=self._reqs_to_load,
|
||||
reqs_to_store=self._get_reqs_to_store(scheduler_output),
|
||||
reqs_to_flush=scheduler_output.preempted_req_ids,
|
||||
)
|
||||
self._reqs_to_load = {}
|
||||
|
||||
# NOTE (orozery): we should move this logic to update_connector_output
|
||||
# once KVConnectorOutput allows us to report completed transfers
|
||||
for req_id in scheduler_output.preempted_req_ids or ():
|
||||
keys = self._reqs_being_stored.get(req_id)
|
||||
if keys:
|
||||
self.manager.complete_store(keys)
|
||||
keys.clear()
|
||||
req_status = self._req_status.get(req_id)
|
||||
if req_status is None or not req_status.transfer_jobs:
|
||||
continue
|
||||
any_jid = next(iter(req_status.transfer_jobs))
|
||||
assert self._jobs[any_jid].is_store
|
||||
self._current_batch_jobs_to_flush.update(req_status.transfer_jobs)
|
||||
|
||||
meta = OffloadingConnectorMetadata(
|
||||
load_jobs=self._current_batch_load_jobs,
|
||||
store_jobs=self._build_store_jobs(scheduler_output),
|
||||
jobs_to_flush=self._current_batch_jobs_to_flush,
|
||||
)
|
||||
self._current_batch_load_jobs = {}
|
||||
self._current_batch_jobs_to_flush = set()
|
||||
return meta
|
||||
|
||||
def update_connector_output(self, connector_output: KVConnectorOutput):
|
||||
@@ -531,17 +612,37 @@ class OffloadingConnectorScheduler:
|
||||
connector_output (KVConnectorOutput): the worker-side
|
||||
connectors output.
|
||||
"""
|
||||
for req_id in connector_output.finished_sending or []:
|
||||
keys = self._reqs_being_stored.pop(req_id, None)
|
||||
if keys:
|
||||
self.manager.complete_store(keys)
|
||||
meta = connector_output.kv_connector_worker_meta
|
||||
if not isinstance(meta, OffloadingWorkerMetadata):
|
||||
assert meta is None
|
||||
meta = OffloadingWorkerMetadata()
|
||||
for job_id, count in meta.completed_jobs.items():
|
||||
assert count > 0
|
||||
job_status = self._jobs[job_id]
|
||||
job_status.pending_count -= count
|
||||
if job_status.pending_count > 0:
|
||||
continue
|
||||
assert job_status.pending_count == 0
|
||||
|
||||
for req_id in connector_output.finished_recving or []:
|
||||
keys = self._reqs_being_loaded.pop(req_id, None)
|
||||
if keys:
|
||||
if job_status.is_store:
|
||||
self.manager.complete_store(job_status.keys)
|
||||
else:
|
||||
self.manager.complete_load(job_status.keys)
|
||||
if self._blocks_being_loaded:
|
||||
self._blocks_being_loaded.difference_update(keys)
|
||||
self.manager.complete_load(keys)
|
||||
self._blocks_being_loaded.difference_update(job_status.keys)
|
||||
|
||||
req_status = self._req_status[job_status.req_id]
|
||||
if self._block_id_to_pending_jobs and req_status.req.is_finished():
|
||||
for bid in job_status.gpu_block_ids or ():
|
||||
pending = self._block_id_to_pending_jobs[bid]
|
||||
pending.remove(job_id)
|
||||
if not pending:
|
||||
del self._block_id_to_pending_jobs[bid]
|
||||
|
||||
del self._jobs[job_id]
|
||||
req_status.transfer_jobs.remove(job_id)
|
||||
if not req_status.transfer_jobs and req_status.req.is_finished():
|
||||
del self._req_status[job_status.req_id]
|
||||
|
||||
def request_finished(
|
||||
self,
|
||||
@@ -558,14 +659,21 @@ class OffloadingConnectorScheduler:
|
||||
Optional KVTransferParams to be included in the request outputs
|
||||
returned by the engine.
|
||||
"""
|
||||
req_id = request.request_id
|
||||
|
||||
# TODO(orozery): possibly kickoff offload for last block
|
||||
# which may have been deferred due to async scheduling
|
||||
self._req_status.pop(req_id, None)
|
||||
|
||||
request_being_stored = req_id in self._reqs_being_stored
|
||||
return request_being_stored, None
|
||||
req_status = self._req_status.get(request.request_id)
|
||||
if req_status is None:
|
||||
return False, None
|
||||
if not req_status.transfer_jobs:
|
||||
del self._req_status[request.request_id]
|
||||
return False, None
|
||||
# Pending stores will outlive the request's block ownership.
|
||||
# Register them so future block reuse triggers a flush.
|
||||
for job_id in req_status.transfer_jobs:
|
||||
job_status = self._jobs[job_id]
|
||||
for bid in job_status.gpu_block_ids or ():
|
||||
self._block_id_to_pending_jobs.setdefault(bid, set()).add(job_id)
|
||||
return False, None
|
||||
|
||||
def take_events(self) -> Iterable[KVCacheEvent]:
|
||||
"""Take the KV cache events from the connector.
|
||||
|
||||
@@ -11,6 +11,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.metrics import (
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.offloading.common import (
|
||||
OffloadingConnectorMetadata,
|
||||
OffloadingWorkerMetadata,
|
||||
ReqId,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.offloading.metrics import (
|
||||
@@ -45,24 +46,11 @@ class OffloadingConnectorWorker:
|
||||
self.spec = spec
|
||||
self.worker = OffloadingWorker()
|
||||
|
||||
self._job_counter = 0
|
||||
|
||||
self.kv_connector_stats = OffloadingConnectorStats()
|
||||
# req_id -> (job_id, store)
|
||||
self._jobs: dict[int, tuple[ReqId, bool]] = {}
|
||||
# req_id -> active job IDs
|
||||
self._load_job: dict[ReqId, int] = {}
|
||||
# req_id -> set(active job IDs)
|
||||
self._store_jobs = defaultdict[ReqId, set[int]](set)
|
||||
# list of store jobs pending submission (job_id, transfer_spec)
|
||||
# job_id -> req_id for in-flight loads.
|
||||
self._load_jobs: dict[int, ReqId] = {}
|
||||
self._unsubmitted_store_jobs: list[tuple[int, TransferSpec]] = []
|
||||
|
||||
self._finished_reqs_waiting_for_store: set[ReqId] = set()
|
||||
|
||||
def _generate_job_id(self) -> int:
|
||||
job_id = self._job_counter
|
||||
self._job_counter = job_id + 1
|
||||
return job_id
|
||||
self._connector_worker_meta = OffloadingWorkerMetadata()
|
||||
|
||||
def _register_handlers(self, kv_caches: CanonicalKVCaches):
|
||||
for src_cls, dst_cls, handler in self.spec.get_handlers(kv_caches):
|
||||
@@ -301,10 +289,8 @@ class OffloadingConnectorWorker:
|
||||
assert success
|
||||
self._unsubmitted_store_jobs.clear()
|
||||
|
||||
for req_id in kv_connector_metadata.reqs_to_flush or ():
|
||||
job_ids = self._store_jobs.get(req_id)
|
||||
if job_ids:
|
||||
self.worker.wait(job_ids)
|
||||
if kv_connector_metadata.jobs_to_flush:
|
||||
self.worker.wait(kv_connector_metadata.jobs_to_flush)
|
||||
|
||||
def start_kv_transfers(self, metadata: OffloadingConnectorMetadata):
|
||||
for job_id, transfer_spec in self._unsubmitted_store_jobs:
|
||||
@@ -312,41 +298,33 @@ class OffloadingConnectorWorker:
|
||||
assert success
|
||||
self._unsubmitted_store_jobs.clear()
|
||||
|
||||
for req_id, transfer_spec in metadata.reqs_to_load.items():
|
||||
job_id = self._generate_job_id()
|
||||
self._jobs[job_id] = (req_id, False)
|
||||
assert req_id not in self._load_job
|
||||
self._load_job[req_id] = job_id
|
||||
success = self.worker.transfer_async(job_id, transfer_spec)
|
||||
for job_id, entry in metadata.load_jobs.items():
|
||||
self._load_jobs[job_id] = entry.req_id
|
||||
success = self.worker.transfer_async(job_id, entry.transfer_spec)
|
||||
assert success
|
||||
|
||||
def prepare_store_kv(self, metadata: OffloadingConnectorMetadata):
|
||||
for req_id, transfer_spec in metadata.reqs_to_store.items():
|
||||
job_id = self._generate_job_id()
|
||||
self._jobs[job_id] = (req_id, True)
|
||||
self._store_jobs[req_id].add(job_id)
|
||||
# NOTE(orozery): defer the store to the beginning of the next engine step,
|
||||
# so that offloading starts AFTER transfers related to token sampling,
|
||||
# thereby avoiding delays to token generation due to offloading.
|
||||
self._unsubmitted_store_jobs.append((job_id, transfer_spec))
|
||||
for job_id, entry in metadata.store_jobs.items():
|
||||
# NOTE(orozery): defer the store to the beginning of the next
|
||||
# engine step, so that offloading starts AFTER transfers related
|
||||
# to token sampling, thereby avoiding delays to token generation.
|
||||
self._unsubmitted_store_jobs.append((job_id, entry.transfer_spec))
|
||||
|
||||
def get_finished(self, finished_req_ids: set[str]) -> tuple[set[str], set[str]]:
|
||||
"""
|
||||
Notifies worker-side connector ids of requests that have
|
||||
finished generating tokens.
|
||||
Returns a list of request IDs that finished loading or storing.
|
||||
|
||||
Returns:
|
||||
ids of requests that have finished asynchronous transfer
|
||||
tuple of (sending/saving ids, recving/loading ids).
|
||||
tuple of (finished_sending, finished_recving). Stores never
|
||||
emit finished_sending — the scheduler tracks store completion
|
||||
via kv_connector_worker_meta.completed_jobs and fences any
|
||||
block reuse via jobs_to_flush. Loads still emit
|
||||
finished_recving so the base scheduler can resume requests
|
||||
blocked on remote KV (and free aborted-during-load reqs).
|
||||
"""
|
||||
finished_sending = set()
|
||||
finished_recving = set()
|
||||
finished_recving: set[str] = set()
|
||||
for transfer_result in self.worker.get_finished():
|
||||
# we currently do not support job failures
|
||||
job_id = transfer_result.job_id
|
||||
assert transfer_result.success
|
||||
req_id, store = self._jobs.pop(job_id)
|
||||
if (
|
||||
transfer_result.transfer_time
|
||||
and transfer_result.transfer_size is not None
|
||||
@@ -357,31 +335,21 @@ class OffloadingConnectorWorker:
|
||||
time=transfer_result.transfer_time,
|
||||
transfer_type=transfer_result.transfer_type,
|
||||
)
|
||||
if store:
|
||||
req_jobs = self._store_jobs[req_id]
|
||||
req_jobs.remove(job_id)
|
||||
if req_jobs:
|
||||
continue
|
||||
|
||||
if req_id in self._finished_reqs_waiting_for_store:
|
||||
self._finished_reqs_waiting_for_store.remove(req_id)
|
||||
finished_sending.add(req_id)
|
||||
del self._store_jobs[req_id]
|
||||
else:
|
||||
req_job = self._load_job[req_id]
|
||||
assert job_id == req_job
|
||||
del self._load_job[req_id]
|
||||
self._connector_worker_meta.mark_completed(job_id)
|
||||
req_id = self._load_jobs.pop(job_id, None)
|
||||
if req_id is not None:
|
||||
finished_recving.add(req_id)
|
||||
|
||||
for req_id in finished_req_ids:
|
||||
pending_req_jobs = self._store_jobs.get(req_id)
|
||||
if pending_req_jobs:
|
||||
self._finished_reqs_waiting_for_store.add(req_id)
|
||||
elif pending_req_jobs is not None:
|
||||
finished_sending.add(req_id)
|
||||
del self._store_jobs[req_id]
|
||||
return set(), finished_recving
|
||||
|
||||
return finished_sending, finished_recving
|
||||
def build_connector_worker_meta(self) -> OffloadingWorkerMetadata | None:
|
||||
"""Return completed transfer job IDs since the last call."""
|
||||
if not self._connector_worker_meta.completed_jobs:
|
||||
return None
|
||||
meta = self._connector_worker_meta
|
||||
self._connector_worker_meta = OffloadingWorkerMetadata()
|
||||
return meta
|
||||
|
||||
def get_kv_connector_stats(self) -> KVConnectorStats | None:
|
||||
"""
|
||||
@@ -396,11 +364,7 @@ class OffloadingConnectorWorker:
|
||||
return kv_connector_stats
|
||||
|
||||
def shutdown(self) -> None:
|
||||
# Drop deferred store jobs: there is no point in submitting
|
||||
# them during shutdown.
|
||||
self._unsubmitted_store_jobs.clear()
|
||||
self._jobs.clear()
|
||||
self._load_job.clear()
|
||||
self._store_jobs.clear()
|
||||
self._finished_reqs_waiting_for_store.clear()
|
||||
self._load_jobs.clear()
|
||||
self._connector_worker_meta = OffloadingWorkerMetadata()
|
||||
self.worker.shutdown()
|
||||
|
||||
@@ -20,6 +20,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.metrics import (
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.offloading.common import (
|
||||
OffloadingConnectorMetadata,
|
||||
OffloadingWorkerMetadata,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.offloading.metrics import (
|
||||
OffloadingConnectorStats,
|
||||
@@ -111,6 +112,11 @@ class OffloadingConnector(KVConnectorBase_V1):
|
||||
assert self.connector_worker is not None
|
||||
return self.connector_worker.get_finished(finished_req_ids)
|
||||
|
||||
def build_connector_worker_meta(self) -> OffloadingWorkerMetadata | None:
|
||||
if self.connector_worker is not None:
|
||||
return self.connector_worker.build_connector_worker_meta()
|
||||
return None
|
||||
|
||||
def get_num_new_matched_tokens(
|
||||
self, request: "Request", num_computed_tokens: int
|
||||
) -> tuple[int | None, bool]:
|
||||
|
||||
Reference in New Issue
Block a user