[kv_offload+HMA][0/N]: Support block-level preemption handling (#34805)
Signed-off-by: Or Ozeri <oro@il.ibm.com> Co-authored-by: Nicolò Lucchesi <nlucches@redhat.com>
This commit is contained in:
co-authored by
Nicolò Lucchesi
parent
86b7e3c95a
commit
fcf0687b27
@@ -25,8 +25,8 @@ The class provides the following primitives:
|
||||
|
||||
Worker-side: runs in each worker, loads/saves KV cache to/from
|
||||
the Connector based on the metadata.
|
||||
handle_preemptions() - called if there are preempted requests,
|
||||
before their blocks are overwritten
|
||||
handle_preemptions() - called for handling preempted requests
|
||||
or request evicted blocks before they are overwritten
|
||||
|
||||
start_load_kv() - starts loading all KVs (maybe async)
|
||||
wait_for_layer_load() - blocks until layer i load is done
|
||||
@@ -288,9 +288,9 @@ class KVConnectorBase_V1(ABC):
|
||||
"""
|
||||
return
|
||||
|
||||
def handle_preemptions(self, preempted_req_ids: set[str]):
|
||||
def handle_preemptions(self, kv_connector_metadata: KVConnectorMetadata):
|
||||
"""
|
||||
Handle preempted requests BEFORE their blocks are overwritten.
|
||||
Handle preempted requests or evicted blocks BEFORE they are overwritten.
|
||||
Needed for connectors which use async saves (e.g., OffloadingConnector)
|
||||
"""
|
||||
return
|
||||
|
||||
@@ -315,10 +315,11 @@ class MultiConnector(KVConnectorBase_V1):
|
||||
for c in self._connectors:
|
||||
c.set_host_xfer_buffer_ops(copy_operation)
|
||||
|
||||
def handle_preemptions(self, preempted_req_ids: set[str]):
|
||||
def handle_preemptions(self, kv_connector_metadata: KVConnectorMetadata):
|
||||
"""Handle preempted requests for all sub-connectors."""
|
||||
for c in self._connectors:
|
||||
c.handle_preemptions(preempted_req_ids)
|
||||
assert isinstance(kv_connector_metadata, MultiKVConnectorMetadata)
|
||||
for c, cm in zip(self._connectors, kv_connector_metadata.metadata):
|
||||
c.handle_preemptions(cm)
|
||||
|
||||
def get_finished_count(self) -> int | None:
|
||||
# TODO(https://github.com/vllm-project/vllm/issues/33400)
|
||||
|
||||
@@ -111,6 +111,7 @@ class OffloadingConnectorStats(KVConnectorStats):
|
||||
class OffloadingConnectorMetadata(KVConnectorMetadata):
|
||||
reqs_to_load: dict[ReqId, TransferSpec]
|
||||
reqs_to_store: dict[ReqId, TransferSpec]
|
||||
reqs_to_flush: set[str] | None = None
|
||||
|
||||
|
||||
class OffloadingConnector(KVConnectorBase_V1):
|
||||
@@ -146,9 +147,10 @@ class OffloadingConnector(KVConnectorBase_V1):
|
||||
assert self.connector_worker is not None
|
||||
self.connector_worker.register_cross_layers_kv_cache(kv_cache, attn_backend)
|
||||
|
||||
def handle_preemptions(self, preempted_req_ids: set[str]):
|
||||
def handle_preemptions(self, kv_connector_metadata: KVConnectorMetadata):
|
||||
assert self.connector_worker is not None
|
||||
self.connector_worker.handle_preemptions(preempted_req_ids)
|
||||
assert isinstance(kv_connector_metadata, OffloadingConnectorMetadata)
|
||||
self.connector_worker.handle_preemptions(kv_connector_metadata)
|
||||
|
||||
def start_load_kv(self, forward_context: "ForwardContext", **kwargs) -> None:
|
||||
assert self.connector_worker is not None
|
||||
@@ -482,6 +484,7 @@ class OffloadingConnectorScheduler:
|
||||
meta = OffloadingConnectorMetadata(
|
||||
reqs_to_load=self._reqs_to_load,
|
||||
reqs_to_store=self._get_reqs_to_store(scheduler_output),
|
||||
reqs_to_flush=scheduler_output.preempted_req_ids,
|
||||
)
|
||||
self._reqs_to_load = {}
|
||||
|
||||
@@ -619,13 +622,13 @@ class OffloadingConnectorWorker:
|
||||
attn_backends = {cross_layer_name: attn_backend}
|
||||
self._register_handlers(kv_caches, attn_backends)
|
||||
|
||||
def handle_preemptions(self, preempted_req_ids: set[str]):
|
||||
def handle_preemptions(self, kv_connector_metadata: OffloadingConnectorMetadata):
|
||||
for job_id, transfer_spec in self._unsubmitted_store_jobs:
|
||||
success = self.worker.transfer_async(job_id, transfer_spec)
|
||||
assert success
|
||||
self._unsubmitted_store_jobs.clear()
|
||||
|
||||
for req_id in preempted_req_ids:
|
||||
for req_id in kv_connector_metadata.reqs_to_flush or ():
|
||||
job_ids = self._store_jobs.get(req_id)
|
||||
if job_ids:
|
||||
self.worker.wait(job_ids)
|
||||
|
||||
Reference in New Issue
Block a user