[KVConnector] OffloadingConnector: Fix bug in handling of preemptions (#29870)
Signed-off-by: Or Ozeri <oro@il.ibm.com>
This commit is contained in:
@@ -25,6 +25,9 @@ The class provides the following primitives:
|
||||
|
||||
Worker-side: runs in each worker, loads/saves KV cache to/from
|
||||
the Connector based on the metadata.
|
||||
handle_preemptions() - called if there are preempted requests,
|
||||
before their blocks are overwritten
|
||||
|
||||
start_load_kv() - starts loading all KVs (maybe async)
|
||||
wait_for_layer_load() - blocks until layer i load is done
|
||||
|
||||
@@ -262,6 +265,13 @@ class KVConnectorBase_V1(ABC):
|
||||
"""
|
||||
return
|
||||
|
||||
def handle_preemptions(self, preempted_req_ids: set[str]):
|
||||
"""
|
||||
Handle preempted requests BEFORE their blocks are overwritten.
|
||||
Needed for connectors which use async saves (e.g., OffloadingConnector)
|
||||
"""
|
||||
return
|
||||
|
||||
@abstractmethod
|
||||
def start_load_kv(self, forward_context: "ForwardContext", **kwargs: Any) -> None:
|
||||
"""
|
||||
|
||||
@@ -75,6 +75,10 @@ class OffloadingConnector(KVConnectorBase_V1):
|
||||
assert self.connector_worker is not None
|
||||
self.connector_worker.register_cross_layers_kv_cache(kv_cache, attn_backend)
|
||||
|
||||
def handle_preemptions(self, preempted_req_ids: set[str]):
|
||||
assert self.connector_worker is not None
|
||||
self.connector_worker.handle_preemptions(preempted_req_ids)
|
||||
|
||||
def start_load_kv(self, forward_context: "ForwardContext", **kwargs) -> None:
|
||||
assert self.connector_worker is not None
|
||||
assert isinstance(self._connector_metadata, OffloadingConnectorMetadata)
|
||||
@@ -348,6 +352,15 @@ class OffloadingConnectorScheduler:
|
||||
reqs_to_store=self._get_reqs_to_store(scheduler_output),
|
||||
)
|
||||
self._reqs_to_load = {}
|
||||
|
||||
# NOTE (orozery): we should move this logic to update_connector_output
|
||||
# once KVConnectorOutput allows us to report completed transfers
|
||||
for req_id in scheduler_output.preempted_req_ids or ():
|
||||
block_hashes = self._reqs_being_stored.get(req_id)
|
||||
if block_hashes:
|
||||
self.manager.complete_store(block_hashes)
|
||||
block_hashes.clear()
|
||||
|
||||
return meta
|
||||
|
||||
def update_connector_output(self, connector_output: KVConnectorOutput):
|
||||
@@ -466,6 +479,17 @@ class OffloadingConnectorWorker:
|
||||
attn_backends = {cross_layer_name: attn_backend}
|
||||
self._register_handlers(kv_caches, attn_backends)
|
||||
|
||||
def handle_preemptions(self, preempted_req_ids: set[str]):
|
||||
for job_id, transfer_spec in self._unsubmitted_store_jobs:
|
||||
success = self.worker.transfer_async(job_id, transfer_spec)
|
||||
assert success
|
||||
self._unsubmitted_store_jobs.clear()
|
||||
|
||||
for req_id in preempted_req_ids:
|
||||
job_ids = self._store_jobs.get(req_id)
|
||||
if job_ids:
|
||||
self.worker.wait(job_ids)
|
||||
|
||||
def start_kv_transfers(self, metadata: OffloadingConnectorMetadata):
|
||||
for job_id, transfer_spec in self._unsubmitted_store_jobs:
|
||||
success = self.worker.transfer_async(job_id, transfer_spec)
|
||||
|
||||
@@ -96,6 +96,8 @@ class SingleDirectionOffloadingHandler(OffloadingHandler):
|
||||
assert len(src_tensors) > 0
|
||||
self.gpu_to_cpu: bool = self.src_tensors[0].is_cuda
|
||||
|
||||
# job_id -> event
|
||||
self._transfer_events: dict[int, torch.Event] = {}
|
||||
# queue of transfers (job_id, stream, event)
|
||||
self._transfers: deque[tuple[int, torch.cuda.Stream, torch.Event]] = deque()
|
||||
# list of CUDA streams available for re-use
|
||||
@@ -152,6 +154,7 @@ class SingleDirectionOffloadingHandler(OffloadingHandler):
|
||||
ops.swap_blocks(src_tensor, dst_tensor, src_to_dst_tensor)
|
||||
event.record(stream)
|
||||
|
||||
self._transfer_events[job_id] = event
|
||||
self._transfers.append((job_id, stream, event))
|
||||
|
||||
# success
|
||||
@@ -164,8 +167,15 @@ class SingleDirectionOffloadingHandler(OffloadingHandler):
|
||||
results.append((job_id, True))
|
||||
self._stream_pool.append(stream)
|
||||
self._event_pool.append(event)
|
||||
del self._transfer_events[job_id]
|
||||
return results
|
||||
|
||||
def wait(self, job_ids: set[int]):
|
||||
for job_id in job_ids:
|
||||
event = self._transfer_events.get(job_id)
|
||||
if event is not None:
|
||||
event.synchronize()
|
||||
|
||||
|
||||
class CpuGpuOffloadingHandlers:
|
||||
def __init__(
|
||||
|
||||
@@ -53,6 +53,15 @@ class OffloadingHandler(ABC):
|
||||
"""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def wait(self, job_ids: set[int]) -> None:
|
||||
"""
|
||||
Wait for jobs to finish (blocking).
|
||||
|
||||
Args:
|
||||
job_ids: The set of job IDs to wait for.
|
||||
"""
|
||||
|
||||
|
||||
class OffloadingWorker:
|
||||
"""
|
||||
@@ -142,3 +151,13 @@ class OffloadingWorker:
|
||||
for handler in self.handlers:
|
||||
finished.extend(handler.get_finished())
|
||||
return finished
|
||||
|
||||
def wait(self, job_ids: set[int]) -> None:
|
||||
"""
|
||||
Wait for jobs to finish (blocking).
|
||||
|
||||
Args:
|
||||
job_ids: The set of job IDs to wait for.
|
||||
"""
|
||||
for handler in self.handlers:
|
||||
handler.wait(job_ids)
|
||||
|
||||
@@ -3112,6 +3112,11 @@ class GPUModelRunner(
|
||||
"after execute_model() returns None."
|
||||
)
|
||||
|
||||
if scheduler_output.preempted_req_ids and has_kv_transfer_group():
|
||||
get_kv_transfer_group().handle_preemptions(
|
||||
scheduler_output.preempted_req_ids
|
||||
)
|
||||
|
||||
num_scheduled_tokens = scheduler_output.total_num_scheduled_tokens
|
||||
with (
|
||||
record_function_or_nullcontext("gpu_model_runner: preprocess"),
|
||||
|
||||
Reference in New Issue
Block a user