[V1] Exception Handling when Loading KV Cache from Remote Store (#21534)

Signed-off-by: liuyumoye <adeline_ly2023@outlook.com>
Co-authored-by: liuyumoye <adeline_ly2023@outlook.com>
This commit is contained in:
Adeline
2025-07-27 20:34:17 -07:00
committed by GitHub
co-authored by liuyumoye
parent 04ff4be310
commit 15a72ac478
10 changed files with 229 additions and 5 deletions
@@ -139,13 +139,27 @@ class KVOutputAggregator:
finished_set.add(req_id)
del remaining_count_dict[req_id]
def update_finished_load_dict(worker_finished_loading_dict: dict[str,
int],
finished_loading_dict: dict[str, int]):
for req_id, num_actual_load_tokens in (worker_finished_loading_dict
or {}).items():
if req_id in finished_loading_dict:
finished_loading_dict[req_id] = min(
finished_loading_dict[req_id], num_actual_load_tokens)
else:
finished_loading_dict[req_id] = num_actual_load_tokens
finished_sending = set[str]()
finished_recving = set[str]()
finished_loading_dict: dict[str, int] = {}
for output in outputs:
update_finished_set(output.finished_sending,
self._send_remaining_count, finished_sending)
update_finished_set(output.finished_recving,
self._recv_remaining_count, finished_recving)
update_finished_load_dict(output.finished_loading_dict,
finished_loading_dict)
# select output of the worker specified by output_rank
output = outputs[output_rank]
@@ -157,7 +171,7 @@ class KVOutputAggregator:
# send/recv
output.finished_sending = finished_sending if finished_sending else None
output.finished_recving = finished_recving if finished_recving else None
output.finished_loading_dict = finished_loading_dict or None
return output
def async_aggregate(self,
@@ -28,6 +28,9 @@ The class provides the following primitives:
get_finished() - called with ids of finished requests, returns
ids of requests that have completed async sending/recving.
get_finished_loading() - called with scheduler outputs, returns
a dictionary that the keys are request IDs and the values are
the actual number of tokens loaded from the remote KV cache
"""
import enum
@@ -219,6 +222,23 @@ class KVConnectorBase_V1(ABC):
"""
return None, None
def get_finished_loading(
self, scheduler_output: "SchedulerOutput") -> dict[str, int]:
"""
Retrieves the actual number of tokens loaded for requests that have
completed the asynchronous loading process from the remote KV cache.
This function is used by the scheduler process (via the Executors)
to track the progress of requests and determine which requests have
successfully finished loading their KV cache data.
Returns:
A dictionary where the keys are request IDs and the values are the
corresponding number of tokens that have been successfully loaded
for each request.
"""
return {}
# ==============================
# Scheduler-side methods
# ==============================
+2
View File
@@ -1167,6 +1167,8 @@ class IntermediateTensors:
# [req_ids]
finished_sending: Optional[set[str]] = None
finished_recving: Optional[set[str]] = None
#req_id -> num_actual_load_tokens
finished_loading_dict: Optional[dict[str, int]] = None
def __init__(self, tensors):
# manually define this function, so that
+30
View File
@@ -118,6 +118,9 @@ class Scheduler(SchedulerInterface):
# KV Connector: requests in process of async KV loading or recving
self.finished_recving_kv_req_ids: set[str] = set()
# The keys are request IDs, and the values are corresponding token
# count that have been successfully loaded from the remote KV store
self.finished_loading_dict: dict[str, int] = {}
# Encoder-related.
# Calculate encoder cache size if applicable
@@ -1094,6 +1097,27 @@ class Scheduler(SchedulerInterface):
(block_ids, ) = self.kv_cache_manager.get_block_ids(request.request_id)
return self.connector.request_finished(request, block_ids)
def _update_actual_load_token_num_from_remote_kv(self,
request: Request) -> bool:
num_actual_load_tokens = self.finished_loading_dict.pop(
request.request_id)
num_computed_tokens = num_actual_load_tokens
assert self.connector is not None
if num_actual_load_tokens <= 0 and hasattr(self.connector,
"add_failure_request"):
self.connector.add_failure_request(request)
return True
if num_actual_load_tokens == request.num_tokens:
num_computed_tokens -= 1
self.kv_cache_manager.cache_blocks(request, num_computed_tokens)
# Update the request state for scheduling.
request.num_computed_tokens = num_computed_tokens
return True
def _update_waiting_for_remote_kv(self, request: Request) -> bool:
"""
KV Connector: check if the request_id is finished_recving.
@@ -1107,6 +1131,9 @@ class Scheduler(SchedulerInterface):
WAITING_FOR_REMOTE_KV.
"""
assert self.connector is not None
if request.request_id in self.finished_loading_dict:
return self._update_actual_load_token_num_from_remote_kv(request)
if request.request_id not in self.finished_recving_kv_req_ids:
return False
@@ -1145,3 +1172,6 @@ class Scheduler(SchedulerInterface):
for req_id in (model_runner_output.finished_sending or ()):
logger.debug("Finished sending KV transfer for request %s", req_id)
self._free_blocks(self.requests[req_id])
if model_runner_output.finished_loading_dict:
self.finished_loading_dict.update(
model_runner_output.finished_loading_dict)
+3
View File
@@ -107,6 +107,8 @@ class ModelRunnerOutput:
# [req_ids]
finished_sending: Optional[set[str]] = None
finished_recving: Optional[set[str]] = None
# req_id -> actual_load_token from connector
finished_loading_dict: Optional[dict[str, int]] = None
# req_id -> num_nans_in_logits
num_nans_in_logits: Optional[dict[str, int]] = None
@@ -121,4 +123,5 @@ EMPTY_MODEL_RUNNER_OUTPUT = ModelRunnerOutput(req_ids=[],
pooler_output=[],
finished_sending=None,
finished_recving=None,
finished_loading_dict=None,
num_nans_in_logits=None)
+8 -2
View File
@@ -1350,6 +1350,7 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
num_scheduled_tokens_np: np.ndarray,
finished_sending: Optional[set[str]],
finished_recving: Optional[set[str]],
finished_loading_dict: Optional[dict[str, int]],
) -> ModelRunnerOutput:
assert self.input_batch.num_reqs ==\
len(self.input_batch.pooling_params), \
@@ -1386,6 +1387,7 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
pooler_output=pooler_output,
finished_sending=finished_sending,
finished_recving=finished_recving,
finished_loading_dict=finished_loading_dict,
)
@torch.inference_mode()
@@ -1505,6 +1507,7 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
self.maybe_wait_for_kv_save()
finished_sending, finished_recving = (
self.get_finished_kv_transfers(scheduler_output))
finished_loading_dict = self.get_finished_loading(scheduler_output)
if self.use_aux_hidden_state_outputs:
hidden_states, aux_hidden_states = model_output
@@ -1522,9 +1525,11 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
if not get_pp_group().is_last_rank:
# For mid-pipeline stages, return the hidden states.
if not broadcast_pp_output:
if finished_sending or finished_recving:
if (finished_sending or finished_recving
or finished_loading_dict):
hidden_states.finished_sending = finished_sending
hidden_states.finished_recving = finished_recving
hidden_states.finished_loading_dict = finished_loading_dict
return hidden_states
assert isinstance(hidden_states, IntermediateTensors)
get_pp_group().send_tensor_dict(hidden_states.tensors,
@@ -1534,7 +1539,7 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
if self.input_batch.pooling_params:
return self._pool(hidden_states, num_scheduled_tokens,
num_scheduled_tokens_np, finished_sending,
finished_recving)
finished_recving, finished_loading_dict)
sample_hidden_states = hidden_states[logits_indices]
logits = self.model.compute_logits(sample_hidden_states, None)
@@ -1686,6 +1691,7 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
pooler_output=[],
finished_sending=finished_sending,
finished_recving=finished_recving,
finished_loading_dict=finished_loading_dict,
num_nans_in_logits=num_nans_in_logits,
)
+3 -1
View File
@@ -359,10 +359,12 @@ class Worker(WorkerBase):
# In case of PP with kv transfer, we need to pass through the
# finished_sending and finished_recving buffers.
new_output = EMPTY_MODEL_RUNNER_OUTPUT
if output.finished_sending or output.finished_recving:
if (output.finished_sending or output.finished_recving
or output.finished_loading_dict):
new_output = copy.copy(new_output)
new_output.finished_sending = output.finished_sending
new_output.finished_recving = output.finished_recving
new_output.finished_loading_dict = output.finished_loading_dict
output = new_output
assert isinstance(output, ModelRunnerOutput)
@@ -53,6 +53,14 @@ class KVConnectorModelRunnerMixin:
scheduler_output.finished_req_ids)
return None, None
@staticmethod
def get_finished_loading(
scheduler_output: "SchedulerOutput", ) -> dict[str, int]:
if has_kv_transfer_group():
return get_kv_transfer_group().get_finished_loading(
scheduler_output)
return {}
def kv_connector_no_forward(self, scheduler_output: "SchedulerOutput",
vllm_config: VllmConfig) -> ModelRunnerOutput:
# KV send/recv even if no work to do.
@@ -60,11 +68,14 @@ class KVConnectorModelRunnerMixin:
self.maybe_setup_kv_connector(scheduler_output)
finished_sending, finished_recving = (
self.get_finished_kv_transfers(scheduler_output))
finished_loading_dict = self.get_finished_loading(scheduler_output)
if not finished_sending and not finished_recving:
if (not finished_sending and not finished_recving
and not finished_loading_dict):
return EMPTY_MODEL_RUNNER_OUTPUT
output = copy.copy(EMPTY_MODEL_RUNNER_OUTPUT)
output.finished_sending = finished_sending
output.finished_recving = finished_recving
output.finished_loading_dict = finished_loading_dict
return output