[V1] Exception Handling when Loading KV Cache from Remote Store (#21534)

Signed-off-by: liuyumoye <adeline_ly2023@outlook.com>
Co-authored-by: liuyumoye <adeline_ly2023@outlook.com>
This commit is contained in:
Adeline
2025-07-27 20:34:17 -07:00
committed by GitHub
co-authored by liuyumoye
parent 04ff4be310
commit 15a72ac478
10 changed files with 229 additions and 5 deletions
+8 -2
View File
@@ -1350,6 +1350,7 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
num_scheduled_tokens_np: np.ndarray,
finished_sending: Optional[set[str]],
finished_recving: Optional[set[str]],
finished_loading_dict: Optional[dict[str, int]],
) -> ModelRunnerOutput:
assert self.input_batch.num_reqs ==\
len(self.input_batch.pooling_params), \
@@ -1386,6 +1387,7 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
pooler_output=pooler_output,
finished_sending=finished_sending,
finished_recving=finished_recving,
finished_loading_dict=finished_loading_dict,
)
@torch.inference_mode()
@@ -1505,6 +1507,7 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
self.maybe_wait_for_kv_save()
finished_sending, finished_recving = (
self.get_finished_kv_transfers(scheduler_output))
finished_loading_dict = self.get_finished_loading(scheduler_output)
if self.use_aux_hidden_state_outputs:
hidden_states, aux_hidden_states = model_output
@@ -1522,9 +1525,11 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
if not get_pp_group().is_last_rank:
# For mid-pipeline stages, return the hidden states.
if not broadcast_pp_output:
if finished_sending or finished_recving:
if (finished_sending or finished_recving
or finished_loading_dict):
hidden_states.finished_sending = finished_sending
hidden_states.finished_recving = finished_recving
hidden_states.finished_loading_dict = finished_loading_dict
return hidden_states
assert isinstance(hidden_states, IntermediateTensors)
get_pp_group().send_tensor_dict(hidden_states.tensors,
@@ -1534,7 +1539,7 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
if self.input_batch.pooling_params:
return self._pool(hidden_states, num_scheduled_tokens,
num_scheduled_tokens_np, finished_sending,
finished_recving)
finished_recving, finished_loading_dict)
sample_hidden_states = hidden_states[logits_indices]
logits = self.model.compute_logits(sample_hidden_states, None)
@@ -1686,6 +1691,7 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
pooler_output=[],
finished_sending=finished_sending,
finished_recving=finished_recving,
finished_loading_dict=finished_loading_dict,
num_nans_in_logits=num_nans_in_logits,
)