From 8c94938cfb92cc00b244ae4a933c5f60dbc1139f Mon Sep 17 00:00:00 2001 From: Nick Hill Date: Tue, 26 May 2026 23:37:56 -0700 Subject: [PATCH] [MRV2][BugFix] Fix KV connector handling in spec decode case (#43719) Signed-off-by: Nick Hill Co-authored-by: Wentao Ye <44945378+yewentao256@users.noreply.github.com> --- .../worker/test_gpu_model_runner_v2_eplb.py | 2 +- vllm/v1/worker/gpu/kv_connector.py | 16 +++++--------- vllm/v1/worker/gpu/model_runner.py | 22 +++++++++++++------ 3 files changed, 22 insertions(+), 18 deletions(-) diff --git a/tests/v1/worker/test_gpu_model_runner_v2_eplb.py b/tests/v1/worker/test_gpu_model_runner_v2_eplb.py index 27bfc691ae..d68ff83c40 100644 --- a/tests/v1/worker/test_gpu_model_runner_v2_eplb.py +++ b/tests/v1/worker/test_gpu_model_runner_v2_eplb.py @@ -168,7 +168,7 @@ def test_v2_sample_tokens_runs_eplb_on_non_last_pp_rank(monkeypatch): slot_mappings_by_layer=None, hidden_states=None, aux_hidden_states=None, - kv_connector_output=None, + finished_req_ids=set(), num_tokens_across_dp=None, ) runner.postprocess = lambda *args, **kwargs: events.append("postprocess") diff --git a/vllm/v1/worker/gpu/kv_connector.py b/vllm/v1/worker/gpu/kv_connector.py index 3c29ea132d..8476331098 100644 --- a/vllm/v1/worker/gpu/kv_connector.py +++ b/vllm/v1/worker/gpu/kv_connector.py @@ -34,7 +34,7 @@ class KVConnector: pass def post_forward( - self, scheduler_output: "SchedulerOutput", wait_for_save: bool = True + self, finished_req_ids: set[str], wait_for_save: bool = True ) -> KVConnectorOutput | None: return None @@ -76,10 +76,7 @@ class ActiveKVConnector(KVConnector): self.kv_connector.start_load_kv(get_forward_context()) def post_forward( - self, - scheduler_output: "SchedulerOutput", - wait_for_save: bool = True, - clear_metadata: bool = True, + self, finished_req_ids: set[str], wait_for_save: bool = True ) -> KVConnectorOutput | None: if self._disabled: return None @@ -88,7 +85,7 @@ class ActiveKVConnector(KVConnector): if wait_for_save: self.kv_connector.wait_for_save() output.finished_sending, output.finished_recving = ( - self.kv_connector.get_finished(scheduler_output.finished_req_ids) + self.kv_connector.get_finished(finished_req_ids) ) output.invalid_block_ids = self.kv_connector.get_block_ids_with_load_errors() output.kv_connector_stats = self.kv_connector.get_kv_connector_stats() @@ -96,9 +93,7 @@ class ActiveKVConnector(KVConnector): output.kv_connector_worker_meta = ( self.kv_connector.build_connector_worker_meta() ) - - if clear_metadata: - self.kv_connector.clear_connector_metadata() + self.kv_connector.clear_connector_metadata() return output def no_forward(self, scheduler_output: "SchedulerOutput") -> ModelRunnerOutput: @@ -106,7 +101,8 @@ class ActiveKVConnector(KVConnector): return EMPTY_MODEL_RUNNER_OUTPUT self.pre_forward(scheduler_output) - kv_connector_output = self.post_forward(scheduler_output, wait_for_save=False) + finished_req_ids = scheduler_output.finished_req_ids + kv_connector_output = self.post_forward(finished_req_ids, wait_for_save=False) if kv_connector_output is None or kv_connector_output.is_empty(): return EMPTY_MODEL_RUNNER_OUTPUT output = copy.copy(EMPTY_MODEL_RUNNER_OUTPUT) diff --git a/vllm/v1/worker/gpu/model_runner.py b/vllm/v1/worker/gpu/model_runner.py index c0a95618c1..52b4475508 100644 --- a/vllm/v1/worker/gpu/model_runner.py +++ b/vllm/v1/worker/gpu/model_runner.py @@ -50,7 +50,7 @@ from vllm.utils.mem_utils import DeviceMemoryProfiler, format_gib from vllm.utils.torch_utils import STR_DTYPE_TO_TORCH_DTYPE from vllm.v1.core.sched.output import GrammarOutput, SchedulerOutput from vllm.v1.kv_cache_interface import KVCacheConfig, MambaSpec -from vllm.v1.outputs import DraftTokenIds, KVConnectorOutput, ModelRunnerOutput +from vllm.v1.outputs import DraftTokenIds, ModelRunnerOutput from vllm.v1.worker.cp_utils import check_attention_cp_compatibility from vllm.v1.worker.gpu.async_utils import AsyncOutput, AsyncPoolingOutput from vllm.v1.worker.gpu.attn_utils import ( @@ -1207,19 +1207,20 @@ class GPUModelRunner(LoRAModelRunnerMixin): aux_hidden_states = None output_intermediate_tensors = model_output - kv_connector_output = self.kv_connector.post_forward(scheduler_output) + finished_req_ids = scheduler_output.finished_req_ids self.execute_model_state = ExecuteModelState( input_batch=input_batch, attn_metadata=attn_metadata, slot_mappings_by_layer=slot_mappings_by_layer, hidden_states=hidden_states, aux_hidden_states=aux_hidden_states, - kv_connector_output=kv_connector_output, + finished_req_ids=finished_req_ids, ) if not self.is_last_pp_rank: # Non-last PP rank: return IntermediateTensors for sending. assert output_intermediate_tensors is not None + kv_connector_output = self.kv_connector.post_forward(finished_req_ids) output_intermediate_tensors.kv_connector_output = kv_connector_output return output_intermediate_tensors return None @@ -1238,7 +1239,7 @@ class GPUModelRunner(LoRAModelRunnerMixin): slot_mappings_by_layer = self.execute_model_state.slot_mappings_by_layer hidden_states = self.execute_model_state.hidden_states aux_hidden_states = self.execute_model_state.aux_hidden_states - kv_connector_output = self.execute_model_state.kv_connector_output + finished_req_ids = self.execute_model_state.finished_req_ids self.execute_model_state = None if not self.is_last_pp_rank: @@ -1280,8 +1281,8 @@ class GPUModelRunner(LoRAModelRunnerMixin): req_id_to_index={req_id: i for i, req_id in enumerate(input_batch.req_ids)}, sampled_token_ids=None, # type: ignore prompt_logprobs_dict=prompt_logprobs_dict, # type: ignore[arg-type] - kv_connector_output=kv_connector_output, ) + # Start async output copy here so that it can overlap with speculator proposal. async_output = AsyncOutput( model_runner_output=model_runner_output, sampler_output=sampler_output, @@ -1344,6 +1345,10 @@ class GPUModelRunner(LoRAModelRunnerMixin): self.req_states.draft_tokens[input_batch.idx_mapping] = draft_tokens self.draft_tokens_handler.set_draft_tokens(input_batch, draft_tokens) + # Post-step KV connector related operations. + kv_connector_output = self.kv_connector.post_forward(finished_req_ids) + model_runner_output.kv_connector_output = kv_connector_output + if self.use_async_scheduling: return async_output return async_output.get_output() @@ -1360,7 +1365,7 @@ class GPUModelRunner(LoRAModelRunnerMixin): input_batch = self.execute_model_state.input_batch hidden_states = self.execute_model_state.hidden_states - kv_connector_output = self.execute_model_state.kv_connector_output + finished_req_ids = self.execute_model_state.finished_req_ids self.execute_model_state = None if not self.is_last_pp_rank: @@ -1372,6 +1377,9 @@ class GPUModelRunner(LoRAModelRunnerMixin): hidden_states, input_batch, self.req_states ) + # Post-step KV connector related operations. + kv_connector_output = self.kv_connector.post_forward(finished_req_ids) + # Build the model runner output. model_runner_output = ModelRunnerOutput( req_ids=input_batch.req_ids, @@ -1455,4 +1463,4 @@ class ExecuteModelState(NamedTuple): slot_mappings_by_layer: dict[str, torch.Tensor] | None hidden_states: torch.Tensor | None aux_hidden_states: list[torch.Tensor] | None - kv_connector_output: KVConnectorOutput | None + finished_req_ids: set[str]