[MRV2][BugFix] Fix KV connector handling in spec decode case (#43719)

Signed-off-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: Wentao Ye <44945378+yewentao256@users.noreply.github.com>
This commit is contained in:
Nick Hill
2026-05-27 06:37:56 +00:00
committed by GitHub
co-authored by Wentao Ye
parent 7b54690244
commit 8c94938cfb
3 changed files with 22 additions and 18 deletions
@@ -168,7 +168,7 @@ def test_v2_sample_tokens_runs_eplb_on_non_last_pp_rank(monkeypatch):
slot_mappings_by_layer=None,
hidden_states=None,
aux_hidden_states=None,
kv_connector_output=None,
finished_req_ids=set(),
num_tokens_across_dp=None,
)
runner.postprocess = lambda *args, **kwargs: events.append("postprocess")
+6 -10
View File
@@ -34,7 +34,7 @@ class KVConnector:
pass
def post_forward(
self, scheduler_output: "SchedulerOutput", wait_for_save: bool = True
self, finished_req_ids: set[str], wait_for_save: bool = True
) -> KVConnectorOutput | None:
return None
@@ -76,10 +76,7 @@ class ActiveKVConnector(KVConnector):
self.kv_connector.start_load_kv(get_forward_context())
def post_forward(
self,
scheduler_output: "SchedulerOutput",
wait_for_save: bool = True,
clear_metadata: bool = True,
self, finished_req_ids: set[str], wait_for_save: bool = True
) -> KVConnectorOutput | None:
if self._disabled:
return None
@@ -88,7 +85,7 @@ class ActiveKVConnector(KVConnector):
if wait_for_save:
self.kv_connector.wait_for_save()
output.finished_sending, output.finished_recving = (
self.kv_connector.get_finished(scheduler_output.finished_req_ids)
self.kv_connector.get_finished(finished_req_ids)
)
output.invalid_block_ids = self.kv_connector.get_block_ids_with_load_errors()
output.kv_connector_stats = self.kv_connector.get_kv_connector_stats()
@@ -96,9 +93,7 @@ class ActiveKVConnector(KVConnector):
output.kv_connector_worker_meta = (
self.kv_connector.build_connector_worker_meta()
)
if clear_metadata:
self.kv_connector.clear_connector_metadata()
self.kv_connector.clear_connector_metadata()
return output
def no_forward(self, scheduler_output: "SchedulerOutput") -> ModelRunnerOutput:
@@ -106,7 +101,8 @@ class ActiveKVConnector(KVConnector):
return EMPTY_MODEL_RUNNER_OUTPUT
self.pre_forward(scheduler_output)
kv_connector_output = self.post_forward(scheduler_output, wait_for_save=False)
finished_req_ids = scheduler_output.finished_req_ids
kv_connector_output = self.post_forward(finished_req_ids, wait_for_save=False)
if kv_connector_output is None or kv_connector_output.is_empty():
return EMPTY_MODEL_RUNNER_OUTPUT
output = copy.copy(EMPTY_MODEL_RUNNER_OUTPUT)
+15 -7
View File
@@ -50,7 +50,7 @@ from vllm.utils.mem_utils import DeviceMemoryProfiler, format_gib
from vllm.utils.torch_utils import STR_DTYPE_TO_TORCH_DTYPE
from vllm.v1.core.sched.output import GrammarOutput, SchedulerOutput
from vllm.v1.kv_cache_interface import KVCacheConfig, MambaSpec
from vllm.v1.outputs import DraftTokenIds, KVConnectorOutput, ModelRunnerOutput
from vllm.v1.outputs import DraftTokenIds, ModelRunnerOutput
from vllm.v1.worker.cp_utils import check_attention_cp_compatibility
from vllm.v1.worker.gpu.async_utils import AsyncOutput, AsyncPoolingOutput
from vllm.v1.worker.gpu.attn_utils import (
@@ -1207,19 +1207,20 @@ class GPUModelRunner(LoRAModelRunnerMixin):
aux_hidden_states = None
output_intermediate_tensors = model_output
kv_connector_output = self.kv_connector.post_forward(scheduler_output)
finished_req_ids = scheduler_output.finished_req_ids
self.execute_model_state = ExecuteModelState(
input_batch=input_batch,
attn_metadata=attn_metadata,
slot_mappings_by_layer=slot_mappings_by_layer,
hidden_states=hidden_states,
aux_hidden_states=aux_hidden_states,
kv_connector_output=kv_connector_output,
finished_req_ids=finished_req_ids,
)
if not self.is_last_pp_rank:
# Non-last PP rank: return IntermediateTensors for sending.
assert output_intermediate_tensors is not None
kv_connector_output = self.kv_connector.post_forward(finished_req_ids)
output_intermediate_tensors.kv_connector_output = kv_connector_output
return output_intermediate_tensors
return None
@@ -1238,7 +1239,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
slot_mappings_by_layer = self.execute_model_state.slot_mappings_by_layer
hidden_states = self.execute_model_state.hidden_states
aux_hidden_states = self.execute_model_state.aux_hidden_states
kv_connector_output = self.execute_model_state.kv_connector_output
finished_req_ids = self.execute_model_state.finished_req_ids
self.execute_model_state = None
if not self.is_last_pp_rank:
@@ -1280,8 +1281,8 @@ class GPUModelRunner(LoRAModelRunnerMixin):
req_id_to_index={req_id: i for i, req_id in enumerate(input_batch.req_ids)},
sampled_token_ids=None, # type: ignore
prompt_logprobs_dict=prompt_logprobs_dict, # type: ignore[arg-type]
kv_connector_output=kv_connector_output,
)
# Start async output copy here so that it can overlap with speculator proposal.
async_output = AsyncOutput(
model_runner_output=model_runner_output,
sampler_output=sampler_output,
@@ -1344,6 +1345,10 @@ class GPUModelRunner(LoRAModelRunnerMixin):
self.req_states.draft_tokens[input_batch.idx_mapping] = draft_tokens
self.draft_tokens_handler.set_draft_tokens(input_batch, draft_tokens)
# Post-step KV connector related operations.
kv_connector_output = self.kv_connector.post_forward(finished_req_ids)
model_runner_output.kv_connector_output = kv_connector_output
if self.use_async_scheduling:
return async_output
return async_output.get_output()
@@ -1360,7 +1365,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
input_batch = self.execute_model_state.input_batch
hidden_states = self.execute_model_state.hidden_states
kv_connector_output = self.execute_model_state.kv_connector_output
finished_req_ids = self.execute_model_state.finished_req_ids
self.execute_model_state = None
if not self.is_last_pp_rank:
@@ -1372,6 +1377,9 @@ class GPUModelRunner(LoRAModelRunnerMixin):
hidden_states, input_batch, self.req_states
)
# Post-step KV connector related operations.
kv_connector_output = self.kv_connector.post_forward(finished_req_ids)
# Build the model runner output.
model_runner_output = ModelRunnerOutput(
req_ids=input_batch.req_ids,
@@ -1455,4 +1463,4 @@ class ExecuteModelState(NamedTuple):
slot_mappings_by_layer: dict[str, torch.Tensor] | None
hidden_states: torch.Tensor | None
aux_hidden_states: list[torch.Tensor] | None
kv_connector_output: KVConnectorOutput | None
finished_req_ids: set[str]