[Core] Cleanup KVConnector handling with PP + fix MRV2 (#43732)

Signed-off-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Nick Hill
2026-05-28 13:12:03 -07:00
committed by GitHub
co-authored by mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 9090368b65
commit 7e53283b1c
9 changed files with 38 additions and 48 deletions
-11
View File
@@ -3,15 +3,9 @@
"""Sequence and its related classes."""
from dataclasses import dataclass
from typing import TYPE_CHECKING, Any
import torch
if TYPE_CHECKING:
from vllm.v1.worker.kv_connector_model_runner_mixin import KVConnectorOutput
else:
KVConnectorOutput = Any
# cannot use msgspec.Struct here because Dynamo does not support it
@dataclass
@@ -19,24 +13,19 @@ class IntermediateTensors:
"""For all pipeline stages except the last, we need to return the hidden
states and residuals to be sent to the next stage. This data structure
contains the hidden states and residuals for a request.
Each stage also needs to handle its own kv_connector_output.
"""
tensors: dict[str, torch.Tensor]
kv_connector_output: KVConnectorOutput | None
def __init__(
self,
tensors: dict[str, torch.Tensor],
kv_connector_output: KVConnectorOutput | None = None,
) -> None:
# manually define this function, so that
# Dynamo knows `IntermediateTensors()` comes from this file.
# Otherwise, dataclass will generate this function by evaluating
# a string, and we will lose the information about the source file.
self.tensors = tensors
self.kv_connector_output = kv_connector_output
def __getitem__(self, key: str | slice):
if isinstance(key, str):
+14
View File
@@ -2,6 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from abc import ABC, abstractmethod
from copy import copy
from dataclasses import dataclass, field
from typing import TYPE_CHECKING, NamedTuple, TypeAlias
@@ -279,6 +280,19 @@ class ModelRunnerOutput:
# ``None`` when ``enable_return_routed_experts`` is off.
routed_experts: RoutedExpertsLists | None = None
@staticmethod
def with_kv_conn_output_only(
kv_connector_output: KVConnectorOutput | None,
) -> "ModelRunnerOutput":
"""Return ModelRunnerOutput containing the provided KVConnectorOutput,
otherwise empty. Returns None if kv_connector_output is passed as None.
"""
if kv_connector_output is None or kv_connector_output.is_empty():
return EMPTY_MODEL_RUNNER_OUTPUT
output = copy(EMPTY_MODEL_RUNNER_OUTPUT)
output.kv_connector_output = kv_connector_output
return output
# ModelRunnerOutput wrapper for async scheduling.
class AsyncModelRunnerOutput(ABC):
+1 -6
View File
@@ -1,6 +1,5 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import copy
from typing import TYPE_CHECKING
import torch
@@ -103,11 +102,7 @@ class ActiveKVConnector(KVConnector):
self.pre_forward(scheduler_output)
finished_req_ids = scheduler_output.finished_req_ids
kv_connector_output = self.post_forward(finished_req_ids, wait_for_save=False)
if kv_connector_output is None or kv_connector_output.is_empty():
return EMPTY_MODEL_RUNNER_OUTPUT
output = copy.copy(EMPTY_MODEL_RUNNER_OUTPUT)
output.kv_connector_output = kv_connector_output
return output
return ModelRunnerOutput.with_kv_conn_output_only(kv_connector_output)
def set_disabled(self, disabled: bool) -> None:
# Ensure that layer-wise connector hooks aren't called when disabled.
+8 -8
View File
@@ -1218,9 +1218,6 @@ class GPUModelRunner(LoRAModelRunnerMixin):
if not self.is_last_pp_rank:
# Non-last PP rank: return IntermediateTensors for sending.
assert output_intermediate_tensors is not None
kv_connector_output = self.kv_connector.post_forward(finished_req_ids)
output_intermediate_tensors.kv_connector_output = kv_connector_output
return output_intermediate_tensors
return None
@@ -1249,7 +1246,10 @@ class GPUModelRunner(LoRAModelRunnerMixin):
input_batch.num_reqs, max_sample_len=self.num_speculative_steps + 1
)
self.postprocess(input_batch, sampled, num_sampled, num_rejected)
return None
# Post-step KV connector related operations.
kv_connector_output = self.kv_connector.post_forward(finished_req_ids)
return ModelRunnerOutput.with_kv_conn_output_only(kv_connector_output)
# Last rank: sample tokens
sampler_output, num_sampled, num_rejected = self.sample(
@@ -1367,18 +1367,18 @@ class GPUModelRunner(LoRAModelRunnerMixin):
finished_req_ids = self.execute_model_state.finished_req_ids
self.execute_model_state = None
# Post-step KV connector related operations.
kv_connector_output = self.kv_connector.post_forward(finished_req_ids)
if not self.is_last_pp_rank:
self.postprocess_pool(input_batch)
return None
return ModelRunnerOutput.with_kv_conn_output_only(kv_connector_output)
assert self.pooling_runner is not None
pooler_output, is_valid = self.pooling_runner.pool(
hidden_states, input_batch, self.req_states
)
# Post-step KV connector related operations.
kv_connector_output = self.kv_connector.post_forward(finished_req_ids)
# Build the model runner output.
model_runner_output = ModelRunnerOutput(
req_ids=input_batch.req_ids,
+1 -10
View File
@@ -4250,7 +4250,6 @@ class GPUModelRunner(
if not get_pp_group().is_last_rank:
# Return the intermediate tensors.
assert isinstance(hidden_states, IntermediateTensors)
hidden_states.kv_connector_output = kv_connector_output
self.kv_connector_output = kv_connector_output
return hidden_states
@@ -4326,17 +4325,9 @@ class GPUModelRunner(
# receive sampled token ids from the last PP rank.
if self.use_async_scheduling and not get_pp_group().is_last_rank:
self._pp_receive_prev_sampled_token_ids_to_input_batch()
if not kv_connector_output:
return None # type: ignore[return-value]
# In case of PP with kv transfer, we need to pass through the
# kv_connector_output
if kv_connector_output.is_empty():
return EMPTY_MODEL_RUNNER_OUTPUT
output = copy(EMPTY_MODEL_RUNNER_OUTPUT)
output.kv_connector_output = kv_connector_output
return output
return ModelRunnerOutput.with_kv_conn_output_only(kv_connector_output)
# Unpack ephemeral state.
(
@@ -4,7 +4,6 @@
Define KV connector functionality mixin for model runners.
"""
import copy
from collections.abc import Generator
from contextlib import AbstractContextManager, contextmanager, nullcontext
from typing import TYPE_CHECKING
@@ -20,7 +19,6 @@ from vllm.logger import init_logger
from vllm.v1.attention.backend import AttentionBackend
from vllm.v1.kv_cache_interface import AttentionSpec, KVCacheConfig
from vllm.v1.outputs import (
EMPTY_MODEL_RUNNER_OUTPUT,
KVConnectorOutput,
ModelRunnerOutput,
)
@@ -47,12 +45,7 @@ class KVConnectorModelRunnerMixin:
):
pass
if kv_connector_output.is_empty():
return EMPTY_MODEL_RUNNER_OUTPUT
output = copy.copy(EMPTY_MODEL_RUNNER_OUTPUT)
output.kv_connector_output = kv_connector_output
return output
return ModelRunnerOutput.with_kv_conn_output_only(kv_connector_output)
@staticmethod
def maybe_get_kv_connector_output(