[Core] Cleanup KVConnector handling with PP + fix MRV2 (#43732)
Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
9090368b65
commit
7e53283b1c
@@ -3,15 +3,9 @@
|
||||
"""Sequence and its related classes."""
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import torch
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.v1.worker.kv_connector_model_runner_mixin import KVConnectorOutput
|
||||
else:
|
||||
KVConnectorOutput = Any
|
||||
|
||||
|
||||
# cannot use msgspec.Struct here because Dynamo does not support it
|
||||
@dataclass
|
||||
@@ -19,24 +13,19 @@ class IntermediateTensors:
|
||||
"""For all pipeline stages except the last, we need to return the hidden
|
||||
states and residuals to be sent to the next stage. This data structure
|
||||
contains the hidden states and residuals for a request.
|
||||
|
||||
Each stage also needs to handle its own kv_connector_output.
|
||||
"""
|
||||
|
||||
tensors: dict[str, torch.Tensor]
|
||||
kv_connector_output: KVConnectorOutput | None
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
tensors: dict[str, torch.Tensor],
|
||||
kv_connector_output: KVConnectorOutput | None = None,
|
||||
) -> None:
|
||||
# manually define this function, so that
|
||||
# Dynamo knows `IntermediateTensors()` comes from this file.
|
||||
# Otherwise, dataclass will generate this function by evaluating
|
||||
# a string, and we will lose the information about the source file.
|
||||
self.tensors = tensors
|
||||
self.kv_connector_output = kv_connector_output
|
||||
|
||||
def __getitem__(self, key: str | slice):
|
||||
if isinstance(key, str):
|
||||
|
||||
Reference in New Issue
Block a user