[Core] Cleanup KVConnector handling with PP + fix MRV2 (#43732)
Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
9090368b65
commit
7e53283b1c
@@ -292,7 +292,7 @@ def test_async_intermediate_tensors_lazy_wait() -> None:
|
|||||||
)
|
)
|
||||||
|
|
||||||
# accessing non-tensor attributes should not trigger wait.
|
# accessing non-tensor attributes should not trigger wait.
|
||||||
assert it.kv_connector_output is None
|
assert it._comm_handles is not None
|
||||||
assert work.wait_calls == 0
|
assert work.wait_calls == 0
|
||||||
assert post_calls["n"] == 0
|
assert post_calls["n"] == 0
|
||||||
|
|
||||||
|
|||||||
@@ -39,6 +39,7 @@ from vllm.v1.kv_cache_interface import (
|
|||||||
KVCacheGroupSpec,
|
KVCacheGroupSpec,
|
||||||
KVCacheTensor,
|
KVCacheTensor,
|
||||||
)
|
)
|
||||||
|
from vllm.v1.outputs import EMPTY_MODEL_RUNNER_OUTPUT
|
||||||
from vllm.v1.sample.metadata import SamplingMetadata
|
from vllm.v1.sample.metadata import SamplingMetadata
|
||||||
from vllm.v1.spec_decode.metadata import SpecDecodeMetadata
|
from vllm.v1.spec_decode.metadata import SpecDecodeMetadata
|
||||||
from vllm.v1.worker.gpu_input_batch import InputBatch
|
from vllm.v1.worker.gpu_input_batch import InputBatch
|
||||||
@@ -279,7 +280,8 @@ def test_sample_tokens_receives_pp_sampled_ids_only_on_non_last_rank(
|
|||||||
lambda: SimpleNamespace(world_size=world_size, is_last_rank=is_last_rank),
|
lambda: SimpleNamespace(world_size=world_size, is_last_rank=is_last_rank),
|
||||||
)
|
)
|
||||||
|
|
||||||
assert GPUModelRunner.sample_tokens(runner, None) is None
|
output = GPUModelRunner.sample_tokens(runner, None)
|
||||||
|
assert output in (EMPTY_MODEL_RUNNER_OUTPUT, None)
|
||||||
assert receive_calls == expected_calls
|
assert receive_calls == expected_calls
|
||||||
|
|
||||||
|
|
||||||
@@ -298,7 +300,8 @@ def test_sample_tokens_skips_pp_group_lookup_without_async_scheduling(
|
|||||||
pytest.fail,
|
pytest.fail,
|
||||||
)
|
)
|
||||||
|
|
||||||
assert GPUModelRunner.sample_tokens(runner, None) is None
|
output = GPUModelRunner.sample_tokens(runner, None)
|
||||||
|
assert output in (EMPTY_MODEL_RUNNER_OUTPUT, None)
|
||||||
|
|
||||||
|
|
||||||
def test_select_common_block_size_no_valid_option():
|
def test_select_common_block_size_no_valid_option():
|
||||||
|
|||||||
@@ -7,6 +7,7 @@ from typing import Any
|
|||||||
|
|
||||||
import torch
|
import torch
|
||||||
|
|
||||||
|
from vllm.v1.outputs import EMPTY_MODEL_RUNNER_OUTPUT
|
||||||
from vllm.v1.worker.gpu import eplb_utils as eplb
|
from vllm.v1.worker.gpu import eplb_utils as eplb
|
||||||
from vllm.v1.worker.gpu import model_runner as mrv2
|
from vllm.v1.worker.gpu import model_runner as mrv2
|
||||||
|
|
||||||
@@ -76,7 +77,10 @@ def _make_runner(**overrides: Any) -> Any:
|
|||||||
runner.max_num_reqs = 8
|
runner.max_num_reqs = 8
|
||||||
runner.max_num_tokens = 16
|
runner.max_num_tokens = 16
|
||||||
runner.decode_query_len = 1
|
runner.decode_query_len = 1
|
||||||
runner.kv_connector = SimpleNamespace(set_disabled=lambda *_: None)
|
runner.kv_connector = SimpleNamespace(
|
||||||
|
set_disabled=lambda *_: None,
|
||||||
|
post_forward=lambda *_, **__: None,
|
||||||
|
)
|
||||||
runner.eplb = eplb.EPLBController(runner.parallel_config, runner.device)
|
runner.eplb = eplb.EPLBController(runner.parallel_config, runner.device)
|
||||||
runner.pooling_runner = None
|
runner.pooling_runner = None
|
||||||
runner.execute_model_state = None
|
runner.execute_model_state = None
|
||||||
@@ -183,5 +187,6 @@ def test_v2_sample_tokens_runs_eplb_on_non_last_pp_rank(monkeypatch):
|
|||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
assert mrv2.GPUModelRunner.sample_tokens(runner, None) is None
|
output = mrv2.GPUModelRunner.sample_tokens(runner, None)
|
||||||
|
assert output in (EMPTY_MODEL_RUNNER_OUTPUT, None)
|
||||||
assert events == ["postprocess", "eplb"]
|
assert events == ["postprocess", "eplb"]
|
||||||
|
|||||||
@@ -3,15 +3,9 @@
|
|||||||
"""Sequence and its related classes."""
|
"""Sequence and its related classes."""
|
||||||
|
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
from typing import TYPE_CHECKING, Any
|
|
||||||
|
|
||||||
import torch
|
import torch
|
||||||
|
|
||||||
if TYPE_CHECKING:
|
|
||||||
from vllm.v1.worker.kv_connector_model_runner_mixin import KVConnectorOutput
|
|
||||||
else:
|
|
||||||
KVConnectorOutput = Any
|
|
||||||
|
|
||||||
|
|
||||||
# cannot use msgspec.Struct here because Dynamo does not support it
|
# cannot use msgspec.Struct here because Dynamo does not support it
|
||||||
@dataclass
|
@dataclass
|
||||||
@@ -19,24 +13,19 @@ class IntermediateTensors:
|
|||||||
"""For all pipeline stages except the last, we need to return the hidden
|
"""For all pipeline stages except the last, we need to return the hidden
|
||||||
states and residuals to be sent to the next stage. This data structure
|
states and residuals to be sent to the next stage. This data structure
|
||||||
contains the hidden states and residuals for a request.
|
contains the hidden states and residuals for a request.
|
||||||
|
|
||||||
Each stage also needs to handle its own kv_connector_output.
|
|
||||||
"""
|
"""
|
||||||
|
|
||||||
tensors: dict[str, torch.Tensor]
|
tensors: dict[str, torch.Tensor]
|
||||||
kv_connector_output: KVConnectorOutput | None
|
|
||||||
|
|
||||||
def __init__(
|
def __init__(
|
||||||
self,
|
self,
|
||||||
tensors: dict[str, torch.Tensor],
|
tensors: dict[str, torch.Tensor],
|
||||||
kv_connector_output: KVConnectorOutput | None = None,
|
|
||||||
) -> None:
|
) -> None:
|
||||||
# manually define this function, so that
|
# manually define this function, so that
|
||||||
# Dynamo knows `IntermediateTensors()` comes from this file.
|
# Dynamo knows `IntermediateTensors()` comes from this file.
|
||||||
# Otherwise, dataclass will generate this function by evaluating
|
# Otherwise, dataclass will generate this function by evaluating
|
||||||
# a string, and we will lose the information about the source file.
|
# a string, and we will lose the information about the source file.
|
||||||
self.tensors = tensors
|
self.tensors = tensors
|
||||||
self.kv_connector_output = kv_connector_output
|
|
||||||
|
|
||||||
def __getitem__(self, key: str | slice):
|
def __getitem__(self, key: str | slice):
|
||||||
if isinstance(key, str):
|
if isinstance(key, str):
|
||||||
|
|||||||
@@ -2,6 +2,7 @@
|
|||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
|
||||||
from abc import ABC, abstractmethod
|
from abc import ABC, abstractmethod
|
||||||
|
from copy import copy
|
||||||
from dataclasses import dataclass, field
|
from dataclasses import dataclass, field
|
||||||
from typing import TYPE_CHECKING, NamedTuple, TypeAlias
|
from typing import TYPE_CHECKING, NamedTuple, TypeAlias
|
||||||
|
|
||||||
@@ -279,6 +280,19 @@ class ModelRunnerOutput:
|
|||||||
# ``None`` when ``enable_return_routed_experts`` is off.
|
# ``None`` when ``enable_return_routed_experts`` is off.
|
||||||
routed_experts: RoutedExpertsLists | None = None
|
routed_experts: RoutedExpertsLists | None = None
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def with_kv_conn_output_only(
|
||||||
|
kv_connector_output: KVConnectorOutput | None,
|
||||||
|
) -> "ModelRunnerOutput":
|
||||||
|
"""Return ModelRunnerOutput containing the provided KVConnectorOutput,
|
||||||
|
otherwise empty. Returns None if kv_connector_output is passed as None.
|
||||||
|
"""
|
||||||
|
if kv_connector_output is None or kv_connector_output.is_empty():
|
||||||
|
return EMPTY_MODEL_RUNNER_OUTPUT
|
||||||
|
output = copy(EMPTY_MODEL_RUNNER_OUTPUT)
|
||||||
|
output.kv_connector_output = kv_connector_output
|
||||||
|
return output
|
||||||
|
|
||||||
|
|
||||||
# ModelRunnerOutput wrapper for async scheduling.
|
# ModelRunnerOutput wrapper for async scheduling.
|
||||||
class AsyncModelRunnerOutput(ABC):
|
class AsyncModelRunnerOutput(ABC):
|
||||||
|
|||||||
@@ -1,6 +1,5 @@
|
|||||||
# SPDX-License-Identifier: Apache-2.0
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
import copy
|
|
||||||
from typing import TYPE_CHECKING
|
from typing import TYPE_CHECKING
|
||||||
|
|
||||||
import torch
|
import torch
|
||||||
@@ -103,11 +102,7 @@ class ActiveKVConnector(KVConnector):
|
|||||||
self.pre_forward(scheduler_output)
|
self.pre_forward(scheduler_output)
|
||||||
finished_req_ids = scheduler_output.finished_req_ids
|
finished_req_ids = scheduler_output.finished_req_ids
|
||||||
kv_connector_output = self.post_forward(finished_req_ids, wait_for_save=False)
|
kv_connector_output = self.post_forward(finished_req_ids, wait_for_save=False)
|
||||||
if kv_connector_output is None or kv_connector_output.is_empty():
|
return ModelRunnerOutput.with_kv_conn_output_only(kv_connector_output)
|
||||||
return EMPTY_MODEL_RUNNER_OUTPUT
|
|
||||||
output = copy.copy(EMPTY_MODEL_RUNNER_OUTPUT)
|
|
||||||
output.kv_connector_output = kv_connector_output
|
|
||||||
return output
|
|
||||||
|
|
||||||
def set_disabled(self, disabled: bool) -> None:
|
def set_disabled(self, disabled: bool) -> None:
|
||||||
# Ensure that layer-wise connector hooks aren't called when disabled.
|
# Ensure that layer-wise connector hooks aren't called when disabled.
|
||||||
|
|||||||
@@ -1218,9 +1218,6 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
|||||||
|
|
||||||
if not self.is_last_pp_rank:
|
if not self.is_last_pp_rank:
|
||||||
# Non-last PP rank: return IntermediateTensors for sending.
|
# Non-last PP rank: return IntermediateTensors for sending.
|
||||||
assert output_intermediate_tensors is not None
|
|
||||||
kv_connector_output = self.kv_connector.post_forward(finished_req_ids)
|
|
||||||
output_intermediate_tensors.kv_connector_output = kv_connector_output
|
|
||||||
return output_intermediate_tensors
|
return output_intermediate_tensors
|
||||||
return None
|
return None
|
||||||
|
|
||||||
@@ -1249,7 +1246,10 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
|||||||
input_batch.num_reqs, max_sample_len=self.num_speculative_steps + 1
|
input_batch.num_reqs, max_sample_len=self.num_speculative_steps + 1
|
||||||
)
|
)
|
||||||
self.postprocess(input_batch, sampled, num_sampled, num_rejected)
|
self.postprocess(input_batch, sampled, num_sampled, num_rejected)
|
||||||
return None
|
|
||||||
|
# Post-step KV connector related operations.
|
||||||
|
kv_connector_output = self.kv_connector.post_forward(finished_req_ids)
|
||||||
|
return ModelRunnerOutput.with_kv_conn_output_only(kv_connector_output)
|
||||||
|
|
||||||
# Last rank: sample tokens
|
# Last rank: sample tokens
|
||||||
sampler_output, num_sampled, num_rejected = self.sample(
|
sampler_output, num_sampled, num_rejected = self.sample(
|
||||||
@@ -1367,18 +1367,18 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
|||||||
finished_req_ids = self.execute_model_state.finished_req_ids
|
finished_req_ids = self.execute_model_state.finished_req_ids
|
||||||
self.execute_model_state = None
|
self.execute_model_state = None
|
||||||
|
|
||||||
|
# Post-step KV connector related operations.
|
||||||
|
kv_connector_output = self.kv_connector.post_forward(finished_req_ids)
|
||||||
|
|
||||||
if not self.is_last_pp_rank:
|
if not self.is_last_pp_rank:
|
||||||
self.postprocess_pool(input_batch)
|
self.postprocess_pool(input_batch)
|
||||||
return None
|
return ModelRunnerOutput.with_kv_conn_output_only(kv_connector_output)
|
||||||
|
|
||||||
assert self.pooling_runner is not None
|
assert self.pooling_runner is not None
|
||||||
pooler_output, is_valid = self.pooling_runner.pool(
|
pooler_output, is_valid = self.pooling_runner.pool(
|
||||||
hidden_states, input_batch, self.req_states
|
hidden_states, input_batch, self.req_states
|
||||||
)
|
)
|
||||||
|
|
||||||
# Post-step KV connector related operations.
|
|
||||||
kv_connector_output = self.kv_connector.post_forward(finished_req_ids)
|
|
||||||
|
|
||||||
# Build the model runner output.
|
# Build the model runner output.
|
||||||
model_runner_output = ModelRunnerOutput(
|
model_runner_output = ModelRunnerOutput(
|
||||||
req_ids=input_batch.req_ids,
|
req_ids=input_batch.req_ids,
|
||||||
|
|||||||
@@ -4250,7 +4250,6 @@ class GPUModelRunner(
|
|||||||
if not get_pp_group().is_last_rank:
|
if not get_pp_group().is_last_rank:
|
||||||
# Return the intermediate tensors.
|
# Return the intermediate tensors.
|
||||||
assert isinstance(hidden_states, IntermediateTensors)
|
assert isinstance(hidden_states, IntermediateTensors)
|
||||||
hidden_states.kv_connector_output = kv_connector_output
|
|
||||||
self.kv_connector_output = kv_connector_output
|
self.kv_connector_output = kv_connector_output
|
||||||
return hidden_states
|
return hidden_states
|
||||||
|
|
||||||
@@ -4326,17 +4325,9 @@ class GPUModelRunner(
|
|||||||
# receive sampled token ids from the last PP rank.
|
# receive sampled token ids from the last PP rank.
|
||||||
if self.use_async_scheduling and not get_pp_group().is_last_rank:
|
if self.use_async_scheduling and not get_pp_group().is_last_rank:
|
||||||
self._pp_receive_prev_sampled_token_ids_to_input_batch()
|
self._pp_receive_prev_sampled_token_ids_to_input_batch()
|
||||||
if not kv_connector_output:
|
|
||||||
return None # type: ignore[return-value]
|
|
||||||
|
|
||||||
# In case of PP with kv transfer, we need to pass through the
|
# In case of PP with kv transfer, we need to pass through the
|
||||||
# kv_connector_output
|
# kv_connector_output
|
||||||
if kv_connector_output.is_empty():
|
return ModelRunnerOutput.with_kv_conn_output_only(kv_connector_output)
|
||||||
return EMPTY_MODEL_RUNNER_OUTPUT
|
|
||||||
|
|
||||||
output = copy(EMPTY_MODEL_RUNNER_OUTPUT)
|
|
||||||
output.kv_connector_output = kv_connector_output
|
|
||||||
return output
|
|
||||||
|
|
||||||
# Unpack ephemeral state.
|
# Unpack ephemeral state.
|
||||||
(
|
(
|
||||||
|
|||||||
@@ -4,7 +4,6 @@
|
|||||||
Define KV connector functionality mixin for model runners.
|
Define KV connector functionality mixin for model runners.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import copy
|
|
||||||
from collections.abc import Generator
|
from collections.abc import Generator
|
||||||
from contextlib import AbstractContextManager, contextmanager, nullcontext
|
from contextlib import AbstractContextManager, contextmanager, nullcontext
|
||||||
from typing import TYPE_CHECKING
|
from typing import TYPE_CHECKING
|
||||||
@@ -20,7 +19,6 @@ from vllm.logger import init_logger
|
|||||||
from vllm.v1.attention.backend import AttentionBackend
|
from vllm.v1.attention.backend import AttentionBackend
|
||||||
from vllm.v1.kv_cache_interface import AttentionSpec, KVCacheConfig
|
from vllm.v1.kv_cache_interface import AttentionSpec, KVCacheConfig
|
||||||
from vllm.v1.outputs import (
|
from vllm.v1.outputs import (
|
||||||
EMPTY_MODEL_RUNNER_OUTPUT,
|
|
||||||
KVConnectorOutput,
|
KVConnectorOutput,
|
||||||
ModelRunnerOutput,
|
ModelRunnerOutput,
|
||||||
)
|
)
|
||||||
@@ -47,12 +45,7 @@ class KVConnectorModelRunnerMixin:
|
|||||||
):
|
):
|
||||||
pass
|
pass
|
||||||
|
|
||||||
if kv_connector_output.is_empty():
|
return ModelRunnerOutput.with_kv_conn_output_only(kv_connector_output)
|
||||||
return EMPTY_MODEL_RUNNER_OUTPUT
|
|
||||||
|
|
||||||
output = copy.copy(EMPTY_MODEL_RUNNER_OUTPUT)
|
|
||||||
output.kv_connector_output = kv_connector_output
|
|
||||||
return output
|
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def maybe_get_kv_connector_output(
|
def maybe_get_kv_connector_output(
|
||||||
|
|||||||
Reference in New Issue
Block a user