[Elastic EP] Support Model Runner V2 (#53934)
Signed-off-by: almogtavor <almogtavor@gmail.com> Signed-off-by: Almog Tavor <70065337+almogtavor@users.noreply.github.com>
This commit is contained in:
@@ -1,9 +1,10 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from collections.abc import Callable
|
||||
from collections.abc import Callable, Iterator
|
||||
from contextlib import contextmanager
|
||||
from functools import wraps
|
||||
from typing import Any
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
@@ -15,6 +16,9 @@ from vllm.model_executor.models.interfaces import (
|
||||
get_mixture_of_experts_model,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.v1.worker.gpu.model_runner import GPUModelRunner
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
@@ -37,6 +41,27 @@ def step_eplb_after(*, is_dummy: bool = False) -> Callable:
|
||||
return decorator
|
||||
|
||||
|
||||
@contextmanager
|
||||
def preserve_serving_state(model_runner: "GPUModelRunner") -> Iterator[None]:
|
||||
"""Keep the elastic EP warmup out of the request pool and the KV cache."""
|
||||
# After the drain only parked streaming sessions still hold a slot, and
|
||||
# those are re-added from NewRequestData on their next chunk.
|
||||
_remove_all_requests(model_runner)
|
||||
model_runner.block_tables.redirect_writes_to_null_block = True
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
_remove_all_requests(model_runner)
|
||||
model_runner.block_tables.redirect_writes_to_null_block = False
|
||||
if model_runner.kv_block_zeroer is not None:
|
||||
model_runner.kv_block_zeroer.zero_block_ids([0])
|
||||
|
||||
|
||||
def _remove_all_requests(model_runner: "GPUModelRunner") -> None:
|
||||
for req_id in list(model_runner.req_states.req_id_to_index):
|
||||
model_runner._remove_request(req_id)
|
||||
|
||||
|
||||
class EPLBController:
|
||||
def __init__(self, parallel_config: Any, device: torch.device):
|
||||
self.parallel_config = parallel_config
|
||||
@@ -82,10 +107,8 @@ class EPLBController:
|
||||
self._has_registered_models = True
|
||||
return True
|
||||
|
||||
def maybe_register_model(
|
||||
self, model: nn.Module, model_config: Any, load_dummy_weights: bool
|
||||
) -> bool:
|
||||
if not self.parallel_config.enable_eplb or load_dummy_weights:
|
||||
def maybe_register_model(self, model: nn.Module, model_config: ModelConfig) -> bool:
|
||||
if not self.parallel_config.enable_eplb:
|
||||
return False
|
||||
|
||||
moe_model = get_mixture_of_experts_model(model)
|
||||
@@ -100,7 +123,13 @@ class EPLBController:
|
||||
self._has_registered_models = True
|
||||
return True
|
||||
|
||||
def maybe_start_async_loop(self, eplb_models_added: bool) -> None:
|
||||
def maybe_start_async_loop(
|
||||
self, eplb_models_added: bool, load_dummy_weights: bool = False
|
||||
) -> None:
|
||||
# A scaling-up worker registers to join the communicator handshake, but
|
||||
# its groups are not live yet, so _perform_eplb_reshuffle() starts this.
|
||||
if load_dummy_weights:
|
||||
return
|
||||
if eplb_models_added and self.state is not None and self.state.is_async:
|
||||
self.state.start_async_loop()
|
||||
|
||||
@@ -128,18 +157,10 @@ class EPLBController:
|
||||
|
||||
def setup_from_mapping(
|
||||
self,
|
||||
model: nn.Module,
|
||||
model_config: Any,
|
||||
model_config: ModelConfig,
|
||||
expanded_physical_to_logical: torch.Tensor,
|
||||
) -> None:
|
||||
moe_model = get_mixture_of_experts_model(model)
|
||||
assert moe_model is not None
|
||||
|
||||
self.state = EplbState.from_mapping(
|
||||
model=moe_model,
|
||||
model_config=model_config,
|
||||
device=self.device,
|
||||
parallel_config=self.parallel_config,
|
||||
expanded_physical_to_logical=expanded_physical_to_logical,
|
||||
)
|
||||
self._has_registered_models = True
|
||||
# Update in place: a fresh EplbState would build a second communicator
|
||||
# over a group the other ranks have already left.
|
||||
assert self.state is not None
|
||||
self.state.update_mapping(model_config, expanded_physical_to_logical)
|
||||
|
||||
Reference in New Issue
Block a user