[Elastic EP] Support Model Runner V2 (#53934)

Signed-off-by: almogtavor <almogtavor@gmail.com>
Signed-off-by: Almog Tavor <70065337+almogtavor@users.noreply.github.com>
This commit is contained in:
Almog Tavor
2026-09-29 00:20:40 +00:00
committed by GitHub
parent c68eb98b69
commit 171d1b9ef6
12 changed files with 171 additions and 80 deletions
+41 -20
View File
@@ -1,9 +1,10 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Callable
from collections.abc import Callable, Iterator
from contextlib import contextmanager
from functools import wraps
from typing import Any
from typing import TYPE_CHECKING, Any
import torch
import torch.nn as nn
@@ -15,6 +16,9 @@ from vllm.model_executor.models.interfaces import (
get_mixture_of_experts_model,
)
if TYPE_CHECKING:
from vllm.v1.worker.gpu.model_runner import GPUModelRunner
logger = init_logger(__name__)
@@ -37,6 +41,27 @@ def step_eplb_after(*, is_dummy: bool = False) -> Callable:
return decorator
@contextmanager
def preserve_serving_state(model_runner: "GPUModelRunner") -> Iterator[None]:
"""Keep the elastic EP warmup out of the request pool and the KV cache."""
# After the drain only parked streaming sessions still hold a slot, and
# those are re-added from NewRequestData on their next chunk.
_remove_all_requests(model_runner)
model_runner.block_tables.redirect_writes_to_null_block = True
try:
yield
finally:
_remove_all_requests(model_runner)
model_runner.block_tables.redirect_writes_to_null_block = False
if model_runner.kv_block_zeroer is not None:
model_runner.kv_block_zeroer.zero_block_ids([0])
def _remove_all_requests(model_runner: "GPUModelRunner") -> None:
for req_id in list(model_runner.req_states.req_id_to_index):
model_runner._remove_request(req_id)
class EPLBController:
def __init__(self, parallel_config: Any, device: torch.device):
self.parallel_config = parallel_config
@@ -82,10 +107,8 @@ class EPLBController:
self._has_registered_models = True
return True
def maybe_register_model(
self, model: nn.Module, model_config: Any, load_dummy_weights: bool
) -> bool:
if not self.parallel_config.enable_eplb or load_dummy_weights:
def maybe_register_model(self, model: nn.Module, model_config: ModelConfig) -> bool:
if not self.parallel_config.enable_eplb:
return False
moe_model = get_mixture_of_experts_model(model)
@@ -100,7 +123,13 @@ class EPLBController:
self._has_registered_models = True
return True
def maybe_start_async_loop(self, eplb_models_added: bool) -> None:
def maybe_start_async_loop(
self, eplb_models_added: bool, load_dummy_weights: bool = False
) -> None:
# A scaling-up worker registers to join the communicator handshake, but
# its groups are not live yet, so _perform_eplb_reshuffle() starts this.
if load_dummy_weights:
return
if eplb_models_added and self.state is not None and self.state.is_async:
self.state.start_async_loop()
@@ -128,18 +157,10 @@ class EPLBController:
def setup_from_mapping(
self,
model: nn.Module,
model_config: Any,
model_config: ModelConfig,
expanded_physical_to_logical: torch.Tensor,
) -> None:
moe_model = get_mixture_of_experts_model(model)
assert moe_model is not None
self.state = EplbState.from_mapping(
model=moe_model,
model_config=model_config,
device=self.device,
parallel_config=self.parallel_config,
expanded_physical_to_logical=expanded_physical_to_logical,
)
self._has_registered_models = True
# Update in place: a fresh EplbState would build a second communicator
# over a group the other ranks have already left.
assert self.state is not None
self.state.update_mapping(model_config, expanded_physical_to_logical)