[Elastic EP] Support Model Runner V2 (#53934)

Signed-off-by: almogtavor <almogtavor@gmail.com>
Signed-off-by: Almog Tavor <70065337+almogtavor@users.noreply.github.com>
This commit is contained in:
Almog Tavor
2026-09-29 00:20:40 +00:00
committed by GitHub
parent c68eb98b69
commit 171d1b9ef6
12 changed files with 171 additions and 80 deletions
+33 -22
View File
@@ -54,6 +54,8 @@ from vllm.platforms import current_platform
from vllm.utils import is_moe_layer
from vllm.v1.engine import ReconfigureDistributedRequest, ReconfigureRankType
from vllm.v1.worker.dp_utils import skip_dp_coordination
from vllm.v1.worker.gpu.cudagraph_utils import CudaGraphManager
from vllm.v1.worker.gpu.eplb_utils import preserve_serving_state
from vllm.v1.worker.gpu_ubatch_wrapper import UBatchWrapper
from vllm.v1.worker.workspace import lock_workspace, unlock_workspace
@@ -171,6 +173,17 @@ class ElasticEPScalingExecutor:
self.worker.vllm_config.parallel_config,
)
@contextmanager
def _suppress_eplb(self) -> Iterator[None]:
# A rank warming up alone must not advance the EPLB step counter.
runner = self.worker.model_runner
was_suppressed = runner.eep_eplb_suppressed
runner.eep_eplb_suppressed = True
try:
yield
finally:
runner.eep_eplb_suppressed = was_suppressed
@contextmanager
def _disable_flashinfer_autotune(self) -> Iterator[None]:
kernel_config = self.worker.vllm_config.kernel_config
@@ -421,7 +434,17 @@ class ElasticEPScalingExecutor:
self._staged_moe_quant_methods.clear()
def _release_cuda_graphs(self) -> None:
if isinstance(self.worker.model_runner.model, CUDAGraphWrapper):
manager = getattr(self.worker.model_runner, "cudagraph_manager", None)
speculator = getattr(self.worker.model_runner, "speculator", None)
if manager is not None:
# MRV2 captures through CudaGraphManager instead of wrapping the
# model, so neither wrapper branch below ever fires.
manager.release_graphs()
for attr in vars(speculator).values() if speculator else ():
if isinstance(attr, CudaGraphManager):
attr.release_graphs()
elif isinstance(self.worker.model_runner.model, CUDAGraphWrapper):
wrapper = self.worker.model_runner.model
wrapper.concrete_cudagraph_entries = {}
@@ -431,6 +454,8 @@ class ElasticEPScalingExecutor:
torch.compiler.reset()
with set_current_vllm_config(self.worker.vllm_config):
reset_compile_wrapper(self.worker.model_runner.get_model())
if speculator is not None:
reset_compile_wrapper(speculator.model)
gc.collect()
torch.accelerator.synchronize()
@@ -675,42 +700,28 @@ class ElasticEPScalingExecutor:
self.warm_and_capture()
def warm_and_capture(self) -> None:
# Save and clear block tables so the dummy MoE forward doesn't
# write dummy slot mappings into real KV-cache blocks.
multi_block_table = self.worker.model_runner.input_batch.block_table
saved_block_tables: list[tuple[torch.Tensor, torch.Tensor]] = []
for bt in multi_block_table.block_tables:
saved_block_tables.append(
(bt.block_table.gpu.clone(), bt.block_table.cpu.clone())
)
multi_block_table.clear()
# _ensure_workspace_size allocates a fresh tensor on grow, leaving
# any captured CUDA graph with a stale data pointer; drop graphs
# before re-warm so captures realign with the resized buffer.
self._release_cuda_graphs()
unlock_workspace()
# Grow the MoE workspace at max_num_tokens. compile_or_warm_up_model
# alone only exercises cudagraph-capture sizes and can leave the
# workspace too small for post-reshuffle routing. Use _dummy_run
# directly with skip_eplb=True so dummy routing doesn't pollute the
# just-rebalanced EPLB stats.
runner = self.worker.model_runner
all2all_manager = get_ep_all2all_manager()
reuse_kernel = self._can_reuse_fused_moe_kernel()
serving_state = (
preserve_serving_state(runner)
if self.worker.use_v2_model_runner
else runner.preserve_serving_state()
)
with (
skip_dp_coordination() if reuse_kernel else nullcontext(),
all2all_manager.mask_remote_ranks() if reuse_kernel else nullcontext(),
self._disable_flashinfer_autotune() if reuse_kernel else nullcontext(),
self._suppress_eplb(),
serving_state,
):
runner._dummy_run(runner.max_num_tokens, is_profile=True, skip_eplb=True)
self.worker.compile_or_warm_up_model()
lock_workspace()
for bt, (saved_gpu, saved_cpu) in zip(
multi_block_table.block_tables, saved_block_tables
):
bt.block_table.gpu.copy_(saved_gpu)
bt.block_table.cpu.copy_(saved_cpu)