[Elastic EP] Reuse CUDA graphs across reconfiguration (#54985)

This commit is contained in:
Itay Alroy
2026-09-12 10:26:39 -04:00
committed by GitHub
parent 13e221f830
commit 658c8131c7
22 changed files with 372 additions and 270 deletions
@@ -8,6 +8,7 @@ import torch
from vllm.config import get_current_vllm_config
from vllm.distributed import (
GroupCoordinator,
get_ep_group,
)
from vllm.logger import init_logger
@@ -104,16 +105,12 @@ if current_platform.is_cuda_alike():
)
def get_ep_all2all_manager(eep_stage: bool = False) -> Any:
if eep_stage:
from vllm.distributed.elastic_ep.standby_state import get_standby_ep_group
ep_group = get_standby_ep_group()
assert ep_group is not None
device_communicator = ep_group.device_communicator
else:
device_communicator = get_ep_group().device_communicator
def get_ep_all2all_manager(
ep_group: GroupCoordinator | None = None,
) -> Any:
if ep_group is None:
ep_group = get_ep_group()
device_communicator = ep_group.device_communicator
assert device_communicator is not None
all2all_manager = device_communicator.all2all_manager
assert all2all_manager is not None
@@ -168,7 +165,7 @@ def maybe_make_prepare_finalize(
routing_tables: tuple[torch.Tensor, torch.Tensor, torch.Tensor] | None = None,
allow_new_interface: bool = False,
use_monolithic: bool = False,
eep_stage: bool = False,
all2all_manager: Any | None = None,
) -> FusedMoEPrepareAndFinalize | None:
if not moe.moe_parallel_config.use_all2all_kernels:
if not allow_new_interface:
@@ -190,7 +187,8 @@ def maybe_make_prepare_finalize(
"Detected DP deployment with no --enable-expert-parallel. "
"Falling back to AllGather+ReduceScatter dispatch/combine."
)
all2all_manager = get_ep_all2all_manager(eep_stage)
if all2all_manager is None:
all2all_manager = get_ep_all2all_manager()
return make_moe_prepare_and_finalize_naive_dp_ep(
is_sequence_parallel=moe.moe_parallel_config.is_sequence_parallel,
num_dispatchers=all2all_manager.world_size,
@@ -199,7 +197,8 @@ def maybe_make_prepare_finalize(
else:
return make_moe_prepare_and_finalize_no_dp_ep(use_monolithic)
all2all_manager = get_ep_all2all_manager(eep_stage)
if all2all_manager is None:
all2all_manager = get_ep_all2all_manager()
prepare_finalize: FusedMoEPrepareAndFinalize | None = None
@@ -361,10 +360,7 @@ def maybe_make_prepare_finalize(
all_to_all_args = dict(
max_num_tokens_per_dp_rank=moe.max_num_tokens,
token_hidden_size=moe.hidden_dim,
num_ep_ranks=all2all_manager.world_size,
num_global_experts=moe.num_experts,
num_local_experts=moe.num_experts // all2all_manager.world_size,
stage=eep_stage,
num_local_experts=moe.num_local_experts,
)
handle = all2all_manager.get_handle(all_to_all_args)
@@ -378,7 +374,8 @@ def maybe_make_prepare_finalize(
prepare_finalize = NixlEPPrepareAndFinalize(
handle,
max_tokens_per_rank=moe.max_num_tokens,
num_dispatchers=all2all_manager.world_size,
num_dispatchers=all2all_manager.max_num_ep_ranks,
expert_capacity=(moe.num_local_experts * all2all_manager.max_num_ep_ranks),
use_fp8_dispatch=use_fp8_dispatch,
global_to_physical=global_to_physical,
physical_to_global=physical_to_global,
@@ -1306,6 +1306,7 @@ class FusedMoEConfig:
moe_backend: MoEBackend = "auto"
max_num_tokens: int = SchedulerConfig.DEFAULT_MAX_NUM_BATCHED_TOKENS_FOR_BATCHED_DP
elastic_ep_max_dp_size: int | None = None
has_bias: bool = False
is_lora_enabled: bool = False
@@ -60,6 +60,8 @@ def _make_eep_experts(
def make_eep_staged_quant_method(
module: "MoERunner",
moe_config: FusedMoEConfig,
*,
all2all_manager: Any,
) -> FusedMoEMethodBase | None:
quant_method = module._quant_method
if not quant_method.supports_internal_mk:
@@ -93,7 +95,7 @@ def make_eep_staged_quant_method(
routing_tables=None,
allow_new_interface=True,
use_monolithic=quant_method.is_monolithic,
eep_stage=True,
all2all_manager=all2all_manager,
)
assert prepare_finalize is not None
assert isinstance(prepare_finalize, FusedMoEPrepareAndFinalizeModular)
@@ -348,6 +348,7 @@ def FusedMoEFactory(
moe_backend=vllm_config.kernel_config.moe_backend,
router_logits_dtype=router_logits_dtype,
max_num_tokens=max_num_batched_tokens,
elastic_ep_max_dp_size=vllm_config.parallel_config.elastic_ep_max_dp_size,
has_bias=has_bias,
is_lora_enabled=vllm_config.lora_config is not None,
activation=moe_activation,
@@ -252,13 +252,6 @@ class FusedMoEPrepareAndFinalize(ABC):
"""
return False
def on_commit(self) -> None:
"""
Runs after this prepare/finalize has been committed to the active
MoE kernel.
"""
return
# TODO: pass FusedMoEParallelConfig in as ctor parameter?
class FusedMoEPrepareAndFinalizeModular(FusedMoEPrepareAndFinalize):
@@ -6,8 +6,6 @@ import nixl_ep
import torch
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
from vllm.distributed import get_ep_group
from vllm.distributed.device_communicators.all2all import NixlEPAll2AllManager
from vllm.logger import init_logger
from vllm.model_executor.layers.fused_moe.config import FusedMoEQuantConfig
from vllm.model_executor.layers.fused_moe.topk_weight_and_reduce import (
@@ -79,6 +77,7 @@ class NixlEPPrepareAndFinalize(mk.FusedMoEPrepareAndFinalizeModular):
buffer: nixl_ep.Buffer,
max_tokens_per_rank: int,
num_dispatchers: int,
expert_capacity: int,
use_fp8_dispatch: bool = False,
global_to_physical: torch.Tensor | None = None,
physical_to_global: torch.Tensor | None = None,
@@ -94,6 +93,7 @@ class NixlEPPrepareAndFinalize(mk.FusedMoEPrepareAndFinalizeModular):
# combine function.
self.handles: list[tuple | None] = [None, None]
self.num_dispatchers_ = num_dispatchers
self.expert_capacity = expert_capacity
topk_indices_dtype = self.topk_indices_dtype()
@@ -141,17 +141,6 @@ class NixlEPPrepareAndFinalize(mk.FusedMoEPrepareAndFinalizeModular):
def max_num_tokens_per_rank(self) -> int | None:
return self.max_tokens_per_rank
def on_commit(self) -> None:
device_communicator = get_ep_group().device_communicator
assert device_communicator is not None
all2all_manager = device_communicator.all2all_manager
assert isinstance(all2all_manager, NixlEPAll2AllManager)
# maybe_make_prepare_finalize(..., eep_stage=True) initializes self.buffer
# with get_handle(..., stage=True), which stages global NIXL state for the
# new config but leaves it inactive while the old config remains active.
# When EEP commit switches to this P/F, this P/F needs to commit that state.
all2all_manager.commit_staged_state()
def topk_indices_dtype(self) -> torch.dtype | None:
return NIXL_EP_TOPK_INDICES_DTYPE
@@ -267,7 +256,7 @@ class NixlEPPrepareAndFinalize(mk.FusedMoEPrepareAndFinalizeModular):
a1,
dispatch_topk_ids,
self.max_tokens_per_rank,
num_experts,
self.expert_capacity,
use_fp8=self.use_fp8_dispatch,
# round_scale needs to be set to dispatch in ue8m0
round_scale=self.use_ue8m0_dispatch,
@@ -500,7 +500,8 @@ def fi_moe_largest_bucket(moe_config: "FusedMoEConfig") -> int:
For a detailed explanation, see: `docs/serving/data_parallel_deployment.md`
"""
return max(moe_config.max_num_tokens * moe_config.dp_size, 8192)
dp_size = moe_config.elastic_ep_max_dp_size or moe_config.dp_size
return max(moe_config.max_num_tokens * dp_size, 8192)
@torch.compile(dynamic=True, backend=current_platform.simple_compile_backend)