[Elastic EP] Reuse CUDA graphs across reconfiguration (#54985)

This commit is contained in:
Itay Alroy
2026-09-12 10:26:39 -04:00
committed by GitHub
parent 13e221f830
commit 658c8131c7
22 changed files with 372 additions and 270 deletions
@@ -1,6 +1,8 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import threading
from collections.abc import Iterator
from contextlib import contextmanager
from dataclasses import dataclass
from typing import Any
@@ -377,6 +379,7 @@ class DeepEPLLAll2AllManager(DeepEPAll2AllManagerBase):
@dataclass
class _NixlEPBufferState:
buffer: Any
max_num_ep_ranks: int
connected_ep_size: int
active_ep_size: int
@@ -402,7 +405,10 @@ class NixlEPAll2AllManager(All2AllManagerBase):
super().__init__(cpu_group, tcp_store_group)
self.support_fault_tolerance = True
self.max_num_ep_ranks = envs.VLLM_NIXL_EP_MAX_NUM_RANKS
@property
def max_num_ep_ranks(self) -> int:
assert NixlEPAll2AllManager._buffer is not None
return NixlEPAll2AllManager._buffer.max_num_ep_ranks
def _init_buffer(
self,
@@ -412,11 +418,19 @@ class NixlEPAll2AllManager(All2AllManagerBase):
) -> None:
from nixl_ep import Buffer # type: ignore[import-not-found]
max_num_global_experts = self.max_num_ep_ranks * num_experts_per_rank
parallel_config = get_current_vllm_config().parallel_config
max_num_ep_ranks = (
self.world_size
* parallel_config.elastic_ep_max_dp_size
// parallel_config.data_parallel_size
if parallel_config.enable_elastic_ep
else self.world_size
)
max_num_global_experts = max_num_ep_ranks * num_experts_per_rank
num_rdma_bytes = Buffer.get_rdma_size_hint(
num_max_dispatch_tokens_per_rank=max_num_tokens_per_dp_rank,
hidden=token_hidden_size,
num_ranks=self.max_num_ep_ranks,
num_ranks=max_num_ep_ranks,
num_experts=max_num_global_experts,
)
assert NixlEPAll2AllManager._buffer is None, (
@@ -427,7 +441,7 @@ class NixlEPAll2AllManager(All2AllManagerBase):
tcp_store_group=self.tcp_store_group.store,
)
buffer.update_memory_buffers(
num_ranks=self.max_num_ep_ranks,
num_ranks=max_num_ep_ranks,
num_experts_per_rank=num_experts_per_rank,
num_rdma_bytes=num_rdma_bytes,
)
@@ -435,6 +449,7 @@ class NixlEPAll2AllManager(All2AllManagerBase):
buffer.connect_ranks(ranks_to_connect)
NixlEPAll2AllManager._buffer = _NixlEPBufferState(
buffer=buffer,
max_num_ep_ranks=max_num_ep_ranks,
connected_ep_size=self.world_size,
active_ep_size=self.world_size,
)
@@ -474,6 +489,7 @@ class NixlEPAll2AllManager(All2AllManagerBase):
for rank in range(state.active_ep_size, target_ep_size):
state.buffer.update_mask_buffer(rank, mask=False)
torch.accelerator.synchronize()
state.active_ep_size = target_ep_size
def _stage_ep_size(self) -> None:
@@ -486,8 +502,12 @@ class NixlEPAll2AllManager(All2AllManagerBase):
if target_ep_size > state.connected_ep_size:
self._connect_to_ep_size(target_ep_size, make_active=False)
def commit_staged_state(self) -> None:
"""Commit staged NIXL EP state to the active communication set."""
def stage_ep_size(self) -> None:
with NixlEPAll2AllManager._lock:
self._stage_ep_size()
def commit_ep_size(self) -> None:
"""Commit the staged EP size to the active communication set."""
with NixlEPAll2AllManager._lock:
assert NixlEPAll2AllManager._buffer is not None
state = NixlEPAll2AllManager._buffer
@@ -500,31 +520,31 @@ class NixlEPAll2AllManager(All2AllManagerBase):
self._unmask_connected_ranks(target_ep_size)
def _ensure_ep_size(self, *, stage: bool) -> None:
if stage:
self._stage_ep_size()
else:
self.commit_staged_state()
@contextmanager
def mask_remote_ranks(self) -> Iterator[None]:
peers = [rank for rank in range(self.world_size) if rank != self.rank]
assert NixlEPAll2AllManager._buffer is not None
buffer = NixlEPAll2AllManager._buffer.buffer
for rank in peers:
buffer.update_mask_buffer(rank, mask=True)
torch.accelerator.synchronize()
try:
yield
finally:
for rank in peers:
buffer.update_mask_buffer(rank, mask=False)
torch.accelerator.synchronize()
def get_handle(self, kwargs):
with NixlEPAll2AllManager._lock:
stage = bool(kwargs.get("stage", False))
state = NixlEPAll2AllManager._buffer
if state is None:
assert not stage, (
"NIXL EP staged initialization requires an existing buffer"
)
max_num_tokens_per_dp_rank = kwargs["max_num_tokens_per_dp_rank"]
num_experts_per_rank = (
kwargs["num_global_experts"] // kwargs["num_ep_ranks"]
)
self._init_buffer(
max_num_tokens_per_dp_rank=max_num_tokens_per_dp_rank,
token_hidden_size=kwargs["token_hidden_size"],
num_experts_per_rank=num_experts_per_rank,
num_experts_per_rank=kwargs["num_local_experts"],
)
else:
self._ensure_ep_size(stage=stage)
assert NixlEPAll2AllManager._buffer is not None
handle = NixlEPAll2AllManager._buffer.buffer
@@ -564,7 +584,7 @@ class NixlEPAll2AllManager(All2AllManagerBase):
assert state is not None
if NixlEPAll2AllManager._mask is None:
NixlEPAll2AllManager._mask = torch.zeros(
self.max_num_ep_ranks, device="cuda", dtype=torch.int32
state.max_num_ep_ranks, device="cuda", dtype=torch.int32
)
state.buffer.query_mask_buffer(NixlEPAll2AllManager._mask)
return NixlEPAll2AllManager._mask[: state.active_ep_size]
@@ -1,6 +1,8 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import threading
from collections.abc import Iterator
from contextlib import contextmanager
from weakref import WeakValueDictionary
import torch
@@ -157,6 +159,16 @@ class All2AllManagerBase:
def destroy(self):
pass
def stage_ep_size(self) -> None:
pass
def commit_ep_size(self) -> None:
pass
@contextmanager
def mask_remote_ranks(self) -> Iterator[None]:
yield
class DeviceCommunicatorBase:
"""
+77 -116
View File
@@ -2,34 +2,32 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import gc
import weakref
from collections.abc import Iterable, Sequence
from collections.abc import Iterable, Iterator, Sequence
from concurrent.futures import Future, ThreadPoolExecutor
from contextlib import contextmanager, nullcontext
from dataclasses import replace
from typing import TYPE_CHECKING
import torch
import torch.nn as nn
import torch.nn.functional as F
from torch.distributed import P2POp
from vllm.compilation.counter import compilation_counter
from vllm.compilation.cuda_graph import CUDAGraphWrapper
from vllm.compilation.wrapper import reset_compile_wrapper
from vllm.config import (
CompilationMode,
set_current_vllm_config,
)
from vllm.config import ParallelConfig, set_current_vllm_config
from vllm.distributed import (
get_dp_group,
get_ep_group,
get_pcp_group,
get_tp_group,
get_world_group,
)
from vllm.distributed.elastic_ep.standby_state import (
create_standby_groups,
get_standby_dp_group,
get_standby_ep_group,
get_standby_eplb_group,
get_standby_world_group,
pop_standby_groups,
)
from vllm.distributed.eplb.eplb_communicator import (
@@ -41,13 +39,20 @@ from vllm.distributed.parallel_state import (
)
from vllm.distributed.stateless_coordinator import StatelessGroupCoordinator
from vllm.logger import init_logger
from vllm.model_executor.layers.fused_moe.all2all_utils import (
get_ep_all2all_manager,
)
from vllm.model_executor.layers.fused_moe.config import FusedMoEParallelConfig
from vllm.model_executor.layers.fused_moe.eep_reconfigure import (
make_eep_staged_quant_method,
)
from vllm.model_executor.warmup.flashinfer_autotune_cache import (
sync_flashinfer_autotune_cache,
)
from vllm.model_executor.warmup.kernel_warmup import kernel_warmup
from vllm.utils import is_moe_layer
from vllm.v1.engine import ReconfigureDistributedRequest, ReconfigureRankType
from vllm.v1.worker.dp_utils import skip_dp_coordination
from vllm.v1.worker.gpu_ubatch_wrapper import UBatchWrapper
from vllm.v1.worker.workspace import lock_workspace, unlock_workspace
@@ -59,6 +64,15 @@ if TYPE_CHECKING:
)
def can_reuse_fused_moe_kernel(parallel_config: ParallelConfig) -> bool:
"""Whether fused MoE kernels can be reused across EP rank changes.
The EP backend must preserve its CUDA graph entries when ranks are connected,
disconnected, masked, or unmasked.
"""
return parallel_config.all2all_backend == "nixl_ep"
def batch_transfer_weights(
model: nn.Module,
is_sender: bool,
@@ -105,31 +119,20 @@ def batch_transfer_weights(
def broadcast_expert_mapping(
physical_to_logical: torch.Tensor | None,
num_local_physical_experts: int | None,
num_logical_experts: int | None,
dp_group: StatelessGroupCoordinator,
device: torch.device,
src_rank: int = 0,
) -> tuple[torch.Tensor, int, int]:
) -> torch.Tensor:
if dp_group.rank_in_group == src_rank:
assert physical_to_logical is not None
assert num_local_physical_experts is not None
assert num_logical_experts is not None
assert physical_to_logical.dtype == torch.int64
shape_tensor = torch.tensor(
list(physical_to_logical.shape), dtype=torch.int64, device="cpu"
)
metadata_tensor = torch.tensor(
[num_local_physical_experts, num_logical_experts],
dtype=torch.int64,
device="cpu",
)
else:
shape_tensor = torch.empty(2, dtype=torch.int64, device="cpu")
metadata_tensor = torch.empty(2, dtype=torch.int64, device="cpu")
shape_tensor = dp_group.tcp_store_group.broadcast(shape_tensor, src_rank)
metadata_tensor = dp_group.tcp_store_group.broadcast(metadata_tensor, src_rank)
if dp_group.rank_in_group != src_rank:
assert device is not None
@@ -140,11 +143,7 @@ def broadcast_expert_mapping(
)
assert physical_to_logical is not None
physical_to_logical = dp_group.broadcast(physical_to_logical, src_rank)
num_local_physical_experts = int(metadata_tensor[0].item())
num_logical_experts = int(metadata_tensor[1].item())
return physical_to_logical, num_local_physical_experts, num_logical_experts
return dp_group.broadcast(physical_to_logical, src_rank)
class ElasticEPScalingExecutor:
@@ -166,6 +165,21 @@ class ElasticEPScalingExecutor:
raise RuntimeError("Worker has been garbage collected")
return worker
def _can_reuse_fused_moe_kernel(self) -> bool:
return can_reuse_fused_moe_kernel(
self.worker.vllm_config.parallel_config,
)
@contextmanager
def _disable_flashinfer_autotune(self) -> Iterator[None]:
kernel_config = self.worker.vllm_config.kernel_config
enabled = kernel_config.enable_flashinfer_autotune
kernel_config.enable_flashinfer_autotune = False
try:
yield
finally:
kernel_config.enable_flashinfer_autotune = enabled
def execute(self, execute_method: str, *args, **kwargs):
method = getattr(self, execute_method, None)
if method is None:
@@ -260,11 +274,20 @@ class ElasticEPScalingExecutor:
use_all2all=use_all2all,
enable_eplb=parallel_config.enable_eplb,
)
self.stage_standby_moe_quant_methods()
standby_ep_group = get_standby_ep_group()
assert standby_ep_group is not None
all2all_manager = get_ep_all2all_manager(standby_ep_group)
all2all_manager.stage_ep_size()
if not self._can_reuse_fused_moe_kernel():
self.stage_standby_moe_quant_methods(all2all_manager)
self._prepare_eplb_communicator(get_standby_eplb_group())
if new_dp_size > old_dp_size:
self.transfer_weights(old_dp_size, new_dp_size)
self._warm_target_groups(get_standby_dp_group(), get_standby_ep_group())
self._warm_target_groups(get_standby_dp_group(), standby_ep_group)
if new_dp_size > old_dp_size and self._can_reuse_fused_moe_kernel():
target_world_group = get_standby_world_group()
assert target_world_group is not None
sync_flashinfer_autotune_cache(self.worker.model_runner, target_world_group)
def _prepare_eplb_communicator(self, eplb_group) -> None:
assert eplb_group is not None
@@ -338,14 +361,9 @@ class ElasticEPScalingExecutor:
assert eplb_state is not None
eplb_state.drain_async()
eplb_model_state = eplb_state.model_states[model_config.compute_hash()]
physical_to_logical = eplb_model_state.physical_to_logical_map
num_physical_experts = physical_to_logical.shape[1]
num_local_physical_experts = num_physical_experts // get_ep_group().world_size
num_logical_experts = eplb_model_state.logical_replica_count.shape[1]
physical_to_logical = eplb_model_state.physical_to_logical_map_buffer
broadcast_expert_mapping(
physical_to_logical=physical_to_logical,
num_local_physical_experts=num_local_physical_experts,
num_logical_experts=num_logical_experts,
dp_group=standby_dp_group,
src_rank=0,
device=self.worker.device,
@@ -368,7 +386,7 @@ class ElasticEPScalingExecutor:
moe_parallel_config=moe_parallel_config,
)
def stage_standby_moe_quant_methods(self) -> None:
def stage_standby_moe_quant_methods(self, all2all_manager) -> None:
standby_dp_group = get_standby_dp_group()
standby_ep_group = get_standby_ep_group()
model = self.worker.model_runner.get_model()
@@ -383,6 +401,7 @@ class ElasticEPScalingExecutor:
standby_dp_group,
standby_ep_group,
),
all2all_manager=all2all_manager,
)
if staged_quant_method is not None:
self._staged_moe_quant_methods[module] = staged_quant_method
@@ -396,7 +415,6 @@ class ElasticEPScalingExecutor:
continue
assert staged_quant_method.moe_kernel is not None
module._replace_quant_method(staged_quant_method)
staged_quant_method.moe_kernel.prepare_finalize.on_commit()
self._staged_moe_quant_methods.clear()
def _release_cuda_graphs(self) -> None:
@@ -405,7 +423,7 @@ class ElasticEPScalingExecutor:
wrapper.concrete_cudagraph_entries = {}
elif isinstance(self.worker.model_runner.model, UBatchWrapper):
raise RuntimeError("DBO is not yet supported in elastic EP")
self.worker.model_runner.model.clear_graphs()
torch.compiler.reset()
with set_current_vllm_config(self.worker.vllm_config):
@@ -427,10 +445,8 @@ class ElasticEPScalingExecutor:
self._wait_for_group_cleanup()
def switch_and_prepare(self) -> tuple[GroupCoordinator | None, ...]:
old_dp_size = get_dp_group().world_size
old_ep_size = get_ep_group().world_size
self._release_cuda_graphs()
if not self._can_reuse_fused_moe_kernel():
self._release_cuda_graphs()
retired_groups = _replace_active_groups(**pop_standby_groups())
parallel_config = self.worker.vllm_config.parallel_config
@@ -487,47 +503,13 @@ class ElasticEPScalingExecutor:
parallel_config.eplb_config.num_redundant_experts = (
num_physical_experts - num_logical_experts
)
old_physical_to_logical = eplb_model_state.physical_to_logical_map
num_moe_layers = old_physical_to_logical.shape[0]
num_local_experts = eplb_model_state.expert_load_pass.shape[1] // old_ep_size
if new_dp_size > old_dp_size:
expanded_physical_to_logical = torch.full(
(num_moe_layers, num_local_experts * new_ep_size),
-1,
dtype=old_physical_to_logical.dtype,
device=old_physical_to_logical.device,
)
expanded_physical_to_logical[:, : num_local_experts * old_ep_size] = (
old_physical_to_logical
)
eplb_model_state.physical_to_logical_map = expanded_physical_to_logical
old_num_physical_experts = eplb_model_state.expert_load_pass.shape[1]
pad_size = num_physical_experts - old_num_physical_experts
if new_dp_size > old_dp_size:
assert pad_size > 0
expanded_expert_load_pass = F.pad(
eplb_model_state.expert_load_pass, (0, pad_size), value=0
)
expanded_expert_load_window = F.pad(
eplb_model_state.expert_load_window, (0, pad_size), value=0
)
eplb_model_state.expert_load_pass = expanded_expert_load_pass
eplb_model_state.expert_load_window = expanded_expert_load_window
else:
assert pad_size < 0
eplb_model_state.expert_load_pass = eplb_model_state.expert_load_pass[
:, :num_physical_experts
]
eplb_model_state.expert_load_window = eplb_model_state.expert_load_window[
:, :, :num_physical_experts
]
eplb_state.reconfigure_physical_expert_slots(model_config, num_physical_experts)
model = self.worker.model_runner.get_model()
model.expert_weights = []
with set_current_vllm_config(self.worker.vllm_config):
model.set_eplb_state(
eplb_model_state.expert_load_pass,
eplb_model_state.expert_load_pass_buffer,
eplb_model_state.logical_to_physical_map,
eplb_model_state.logical_replica_count,
)
@@ -539,6 +521,7 @@ class ElasticEPScalingExecutor:
num_local_physical_experts=num_local_experts,
)
self._commit_staged_moe_quant_methods()
get_ep_all2all_manager().commit_ep_size()
# Legacy modular methods need to be recreated for the new EP size.
for module in moe_modules:
if getattr(module._quant_method, "wraps_legacy_quant_method", False):
@@ -548,21 +531,6 @@ class ElasticEPScalingExecutor:
eplb_state.update_communicator(model_config, self._prepared_eplb_communicator)
self._prepared_eplb_communicator = None
if (
self.worker.vllm_config.compilation_config.mode
== CompilationMode.STOCK_TORCH_COMPILE
):
# NOTE(yongji): when using stock torch.compile,
# torch.compile is triggered during GPUModelRunner's load_model()
# TODO(yongji):check do we need to re-trigger torch.compile here?
# any changes to the tensor shapes in execution should already
# be handled internally by torch.compile.
backend = self.worker.vllm_config.compilation_config.init_backend(
self.worker.vllm_config
)
compilation_counter.stock_torch_compile_count += 1
self.worker.model_runner.model.compile(fullgraph=True, backend=backend)
return retired_groups
def _perform_eplb_reshuffle(
@@ -608,7 +576,8 @@ class ElasticEPScalingExecutor:
else:
mapping = self.receive_expert_mapping()
self.worker.model_runner.setup_eplb_from_mapping(mapping)
self.warm_and_capture()
if not self._can_reuse_fused_moe_kernel():
self.warm_and_capture()
self._perform_eplb_reshuffle(async_op=True)
if is_existing_worker:
self._start_group_cleanup(retired_groups)
@@ -619,7 +588,8 @@ class ElasticEPScalingExecutor:
self.switch_and_remove()
else:
retired_groups = self.switch_and_prepare()
self.warm_and_capture()
if not self._can_reuse_fused_moe_kernel():
self.warm_and_capture()
self._start_group_cleanup(retired_groups)
def perform_scale_down_eplb_reshuffle(self, new_dp_size: int) -> None:
@@ -677,42 +647,26 @@ class ElasticEPScalingExecutor:
)
torch.accelerator.synchronize()
self._warm_target_groups(get_dp_group(), get_ep_group())
if self._can_reuse_fused_moe_kernel():
sync_flashinfer_autotune_cache(self.worker.model_runner, get_world_group())
def receive_expert_mapping(self) -> torch.Tensor:
dp_group = get_dp_group()
assert isinstance(dp_group, StatelessGroupCoordinator)
physical_to_logical, num_local_physical_experts, _ = broadcast_expert_mapping(
return broadcast_expert_mapping(
physical_to_logical=None,
num_local_physical_experts=None,
num_logical_experts=None,
dp_group=dp_group,
src_rank=0,
device=self.worker.device,
)
num_moe_layers = physical_to_logical.shape[0]
new_dp_size = get_dp_group().world_size
tp_size = self.worker.vllm_config.parallel_config.tensor_parallel_size
new_ep_size = new_dp_size * tp_size
expanded_physical_to_logical = torch.full(
(num_moe_layers, num_local_physical_experts * new_ep_size),
-1,
dtype=physical_to_logical.dtype,
device=physical_to_logical.device,
)
expanded_physical_to_logical[:, : physical_to_logical.shape[1]] = (
physical_to_logical
)
return expanded_physical_to_logical
def warmup_local_kernels(self) -> None:
def warmup_new_worker(self) -> None:
with set_current_vllm_config(self.worker.vllm_config):
kernel_warmup(self.worker, process_local_only=True)
if self._can_reuse_fused_moe_kernel():
self.warm_and_capture()
def warm_and_capture(self) -> None:
# Must run on every DP sibling in lockstep: _dummy_run calls
# coordinate_batch_across_dp whenever data_parallel_size > 1
# (gpu_model_runner.py:3663), which deadlocks if any rank skips it.
# Save and clear block tables so the dummy MoE forward doesn't
# write dummy slot mappings into real KV-cache blocks.
multi_block_table = self.worker.model_runner.input_batch.block_table
@@ -735,8 +689,15 @@ class ElasticEPScalingExecutor:
# directly with skip_eplb=True so dummy routing doesn't pollute the
# just-rebalanced EPLB stats.
runner = self.worker.model_runner
runner._dummy_run(runner.max_num_tokens, is_profile=True, skip_eplb=True)
self.worker.compile_or_warm_up_model()
all2all_manager = get_ep_all2all_manager()
reuse_kernel = self._can_reuse_fused_moe_kernel()
with (
skip_dp_coordination() if reuse_kernel else nullcontext(),
all2all_manager.mask_remote_ranks() if reuse_kernel else nullcontext(),
self._disable_flashinfer_autotune() if reuse_kernel else nullcontext(),
):
runner._dummy_run(runner.max_num_tokens, is_profile=True, skip_eplb=True)
self.worker.compile_or_warm_up_model()
lock_workspace()
+1 -1
View File
@@ -204,7 +204,7 @@ class ElasticEPScalingState:
return True
elif state == ScaleUpNewEngineState.PREPARE:
self._collective_rpc("elastic_ep_execute", args=("warmup_local_kernels",))
self._collective_rpc("elastic_ep_execute", args=("warmup_new_worker",))
self._mark_ready_for_switch()
tensor = torch.tensor([0, 0, 0], dtype=torch.int32, device="cpu")
torch.distributed.all_reduce(
+66 -12
View File
@@ -121,6 +121,8 @@ class EplbModelState:
[0, 2, 0, 1, 0, 3]]
```
"""
physical_to_logical_map_buffer: torch.Tensor
"""Maximum-capacity buffer backing ``physical_to_logical_map``."""
logical_to_physical_map: torch.Tensor
"""
Mapping from logical experts to physical experts.
@@ -168,6 +170,8 @@ class EplbModelState:
Shape: (num_moe_layers, num_physical_experts)
"""
expert_load_pass_buffer: torch.Tensor
"""Maximum-capacity buffer backing ``expert_load_pass``."""
expert_load_window: torch.Tensor
"""
A sliding window of expert load.
@@ -373,6 +377,19 @@ class EplbState:
physical_to_logical_map_list,
device=self.device,
)
physical_expert_capacity = model.num_physical_experts
if self.parallel_config.enable_elastic_ep:
physical_expert_capacity = (
model.num_local_physical_experts
* self.parallel_config.elastic_ep_max_dp_size
* get_ep_group().world_size
// self.parallel_config.data_parallel_size
)
physical_to_logical_map = torch.nn.functional.pad(
physical_to_logical_map,
(0, physical_expert_capacity - model.num_physical_experts),
value=-1,
)
# Assuming 8 GPUs per node, this supports up to
# (1023 + 1) / 8 = 128 nodes for now.
# TODO(rui): make this configurable
@@ -407,6 +424,10 @@ class EplbState:
)
.contiguous()
)
physical_to_logical_map_buffer = physical_to_logical_map
physical_to_logical_map = physical_to_logical_map_buffer[
:, : model.num_physical_experts
]
logical_to_physical_map = (
logical_to_physical_map.unsqueeze(0)
.expand(
@@ -425,11 +446,12 @@ class EplbState:
.contiguous()
)
expert_load_pass = torch.zeros(
(model.num_moe_layers, model.num_physical_experts),
expert_load_pass_buffer = torch.zeros(
(model.num_moe_layers, physical_expert_capacity),
dtype=torch.int32,
device=self.device,
)
expert_load_pass = expert_load_pass_buffer[:, : model.num_physical_experts]
self.expert_load_window_size = self.parallel_config.eplb_config.window_size
expert_load_window = torch.zeros(
(
@@ -460,7 +482,7 @@ class EplbState:
]
model.set_eplb_state(
expert_load_pass,
expert_load_pass_buffer,
logical_to_physical_map,
logical_replica_count,
)
@@ -479,9 +501,11 @@ class EplbState:
model_state = EplbModelState(
physical_to_logical_map=physical_to_logical_map,
physical_to_logical_map_buffer=physical_to_logical_map_buffer,
logical_to_physical_map=logical_to_physical_map,
logical_replica_count=logical_replica_count,
expert_load_pass=expert_load_pass,
expert_load_pass_buffer=expert_load_pass_buffer,
expert_load_window=expert_load_window,
model_name=model_config.model,
model=model,
@@ -1075,10 +1099,14 @@ class EplbState:
expanded_physical_to_logical: torch.Tensor,
) -> None:
eplb_model_state = self.model_states[model_config.compute_hash()]
eplb_model_state.physical_to_logical_map.copy_(expanded_physical_to_logical)
eplb_model_state.physical_to_logical_map_buffer.copy_(
expanded_physical_to_logical
)
eplb_model_state.expert_load_pass.zero_()
physical_to_logical_map = eplb_model_state.physical_to_logical_map
(logical_to_physical_map_cpu, logical_replica_count_cpu) = compute_logical_maps(
expanded_physical_to_logical.cpu(),
physical_to_logical_map.cpu(),
eplb_model_state.model.num_logical_experts,
)
@@ -1097,6 +1125,33 @@ class EplbState:
eplb_model_state.logical_to_physical_map.copy_(logical_to_physical_map)
eplb_model_state.logical_replica_count.copy_(logical_replica_count)
def reconfigure_physical_expert_slots(
self,
model_config: ModelConfig,
num_physical_experts: int,
) -> None:
"""Replace physical_to_logical_map and expert_load_pass with views
covering the new active size.
"""
model_state = self.model_states[model_config.compute_hash()]
old_num_physical_experts = model_state.model.num_physical_experts
first_slot, last_slot = sorted((old_num_physical_experts, num_physical_experts))
physical_to_logical_map_buffer = model_state.physical_to_logical_map_buffer
expert_load_pass_buffer = model_state.expert_load_pass_buffer
assert last_slot <= physical_to_logical_map_buffer.shape[1]
expert_slots = slice(first_slot, last_slot)
physical_to_logical_map_buffer[:, expert_slots].fill_(-1)
expert_load_pass_buffer[:, expert_slots].zero_()
model_state.physical_to_logical_map = physical_to_logical_map_buffer[
:, :num_physical_experts
]
model_state.expert_load_pass = expert_load_pass_buffer[:, :num_physical_experts]
pad_size = num_physical_experts - model_state.expert_load_window.shape[-1]
model_state.expert_load_window = torch.nn.functional.pad(
model_state.expert_load_window, (0, pad_size)
)
def create_communicator(
self, model_config: ModelConfig, group_coordinator: GroupCoordinator
) -> EplbCommunicator:
@@ -1333,14 +1388,13 @@ def _commit_eplb_maps(
if PIN_MEMORY and src.is_cpu:
src = src.new_empty(src.shape, pin_memory=True).copy_(src)
# Rare Case: When the number of physical experts has changed, discard the old
# physical to logical expert map and use the new one. This only happens when the
# number of GPUs available to vLLM changes while vLLM is running. Otherwise copy the
# new map into the old one.
# When the number of physical experts changes, refresh the active map view to
# cover the new size before copying it.
if src.shape[1] != dst.shape[1]:
model_state.physical_to_logical_map = src.to(dst.device, non_blocking=True)
else:
dst.copy_(src, non_blocking=True)
dst = model_state.physical_to_logical_map_buffer[:, : src.shape[1]]
assert dst.shape == src.shape
model_state.physical_to_logical_map = dst
dst.copy_(src, non_blocking=True)
num_logical_experts = model_state.logical_to_physical_map.shape[1]
new_logical, new_replica_count = compute_logical_maps(src, num_logical_experts)