elastic_ep: stage/commit MoE quant method on reconfigure (#40881)
Signed-off-by: Itay Alroy <ialroy@nvidia.com>
This commit is contained in:
@@ -1,6 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import threading
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
@@ -324,15 +325,21 @@ class DeepEPLLAll2AllManager(DeepEPAll2AllManagerBase):
|
||||
return 0
|
||||
|
||||
|
||||
@dataclass
|
||||
class _NixlEPBufferState:
|
||||
buffer: Any
|
||||
connected_ep_size: int
|
||||
active_ep_size: int
|
||||
|
||||
|
||||
class NixlEPAll2AllManager(All2AllManagerBase):
|
||||
"""
|
||||
All2All communication based on NIXL EP kernels.
|
||||
This backend supports elastic EP with dynamic rank connection/disconnection.
|
||||
"""
|
||||
|
||||
# (nixl_ep_buffer, ep_size)
|
||||
_buffer: tuple[Any, int] | None = None
|
||||
_lock = threading.Lock()
|
||||
_buffer: _NixlEPBufferState | None = None
|
||||
_lock = threading.RLock()
|
||||
|
||||
def __init__(self, cpu_group, tcp_store_group=None):
|
||||
assert tcp_store_group is not None
|
||||
@@ -367,47 +374,103 @@ class NixlEPAll2AllManager(All2AllManagerBase):
|
||||
num_experts_per_rank=num_experts_per_rank,
|
||||
num_rdma_bytes=num_rdma_bytes,
|
||||
)
|
||||
ranks_to_connect = list(range(self.cpu_group.size()))
|
||||
ranks_to_connect = list(range(self.world_size))
|
||||
buffer.connect_ranks(ranks_to_connect)
|
||||
NixlEPAll2AllManager._buffer = (buffer, self.cpu_group.size())
|
||||
NixlEPAll2AllManager._buffer = _NixlEPBufferState(
|
||||
buffer=buffer,
|
||||
connected_ep_size=self.world_size,
|
||||
active_ep_size=self.world_size,
|
||||
)
|
||||
|
||||
def _update_buffer(self):
|
||||
def _connect_to_ep_size(self, ep_size: int, *, make_active: bool) -> None:
|
||||
assert NixlEPAll2AllManager._buffer is not None
|
||||
buffer, current_ep_size = NixlEPAll2AllManager._buffer
|
||||
current_ranks = list(range(current_ep_size))
|
||||
new_ep_size = self.cpu_group.size()
|
||||
buffer.set_tcp_store_group(self.tcp_store_group.store)
|
||||
if new_ep_size > len(current_ranks):
|
||||
ranks_to_connect = list(range(len(current_ranks), new_ep_size))
|
||||
buffer.connect_ranks(ranks_to_connect)
|
||||
state = NixlEPAll2AllManager._buffer
|
||||
if ep_size <= state.connected_ep_size:
|
||||
return
|
||||
|
||||
state.buffer.set_tcp_store_group(self.tcp_store_group.store)
|
||||
ranks_to_connect = list(range(state.connected_ep_size, ep_size))
|
||||
state.buffer.connect_ranks(ranks_to_connect, activate=make_active)
|
||||
state.connected_ep_size = ep_size
|
||||
if make_active:
|
||||
state.active_ep_size = ep_size
|
||||
|
||||
def _disconnect_to_ep_size(self, ep_size: int) -> None:
|
||||
assert NixlEPAll2AllManager._buffer is not None
|
||||
state = NixlEPAll2AllManager._buffer
|
||||
if ep_size >= state.connected_ep_size:
|
||||
return
|
||||
|
||||
state.buffer.set_tcp_store_group(self.tcp_store_group.store)
|
||||
ranks_to_disconnect = list(range(ep_size, state.connected_ep_size))
|
||||
state.buffer.disconnect_ranks(ranks_to_disconnect)
|
||||
state.connected_ep_size = ep_size
|
||||
state.active_ep_size = min(state.active_ep_size, ep_size)
|
||||
|
||||
def _unmask_connected_ranks(self, target_ep_size: int) -> None:
|
||||
assert NixlEPAll2AllManager._buffer is not None
|
||||
state = NixlEPAll2AllManager._buffer
|
||||
state.buffer.set_tcp_store_group(self.tcp_store_group.store)
|
||||
if target_ep_size <= state.active_ep_size:
|
||||
return
|
||||
assert state.connected_ep_size >= target_ep_size
|
||||
|
||||
for rank in range(state.active_ep_size, target_ep_size):
|
||||
state.buffer.update_mask_buffer(rank, mask=False)
|
||||
state.active_ep_size = target_ep_size
|
||||
|
||||
def _stage_ep_size(self) -> None:
|
||||
assert NixlEPAll2AllManager._buffer is not None
|
||||
state = NixlEPAll2AllManager._buffer
|
||||
target_ep_size = self.world_size
|
||||
|
||||
# Scale-up can safely connect standby ranks while leaving them masked.
|
||||
# Scale-down must not disconnect active ranks until commit.
|
||||
if target_ep_size > state.connected_ep_size:
|
||||
self._connect_to_ep_size(target_ep_size, make_active=False)
|
||||
|
||||
def commit_staged_state(self) -> None:
|
||||
"""Commit staged NIXL EP state to the active communication set."""
|
||||
with NixlEPAll2AllManager._lock:
|
||||
assert NixlEPAll2AllManager._buffer is not None
|
||||
state = NixlEPAll2AllManager._buffer
|
||||
target_ep_size = self.world_size
|
||||
|
||||
if target_ep_size < state.connected_ep_size:
|
||||
self._disconnect_to_ep_size(target_ep_size)
|
||||
elif target_ep_size > state.connected_ep_size:
|
||||
self._connect_to_ep_size(target_ep_size, make_active=True)
|
||||
|
||||
self._unmask_connected_ranks(target_ep_size)
|
||||
|
||||
def _ensure_ep_size(self, *, stage: bool) -> None:
|
||||
if stage:
|
||||
self._stage_ep_size()
|
||||
else:
|
||||
ranks_to_disconnect = current_ranks[new_ep_size:]
|
||||
buffer.disconnect_ranks(ranks_to_disconnect)
|
||||
NixlEPAll2AllManager._buffer = (buffer, new_ep_size)
|
||||
self.commit_staged_state()
|
||||
|
||||
def get_handle(self, kwargs):
|
||||
with NixlEPAll2AllManager._lock:
|
||||
if (
|
||||
NixlEPAll2AllManager._buffer is not None
|
||||
and NixlEPAll2AllManager._buffer[1] == self.cpu_group.size()
|
||||
):
|
||||
return NixlEPAll2AllManager._buffer[0]
|
||||
|
||||
num_experts_per_rank = (
|
||||
kwargs["num_global_experts"] // kwargs["num_ep_ranks"]
|
||||
)
|
||||
nixl_kwargs = dict(
|
||||
max_num_tokens_per_dp_rank=kwargs["max_num_tokens_per_dp_rank"],
|
||||
token_hidden_size=kwargs["token_hidden_size"],
|
||||
num_experts_per_rank=num_experts_per_rank,
|
||||
)
|
||||
if NixlEPAll2AllManager._buffer is None:
|
||||
self._init_buffer(**nixl_kwargs)
|
||||
stage = bool(kwargs.get("stage", False))
|
||||
state = NixlEPAll2AllManager._buffer
|
||||
if state is None:
|
||||
assert not stage, (
|
||||
"NIXL EP staged initialization requires an existing buffer"
|
||||
)
|
||||
max_num_tokens_per_dp_rank = kwargs["max_num_tokens_per_dp_rank"]
|
||||
num_experts_per_rank = (
|
||||
kwargs["num_global_experts"] // kwargs["num_ep_ranks"]
|
||||
)
|
||||
self._init_buffer(
|
||||
max_num_tokens_per_dp_rank=max_num_tokens_per_dp_rank,
|
||||
token_hidden_size=kwargs["token_hidden_size"],
|
||||
num_experts_per_rank=num_experts_per_rank,
|
||||
)
|
||||
else:
|
||||
self._update_buffer()
|
||||
self._ensure_ep_size(stage=stage)
|
||||
|
||||
assert NixlEPAll2AllManager._buffer is not None
|
||||
handle = NixlEPAll2AllManager._buffer[0]
|
||||
handle = NixlEPAll2AllManager._buffer.buffer
|
||||
return handle
|
||||
|
||||
def dispatch(
|
||||
@@ -432,7 +495,7 @@ class NixlEPAll2AllManager(All2AllManagerBase):
|
||||
# NOTE(yongji): NIXLEPAll2AllManager instance is recreated during
|
||||
# scale-up/down, so we cannot destroy the persistent buffer here.
|
||||
assert NixlEPAll2AllManager._buffer is not None
|
||||
buffer = NixlEPAll2AllManager._buffer[0]
|
||||
buffer = NixlEPAll2AllManager._buffer.buffer
|
||||
buffer.set_tcp_store_group(None)
|
||||
|
||||
# NIXL EP uses RDMA so no SMs are used for communication
|
||||
|
||||
@@ -4,6 +4,8 @@ import copy
|
||||
import gc
|
||||
import weakref
|
||||
from collections.abc import Iterable, Sequence
|
||||
from dataclasses import replace
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
@@ -37,7 +39,10 @@ from vllm.distributed.parallel_state import (
|
||||
)
|
||||
from vllm.distributed.stateless_coordinator import StatelessGroupCoordinator
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.fused_moe.layer import FusedMoEParallelConfig
|
||||
from vllm.model_executor.layers.fused_moe.config import FusedMoEParallelConfig
|
||||
from vllm.model_executor.layers.fused_moe.eep_reconfigure import (
|
||||
make_eep_staged_quant_method,
|
||||
)
|
||||
from vllm.utils import is_moe_layer
|
||||
from vllm.v1.engine import ReconfigureDistributedRequest, ReconfigureRankType
|
||||
from vllm.v1.worker.gpu_ubatch_wrapper import UBatchWrapper
|
||||
@@ -45,6 +50,11 @@ from vllm.v1.worker.workspace import lock_workspace, unlock_workspace
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.model_executor.layers.fused_moe.fused_moe_method_base import (
|
||||
FusedMoEMethodBase,
|
||||
)
|
||||
|
||||
|
||||
def batch_transfer_weights(
|
||||
model: nn.Module,
|
||||
@@ -134,6 +144,7 @@ class ElasticEPScalingExecutor:
|
||||
def __init__(self, worker):
|
||||
self.worker_ref = weakref.ref(worker)
|
||||
self.reconfig_request = None
|
||||
self._staged_moe_quant_methods: dict[nn.Module, FusedMoEMethodBase] = {}
|
||||
|
||||
@property
|
||||
def worker(self):
|
||||
@@ -196,6 +207,8 @@ class ElasticEPScalingExecutor:
|
||||
)
|
||||
if new_dp_size > old_dp_size:
|
||||
self._set_eplb_suppressed(True)
|
||||
elif new_dp_size < old_dp_size:
|
||||
self._stage_standby_moe_quant_methods()
|
||||
|
||||
def transfer_weights(self, old_dp_size: int, new_dp_size: int) -> None:
|
||||
standby_dp_group = get_standby_dp_group()
|
||||
@@ -262,6 +275,58 @@ class ElasticEPScalingExecutor:
|
||||
src_rank=0,
|
||||
device=self.worker.device,
|
||||
)
|
||||
# New workers enter load_model after receiving the expert mapping.
|
||||
# Stage replacement MoE kernels before returning to the state machine
|
||||
# so existing ranks can participate in collective EP comm creation.
|
||||
self._stage_standby_moe_quant_methods()
|
||||
|
||||
def _make_eep_moe_config(self, module, dp_group, ep_group):
|
||||
parallel_config = self.worker.vllm_config.parallel_config
|
||||
tp_size = get_tp_group().world_size
|
||||
sp_size = tp_size if parallel_config.use_sequence_parallel_moe else 1
|
||||
moe_parallel_config = FusedMoEParallelConfig.make(
|
||||
tp_size_=tp_size,
|
||||
pcp_size_=get_pcp_group().world_size,
|
||||
dp_size_=dp_group.world_size,
|
||||
sp_size_=sp_size,
|
||||
vllm_parallel_config=parallel_config,
|
||||
)
|
||||
return replace(
|
||||
module.moe_config,
|
||||
num_experts=module.moe_config.num_local_experts * ep_group.world_size,
|
||||
moe_parallel_config=moe_parallel_config,
|
||||
)
|
||||
|
||||
def _stage_standby_moe_quant_methods(self) -> None:
|
||||
standby_dp_group = get_standby_dp_group()
|
||||
standby_ep_group = get_standby_ep_group()
|
||||
model = self.worker.model_runner.get_model()
|
||||
moe_modules = [module for module in model.modules() if is_moe_layer(module)]
|
||||
self._staged_moe_quant_methods.clear()
|
||||
with set_current_vllm_config(self.worker.vllm_config):
|
||||
for module in moe_modules:
|
||||
staged_quant_method = make_eep_staged_quant_method(
|
||||
module,
|
||||
self._make_eep_moe_config(
|
||||
module,
|
||||
standby_dp_group,
|
||||
standby_ep_group,
|
||||
),
|
||||
)
|
||||
if staged_quant_method is not None:
|
||||
self._staged_moe_quant_methods[module] = staged_quant_method
|
||||
|
||||
def _commit_staged_moe_quant_methods(self) -> None:
|
||||
model = self.worker.model_runner.get_model()
|
||||
moe_modules = [module for module in model.modules() if is_moe_layer(module)]
|
||||
for module in moe_modules:
|
||||
staged_quant_method = self._staged_moe_quant_methods.pop(module, None)
|
||||
if staged_quant_method is None:
|
||||
continue
|
||||
assert staged_quant_method.moe_kernel is not None
|
||||
module._replace_quant_method(staged_quant_method)
|
||||
staged_quant_method.moe_kernel.prepare_finalize.on_commit()
|
||||
self._staged_moe_quant_methods.clear()
|
||||
|
||||
def _release_cuda_graphs(self) -> None:
|
||||
if isinstance(self.worker.model_runner.model, CUDAGraphWrapper):
|
||||
@@ -327,19 +392,13 @@ class ElasticEPScalingExecutor:
|
||||
module.moe_config.num_local_experts == num_local_experts
|
||||
for module in moe_modules
|
||||
), "All MoE modules must have the same number of experts"
|
||||
dp_group = get_dp_group()
|
||||
ep_group = get_ep_group()
|
||||
for module in moe_modules:
|
||||
module.moe_config.num_experts = num_local_experts * new_ep_size
|
||||
new_moe_config = self._make_eep_moe_config(module, dp_group, ep_group)
|
||||
module.moe_config.num_experts = new_moe_config.num_experts
|
||||
module.global_num_experts = module.moe_config.num_experts
|
||||
tp_size = get_tp_group().world_size
|
||||
is_sequence_parallel = parallel_config.use_sequence_parallel_moe
|
||||
sp_size = tp_size if is_sequence_parallel else 1
|
||||
module.moe_parallel_config = FusedMoEParallelConfig.make(
|
||||
tp_size_=tp_size,
|
||||
pcp_size_=get_pcp_group().world_size,
|
||||
dp_size_=get_dp_group().world_size,
|
||||
sp_size_=sp_size,
|
||||
vllm_parallel_config=parallel_config,
|
||||
)
|
||||
module.moe_parallel_config = new_moe_config.moe_parallel_config
|
||||
module.moe_config.moe_parallel_config = module.moe_parallel_config
|
||||
|
||||
# Update EPLB state
|
||||
@@ -404,10 +463,10 @@ class ElasticEPScalingExecutor:
|
||||
num_physical_experts=num_physical_experts,
|
||||
num_local_physical_experts=num_local_experts,
|
||||
)
|
||||
# Force re-creation of the modular kernel (and all2all manager)
|
||||
# for the new EP size by resetting quant_method to base
|
||||
self._commit_staged_moe_quant_methods()
|
||||
# Legacy modular methods need to be recreated for the new EP size.
|
||||
for module in moe_modules:
|
||||
if hasattr(module.quant_method, "old_quant_method"):
|
||||
if getattr(module.quant_method, "wraps_legacy_quant_method", False):
|
||||
module._replace_quant_method(module.quant_method.old_quant_method)
|
||||
prepare_communication_buffer_for_model(self.worker.model_runner.model)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user