[Distributed][MoonEP] BF16 integration of MoonEP balanced EP backend (#52101)
Signed-off-by: kevinli <306243285+kaijunli-infr@users.noreply.github.com> Co-authored-by: kevinli <306243285+kaijunli-infr@users.noreply.github.com>
This commit is contained in:
@@ -21,7 +21,12 @@ from vllm.utils.flashinfer import (
|
||||
has_flashinfer_nvlink_two_sided,
|
||||
)
|
||||
from vllm.utils.func_utils import supports_kw
|
||||
from vllm.utils.import_utils import has_deep_ep, has_deep_ep_v2, has_mori
|
||||
from vllm.utils.import_utils import (
|
||||
check_moonep_system_support,
|
||||
has_deep_ep,
|
||||
has_deep_ep_v2,
|
||||
has_mori,
|
||||
)
|
||||
|
||||
from .base_device_communicator import All2AllManagerBase, Cache
|
||||
|
||||
@@ -1094,3 +1099,71 @@ class DeepEPV2All2AllManager(All2AllManagerBase):
|
||||
for _, handle in self.handle_cache._cache.items():
|
||||
handle.destroy()
|
||||
self.handle_cache._cache.clear()
|
||||
|
||||
|
||||
class MoonEPAll2AllManager(All2AllManagerBase):
|
||||
"""All2All communication based on MoonEP
|
||||
(https://github.com/MoonshotAI/MoonEP).
|
||||
|
||||
MoonEP keeps token loads perfectly balanced across EP ranks by planning
|
||||
a small number of dynamically redundant experts online and prefetching
|
||||
their weights before expert compute. Every rank receives exactly
|
||||
S x K token slots regardless of router skew, so all communication and
|
||||
compute shapes are static.
|
||||
|
||||
Requires NVLink symmetric-memory / multicast capable topologies
|
||||
(single node NVSwitch, e.g. H100/H200/B200/GB300 class).
|
||||
"""
|
||||
|
||||
def __init__(self, cpu_group, tcp_store_group=None, device_group=None):
|
||||
check_moonep_system_support()
|
||||
super().__init__(cpu_group, tcp_store_group)
|
||||
self._device_group = device_group
|
||||
self.handle_cache = Cache()
|
||||
|
||||
def _make_buffer_kwargs(
|
||||
self,
|
||||
max_num_tokens_per_dp_rank: int,
|
||||
token_hidden_size: int,
|
||||
num_topk: int,
|
||||
num_global_experts: int,
|
||||
num_prefetch_slots: int,
|
||||
token_padding: int,
|
||||
num_sms: int,
|
||||
) -> dict:
|
||||
return dict(
|
||||
S=max_num_tokens_per_dp_rank,
|
||||
H=token_hidden_size,
|
||||
K=num_topk,
|
||||
E=num_global_experts,
|
||||
num_ep_ranks=self.world_size,
|
||||
num_sms=num_sms,
|
||||
token_padding=token_padding,
|
||||
B=num_prefetch_slots,
|
||||
group=self._device_group
|
||||
if self._device_group is not None
|
||||
else self.cpu_group,
|
||||
explicitly_destroy=True,
|
||||
)
|
||||
|
||||
def get_handle(self, kwargs):
|
||||
from vllm.model_executor.layers.fused_moe.prepare_finalize.moonep import (
|
||||
MoonEPBufferPool,
|
||||
)
|
||||
|
||||
buffer_kwargs = self._make_buffer_kwargs(**kwargs)
|
||||
logger.debug("MoonEP all2all args %s", buffer_kwargs)
|
||||
|
||||
def make_pool(**kw):
|
||||
return MoonEPBufferPool(kw, max_tokens_per_rank=kw["S"])
|
||||
|
||||
handle: MoonEPBufferPool = self.handle_cache.get_or_create(
|
||||
buffer_kwargs, make_pool
|
||||
)
|
||||
return handle
|
||||
|
||||
def destroy(self):
|
||||
with self.handle_cache._lock:
|
||||
for _, handle in self.handle_cache._cache.items():
|
||||
handle.destroy()
|
||||
self.handle_cache._cache.clear()
|
||||
|
||||
Reference in New Issue
Block a user