[EPLB] Enable nixl eplb communicator for elastic ep (#45013)
Signed-off-by: Markov Ilya <markovilya197@gmail.com> Signed-off-by: Markov Ilya <markovilya19@gmail.com> Co-authored-by: Markov Ilya <markovilya19@gmail.com>
This commit is contained in:
co-authored by
Markov Ilya
parent
f2069b005b
commit
ac614587f5
@@ -59,9 +59,8 @@ def _run_gsm8k_eval(server: RemoteOpenAIServer, stage: str) -> float:
|
||||
return accuracy
|
||||
|
||||
|
||||
@multi_gpu_test(num_gpus=4)
|
||||
def test_elastic_ep_scaling():
|
||||
vllm_serve_args = [
|
||||
def _base_serve_args(use_async_eplb: bool = False) -> list[str]:
|
||||
args = [
|
||||
"--trust-remote-code",
|
||||
"--tensor-parallel-size",
|
||||
"1",
|
||||
@@ -79,7 +78,11 @@ def test_elastic_ep_scaling():
|
||||
"--eplb-config.num_redundant_experts",
|
||||
"0",
|
||||
"--eplb-config.use_async",
|
||||
"false",
|
||||
"true" if use_async_eplb else "false",
|
||||
"--eplb-config.step_interval",
|
||||
"10",
|
||||
"--eplb-config.window_size",
|
||||
"5",
|
||||
"--data-parallel-backend",
|
||||
"ray",
|
||||
"--data-parallel-size",
|
||||
@@ -90,7 +93,23 @@ def test_elastic_ep_scaling():
|
||||
|
||||
leader_address = os.environ.get("LEADER_ADDRESS")
|
||||
if leader_address:
|
||||
vllm_serve_args.extend(["--data-parallel-address", leader_address])
|
||||
args.extend(["--data-parallel-address", leader_address])
|
||||
|
||||
return args
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"use_async_eplb", [False, True], ids=["sync_eplb", "async_eplb"]
|
||||
)
|
||||
@multi_gpu_test(num_gpus=4)
|
||||
def test_elastic_ep_scaling(use_async_eplb: bool):
|
||||
if use_async_eplb:
|
||||
from vllm.distributed.eplb.eplb_communicator import has_nixl
|
||||
|
||||
if not has_nixl():
|
||||
pytest.skip("Async EPLB with elastic EP requires NIXL (not installed)")
|
||||
|
||||
vllm_serve_args = _base_serve_args(use_async_eplb)
|
||||
|
||||
with RemoteOpenAIServer(
|
||||
MODEL_NAME, vllm_serve_args, env_dict={}, max_wait_seconds=1200
|
||||
@@ -128,44 +147,24 @@ def test_elastic_ep_scaling():
|
||||
print(f" Tolerance: {ACCURACY_TOL:.3f}")
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"use_async_eplb", [False, True], ids=["sync_eplb", "async_eplb"]
|
||||
)
|
||||
@multi_gpu_test(num_gpus=4)
|
||||
def test_elastic_ep_scaling_uneven():
|
||||
def test_elastic_ep_scaling_uneven(use_async_eplb: bool):
|
||||
"""Test scale up with uneven worker distribution.
|
||||
|
||||
This tests the case where num_new_workers % old_dp_size != 0,
|
||||
specifically 2 -> 3 where remainder = 1 % 2 = 1.
|
||||
This exercises the remainder handling in sender-receiver pairing.
|
||||
"""
|
||||
vllm_serve_args = [
|
||||
"--trust-remote-code",
|
||||
"--tensor-parallel-size",
|
||||
"1",
|
||||
"--gpu-memory-utilization",
|
||||
"0.8",
|
||||
"--max-model-len",
|
||||
"4096",
|
||||
"--max-num-seqs",
|
||||
str(MAX_NUM_SEQS),
|
||||
"--enable-expert-parallel",
|
||||
"--all2all-backend",
|
||||
"allgather_reducescatter",
|
||||
"--enable-elastic-ep",
|
||||
"--enable-eplb",
|
||||
"--eplb-config.num_redundant_experts",
|
||||
"0",
|
||||
"--eplb-config.use_async",
|
||||
"false",
|
||||
"--data-parallel-backend",
|
||||
"ray",
|
||||
"--data-parallel-size",
|
||||
"2",
|
||||
"--api-server-count",
|
||||
"1",
|
||||
]
|
||||
if use_async_eplb:
|
||||
from vllm.distributed.eplb.eplb_communicator import has_nixl
|
||||
|
||||
leader_address = os.environ.get("LEADER_ADDRESS")
|
||||
if leader_address:
|
||||
vllm_serve_args.extend(["--data-parallel-address", leader_address])
|
||||
if not has_nixl():
|
||||
pytest.skip("Async EPLB with elastic EP requires NIXL (not installed)")
|
||||
|
||||
vllm_serve_args = _base_serve_args(use_async_eplb)
|
||||
|
||||
with RemoteOpenAIServer(
|
||||
MODEL_NAME, vllm_serve_args, env_dict={}, max_wait_seconds=1200
|
||||
|
||||
@@ -782,3 +782,125 @@ def test_rearrange_expert_weights_profile_mode(world_size):
|
||||
_test_rearrange_expert_weights_profile_mode,
|
||||
world_size,
|
||||
)
|
||||
|
||||
|
||||
def _test_nixl_deferred_init_worker(
|
||||
env,
|
||||
world_size: int,
|
||||
num_layers: int,
|
||||
num_local_experts: int,
|
||||
num_logical_experts: int,
|
||||
) -> None:
|
||||
"""Exercise NixlEplbCommunicator with defer_remote_setup=True (elastic EP path)."""
|
||||
from vllm.distributed.eplb.eplb_communicator import NixlEplbCommunicator
|
||||
|
||||
set_env_vars_and_device(env)
|
||||
|
||||
vllm_config = VllmConfig()
|
||||
vllm_config.parallel_config.tensor_parallel_size = world_size
|
||||
|
||||
with set_current_vllm_config(vllm_config):
|
||||
ensure_model_parallel_initialized(
|
||||
tensor_model_parallel_size=world_size, pipeline_model_parallel_size=1
|
||||
)
|
||||
|
||||
ep_group_coordinator = get_tp_group()
|
||||
ep_group = ep_group_coordinator.cpu_group
|
||||
ep_rank = torch.distributed.get_rank()
|
||||
device = torch.device(f"cuda:{ep_rank}")
|
||||
|
||||
total_physical_experts = world_size * num_local_experts
|
||||
hidden_sizes = [32, 64]
|
||||
|
||||
redundancy_config = create_redundancy_config(
|
||||
num_logical_experts, total_physical_experts
|
||||
)
|
||||
old_indices = create_expert_indices_with_redundancy(
|
||||
num_layers,
|
||||
num_logical_experts,
|
||||
total_physical_experts,
|
||||
redundancy_config,
|
||||
)
|
||||
|
||||
new_redundancy_config = create_redundancy_config(
|
||||
num_logical_experts, total_physical_experts
|
||||
)
|
||||
new_indices = create_expert_indices_with_redundancy(
|
||||
num_layers,
|
||||
num_logical_experts,
|
||||
total_physical_experts,
|
||||
new_redundancy_config,
|
||||
)
|
||||
|
||||
expert_weights = create_expert_weights(
|
||||
num_layers, num_local_experts, hidden_sizes, ep_rank, device, old_indices
|
||||
)
|
||||
|
||||
expert_buffer = [torch.empty_like(w) for w in expert_weights[0]]
|
||||
|
||||
communicator = NixlEplbCommunicator(
|
||||
cpu_group=ep_group_coordinator.cpu_group,
|
||||
all_expert_weights=expert_weights,
|
||||
expert_buffer=expert_buffer,
|
||||
defer_remote_setup=True,
|
||||
)
|
||||
assert not communicator._remote_state_initialized
|
||||
|
||||
rearrange_expert_weights_inplace(
|
||||
old_indices,
|
||||
new_indices,
|
||||
expert_weights,
|
||||
expert_buffer,
|
||||
ep_group,
|
||||
communicator,
|
||||
)
|
||||
|
||||
assert communicator._remote_state_initialized
|
||||
|
||||
local_ok = verify_expert_weights_after_shuffle(
|
||||
expert_weights,
|
||||
new_indices,
|
||||
hidden_sizes,
|
||||
ep_rank,
|
||||
num_local_experts,
|
||||
)
|
||||
|
||||
local_ok = (
|
||||
verify_redundant_experts_have_same_weights(
|
||||
expert_weights,
|
||||
new_indices,
|
||||
hidden_sizes,
|
||||
ep_rank,
|
||||
world_size,
|
||||
num_local_experts,
|
||||
)
|
||||
and local_ok
|
||||
)
|
||||
assert_verification_synced(
|
||||
local_ok,
|
||||
"Deferred NIXL init verification failed on at least one rank.",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not has_nixl(), reason="NIXL is not available")
|
||||
@pytest.mark.parametrize(
|
||||
"world_size,num_layers,num_local_experts,num_logical_experts",
|
||||
[(2, 2, 3, 4)],
|
||||
)
|
||||
def test_nixl_deferred_init(
|
||||
world_size,
|
||||
num_layers,
|
||||
num_local_experts,
|
||||
num_logical_experts,
|
||||
):
|
||||
"""Test NixlEplbCommunicator with defer_remote_setup=True (elastic EP path)."""
|
||||
|
||||
if torch.accelerator.device_count() < world_size:
|
||||
pytest.skip(f"Need at least {world_size} GPUs to run the test")
|
||||
distributed_run(
|
||||
_test_nixl_deferred_init_worker,
|
||||
world_size,
|
||||
num_layers,
|
||||
num_local_experts,
|
||||
num_logical_experts,
|
||||
)
|
||||
|
||||
+22
-22
@@ -803,13 +803,6 @@ class ParallelConfig:
|
||||
if self.enable_elastic_ep:
|
||||
if not self.enable_eplb:
|
||||
raise ValueError("Elastic EP is only supported with enable_eplb=True.")
|
||||
if self.eplb_config.use_async:
|
||||
raise ValueError(
|
||||
"Elastic EP requires the pynccl communicator, which is "
|
||||
"incompatible with async EPLB due to NCCL multi-stream "
|
||||
"conflicts. Disable async EPLB (eplb_config.use_async=False) "
|
||||
"to use elastic EP."
|
||||
)
|
||||
if self.pipeline_parallel_size > 1:
|
||||
raise ValueError(
|
||||
"Elastic EP is not supported with pipeline parallelism "
|
||||
@@ -821,6 +814,15 @@ class ParallelConfig:
|
||||
"or data_parallel_hybrid_lb. Elastic EP relies on a single API "
|
||||
"server and core client to coordinate scale up/down."
|
||||
)
|
||||
if self.eplb_config.use_async:
|
||||
from vllm.distributed.nixl_utils import is_nixl_available
|
||||
|
||||
if not is_nixl_available():
|
||||
raise ValueError(
|
||||
"Elastic EP with async EPLB requires the NIXL "
|
||||
"package. Either install NIXL or set "
|
||||
"--eplb-config.use_async=false."
|
||||
)
|
||||
|
||||
if self.data_parallel_size > 1 or self.data_parallel_size_local == 0:
|
||||
# Data parallel was specified in the engine args.
|
||||
@@ -929,23 +931,21 @@ class ParallelConfig:
|
||||
)
|
||||
|
||||
if self.enable_eplb and self.eplb_config.communicator is None:
|
||||
if self.enable_elastic_ep:
|
||||
# Elastic EP requires stateless mode
|
||||
# (torch.distributed.batch_isend_irecv doesn't
|
||||
# support stateless mode), so we use PyNCCL backend
|
||||
# Prefer NIXL when available: zero-copy RDMA reads, compatible
|
||||
# with both async EPLB and elastic EP (deferred remote setup).
|
||||
# Fallbacks: pynccl for elastic EP (stateless groups need it),
|
||||
# torch_gloo for static EP. torch_nccl is avoided because NCCL
|
||||
# is incompatible with async EPLB (multi-stream conflicts) and
|
||||
# batched isend/irecv hangs under high load.
|
||||
# See https://github.com/pytorch/pytorch/issues/174288
|
||||
from vllm.distributed.nixl_utils import is_nixl_available
|
||||
|
||||
if is_nixl_available():
|
||||
self.eplb_config.communicator = "nixl"
|
||||
elif self.enable_elastic_ep:
|
||||
self.eplb_config.communicator = "pynccl"
|
||||
else:
|
||||
# Avoid torch_nccl: NCCL is fundamentally incompatible
|
||||
# with async EPLB due to multi-stream conflicts, and
|
||||
# batched isend/irecv hangs under high load.
|
||||
# See https://github.com/pytorch/pytorch/issues/174288
|
||||
# Prefer nixl when available; fall back to torch_gloo.
|
||||
from vllm.distributed.nixl_utils import is_nixl_available
|
||||
|
||||
if is_nixl_available():
|
||||
self.eplb_config.communicator = "nixl"
|
||||
else:
|
||||
self.eplb_config.communicator = "torch_gloo"
|
||||
self.eplb_config.communicator = "torch_gloo"
|
||||
|
||||
@property
|
||||
def use_ray(self) -> bool:
|
||||
|
||||
@@ -207,6 +207,9 @@ class ElasticEPScalingExecutor:
|
||||
)
|
||||
if new_dp_size > old_dp_size:
|
||||
self._set_eplb_suppressed(True)
|
||||
eplb_state = self.worker.model_runner.eplb_state
|
||||
if eplb_state is not None:
|
||||
eplb_state.drain_async()
|
||||
elif new_dp_size < old_dp_size:
|
||||
self._stage_standby_moe_quant_methods()
|
||||
|
||||
@@ -540,6 +543,11 @@ class ElasticEPScalingExecutor:
|
||||
eplb_model_state.physical_to_logical_map.shape[1]
|
||||
)
|
||||
eplb_state.is_async = is_async_enabled
|
||||
# Start the async worker thread if it doesn't exist yet (idempotent).
|
||||
# This is needed for new workers after scale-up: they create EplbState
|
||||
# in setup_eplb_from_mapping() but don't start the thread there because
|
||||
# groups aren't ready yet.
|
||||
eplb_state.start_async_loop()
|
||||
if get_ep_group().rank == 0:
|
||||
logger.info("[Elastic EP] Expert resharding completed")
|
||||
|
||||
@@ -549,6 +557,9 @@ class ElasticEPScalingExecutor:
|
||||
|
||||
def perform_scale_down_eplb_reshuffle(self, new_dp_size: int) -> None:
|
||||
self._set_eplb_suppressed(True)
|
||||
eplb_state = self.worker.model_runner.eplb_state
|
||||
if eplb_state is not None:
|
||||
eplb_state.drain_async()
|
||||
parallel_config = self.worker.vllm_config.parallel_config
|
||||
tp_size = parallel_config.tensor_parallel_size
|
||||
old_ep_size = parallel_config.data_parallel_size * tp_size
|
||||
|
||||
@@ -8,7 +8,6 @@ import threading
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import torch
|
||||
from torch.distributed import ProcessGroup
|
||||
|
||||
from vllm.distributed.parallel_state import get_eplb_group
|
||||
from vllm.logger import init_logger
|
||||
@@ -26,8 +25,7 @@ def start_async_worker(
|
||||
state: "EplbState",
|
||||
is_profile: bool = False,
|
||||
) -> threading.Thread:
|
||||
eplb_group = get_eplb_group().device_group
|
||||
rank = eplb_group.rank()
|
||||
rank = get_eplb_group().device_group.rank()
|
||||
device_index = state.cuda_device_index
|
||||
assert state.is_async
|
||||
|
||||
@@ -38,7 +36,6 @@ def start_async_worker(
|
||||
try:
|
||||
transfer_run_periodically(
|
||||
state=state,
|
||||
eplb_group=eplb_group,
|
||||
cuda_stream=cuda_stream,
|
||||
is_profile=is_profile,
|
||||
)
|
||||
@@ -78,13 +75,15 @@ def run_rebalance_experts(
|
||||
|
||||
def transfer_run_periodically(
|
||||
state: "EplbState",
|
||||
eplb_group: ProcessGroup,
|
||||
cuda_stream: torch.cuda.Stream,
|
||||
is_profile: bool = False,
|
||||
) -> None:
|
||||
while True:
|
||||
state.rearrange_event.wait(stream=cuda_stream)
|
||||
logger.info("async worker woke up for EPLB transfer")
|
||||
|
||||
eplb_group = get_eplb_group().device_group
|
||||
eplb_cpu_group = get_eplb_group().cpu_group
|
||||
ep_rank = eplb_group.rank()
|
||||
|
||||
assert state.is_async
|
||||
for model_state in state.model_states.values():
|
||||
@@ -101,16 +100,32 @@ def transfer_run_periodically(
|
||||
new_physical_to_logical_map = run_rebalance_experts(
|
||||
model_state, state, physical_to_logical_map_cpu, cuda_stream
|
||||
)
|
||||
logger.info(
|
||||
"Async worker computed new indices for model %s",
|
||||
model_state.model_name,
|
||||
)
|
||||
|
||||
# Execute one EPLB layer transfer per model forward pass. Each iteration
|
||||
# of this loop will copy the new set of expert weights into
|
||||
# model_state.expert_buffer, which will be consumed by the main thread in
|
||||
# move_to_workspace
|
||||
while model_state.rebalanced and layer_idx < num_layers:
|
||||
# move_to_workspace.
|
||||
# We sync the rebalanced flag across ranks before each iteration so
|
||||
# all ranks make a coordinated decision to continue or stop.
|
||||
while layer_idx < num_layers:
|
||||
flag = torch.tensor(
|
||||
[int(model_state.rebalanced)],
|
||||
dtype=torch.int32,
|
||||
device="cpu",
|
||||
)
|
||||
torch.distributed.all_reduce(flag, group=eplb_cpu_group)
|
||||
if int(flag.item()) != eplb_cpu_group.size():
|
||||
logger.warning(
|
||||
"async worker (rank=%d): layer %d coordinated stop "
|
||||
"(flag_sum=%d, group_size=%d)",
|
||||
ep_rank,
|
||||
layer_idx,
|
||||
int(flag.item()),
|
||||
eplb_cpu_group.size(),
|
||||
)
|
||||
model_state.rebalanced = False
|
||||
break
|
||||
|
||||
transfer_metadata = transfer_layer(
|
||||
old_layer_indices=physical_to_logical_map_cpu[layer_idx],
|
||||
new_layer_indices=new_physical_to_logical_map[layer_idx],
|
||||
@@ -143,6 +158,5 @@ def transfer_run_periodically(
|
||||
# finish copying model_state.expert_buffer into
|
||||
# model_state.model.expert_weights[layer_idx]
|
||||
consumed_event.wait(stream=cuda_stream)
|
||||
logger.debug("Layer %d transfer complete", layer_idx)
|
||||
assert model_state.pending_result is None
|
||||
layer_idx += 1
|
||||
|
||||
@@ -246,7 +246,20 @@ class NixlEplbCommunicator(EplbCommunicator):
|
||||
cpu_group: ProcessGroup,
|
||||
all_expert_weights: Sequence[Sequence[torch.Tensor]],
|
||||
expert_buffer: Sequence[torch.Tensor],
|
||||
defer_remote_setup: bool = False,
|
||||
) -> None:
|
||||
"""Create a NIXL-backed EPLB communicator.
|
||||
|
||||
Args:
|
||||
cpu_group: CPU process group for metadata exchange.
|
||||
all_expert_weights: Expert weight tensors for all MoE layers.
|
||||
expert_buffer: Pre-allocated receive buffer tensors.
|
||||
defer_remote_setup: If True, postpone the collective
|
||||
all-gather of NIXL agent metadata until the first
|
||||
``set_transfer_context`` call. Required for elastic EP
|
||||
where ranks join asynchronously and cannot participate
|
||||
in collectives at construction time.
|
||||
"""
|
||||
assert all_expert_weights, (
|
||||
"NixlEplbCommunicator requires non-empty all_expert_weights."
|
||||
)
|
||||
@@ -302,10 +315,29 @@ class NixlEplbCommunicator(EplbCommunicator):
|
||||
] = {}
|
||||
|
||||
self._cuda_device_id = int(self._device.index or 0)
|
||||
self._remote_state_initialized = False
|
||||
self._init_step("buffers", self._init_registered_buffers)
|
||||
if defer_remote_setup:
|
||||
logger.info_once("NIXL EPLB: deferring remote agent setup (elastic EP).")
|
||||
else:
|
||||
self._init_remote_state()
|
||||
self._log_initialized()
|
||||
|
||||
def _init_remote_state(self) -> None:
|
||||
"""Exchange NIXL agent metadata and RDMA pointer info with all peers.
|
||||
|
||||
This is a collective operation (uses ``all_gather_object`` twice).
|
||||
Under elastic EP the call is deferred to the first
|
||||
``set_transfer_context`` invocation, where all ranks are
|
||||
guaranteed to be synchronized.
|
||||
"""
|
||||
self._init_step("agents", self._init_remote_agents)
|
||||
self._init_step("send meta", self._exchange_remote_send_meta)
|
||||
self._log_initialized()
|
||||
self._remote_state_initialized = True
|
||||
|
||||
def _ensure_remote_state(self) -> None:
|
||||
if not self._remote_state_initialized:
|
||||
self._init_remote_state()
|
||||
|
||||
@property
|
||||
def needs_profile_buffer_reservation(self) -> bool:
|
||||
@@ -339,8 +371,7 @@ class NixlEplbCommunicator(EplbCommunicator):
|
||||
pass
|
||||
|
||||
def set_transfer_context(self, old_indices: np.ndarray, layer_idx: int) -> None:
|
||||
# Pre-compute expert_id -> src_row mapping for every rank so that
|
||||
# add_recv can immediately issue NIXL READs.
|
||||
self._ensure_remote_state()
|
||||
assert not self._xfer_entries, (
|
||||
f"set_transfer_context() called with {len(self._xfer_entries)} "
|
||||
f"pending transfers from layer {self._layer_idx}; "
|
||||
@@ -523,6 +554,21 @@ class NixlEplbCommunicator(EplbCommunicator):
|
||||
)
|
||||
return (local_handle, remote_handle, xfer_handle)
|
||||
|
||||
def _post_read_barrier(self) -> None:
|
||||
"""Correctness fence: prevents overwrite-while-remote-read race.
|
||||
|
||||
We avoid ``torch.distributed.monitored_barrier`` because it
|
||||
calls ``get_backend(group)`` which fails for stateless groups
|
||||
(elastic EP). An async ``all_reduce`` + ``wait(timeout)``
|
||||
works with both regular and stateless groups and provides
|
||||
equivalent timeout detection.
|
||||
"""
|
||||
_dummy = torch.zeros(1, dtype=torch.int32)
|
||||
work = torch.distributed.all_reduce(
|
||||
_dummy, group=self._cpu_group, async_op=True
|
||||
)
|
||||
work.wait(timeout=timedelta(minutes=5))
|
||||
|
||||
def execute(self) -> None:
|
||||
assert self._layer_idx is not None or not self._xfer_entries, (
|
||||
"set_transfer_context() must be called before execute() "
|
||||
@@ -531,13 +577,7 @@ class NixlEplbCommunicator(EplbCommunicator):
|
||||
try:
|
||||
self._wait_for_all_transfers([x[2] for x in self._xfer_entries])
|
||||
|
||||
# Post-READ barrier.
|
||||
# Correctness fence for zero-copy: prevents overwrite-while-
|
||||
# remote-read race.
|
||||
torch.distributed.monitored_barrier(
|
||||
group=self._cpu_group,
|
||||
timeout=timedelta(minutes=5),
|
||||
)
|
||||
self._post_read_barrier()
|
||||
finally:
|
||||
for local_h, remote_h, xfer_h in self._xfer_entries:
|
||||
with contextlib.suppress(Exception):
|
||||
@@ -628,11 +668,13 @@ def create_eplb_communicator(
|
||||
device and CPU communication groups.
|
||||
backend: Communicator backend name (``"torch_nccl"``,
|
||||
``"torch_gloo"``, ``"pynccl"``, or ``"nixl"``).
|
||||
Stateless (elastic EP) groups only support ``"torch_nccl"``
|
||||
and ``"pynccl"``; ``"torch_nccl"`` is silently promoted to
|
||||
``"pynccl"`` in that case. When tensors reside on CPU,
|
||||
``"torch_gloo"`` or ``"torch_nccl"`` are used via the CPU
|
||||
process group.
|
||||
Falls back to ``"torch_nccl"`` when *None*.
|
||||
Stateless (elastic EP) groups support ``"torch_nccl"``,
|
||||
``"pynccl"``, and ``"nixl"``; ``"torch_nccl"`` is silently
|
||||
promoted to ``"pynccl"``. ``"nixl"`` uses deferred remote
|
||||
agent setup to avoid collective deadlocks during elastic
|
||||
scaling. When tensors reside on CPU, ``"torch_gloo"`` or
|
||||
``"torch_nccl"`` are used via the CPU process group.
|
||||
expert_weights: Expert weight tensors for *all* MoE layers.
|
||||
Shape ``(num_layers)(num_tensors_per_layer)``.
|
||||
NixlEplbCommunicator registers all layers with NIXL for
|
||||
@@ -686,18 +728,21 @@ def create_eplb_communicator(
|
||||
|
||||
is_stateless = isinstance(group_coordinator, StatelessGroupCoordinator)
|
||||
if is_stateless:
|
||||
if backend not in ("torch_nccl", "pynccl"):
|
||||
if backend == "nixl":
|
||||
pass # handled below with defer_remote_setup=True
|
||||
elif backend not in ("torch_nccl", "pynccl"):
|
||||
raise ValueError(
|
||||
f"Elastic EP requires 'torch_nccl' or 'pynccl' EPLB communicator "
|
||||
f"(got '{backend}')."
|
||||
f"Elastic EP requires 'torch_nccl', 'pynccl', or 'nixl' "
|
||||
f"EPLB communicator (got '{backend}')."
|
||||
)
|
||||
if backend == "torch_nccl":
|
||||
logger.warning(
|
||||
"Stateless elastic EP requires PyNCCL backend. "
|
||||
"Forcing EPLB communicator to 'pynccl'."
|
||||
)
|
||||
backend = "pynccl"
|
||||
return _create_pynccl()
|
||||
else:
|
||||
if backend == "torch_nccl":
|
||||
logger.warning(
|
||||
"Stateless elastic EP requires PyNCCL backend. "
|
||||
"Forcing EPLB communicator to 'pynccl'."
|
||||
)
|
||||
backend = "pynccl"
|
||||
return _create_pynccl()
|
||||
|
||||
if backend == "nixl":
|
||||
if not has_nixl():
|
||||
@@ -714,6 +759,7 @@ def create_eplb_communicator(
|
||||
cpu_group=group_coordinator.cpu_group,
|
||||
all_expert_weights=expert_weights,
|
||||
expert_buffer=expert_buffer,
|
||||
defer_remote_setup=is_stateless,
|
||||
)
|
||||
except Exception as exc:
|
||||
raise RuntimeError(
|
||||
|
||||
@@ -27,6 +27,7 @@ physical experts.
|
||||
"""
|
||||
|
||||
import threading
|
||||
import time
|
||||
from collections.abc import Sequence
|
||||
from dataclasses import dataclass
|
||||
|
||||
@@ -825,6 +826,45 @@ class EplbState:
|
||||
is_profile=is_profile,
|
||||
)
|
||||
|
||||
def drain_async(self) -> None:
|
||||
"""Drain in-flight async EPLB by consuming all remaining layer results.
|
||||
|
||||
Each pending result is acknowledged (consumed_event recorded) so the
|
||||
async worker can proceed, but the transferred weights are intentionally
|
||||
NOT applied — a full synchronous rearrange is expected to follow.
|
||||
|
||||
Ranks are kept in lockstep via _all_ranks_result_ready (all_reduce
|
||||
on the EP CPU group). The async worker's coordinated-stop collectives
|
||||
use the separate EPLB group, so the two sets of collectives do not
|
||||
interfere.
|
||||
|
||||
No-op when no async cycle is in progress (rebalanced=False).
|
||||
"""
|
||||
if not self.is_async:
|
||||
return
|
||||
for model_key, ms in self.model_states.items():
|
||||
needs_drain = ms.rebalanced
|
||||
if needs_drain:
|
||||
logger.info(
|
||||
"Draining async EPLB worker for model %s",
|
||||
model_key,
|
||||
)
|
||||
while ms.rebalanced:
|
||||
if self._all_ranks_result_ready(ms):
|
||||
result = ms.pending_result
|
||||
assert result is not None
|
||||
if result.layer_idx == ms.model.num_moe_layers - 1:
|
||||
ms.rebalanced = False
|
||||
ms.pending_result = None
|
||||
result.consumed_event.record()
|
||||
else:
|
||||
time.sleep(0.001)
|
||||
if needs_drain:
|
||||
logger.info(
|
||||
"Async EPLB worker drained for model %s",
|
||||
model_key,
|
||||
)
|
||||
|
||||
def _all_ranks_result_ready(self, model_state: EplbModelState) -> bool:
|
||||
parallel_state = get_ep_group()
|
||||
has_result = int(model_state.pending_result is not None)
|
||||
@@ -850,8 +890,9 @@ class EplbState:
|
||||
"""
|
||||
All-reduce a list of tensors.
|
||||
"""
|
||||
ep_group = get_ep_group().device_group
|
||||
if len(tensor_list) == 1:
|
||||
all_reduce(tensor_list[0], group=get_ep_group().device_group)
|
||||
all_reduce(tensor_list[0], group=ep_group)
|
||||
return tensor_list
|
||||
assert all(t.dim() == 2 for t in tensor_list), "All tensors must be 2D."
|
||||
assert all(t.shape[1] == tensor_list[0].shape[1] for t in tensor_list), (
|
||||
@@ -863,7 +904,6 @@ class EplbState:
|
||||
shapes = [t.shape for t in tensor_list]
|
||||
concat_tensor = torch.cat(tensor_list, dim=0)
|
||||
|
||||
ep_group = get_ep_group().device_group
|
||||
all_reduce(concat_tensor, group=ep_group)
|
||||
|
||||
all_reduce_list = []
|
||||
|
||||
Reference in New Issue
Block a user