[Elastic EP] Reuse CUDA graphs across reconfiguration (#54985)
This commit is contained in:
@@ -1,6 +1,8 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import threading
|
||||
from collections.abc import Iterator
|
||||
from contextlib import contextmanager
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
@@ -377,6 +379,7 @@ class DeepEPLLAll2AllManager(DeepEPAll2AllManagerBase):
|
||||
@dataclass
|
||||
class _NixlEPBufferState:
|
||||
buffer: Any
|
||||
max_num_ep_ranks: int
|
||||
connected_ep_size: int
|
||||
active_ep_size: int
|
||||
|
||||
@@ -402,7 +405,10 @@ class NixlEPAll2AllManager(All2AllManagerBase):
|
||||
super().__init__(cpu_group, tcp_store_group)
|
||||
self.support_fault_tolerance = True
|
||||
|
||||
self.max_num_ep_ranks = envs.VLLM_NIXL_EP_MAX_NUM_RANKS
|
||||
@property
|
||||
def max_num_ep_ranks(self) -> int:
|
||||
assert NixlEPAll2AllManager._buffer is not None
|
||||
return NixlEPAll2AllManager._buffer.max_num_ep_ranks
|
||||
|
||||
def _init_buffer(
|
||||
self,
|
||||
@@ -412,11 +418,19 @@ class NixlEPAll2AllManager(All2AllManagerBase):
|
||||
) -> None:
|
||||
from nixl_ep import Buffer # type: ignore[import-not-found]
|
||||
|
||||
max_num_global_experts = self.max_num_ep_ranks * num_experts_per_rank
|
||||
parallel_config = get_current_vllm_config().parallel_config
|
||||
max_num_ep_ranks = (
|
||||
self.world_size
|
||||
* parallel_config.elastic_ep_max_dp_size
|
||||
// parallel_config.data_parallel_size
|
||||
if parallel_config.enable_elastic_ep
|
||||
else self.world_size
|
||||
)
|
||||
max_num_global_experts = max_num_ep_ranks * num_experts_per_rank
|
||||
num_rdma_bytes = Buffer.get_rdma_size_hint(
|
||||
num_max_dispatch_tokens_per_rank=max_num_tokens_per_dp_rank,
|
||||
hidden=token_hidden_size,
|
||||
num_ranks=self.max_num_ep_ranks,
|
||||
num_ranks=max_num_ep_ranks,
|
||||
num_experts=max_num_global_experts,
|
||||
)
|
||||
assert NixlEPAll2AllManager._buffer is None, (
|
||||
@@ -427,7 +441,7 @@ class NixlEPAll2AllManager(All2AllManagerBase):
|
||||
tcp_store_group=self.tcp_store_group.store,
|
||||
)
|
||||
buffer.update_memory_buffers(
|
||||
num_ranks=self.max_num_ep_ranks,
|
||||
num_ranks=max_num_ep_ranks,
|
||||
num_experts_per_rank=num_experts_per_rank,
|
||||
num_rdma_bytes=num_rdma_bytes,
|
||||
)
|
||||
@@ -435,6 +449,7 @@ class NixlEPAll2AllManager(All2AllManagerBase):
|
||||
buffer.connect_ranks(ranks_to_connect)
|
||||
NixlEPAll2AllManager._buffer = _NixlEPBufferState(
|
||||
buffer=buffer,
|
||||
max_num_ep_ranks=max_num_ep_ranks,
|
||||
connected_ep_size=self.world_size,
|
||||
active_ep_size=self.world_size,
|
||||
)
|
||||
@@ -474,6 +489,7 @@ class NixlEPAll2AllManager(All2AllManagerBase):
|
||||
|
||||
for rank in range(state.active_ep_size, target_ep_size):
|
||||
state.buffer.update_mask_buffer(rank, mask=False)
|
||||
torch.accelerator.synchronize()
|
||||
state.active_ep_size = target_ep_size
|
||||
|
||||
def _stage_ep_size(self) -> None:
|
||||
@@ -486,8 +502,12 @@ class NixlEPAll2AllManager(All2AllManagerBase):
|
||||
if target_ep_size > state.connected_ep_size:
|
||||
self._connect_to_ep_size(target_ep_size, make_active=False)
|
||||
|
||||
def commit_staged_state(self) -> None:
|
||||
"""Commit staged NIXL EP state to the active communication set."""
|
||||
def stage_ep_size(self) -> None:
|
||||
with NixlEPAll2AllManager._lock:
|
||||
self._stage_ep_size()
|
||||
|
||||
def commit_ep_size(self) -> None:
|
||||
"""Commit the staged EP size to the active communication set."""
|
||||
with NixlEPAll2AllManager._lock:
|
||||
assert NixlEPAll2AllManager._buffer is not None
|
||||
state = NixlEPAll2AllManager._buffer
|
||||
@@ -500,31 +520,31 @@ class NixlEPAll2AllManager(All2AllManagerBase):
|
||||
|
||||
self._unmask_connected_ranks(target_ep_size)
|
||||
|
||||
def _ensure_ep_size(self, *, stage: bool) -> None:
|
||||
if stage:
|
||||
self._stage_ep_size()
|
||||
else:
|
||||
self.commit_staged_state()
|
||||
@contextmanager
|
||||
def mask_remote_ranks(self) -> Iterator[None]:
|
||||
peers = [rank for rank in range(self.world_size) if rank != self.rank]
|
||||
assert NixlEPAll2AllManager._buffer is not None
|
||||
buffer = NixlEPAll2AllManager._buffer.buffer
|
||||
for rank in peers:
|
||||
buffer.update_mask_buffer(rank, mask=True)
|
||||
torch.accelerator.synchronize()
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
for rank in peers:
|
||||
buffer.update_mask_buffer(rank, mask=False)
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
def get_handle(self, kwargs):
|
||||
with NixlEPAll2AllManager._lock:
|
||||
stage = bool(kwargs.get("stage", False))
|
||||
state = NixlEPAll2AllManager._buffer
|
||||
if state is None:
|
||||
assert not stage, (
|
||||
"NIXL EP staged initialization requires an existing buffer"
|
||||
)
|
||||
max_num_tokens_per_dp_rank = kwargs["max_num_tokens_per_dp_rank"]
|
||||
num_experts_per_rank = (
|
||||
kwargs["num_global_experts"] // kwargs["num_ep_ranks"]
|
||||
)
|
||||
self._init_buffer(
|
||||
max_num_tokens_per_dp_rank=max_num_tokens_per_dp_rank,
|
||||
token_hidden_size=kwargs["token_hidden_size"],
|
||||
num_experts_per_rank=num_experts_per_rank,
|
||||
num_experts_per_rank=kwargs["num_local_experts"],
|
||||
)
|
||||
else:
|
||||
self._ensure_ep_size(stage=stage)
|
||||
|
||||
assert NixlEPAll2AllManager._buffer is not None
|
||||
handle = NixlEPAll2AllManager._buffer.buffer
|
||||
@@ -564,7 +584,7 @@ class NixlEPAll2AllManager(All2AllManagerBase):
|
||||
assert state is not None
|
||||
if NixlEPAll2AllManager._mask is None:
|
||||
NixlEPAll2AllManager._mask = torch.zeros(
|
||||
self.max_num_ep_ranks, device="cuda", dtype=torch.int32
|
||||
state.max_num_ep_ranks, device="cuda", dtype=torch.int32
|
||||
)
|
||||
state.buffer.query_mask_buffer(NixlEPAll2AllManager._mask)
|
||||
return NixlEPAll2AllManager._mask[: state.active_ep_size]
|
||||
|
||||
Reference in New Issue
Block a user