[Elastic EP] Reuse CUDA graphs across reconfiguration (#54985)
This commit is contained in:
@@ -8,6 +8,7 @@ import torch
|
||||
|
||||
from vllm.config import get_current_vllm_config
|
||||
from vllm.distributed import (
|
||||
GroupCoordinator,
|
||||
get_ep_group,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
@@ -104,16 +105,12 @@ if current_platform.is_cuda_alike():
|
||||
)
|
||||
|
||||
|
||||
def get_ep_all2all_manager(eep_stage: bool = False) -> Any:
|
||||
if eep_stage:
|
||||
from vllm.distributed.elastic_ep.standby_state import get_standby_ep_group
|
||||
|
||||
ep_group = get_standby_ep_group()
|
||||
assert ep_group is not None
|
||||
device_communicator = ep_group.device_communicator
|
||||
else:
|
||||
device_communicator = get_ep_group().device_communicator
|
||||
|
||||
def get_ep_all2all_manager(
|
||||
ep_group: GroupCoordinator | None = None,
|
||||
) -> Any:
|
||||
if ep_group is None:
|
||||
ep_group = get_ep_group()
|
||||
device_communicator = ep_group.device_communicator
|
||||
assert device_communicator is not None
|
||||
all2all_manager = device_communicator.all2all_manager
|
||||
assert all2all_manager is not None
|
||||
@@ -168,7 +165,7 @@ def maybe_make_prepare_finalize(
|
||||
routing_tables: tuple[torch.Tensor, torch.Tensor, torch.Tensor] | None = None,
|
||||
allow_new_interface: bool = False,
|
||||
use_monolithic: bool = False,
|
||||
eep_stage: bool = False,
|
||||
all2all_manager: Any | None = None,
|
||||
) -> FusedMoEPrepareAndFinalize | None:
|
||||
if not moe.moe_parallel_config.use_all2all_kernels:
|
||||
if not allow_new_interface:
|
||||
@@ -190,7 +187,8 @@ def maybe_make_prepare_finalize(
|
||||
"Detected DP deployment with no --enable-expert-parallel. "
|
||||
"Falling back to AllGather+ReduceScatter dispatch/combine."
|
||||
)
|
||||
all2all_manager = get_ep_all2all_manager(eep_stage)
|
||||
if all2all_manager is None:
|
||||
all2all_manager = get_ep_all2all_manager()
|
||||
return make_moe_prepare_and_finalize_naive_dp_ep(
|
||||
is_sequence_parallel=moe.moe_parallel_config.is_sequence_parallel,
|
||||
num_dispatchers=all2all_manager.world_size,
|
||||
@@ -199,7 +197,8 @@ def maybe_make_prepare_finalize(
|
||||
else:
|
||||
return make_moe_prepare_and_finalize_no_dp_ep(use_monolithic)
|
||||
|
||||
all2all_manager = get_ep_all2all_manager(eep_stage)
|
||||
if all2all_manager is None:
|
||||
all2all_manager = get_ep_all2all_manager()
|
||||
|
||||
prepare_finalize: FusedMoEPrepareAndFinalize | None = None
|
||||
|
||||
@@ -361,10 +360,7 @@ def maybe_make_prepare_finalize(
|
||||
all_to_all_args = dict(
|
||||
max_num_tokens_per_dp_rank=moe.max_num_tokens,
|
||||
token_hidden_size=moe.hidden_dim,
|
||||
num_ep_ranks=all2all_manager.world_size,
|
||||
num_global_experts=moe.num_experts,
|
||||
num_local_experts=moe.num_experts // all2all_manager.world_size,
|
||||
stage=eep_stage,
|
||||
num_local_experts=moe.num_local_experts,
|
||||
)
|
||||
handle = all2all_manager.get_handle(all_to_all_args)
|
||||
|
||||
@@ -378,7 +374,8 @@ def maybe_make_prepare_finalize(
|
||||
prepare_finalize = NixlEPPrepareAndFinalize(
|
||||
handle,
|
||||
max_tokens_per_rank=moe.max_num_tokens,
|
||||
num_dispatchers=all2all_manager.world_size,
|
||||
num_dispatchers=all2all_manager.max_num_ep_ranks,
|
||||
expert_capacity=(moe.num_local_experts * all2all_manager.max_num_ep_ranks),
|
||||
use_fp8_dispatch=use_fp8_dispatch,
|
||||
global_to_physical=global_to_physical,
|
||||
physical_to_global=physical_to_global,
|
||||
|
||||
@@ -1306,6 +1306,7 @@ class FusedMoEConfig:
|
||||
|
||||
moe_backend: MoEBackend = "auto"
|
||||
max_num_tokens: int = SchedulerConfig.DEFAULT_MAX_NUM_BATCHED_TOKENS_FOR_BATCHED_DP
|
||||
elastic_ep_max_dp_size: int | None = None
|
||||
has_bias: bool = False
|
||||
is_lora_enabled: bool = False
|
||||
|
||||
|
||||
@@ -60,6 +60,8 @@ def _make_eep_experts(
|
||||
def make_eep_staged_quant_method(
|
||||
module: "MoERunner",
|
||||
moe_config: FusedMoEConfig,
|
||||
*,
|
||||
all2all_manager: Any,
|
||||
) -> FusedMoEMethodBase | None:
|
||||
quant_method = module._quant_method
|
||||
if not quant_method.supports_internal_mk:
|
||||
@@ -93,7 +95,7 @@ def make_eep_staged_quant_method(
|
||||
routing_tables=None,
|
||||
allow_new_interface=True,
|
||||
use_monolithic=quant_method.is_monolithic,
|
||||
eep_stage=True,
|
||||
all2all_manager=all2all_manager,
|
||||
)
|
||||
assert prepare_finalize is not None
|
||||
assert isinstance(prepare_finalize, FusedMoEPrepareAndFinalizeModular)
|
||||
|
||||
@@ -348,6 +348,7 @@ def FusedMoEFactory(
|
||||
moe_backend=vllm_config.kernel_config.moe_backend,
|
||||
router_logits_dtype=router_logits_dtype,
|
||||
max_num_tokens=max_num_batched_tokens,
|
||||
elastic_ep_max_dp_size=vllm_config.parallel_config.elastic_ep_max_dp_size,
|
||||
has_bias=has_bias,
|
||||
is_lora_enabled=vllm_config.lora_config is not None,
|
||||
activation=moe_activation,
|
||||
|
||||
@@ -252,13 +252,6 @@ class FusedMoEPrepareAndFinalize(ABC):
|
||||
"""
|
||||
return False
|
||||
|
||||
def on_commit(self) -> None:
|
||||
"""
|
||||
Runs after this prepare/finalize has been committed to the active
|
||||
MoE kernel.
|
||||
"""
|
||||
return
|
||||
|
||||
|
||||
# TODO: pass FusedMoEParallelConfig in as ctor parameter?
|
||||
class FusedMoEPrepareAndFinalizeModular(FusedMoEPrepareAndFinalize):
|
||||
|
||||
@@ -6,8 +6,6 @@ import nixl_ep
|
||||
import torch
|
||||
|
||||
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
|
||||
from vllm.distributed import get_ep_group
|
||||
from vllm.distributed.device_communicators.all2all import NixlEPAll2AllManager
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.fused_moe.config import FusedMoEQuantConfig
|
||||
from vllm.model_executor.layers.fused_moe.topk_weight_and_reduce import (
|
||||
@@ -79,6 +77,7 @@ class NixlEPPrepareAndFinalize(mk.FusedMoEPrepareAndFinalizeModular):
|
||||
buffer: nixl_ep.Buffer,
|
||||
max_tokens_per_rank: int,
|
||||
num_dispatchers: int,
|
||||
expert_capacity: int,
|
||||
use_fp8_dispatch: bool = False,
|
||||
global_to_physical: torch.Tensor | None = None,
|
||||
physical_to_global: torch.Tensor | None = None,
|
||||
@@ -94,6 +93,7 @@ class NixlEPPrepareAndFinalize(mk.FusedMoEPrepareAndFinalizeModular):
|
||||
# combine function.
|
||||
self.handles: list[tuple | None] = [None, None]
|
||||
self.num_dispatchers_ = num_dispatchers
|
||||
self.expert_capacity = expert_capacity
|
||||
|
||||
topk_indices_dtype = self.topk_indices_dtype()
|
||||
|
||||
@@ -141,17 +141,6 @@ class NixlEPPrepareAndFinalize(mk.FusedMoEPrepareAndFinalizeModular):
|
||||
def max_num_tokens_per_rank(self) -> int | None:
|
||||
return self.max_tokens_per_rank
|
||||
|
||||
def on_commit(self) -> None:
|
||||
device_communicator = get_ep_group().device_communicator
|
||||
assert device_communicator is not None
|
||||
all2all_manager = device_communicator.all2all_manager
|
||||
assert isinstance(all2all_manager, NixlEPAll2AllManager)
|
||||
# maybe_make_prepare_finalize(..., eep_stage=True) initializes self.buffer
|
||||
# with get_handle(..., stage=True), which stages global NIXL state for the
|
||||
# new config but leaves it inactive while the old config remains active.
|
||||
# When EEP commit switches to this P/F, this P/F needs to commit that state.
|
||||
all2all_manager.commit_staged_state()
|
||||
|
||||
def topk_indices_dtype(self) -> torch.dtype | None:
|
||||
return NIXL_EP_TOPK_INDICES_DTYPE
|
||||
|
||||
@@ -267,7 +256,7 @@ class NixlEPPrepareAndFinalize(mk.FusedMoEPrepareAndFinalizeModular):
|
||||
a1,
|
||||
dispatch_topk_ids,
|
||||
self.max_tokens_per_rank,
|
||||
num_experts,
|
||||
self.expert_capacity,
|
||||
use_fp8=self.use_fp8_dispatch,
|
||||
# round_scale needs to be set to dispatch in ue8m0
|
||||
round_scale=self.use_ue8m0_dispatch,
|
||||
|
||||
@@ -500,7 +500,8 @@ def fi_moe_largest_bucket(moe_config: "FusedMoEConfig") -> int:
|
||||
|
||||
For a detailed explanation, see: `docs/serving/data_parallel_deployment.md`
|
||||
"""
|
||||
return max(moe_config.max_num_tokens * moe_config.dp_size, 8192)
|
||||
dp_size = moe_config.elastic_ep_max_dp_size or moe_config.dp_size
|
||||
return max(moe_config.max_num_tokens * dp_size, 8192)
|
||||
|
||||
|
||||
@torch.compile(dynamic=True, backend=current_platform.simple_compile_backend)
|
||||
|
||||
@@ -12,6 +12,7 @@ from typing import TYPE_CHECKING
|
||||
import vllm.envs as envs
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.distributed.parallel_state import GroupCoordinator
|
||||
from vllm.v1.worker.gpu_model_runner import GPUModelRunner
|
||||
|
||||
|
||||
@@ -40,6 +41,44 @@ def resolve_flashinfer_autotune_file(runner: "GPUModelRunner") -> Path:
|
||||
return output_dir / "autotune_configs.json"
|
||||
|
||||
|
||||
def sync_flashinfer_autotune_cache(
|
||||
runner: "GPUModelRunner",
|
||||
group: "GroupCoordinator",
|
||||
) -> None:
|
||||
cache: bytes | str | None = None
|
||||
if (
|
||||
group.rank_in_group == 0
|
||||
and runner.vllm_config.kernel_config.enable_flashinfer_autotune
|
||||
):
|
||||
try:
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.flashinfer import has_flashinfer
|
||||
|
||||
if has_flashinfer() and current_platform.has_device_capability(90):
|
||||
from flashinfer.autotuner import AutoTuner
|
||||
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
path = Path(temp_dir) / "autotune_configs.json"
|
||||
AutoTuner.get().save_configs(str(path))
|
||||
cache = path.read_bytes()
|
||||
except Exception as exc:
|
||||
cache = f"{type(exc).__name__}: {exc}"
|
||||
|
||||
cache = group.broadcast_object(cache)
|
||||
if isinstance(cache, str):
|
||||
raise RuntimeError(f"Failed to serialize FlashInfer autotune state: {cache}")
|
||||
if cache is None or group.rank_in_group == 0:
|
||||
return
|
||||
|
||||
from flashinfer.autotuner import AutoTuner
|
||||
|
||||
with tempfile.NamedTemporaryFile() as f:
|
||||
f.write(cache)
|
||||
f.flush()
|
||||
if not AutoTuner.get().load_configs(f.name):
|
||||
raise RuntimeError("FlashInfer autotune cache is incompatible")
|
||||
|
||||
|
||||
def write_flashinfer_autotune_cache(cache_path: Path, contents: bytes) -> None:
|
||||
cache_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
fd, tmp_path = tempfile.mkstemp(
|
||||
|
||||
Reference in New Issue
Block a user