[MoE Refactor] Remove MoE legacy code (#51078)

Signed-off-by: Bill Nell <bnell@redhat.com>
This commit is contained in:
bnellnm
2026-08-05 14:35:36 -07:00
committed by GitHub
parent 4282fe52d5
commit 373fe8b83e
22 changed files with 3 additions and 295 deletions
@@ -93,7 +93,6 @@ def _make_runner(**overrides: Any) -> Any:
def test_v2_load_model_registers_moe_with_eplb(monkeypatch):
FakeEplbState.instances.clear()
model = SimpleNamespace(is_moe=True)
prepared: list[object] = []
monkeypatch.setattr(mrv2, "DeviceMemoryProfiler", FakeMemoryProfiler)
monkeypatch.setattr(eplb, "EplbState", FakeEplbState)
@@ -102,7 +101,6 @@ def test_v2_load_model_registers_moe_with_eplb(monkeypatch):
"get_model_loader",
lambda load_config: SimpleNamespace(load_model=lambda **_: model),
)
monkeypatch.setattr(mrv2, "prepare_communication_buffer_for_model", prepared.append)
monkeypatch.setattr(
mrv2,
"init_model_state",
@@ -119,7 +117,6 @@ def test_v2_load_model_registers_moe_with_eplb(monkeypatch):
assert runner.model is model
assert runner.model_state is not None
assert prepared == [model]
assert runner.eplb_state is not None
assert runner.eplb_state.add_model_calls == [(model, runner.model_config)]
assert runner.eplb_state.async_started is True
@@ -128,7 +125,6 @@ def test_v2_load_model_registers_moe_with_eplb(monkeypatch):
def test_v2_load_model_with_dummy_weights_skips_eplb_registration(monkeypatch):
FakeEplbState.instances.clear()
model = SimpleNamespace(is_moe=True)
prepared: list[object] = []
monkeypatch.setattr(mrv2, "DeviceMemoryProfiler", FakeMemoryProfiler)
monkeypatch.setattr(eplb, "EplbState", FakeEplbState)
@@ -137,7 +133,6 @@ def test_v2_load_model_with_dummy_weights_skips_eplb_registration(monkeypatch):
"get_model_loader",
lambda load_config: SimpleNamespace(load_model=lambda **_: model),
)
monkeypatch.setattr(mrv2, "prepare_communication_buffer_for_model", prepared.append)
monkeypatch.setattr(
mrv2,
"init_model_state",
@@ -149,7 +144,6 @@ def test_v2_load_model_with_dummy_weights_skips_eplb_registration(monkeypatch):
mrv2.GPUModelRunner.load_model(runner, load_dummy_weights=True)
assert runner.load_config.load_format == "dummy"
assert prepared == []
assert runner.eplb_state is not None
assert runner.eplb_state.add_model_calls == []
assert runner.eplb_state.async_started is False
@@ -8,7 +8,6 @@ import torch.distributed as dist
from torch.distributed import ProcessGroup
from vllm.logger import init_logger
from vllm.utils import is_moe_layer
logger = init_logger(__name__)
@@ -358,17 +357,6 @@ class DeviceCommunicatorBase:
def destroy(self):
pass
def prepare_communication_buffer_for_model(self, model: torch.nn.Module) -> None:
"""
Prepare the communication buffer for the model.
"""
if not self.is_ep_communicator:
return
moe_modules = [module for module in model.modules() if is_moe_layer(module)]
for module in moe_modules:
module.maybe_init_modular_kernel()
def dispatch_router_logits(
self,
hidden_states: torch.Tensor,
@@ -35,7 +35,6 @@ from vllm.distributed.eplb.eplb_communicator import create_eplb_communicator
from vllm.distributed.parallel_state import (
_replace_active_groups,
get_eplb_group,
prepare_communication_buffer_for_model,
)
from vllm.distributed.stateless_coordinator import StatelessGroupCoordinator
from vllm.logger import init_logger
@@ -485,7 +484,6 @@ class ElasticEPScalingExecutor:
for module in moe_modules:
if getattr(module._quant_method, "wraps_legacy_quant_method", False):
module._replace_quant_method(module._quant_method.old_quant_method)
prepare_communication_buffer_for_model(self.worker.model_runner.model)
eplb_model_state.expert_buffer = [
torch.empty_like(w) for w in model.expert_weights[0]
@@ -652,10 +650,6 @@ class ElasticEPScalingExecutor:
old_num_physical_experts,
)
def prepare_new_worker(self) -> None:
with set_current_vllm_config(self.worker.vllm_config):
prepare_communication_buffer_for_model(self.worker.model_runner.get_model())
def warmup_local_kernels(self) -> None:
with set_current_vllm_config(self.worker.vllm_config):
kernel_warmup(self.worker, process_local_only=True)
@@ -214,7 +214,6 @@ class ElasticEPScalingState:
self.engine_core.available_gpu_memory_for_kv_cache = (
ParallelConfig.sync_kv_cache_memory_size(self.new_dp_group, -1)
)
self._collective_rpc("elastic_ep_execute", args=("prepare_new_worker",))
self.state = ScaleUpNewEngineState.PREPARE
return True
-25
View File
@@ -1234,10 +1234,6 @@ class GroupCoordinator:
if self.mq_broadcaster is not None:
self.mq_broadcaster = None
def prepare_communication_buffer_for_model(self, model: torch.nn.Module):
if self.device_communicator is not None:
self.device_communicator.prepare_communication_buffer_for_model(model)
def dispatch_router_logits(
self,
hidden_states: torch.Tensor,
@@ -2041,27 +2037,6 @@ def ensure_model_parallel_initialized(
)
def prepare_communication_buffer_for_model(model: torch.nn.Module):
"""Prepare the communication buffer for the model.
Traditional communication libraries like NCCL are almost
model agnostic. However, emerging new communication libraries like
MoE all2all (DeepEP) usually allocate the communication buffer
based on the model shape for optimal performance.
"""
if _TP is not None:
_TP.prepare_communication_buffer_for_model(model)
if _PCP is not None:
_PCP.prepare_communication_buffer_for_model(model)
if _PP is not None:
_PP.prepare_communication_buffer_for_model(model)
if _DP is not None:
_DP.prepare_communication_buffer_for_model(model)
if _EP is not None:
_EP.prepare_communication_buffer_for_model(model)
if _EPLB is not None:
_EPLB.prepare_communication_buffer_for_model(model)
def checkpoint_prepare_distributed_state() -> None:
"""Prepare every device communicator for a process checkpoint."""
torch.accelerator.synchronize()
+2 -11
View File
@@ -19,12 +19,8 @@ from vllm.model_executor.layers.fused_moe.fused_moe_modular_method import (
FusedMoEModularMethod,
)
from vllm.model_executor.layers.fused_moe.modular_kernel import (
FusedMoEKernel,
FusedMoEKernelModularImpl,
)
from vllm.model_executor.layers.fused_moe.prepare_finalize import (
MoEPrepareAndFinalizeNoDPEPModular,
)
from vllm.platforms import current_platform
from .utils import _get_lora_aux_cuda_stream, _get_lora_device
@@ -76,13 +72,8 @@ class FusedMoEWithLoRA(BaseLayerWithLoRA):
assert isinstance(moe_kernel.impl, FusedMoEKernelModularImpl)
moe_kernel.impl.shared_experts = None
else:
prepare_finalize = MoEPrepareAndFinalizeNoDPEPModular()
moe_kernel = FusedMoEKernel(
prepare_finalize,
routed_experts.quant_method.select_gemm_impl(
prepare_finalize, routed_experts
),
)
raise RuntimeError("Old-style modular kernel not supported.")
assert moe_kernel.supports_lora(), (
f"{type(moe_kernel.fused_experts).__name__} does not support LoRA. "
"For unquantized MoE, set moe_backend='triton' or moe_backend='auto' "
@@ -123,20 +123,6 @@ def maybe_make_prepare_finalize(
use_monolithic: bool = False,
eep_stage: bool = False,
) -> FusedMoEPrepareAndFinalize | None:
# NOTE(rob): we are migrating each quant_method to hold the MK
# in all cases. The allow_new_interface=False flag allow us to fall
# back to the old method for methods that have not yet been migrated.
#
# In old method:
# * maybe_init_modular_kernel() calls this function. If we are
# using no Dp/Ep or naive all2all, we return None this function
# returns None and no ModularKernelMethod is created. If non-naive
# all2all is used, this returns a PrepareAndFinalize object and
# a ModularKernelMethod is created.
# In new method:
# * maybe_make_prepare_finalize() is called from the oracle. We
# always return a PrepareAndFinalize object and the quant method
# holds the ModularKernel.
if not moe.moe_parallel_config.use_all2all_kernels:
if not allow_new_interface:
return None
@@ -13,10 +13,6 @@ from vllm.model_executor.layers.fused_moe.config import (
FusedMoEParallelConfig,
FusedMoEQuantConfig,
)
from vllm.model_executor.layers.fused_moe.modular_kernel import (
FusedMoEExpertsModular,
FusedMoEPrepareAndFinalizeModular,
)
from vllm.model_executor.layers.quantization.base_config import (
QuantizeMethodBase,
)
@@ -103,30 +99,6 @@ class FusedMoEMethodBase(QuantizeMethodBase):
hidden_size, act_dtype, moe_parallel_config
), intermediate_size_per_partition
def maybe_make_prepare_finalize(
self,
routing_tables: tuple[torch.Tensor, torch.Tensor, torch.Tensor] | None = None,
) -> FusedMoEPrepareAndFinalizeModular | None:
from .all2all_utils import maybe_make_prepare_finalize
pf = maybe_make_prepare_finalize(
self.moe, self.moe_quant_config, routing_tables
)
assert pf is None or isinstance(pf, FusedMoEPrepareAndFinalizeModular)
return pf
def select_gemm_impl(
self,
prepare_finalize: FusedMoEPrepareAndFinalizeModular,
layer: "RoutedExperts",
) -> FusedMoEExpertsModular:
# based on the all2all implementation, select the appropriate
# gemm implementation
raise ValueError(
f"{self.__class__.__name__} uses the new modular kernel initialization "
"logic. This function should not be called."
)
@abstractmethod
def get_fused_moe_quant_config(
self, layer: "RoutedExperts"
@@ -15,7 +15,6 @@ from vllm.model_executor.layers.fused_moe.fused_moe_method_base import (
)
from vllm.model_executor.layers.fused_moe.modular_kernel import (
FusedMoEKernel,
FusedMoEPrepareAndFinalizeModular,
)
from vllm.model_executor.layers.fused_moe.runner.shared_experts import (
SharedExperts,
@@ -47,20 +46,6 @@ class FusedMoEModularMethod(FusedMoEMethodBase, CustomOp):
def wraps_legacy_quant_method(self) -> bool:
return not self.old_quant_method.supports_internal_mk
@staticmethod
def make(
routed_experts: "RoutedExperts",
old_quant_method: FusedMoEMethodBase,
prepare_finalize: FusedMoEPrepareAndFinalizeModular,
) -> "FusedMoEModularMethod":
return FusedMoEModularMethod(
old_quant_method,
FusedMoEKernel(
prepare_finalize,
old_quant_method.select_gemm_impl(prepare_finalize, routed_experts),
),
)
@property
def skip_forward_padding(self) -> bool:
return self.old_quant_method.skip_forward_padding
@@ -28,9 +28,6 @@ from vllm.model_executor.layers.fused_moe.config import (
from vllm.model_executor.layers.fused_moe.fused_moe_method_base import (
FusedMoEMethodBase,
)
from vllm.model_executor.layers.fused_moe.fused_moe_modular_method import (
FusedMoEModularMethod,
)
from vllm.model_executor.layers.fused_moe.routed_experts import (
RoutedExperts,
)
@@ -896,44 +893,6 @@ class MoERunner(MoERunnerInterface):
#
#########################################################
# Note: maybe_init_modular_kernel should only be called by
# prepare_communication_buffer_for_model.
# This is called after all weight loading and post-processing, so it
# should be safe to swap out the quant_method.
def maybe_init_modular_kernel(self) -> None:
# NOTE(rob): WIP refactor. For quant methods that own the MK
# we create the MK during process_weights_after_loading.
if (
self.routed_experts.quant_method.supports_internal_mk
or self.routed_experts.quant_method.is_monolithic
):
return None
self.routed_experts._ensure_moe_quant_config_init()
# routing_tables only needed for round-robin expert placement with
# DeepEP all2all backend.
routing_tables = self._expert_routing_tables()
if isinstance(self.routed_experts.quant_method, FusedMoEModularMethod):
base_quant_method = self.routed_experts.quant_method.old_quant_method
else:
base_quant_method = self.routed_experts.quant_method
prepare_finalize = base_quant_method.maybe_make_prepare_finalize(
routing_tables=routing_tables
)
if prepare_finalize is not None:
logger.debug(
"%s for %s(%s)", prepare_finalize.__class__.__name__, self, id(self)
)
self._replace_quant_method(
FusedMoEModularMethod.make(
self.routed_experts,
base_quant_method,
prepare_finalize,
)
)
#
# Properties
#
@@ -66,10 +66,6 @@ class MoERunnerInterface(PluggableLayer, ABC):
#
########################################################################
@abstractmethod
def maybe_init_modular_kernel(self) -> None:
raise NotImplementedError
@property
@abstractmethod
def layer_id(self):
@@ -17,10 +17,6 @@ from vllm.model_executor.layers.fused_moe.config import (
from vllm.model_executor.layers.fused_moe.fused_moe_method_base import (
FusedMoEMethodBase,
)
from vllm.model_executor.layers.fused_moe.modular_kernel import (
FusedMoEExpertsModular,
FusedMoEPrepareAndFinalizeModular,
)
from vllm.model_executor.layers.fused_moe.oracle.unquantized import (
UnquantizedMoeBackend,
convert_to_unquantized_kernel_format,
@@ -56,25 +52,6 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp):
def supports_eplb(self) -> bool:
return True
def maybe_make_prepare_finalize(
self,
routing_tables: tuple[torch.Tensor, torch.Tensor, torch.Tensor] | None = None,
):
raise ValueError(
f"{self.__class__.__name__} uses the new modular kernel initialization "
"logic. So this function should not be called."
)
def select_gemm_impl(
self,
prepare_finalize: FusedMoEPrepareAndFinalizeModular,
layer: "RoutedExperts",
) -> FusedMoEExpertsModular:
raise ValueError(
f"{self.__class__.__name__} uses the new modular kernel initialization "
"logic. This function should not be called."
)
def create_weights(
self,
layer: "RoutedExperts",
@@ -8,7 +8,6 @@ from compressed_tensors.quantization import (
QuantizationArgs,
)
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
from vllm.logger import init_logger
from vllm.model_executor.layers.fused_moe import (
RoutedExperts,
@@ -310,13 +309,6 @@ class CompressedTensorsW4A16FlydslMoEMethod(CompressedTensorsMoEMethod):
block_shape=[0, self.group_size],
)
def select_gemm_impl(
self,
prepare_finalize: mk.FusedMoEPrepareAndFinalizeModular,
layer: torch.nn.Module,
) -> mk.FusedMoEExpertsModular:
raise NotImplementedError
def apply(
self,
layer: RoutedExperts,
@@ -4,7 +4,6 @@
import torch
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
from vllm.logger import init_logger
from vllm.model_executor.layers.fused_moe import (
FusedMoeWeightScaleSupported,
@@ -242,15 +241,6 @@ class CompressedTensorsW4A4Nvfp4MoEMethod(CompressedTensorsMoEMethod):
)
self.moe_kernel.fused_experts.process_weights_after_loading(layer)
def maybe_make_prepare_finalize(
self,
routing_tables: tuple[torch.Tensor, torch.Tensor, torch.Tensor] | None = None,
) -> mk.FusedMoEPrepareAndFinalizeModular | None:
raise ValueError(
f"{self.__class__.__name__} uses the new modular kernel initialization "
"logic. This function should not be called."
)
def get_fused_moe_quant_config(self, layer: torch.nn.Module) -> FusedMoEQuantConfig:
return make_nvfp4_moe_quant_config(
backend=self.nvfp4_backend,
@@ -8,7 +8,6 @@ from compressed_tensors.quantization import (
QuantizationStrategy,
)
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
from vllm.distributed import get_tensor_model_parallel_world_size
from vllm.logger import init_logger
from vllm.model_executor.layers.fused_moe import (
@@ -341,15 +340,6 @@ class CompressedTensorsW8A8Fp8MoEMethod(CompressedTensorsMoEMethod):
layer=layer,
)
def maybe_make_prepare_finalize(
self,
routing_tables: tuple[torch.Tensor, torch.Tensor, torch.Tensor] | None = None,
) -> mk.FusedMoEPrepareAndFinalizeModular | None:
raise ValueError(
f"{self.__class__.__name__} uses the new modular kernel initialization "
"logic. This function should not be called."
)
def get_fused_moe_quant_config(self, layer: torch.nn.Module) -> FusedMoEQuantConfig:
is_per_token = self.input_quant.strategy == QuantizationStrategy.TOKEN
return make_fp8_moe_quant_config(
@@ -7,7 +7,6 @@ from compressed_tensors.quantization import (
QuantizationStrategy,
)
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
from vllm.logger import init_logger
from vllm.model_executor.layers.fused_moe import (
FusedMoeWeightScaleSupported,
@@ -164,15 +163,6 @@ class CompressedTensorsW8A8Int8MoEMethod(CompressedTensorsMoEMethod):
)
self.moe_kernel.fused_experts.process_weights_after_loading(layer)
def maybe_make_prepare_finalize(
self,
routing_tables: tuple[torch.Tensor, torch.Tensor, torch.Tensor] | None = None,
) -> mk.FusedMoEPrepareAndFinalizeModular | None:
raise ValueError(
f"{self.__class__.__name__} uses the new modular kernel initialization "
"logic. This function should not be called."
)
def get_fused_moe_quant_config(self, layer: torch.nn.Module) -> FusedMoEQuantConfig:
return make_int8_moe_quant_config(
int8_backend=self.int8_backend,
@@ -3,7 +3,6 @@
import torch
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
from vllm.model_executor.layers.fused_moe import (
FusedMoeWeightScaleSupported,
RoutedExperts,
@@ -159,15 +158,6 @@ class CompressedTensorsW8A8Mxfp8MoEMethod(CompressedTensorsMoEMethod):
layer=layer,
)
def maybe_make_prepare_finalize(
self,
routing_tables: tuple[torch.Tensor, torch.Tensor, torch.Tensor] | None = None,
) -> mk.FusedMoEPrepareAndFinalizeModular | None:
raise ValueError(
f"{self.__class__.__name__} uses the new modular kernel "
"initialization logic. This function should not be called."
)
def apply_monolithic(
self,
layer: RoutedExperts,
@@ -6,7 +6,6 @@ from typing import TYPE_CHECKING, Any
import torch
import vllm.envs as envs
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
from vllm.config import get_current_vllm_config
from vllm.distributed import get_tensor_model_parallel_world_size
from vllm.logger import init_logger
@@ -731,15 +730,6 @@ class Fp8MoEMethod(FusedMoEMethodBase):
layer, w13, w2, w13_scale, w2_scale, w13_input_scale, w2_input_scale
)
def maybe_make_prepare_finalize(
self,
routing_tables: tuple[torch.Tensor, torch.Tensor, torch.Tensor] | None = None,
) -> mk.FusedMoEPrepareAndFinalizeModular | None:
raise ValueError(
f"{self.__class__.__name__} uses the new modular kernel initialization "
"logic. This function should not be called."
)
def get_fused_moe_quant_config(self, layer: RoutedExperts) -> FusedMoEQuantConfig:
w1_scale = getattr(layer, f"w13_{self.weight_scale_name}")
w2_scale = getattr(layer, f"w2_{self.weight_scale_name}")
@@ -8,7 +8,6 @@ import torch
from torch.nn.parameter import Parameter
import vllm.envs as envs
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
from vllm.config import get_current_vllm_config
from vllm.logger import init_logger
from vllm.model_executor.kernels.linear import (
@@ -771,15 +770,6 @@ class ModelOptFp8MoEMethod(FusedMoEMethodBase):
activation_key=kFp8StaticTensorSym,
)
def maybe_make_prepare_finalize(
self,
routing_tables: tuple[torch.Tensor, torch.Tensor, torch.Tensor] | None = None,
) -> mk.FusedMoEPrepareAndFinalizeModular | None:
raise ValueError(
f"{self.__class__.__name__} uses the new modular kernel initialization "
"logic. This function should not be called."
)
def create_weights(
self,
layer: RoutedExperts,
@@ -1409,15 +1399,6 @@ class ModelOptNvFp4FusedMoE(FusedMoEMethodBase):
self.nvfp4_backend
)
def maybe_make_prepare_finalize(
self,
routing_tables: tuple[torch.Tensor, torch.Tensor, torch.Tensor] | None = None,
) -> mk.FusedMoEPrepareAndFinalizeModular | None:
raise ValueError(
f"{self.__class__.__name__} uses the new modular kernel initialization "
"logic. This function should not be called."
)
def uses_weight_scale_2_pattern(self) -> bool:
"""
FP4 variants use 'weight_scale_2' pattern for per-tensor weight scales.
@@ -2087,15 +2068,6 @@ class ModelOptMxFp8FusedMoE(FusedMoEMethodBase):
):
self._dequant_mxfp8_weights_to_bf16(layer)
def maybe_make_prepare_finalize(
self,
routing_tables: tuple[torch.Tensor, torch.Tensor, torch.Tensor] | None = None,
) -> mk.FusedMoEPrepareAndFinalizeModular | None:
raise ValueError(
f"{self.__class__.__name__} uses the new modular kernel initialization "
"logic. This function should not be called."
)
def get_fused_moe_quant_config(
self, layer: RoutedExperts
) -> FusedMoEQuantConfig | None:
@@ -5,7 +5,6 @@ from abc import abstractmethod
import torch
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
from vllm.model_executor.layers.fused_moe import (
FusedMoEMethodBase,
RoutedExperts,
@@ -100,15 +99,6 @@ class OnlineMoEMethodBase(FusedMoEMethodBase):
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
pass
def maybe_make_prepare_finalize(
self,
routing_tables: tuple[torch.Tensor, torch.Tensor, torch.Tensor] | None = None,
) -> mk.FusedMoEPrepareAndFinalizeModular | None:
raise ValueError(
f"{self.__class__.__name__} uses the new modular kernel "
"initialization logic. This function should not be called."
)
@property
def supports_eplb(self) -> bool:
return True
-6
View File
@@ -34,7 +34,6 @@ from vllm.config.compilation import CUDAGraphMode
from vllm.distributed.parallel_state import (
get_dcp_group,
get_pp_group,
prepare_communication_buffer_for_model,
)
from vllm.forward_context import BatchDescriptor, set_forward_context
from vllm.logger import init_logger
@@ -354,11 +353,6 @@ class GPUModelRunner(LoRAModelRunnerMixin):
time_after_load - time_before_load,
)
if not load_dummy_weights:
prepare_communication_buffer_for_model(self.model)
if self.speculator is not None:
prepare_communication_buffer_for_model(self.speculator.model)
# Initialize the components that require the model.
self.model_state = init_model_state(
self.vllm_config, self.model, self.encoder_cache, self.device
+1 -7
View File
@@ -50,7 +50,6 @@ from vllm.distributed.parallel_state import (
get_tp_group,
graph_capture,
is_global_first_rank,
prepare_communication_buffer_for_model,
)
from vllm.forward_context import (
BatchDescriptor,
@@ -5387,12 +5386,7 @@ class GPUModelRunner(
format_gib(self.model_memory_usage),
time_after_load - time_before_load,
)
if not load_dummy_weights:
prepare_communication_buffer_for_model(self.model)
if (drafter := getattr(self, "drafter", None)) and (
drafter_model := getattr(drafter, "model", None)
):
prepare_communication_buffer_for_model(drafter_model)
mm_config = self.model_config.multimodal_config
self.is_multimodal_pruning_enabled = (
supports_multimodal_pruning(self.get_model())