diff --git a/tests/v1/worker/test_gpu_model_runner_v2_eplb.py b/tests/v1/worker/test_gpu_model_runner_v2_eplb.py index 89a8f6878a..c7813b8548 100644 --- a/tests/v1/worker/test_gpu_model_runner_v2_eplb.py +++ b/tests/v1/worker/test_gpu_model_runner_v2_eplb.py @@ -93,7 +93,6 @@ def _make_runner(**overrides: Any) -> Any: def test_v2_load_model_registers_moe_with_eplb(monkeypatch): FakeEplbState.instances.clear() model = SimpleNamespace(is_moe=True) - prepared: list[object] = [] monkeypatch.setattr(mrv2, "DeviceMemoryProfiler", FakeMemoryProfiler) monkeypatch.setattr(eplb, "EplbState", FakeEplbState) @@ -102,7 +101,6 @@ def test_v2_load_model_registers_moe_with_eplb(monkeypatch): "get_model_loader", lambda load_config: SimpleNamespace(load_model=lambda **_: model), ) - monkeypatch.setattr(mrv2, "prepare_communication_buffer_for_model", prepared.append) monkeypatch.setattr( mrv2, "init_model_state", @@ -119,7 +117,6 @@ def test_v2_load_model_registers_moe_with_eplb(monkeypatch): assert runner.model is model assert runner.model_state is not None - assert prepared == [model] assert runner.eplb_state is not None assert runner.eplb_state.add_model_calls == [(model, runner.model_config)] assert runner.eplb_state.async_started is True @@ -128,7 +125,6 @@ def test_v2_load_model_registers_moe_with_eplb(monkeypatch): def test_v2_load_model_with_dummy_weights_skips_eplb_registration(monkeypatch): FakeEplbState.instances.clear() model = SimpleNamespace(is_moe=True) - prepared: list[object] = [] monkeypatch.setattr(mrv2, "DeviceMemoryProfiler", FakeMemoryProfiler) monkeypatch.setattr(eplb, "EplbState", FakeEplbState) @@ -137,7 +133,6 @@ def test_v2_load_model_with_dummy_weights_skips_eplb_registration(monkeypatch): "get_model_loader", lambda load_config: SimpleNamespace(load_model=lambda **_: model), ) - monkeypatch.setattr(mrv2, "prepare_communication_buffer_for_model", prepared.append) monkeypatch.setattr( mrv2, "init_model_state", @@ -149,7 +144,6 @@ def test_v2_load_model_with_dummy_weights_skips_eplb_registration(monkeypatch): mrv2.GPUModelRunner.load_model(runner, load_dummy_weights=True) assert runner.load_config.load_format == "dummy" - assert prepared == [] assert runner.eplb_state is not None assert runner.eplb_state.add_model_calls == [] assert runner.eplb_state.async_started is False diff --git a/vllm/distributed/device_communicators/base_device_communicator.py b/vllm/distributed/device_communicators/base_device_communicator.py index 73fd1331f5..fc4def37f7 100644 --- a/vllm/distributed/device_communicators/base_device_communicator.py +++ b/vllm/distributed/device_communicators/base_device_communicator.py @@ -8,7 +8,6 @@ import torch.distributed as dist from torch.distributed import ProcessGroup from vllm.logger import init_logger -from vllm.utils import is_moe_layer logger = init_logger(__name__) @@ -358,17 +357,6 @@ class DeviceCommunicatorBase: def destroy(self): pass - def prepare_communication_buffer_for_model(self, model: torch.nn.Module) -> None: - """ - Prepare the communication buffer for the model. - """ - if not self.is_ep_communicator: - return - - moe_modules = [module for module in model.modules() if is_moe_layer(module)] - for module in moe_modules: - module.maybe_init_modular_kernel() - def dispatch_router_logits( self, hidden_states: torch.Tensor, diff --git a/vllm/distributed/elastic_ep/elastic_execute.py b/vllm/distributed/elastic_ep/elastic_execute.py index 4ccc4d2c01..99325a1706 100644 --- a/vllm/distributed/elastic_ep/elastic_execute.py +++ b/vllm/distributed/elastic_ep/elastic_execute.py @@ -35,7 +35,6 @@ from vllm.distributed.eplb.eplb_communicator import create_eplb_communicator from vllm.distributed.parallel_state import ( _replace_active_groups, get_eplb_group, - prepare_communication_buffer_for_model, ) from vllm.distributed.stateless_coordinator import StatelessGroupCoordinator from vllm.logger import init_logger @@ -485,7 +484,6 @@ class ElasticEPScalingExecutor: for module in moe_modules: if getattr(module._quant_method, "wraps_legacy_quant_method", False): module._replace_quant_method(module._quant_method.old_quant_method) - prepare_communication_buffer_for_model(self.worker.model_runner.model) eplb_model_state.expert_buffer = [ torch.empty_like(w) for w in model.expert_weights[0] @@ -652,10 +650,6 @@ class ElasticEPScalingExecutor: old_num_physical_experts, ) - def prepare_new_worker(self) -> None: - with set_current_vllm_config(self.worker.vllm_config): - prepare_communication_buffer_for_model(self.worker.model_runner.get_model()) - def warmup_local_kernels(self) -> None: with set_current_vllm_config(self.worker.vllm_config): kernel_warmup(self.worker, process_local_only=True) diff --git a/vllm/distributed/elastic_ep/elastic_state.py b/vllm/distributed/elastic_ep/elastic_state.py index f33fd90e86..1b15ba8dc7 100644 --- a/vllm/distributed/elastic_ep/elastic_state.py +++ b/vllm/distributed/elastic_ep/elastic_state.py @@ -214,7 +214,6 @@ class ElasticEPScalingState: self.engine_core.available_gpu_memory_for_kv_cache = ( ParallelConfig.sync_kv_cache_memory_size(self.new_dp_group, -1) ) - self._collective_rpc("elastic_ep_execute", args=("prepare_new_worker",)) self.state = ScaleUpNewEngineState.PREPARE return True diff --git a/vllm/distributed/parallel_state.py b/vllm/distributed/parallel_state.py index dd3b1f4d2f..1b45692453 100644 --- a/vllm/distributed/parallel_state.py +++ b/vllm/distributed/parallel_state.py @@ -1234,10 +1234,6 @@ class GroupCoordinator: if self.mq_broadcaster is not None: self.mq_broadcaster = None - def prepare_communication_buffer_for_model(self, model: torch.nn.Module): - if self.device_communicator is not None: - self.device_communicator.prepare_communication_buffer_for_model(model) - def dispatch_router_logits( self, hidden_states: torch.Tensor, @@ -2041,27 +2037,6 @@ def ensure_model_parallel_initialized( ) -def prepare_communication_buffer_for_model(model: torch.nn.Module): - """Prepare the communication buffer for the model. - Traditional communication libraries like NCCL are almost - model agnostic. However, emerging new communication libraries like - MoE all2all (DeepEP) usually allocate the communication buffer - based on the model shape for optimal performance. - """ - if _TP is not None: - _TP.prepare_communication_buffer_for_model(model) - if _PCP is not None: - _PCP.prepare_communication_buffer_for_model(model) - if _PP is not None: - _PP.prepare_communication_buffer_for_model(model) - if _DP is not None: - _DP.prepare_communication_buffer_for_model(model) - if _EP is not None: - _EP.prepare_communication_buffer_for_model(model) - if _EPLB is not None: - _EPLB.prepare_communication_buffer_for_model(model) - - def checkpoint_prepare_distributed_state() -> None: """Prepare every device communicator for a process checkpoint.""" torch.accelerator.synchronize() diff --git a/vllm/lora/layers/fused_moe.py b/vllm/lora/layers/fused_moe.py index d80848b14b..3027fe6430 100644 --- a/vllm/lora/layers/fused_moe.py +++ b/vllm/lora/layers/fused_moe.py @@ -19,12 +19,8 @@ from vllm.model_executor.layers.fused_moe.fused_moe_modular_method import ( FusedMoEModularMethod, ) from vllm.model_executor.layers.fused_moe.modular_kernel import ( - FusedMoEKernel, FusedMoEKernelModularImpl, ) -from vllm.model_executor.layers.fused_moe.prepare_finalize import ( - MoEPrepareAndFinalizeNoDPEPModular, -) from vllm.platforms import current_platform from .utils import _get_lora_aux_cuda_stream, _get_lora_device @@ -76,13 +72,8 @@ class FusedMoEWithLoRA(BaseLayerWithLoRA): assert isinstance(moe_kernel.impl, FusedMoEKernelModularImpl) moe_kernel.impl.shared_experts = None else: - prepare_finalize = MoEPrepareAndFinalizeNoDPEPModular() - moe_kernel = FusedMoEKernel( - prepare_finalize, - routed_experts.quant_method.select_gemm_impl( - prepare_finalize, routed_experts - ), - ) + raise RuntimeError("Old-style modular kernel not supported.") + assert moe_kernel.supports_lora(), ( f"{type(moe_kernel.fused_experts).__name__} does not support LoRA. " "For unquantized MoE, set moe_backend='triton' or moe_backend='auto' " diff --git a/vllm/model_executor/layers/fused_moe/all2all_utils.py b/vllm/model_executor/layers/fused_moe/all2all_utils.py index 58c9c8d9f6..29905927c1 100644 --- a/vllm/model_executor/layers/fused_moe/all2all_utils.py +++ b/vllm/model_executor/layers/fused_moe/all2all_utils.py @@ -123,20 +123,6 @@ def maybe_make_prepare_finalize( use_monolithic: bool = False, eep_stage: bool = False, ) -> FusedMoEPrepareAndFinalize | None: - # NOTE(rob): we are migrating each quant_method to hold the MK - # in all cases. The allow_new_interface=False flag allow us to fall - # back to the old method for methods that have not yet been migrated. - # - # In old method: - # * maybe_init_modular_kernel() calls this function. If we are - # using no Dp/Ep or naive all2all, we return None this function - # returns None and no ModularKernelMethod is created. If non-naive - # all2all is used, this returns a PrepareAndFinalize object and - # a ModularKernelMethod is created. - # In new method: - # * maybe_make_prepare_finalize() is called from the oracle. We - # always return a PrepareAndFinalize object and the quant method - # holds the ModularKernel. if not moe.moe_parallel_config.use_all2all_kernels: if not allow_new_interface: return None diff --git a/vllm/model_executor/layers/fused_moe/fused_moe_method_base.py b/vllm/model_executor/layers/fused_moe/fused_moe_method_base.py index 888d064d31..4d68a4ab64 100644 --- a/vllm/model_executor/layers/fused_moe/fused_moe_method_base.py +++ b/vllm/model_executor/layers/fused_moe/fused_moe_method_base.py @@ -13,10 +13,6 @@ from vllm.model_executor.layers.fused_moe.config import ( FusedMoEParallelConfig, FusedMoEQuantConfig, ) -from vllm.model_executor.layers.fused_moe.modular_kernel import ( - FusedMoEExpertsModular, - FusedMoEPrepareAndFinalizeModular, -) from vllm.model_executor.layers.quantization.base_config import ( QuantizeMethodBase, ) @@ -103,30 +99,6 @@ class FusedMoEMethodBase(QuantizeMethodBase): hidden_size, act_dtype, moe_parallel_config ), intermediate_size_per_partition - def maybe_make_prepare_finalize( - self, - routing_tables: tuple[torch.Tensor, torch.Tensor, torch.Tensor] | None = None, - ) -> FusedMoEPrepareAndFinalizeModular | None: - from .all2all_utils import maybe_make_prepare_finalize - - pf = maybe_make_prepare_finalize( - self.moe, self.moe_quant_config, routing_tables - ) - assert pf is None or isinstance(pf, FusedMoEPrepareAndFinalizeModular) - return pf - - def select_gemm_impl( - self, - prepare_finalize: FusedMoEPrepareAndFinalizeModular, - layer: "RoutedExperts", - ) -> FusedMoEExpertsModular: - # based on the all2all implementation, select the appropriate - # gemm implementation - raise ValueError( - f"{self.__class__.__name__} uses the new modular kernel initialization " - "logic. This function should not be called." - ) - @abstractmethod def get_fused_moe_quant_config( self, layer: "RoutedExperts" diff --git a/vllm/model_executor/layers/fused_moe/fused_moe_modular_method.py b/vllm/model_executor/layers/fused_moe/fused_moe_modular_method.py index fb8e17932a..5bf694218d 100644 --- a/vllm/model_executor/layers/fused_moe/fused_moe_modular_method.py +++ b/vllm/model_executor/layers/fused_moe/fused_moe_modular_method.py @@ -15,7 +15,6 @@ from vllm.model_executor.layers.fused_moe.fused_moe_method_base import ( ) from vllm.model_executor.layers.fused_moe.modular_kernel import ( FusedMoEKernel, - FusedMoEPrepareAndFinalizeModular, ) from vllm.model_executor.layers.fused_moe.runner.shared_experts import ( SharedExperts, @@ -47,20 +46,6 @@ class FusedMoEModularMethod(FusedMoEMethodBase, CustomOp): def wraps_legacy_quant_method(self) -> bool: return not self.old_quant_method.supports_internal_mk - @staticmethod - def make( - routed_experts: "RoutedExperts", - old_quant_method: FusedMoEMethodBase, - prepare_finalize: FusedMoEPrepareAndFinalizeModular, - ) -> "FusedMoEModularMethod": - return FusedMoEModularMethod( - old_quant_method, - FusedMoEKernel( - prepare_finalize, - old_quant_method.select_gemm_impl(prepare_finalize, routed_experts), - ), - ) - @property def skip_forward_padding(self) -> bool: return self.old_quant_method.skip_forward_padding diff --git a/vllm/model_executor/layers/fused_moe/runner/moe_runner.py b/vllm/model_executor/layers/fused_moe/runner/moe_runner.py index 202ecac9e5..92b559f95f 100644 --- a/vllm/model_executor/layers/fused_moe/runner/moe_runner.py +++ b/vllm/model_executor/layers/fused_moe/runner/moe_runner.py @@ -28,9 +28,6 @@ from vllm.model_executor.layers.fused_moe.config import ( from vllm.model_executor.layers.fused_moe.fused_moe_method_base import ( FusedMoEMethodBase, ) -from vllm.model_executor.layers.fused_moe.fused_moe_modular_method import ( - FusedMoEModularMethod, -) from vllm.model_executor.layers.fused_moe.routed_experts import ( RoutedExperts, ) @@ -896,44 +893,6 @@ class MoERunner(MoERunnerInterface): # ######################################################### - # Note: maybe_init_modular_kernel should only be called by - # prepare_communication_buffer_for_model. - # This is called after all weight loading and post-processing, so it - # should be safe to swap out the quant_method. - def maybe_init_modular_kernel(self) -> None: - # NOTE(rob): WIP refactor. For quant methods that own the MK - # we create the MK during process_weights_after_loading. - if ( - self.routed_experts.quant_method.supports_internal_mk - or self.routed_experts.quant_method.is_monolithic - ): - return None - - self.routed_experts._ensure_moe_quant_config_init() - # routing_tables only needed for round-robin expert placement with - # DeepEP all2all backend. - routing_tables = self._expert_routing_tables() - - if isinstance(self.routed_experts.quant_method, FusedMoEModularMethod): - base_quant_method = self.routed_experts.quant_method.old_quant_method - else: - base_quant_method = self.routed_experts.quant_method - - prepare_finalize = base_quant_method.maybe_make_prepare_finalize( - routing_tables=routing_tables - ) - if prepare_finalize is not None: - logger.debug( - "%s for %s(%s)", prepare_finalize.__class__.__name__, self, id(self) - ) - self._replace_quant_method( - FusedMoEModularMethod.make( - self.routed_experts, - base_quant_method, - prepare_finalize, - ) - ) - # # Properties # diff --git a/vllm/model_executor/layers/fused_moe/runner/moe_runner_interface.py b/vllm/model_executor/layers/fused_moe/runner/moe_runner_interface.py index 5129c9f22d..199a98f4a0 100644 --- a/vllm/model_executor/layers/fused_moe/runner/moe_runner_interface.py +++ b/vllm/model_executor/layers/fused_moe/runner/moe_runner_interface.py @@ -66,10 +66,6 @@ class MoERunnerInterface(PluggableLayer, ABC): # ######################################################################## - @abstractmethod - def maybe_init_modular_kernel(self) -> None: - raise NotImplementedError - @property @abstractmethod def layer_id(self): diff --git a/vllm/model_executor/layers/fused_moe/unquantized_fused_moe_method.py b/vllm/model_executor/layers/fused_moe/unquantized_fused_moe_method.py index 2f17e147a6..9f7e1b1f7f 100644 --- a/vllm/model_executor/layers/fused_moe/unquantized_fused_moe_method.py +++ b/vllm/model_executor/layers/fused_moe/unquantized_fused_moe_method.py @@ -17,10 +17,6 @@ from vllm.model_executor.layers.fused_moe.config import ( from vllm.model_executor.layers.fused_moe.fused_moe_method_base import ( FusedMoEMethodBase, ) -from vllm.model_executor.layers.fused_moe.modular_kernel import ( - FusedMoEExpertsModular, - FusedMoEPrepareAndFinalizeModular, -) from vllm.model_executor.layers.fused_moe.oracle.unquantized import ( UnquantizedMoeBackend, convert_to_unquantized_kernel_format, @@ -56,25 +52,6 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp): def supports_eplb(self) -> bool: return True - def maybe_make_prepare_finalize( - self, - routing_tables: tuple[torch.Tensor, torch.Tensor, torch.Tensor] | None = None, - ): - raise ValueError( - f"{self.__class__.__name__} uses the new modular kernel initialization " - "logic. So this function should not be called." - ) - - def select_gemm_impl( - self, - prepare_finalize: FusedMoEPrepareAndFinalizeModular, - layer: "RoutedExperts", - ) -> FusedMoEExpertsModular: - raise ValueError( - f"{self.__class__.__name__} uses the new modular kernel initialization " - "logic. This function should not be called." - ) - def create_weights( self, layer: "RoutedExperts", diff --git a/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w4a16_flydsl.py b/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w4a16_flydsl.py index f2159b0eb2..076da0a081 100644 --- a/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w4a16_flydsl.py +++ b/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w4a16_flydsl.py @@ -8,7 +8,6 @@ from compressed_tensors.quantization import ( QuantizationArgs, ) -import vllm.model_executor.layers.fused_moe.modular_kernel as mk from vllm.logger import init_logger from vllm.model_executor.layers.fused_moe import ( RoutedExperts, @@ -310,13 +309,6 @@ class CompressedTensorsW4A16FlydslMoEMethod(CompressedTensorsMoEMethod): block_shape=[0, self.group_size], ) - def select_gemm_impl( - self, - prepare_finalize: mk.FusedMoEPrepareAndFinalizeModular, - layer: torch.nn.Module, - ) -> mk.FusedMoEExpertsModular: - raise NotImplementedError - def apply( self, layer: RoutedExperts, diff --git a/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w4a4_nvfp4.py b/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w4a4_nvfp4.py index f74ed2d7b3..c184d20689 100644 --- a/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w4a4_nvfp4.py +++ b/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w4a4_nvfp4.py @@ -4,7 +4,6 @@ import torch -import vllm.model_executor.layers.fused_moe.modular_kernel as mk from vllm.logger import init_logger from vllm.model_executor.layers.fused_moe import ( FusedMoeWeightScaleSupported, @@ -242,15 +241,6 @@ class CompressedTensorsW4A4Nvfp4MoEMethod(CompressedTensorsMoEMethod): ) self.moe_kernel.fused_experts.process_weights_after_loading(layer) - def maybe_make_prepare_finalize( - self, - routing_tables: tuple[torch.Tensor, torch.Tensor, torch.Tensor] | None = None, - ) -> mk.FusedMoEPrepareAndFinalizeModular | None: - raise ValueError( - f"{self.__class__.__name__} uses the new modular kernel initialization " - "logic. This function should not be called." - ) - def get_fused_moe_quant_config(self, layer: torch.nn.Module) -> FusedMoEQuantConfig: return make_nvfp4_moe_quant_config( backend=self.nvfp4_backend, diff --git a/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w8a8_fp8.py b/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w8a8_fp8.py index 35d2a37842..d4f71bd76f 100644 --- a/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w8a8_fp8.py +++ b/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w8a8_fp8.py @@ -8,7 +8,6 @@ from compressed_tensors.quantization import ( QuantizationStrategy, ) -import vllm.model_executor.layers.fused_moe.modular_kernel as mk from vllm.distributed import get_tensor_model_parallel_world_size from vllm.logger import init_logger from vllm.model_executor.layers.fused_moe import ( @@ -341,15 +340,6 @@ class CompressedTensorsW8A8Fp8MoEMethod(CompressedTensorsMoEMethod): layer=layer, ) - def maybe_make_prepare_finalize( - self, - routing_tables: tuple[torch.Tensor, torch.Tensor, torch.Tensor] | None = None, - ) -> mk.FusedMoEPrepareAndFinalizeModular | None: - raise ValueError( - f"{self.__class__.__name__} uses the new modular kernel initialization " - "logic. This function should not be called." - ) - def get_fused_moe_quant_config(self, layer: torch.nn.Module) -> FusedMoEQuantConfig: is_per_token = self.input_quant.strategy == QuantizationStrategy.TOKEN return make_fp8_moe_quant_config( diff --git a/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w8a8_int8.py b/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w8a8_int8.py index 28100b012e..fd265fa858 100644 --- a/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w8a8_int8.py +++ b/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w8a8_int8.py @@ -7,7 +7,6 @@ from compressed_tensors.quantization import ( QuantizationStrategy, ) -import vllm.model_executor.layers.fused_moe.modular_kernel as mk from vllm.logger import init_logger from vllm.model_executor.layers.fused_moe import ( FusedMoeWeightScaleSupported, @@ -164,15 +163,6 @@ class CompressedTensorsW8A8Int8MoEMethod(CompressedTensorsMoEMethod): ) self.moe_kernel.fused_experts.process_weights_after_loading(layer) - def maybe_make_prepare_finalize( - self, - routing_tables: tuple[torch.Tensor, torch.Tensor, torch.Tensor] | None = None, - ) -> mk.FusedMoEPrepareAndFinalizeModular | None: - raise ValueError( - f"{self.__class__.__name__} uses the new modular kernel initialization " - "logic. This function should not be called." - ) - def get_fused_moe_quant_config(self, layer: torch.nn.Module) -> FusedMoEQuantConfig: return make_int8_moe_quant_config( int8_backend=self.int8_backend, diff --git a/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w8a8_mxfp8.py b/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w8a8_mxfp8.py index 468aed29a9..ee4a42786b 100644 --- a/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w8a8_mxfp8.py +++ b/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w8a8_mxfp8.py @@ -3,7 +3,6 @@ import torch -import vllm.model_executor.layers.fused_moe.modular_kernel as mk from vllm.model_executor.layers.fused_moe import ( FusedMoeWeightScaleSupported, RoutedExperts, @@ -159,15 +158,6 @@ class CompressedTensorsW8A8Mxfp8MoEMethod(CompressedTensorsMoEMethod): layer=layer, ) - def maybe_make_prepare_finalize( - self, - routing_tables: tuple[torch.Tensor, torch.Tensor, torch.Tensor] | None = None, - ) -> mk.FusedMoEPrepareAndFinalizeModular | None: - raise ValueError( - f"{self.__class__.__name__} uses the new modular kernel " - "initialization logic. This function should not be called." - ) - def apply_monolithic( self, layer: RoutedExperts, diff --git a/vllm/model_executor/layers/quantization/fp8.py b/vllm/model_executor/layers/quantization/fp8.py index 251e4b9999..cff4c547df 100644 --- a/vllm/model_executor/layers/quantization/fp8.py +++ b/vllm/model_executor/layers/quantization/fp8.py @@ -6,7 +6,6 @@ from typing import TYPE_CHECKING, Any import torch import vllm.envs as envs -import vllm.model_executor.layers.fused_moe.modular_kernel as mk from vllm.config import get_current_vllm_config from vllm.distributed import get_tensor_model_parallel_world_size from vllm.logger import init_logger @@ -731,15 +730,6 @@ class Fp8MoEMethod(FusedMoEMethodBase): layer, w13, w2, w13_scale, w2_scale, w13_input_scale, w2_input_scale ) - def maybe_make_prepare_finalize( - self, - routing_tables: tuple[torch.Tensor, torch.Tensor, torch.Tensor] | None = None, - ) -> mk.FusedMoEPrepareAndFinalizeModular | None: - raise ValueError( - f"{self.__class__.__name__} uses the new modular kernel initialization " - "logic. This function should not be called." - ) - def get_fused_moe_quant_config(self, layer: RoutedExperts) -> FusedMoEQuantConfig: w1_scale = getattr(layer, f"w13_{self.weight_scale_name}") w2_scale = getattr(layer, f"w2_{self.weight_scale_name}") diff --git a/vllm/model_executor/layers/quantization/modelopt.py b/vllm/model_executor/layers/quantization/modelopt.py index 4c33c19070..05f486493d 100644 --- a/vllm/model_executor/layers/quantization/modelopt.py +++ b/vllm/model_executor/layers/quantization/modelopt.py @@ -8,7 +8,6 @@ import torch from torch.nn.parameter import Parameter import vllm.envs as envs -import vllm.model_executor.layers.fused_moe.modular_kernel as mk from vllm.config import get_current_vllm_config from vllm.logger import init_logger from vllm.model_executor.kernels.linear import ( @@ -771,15 +770,6 @@ class ModelOptFp8MoEMethod(FusedMoEMethodBase): activation_key=kFp8StaticTensorSym, ) - def maybe_make_prepare_finalize( - self, - routing_tables: tuple[torch.Tensor, torch.Tensor, torch.Tensor] | None = None, - ) -> mk.FusedMoEPrepareAndFinalizeModular | None: - raise ValueError( - f"{self.__class__.__name__} uses the new modular kernel initialization " - "logic. This function should not be called." - ) - def create_weights( self, layer: RoutedExperts, @@ -1409,15 +1399,6 @@ class ModelOptNvFp4FusedMoE(FusedMoEMethodBase): self.nvfp4_backend ) - def maybe_make_prepare_finalize( - self, - routing_tables: tuple[torch.Tensor, torch.Tensor, torch.Tensor] | None = None, - ) -> mk.FusedMoEPrepareAndFinalizeModular | None: - raise ValueError( - f"{self.__class__.__name__} uses the new modular kernel initialization " - "logic. This function should not be called." - ) - def uses_weight_scale_2_pattern(self) -> bool: """ FP4 variants use 'weight_scale_2' pattern for per-tensor weight scales. @@ -2087,15 +2068,6 @@ class ModelOptMxFp8FusedMoE(FusedMoEMethodBase): ): self._dequant_mxfp8_weights_to_bf16(layer) - def maybe_make_prepare_finalize( - self, - routing_tables: tuple[torch.Tensor, torch.Tensor, torch.Tensor] | None = None, - ) -> mk.FusedMoEPrepareAndFinalizeModular | None: - raise ValueError( - f"{self.__class__.__name__} uses the new modular kernel initialization " - "logic. This function should not be called." - ) - def get_fused_moe_quant_config( self, layer: RoutedExperts ) -> FusedMoEQuantConfig | None: diff --git a/vllm/model_executor/layers/quantization/online/moe_base.py b/vllm/model_executor/layers/quantization/online/moe_base.py index fe6302fd11..27d3ccc307 100644 --- a/vllm/model_executor/layers/quantization/online/moe_base.py +++ b/vllm/model_executor/layers/quantization/online/moe_base.py @@ -5,7 +5,6 @@ from abc import abstractmethod import torch -import vllm.model_executor.layers.fused_moe.modular_kernel as mk from vllm.model_executor.layers.fused_moe import ( FusedMoEMethodBase, RoutedExperts, @@ -100,15 +99,6 @@ class OnlineMoEMethodBase(FusedMoEMethodBase): def process_weights_after_loading(self, layer: torch.nn.Module) -> None: pass - def maybe_make_prepare_finalize( - self, - routing_tables: tuple[torch.Tensor, torch.Tensor, torch.Tensor] | None = None, - ) -> mk.FusedMoEPrepareAndFinalizeModular | None: - raise ValueError( - f"{self.__class__.__name__} uses the new modular kernel " - "initialization logic. This function should not be called." - ) - @property def supports_eplb(self) -> bool: return True diff --git a/vllm/v1/worker/gpu/model_runner.py b/vllm/v1/worker/gpu/model_runner.py index 0a1b726c0a..ea476d0c13 100644 --- a/vllm/v1/worker/gpu/model_runner.py +++ b/vllm/v1/worker/gpu/model_runner.py @@ -34,7 +34,6 @@ from vllm.config.compilation import CUDAGraphMode from vllm.distributed.parallel_state import ( get_dcp_group, get_pp_group, - prepare_communication_buffer_for_model, ) from vllm.forward_context import BatchDescriptor, set_forward_context from vllm.logger import init_logger @@ -354,11 +353,6 @@ class GPUModelRunner(LoRAModelRunnerMixin): time_after_load - time_before_load, ) - if not load_dummy_weights: - prepare_communication_buffer_for_model(self.model) - if self.speculator is not None: - prepare_communication_buffer_for_model(self.speculator.model) - # Initialize the components that require the model. self.model_state = init_model_state( self.vllm_config, self.model, self.encoder_cache, self.device diff --git a/vllm/v1/worker/gpu_model_runner.py b/vllm/v1/worker/gpu_model_runner.py index 8585133bf0..64fefd7a6a 100644 --- a/vllm/v1/worker/gpu_model_runner.py +++ b/vllm/v1/worker/gpu_model_runner.py @@ -50,7 +50,6 @@ from vllm.distributed.parallel_state import ( get_tp_group, graph_capture, is_global_first_rank, - prepare_communication_buffer_for_model, ) from vllm.forward_context import ( BatchDescriptor, @@ -5387,12 +5386,7 @@ class GPUModelRunner( format_gib(self.model_memory_usage), time_after_load - time_before_load, ) - if not load_dummy_weights: - prepare_communication_buffer_for_model(self.model) - if (drafter := getattr(self, "drafter", None)) and ( - drafter_model := getattr(drafter, "model", None) - ): - prepare_communication_buffer_for_model(drafter_model) + mm_config = self.model_config.multimodal_config self.is_multimodal_pruning_enabled = ( supports_multimodal_pruning(self.get_model())