[MoE Refactor] Remove MoE legacy code (#51078)
Signed-off-by: Bill Nell <bnell@redhat.com>
This commit is contained in:
@@ -93,7 +93,6 @@ def _make_runner(**overrides: Any) -> Any:
|
||||
def test_v2_load_model_registers_moe_with_eplb(monkeypatch):
|
||||
FakeEplbState.instances.clear()
|
||||
model = SimpleNamespace(is_moe=True)
|
||||
prepared: list[object] = []
|
||||
|
||||
monkeypatch.setattr(mrv2, "DeviceMemoryProfiler", FakeMemoryProfiler)
|
||||
monkeypatch.setattr(eplb, "EplbState", FakeEplbState)
|
||||
@@ -102,7 +101,6 @@ def test_v2_load_model_registers_moe_with_eplb(monkeypatch):
|
||||
"get_model_loader",
|
||||
lambda load_config: SimpleNamespace(load_model=lambda **_: model),
|
||||
)
|
||||
monkeypatch.setattr(mrv2, "prepare_communication_buffer_for_model", prepared.append)
|
||||
monkeypatch.setattr(
|
||||
mrv2,
|
||||
"init_model_state",
|
||||
@@ -119,7 +117,6 @@ def test_v2_load_model_registers_moe_with_eplb(monkeypatch):
|
||||
|
||||
assert runner.model is model
|
||||
assert runner.model_state is not None
|
||||
assert prepared == [model]
|
||||
assert runner.eplb_state is not None
|
||||
assert runner.eplb_state.add_model_calls == [(model, runner.model_config)]
|
||||
assert runner.eplb_state.async_started is True
|
||||
@@ -128,7 +125,6 @@ def test_v2_load_model_registers_moe_with_eplb(monkeypatch):
|
||||
def test_v2_load_model_with_dummy_weights_skips_eplb_registration(monkeypatch):
|
||||
FakeEplbState.instances.clear()
|
||||
model = SimpleNamespace(is_moe=True)
|
||||
prepared: list[object] = []
|
||||
|
||||
monkeypatch.setattr(mrv2, "DeviceMemoryProfiler", FakeMemoryProfiler)
|
||||
monkeypatch.setattr(eplb, "EplbState", FakeEplbState)
|
||||
@@ -137,7 +133,6 @@ def test_v2_load_model_with_dummy_weights_skips_eplb_registration(monkeypatch):
|
||||
"get_model_loader",
|
||||
lambda load_config: SimpleNamespace(load_model=lambda **_: model),
|
||||
)
|
||||
monkeypatch.setattr(mrv2, "prepare_communication_buffer_for_model", prepared.append)
|
||||
monkeypatch.setattr(
|
||||
mrv2,
|
||||
"init_model_state",
|
||||
@@ -149,7 +144,6 @@ def test_v2_load_model_with_dummy_weights_skips_eplb_registration(monkeypatch):
|
||||
mrv2.GPUModelRunner.load_model(runner, load_dummy_weights=True)
|
||||
|
||||
assert runner.load_config.load_format == "dummy"
|
||||
assert prepared == []
|
||||
assert runner.eplb_state is not None
|
||||
assert runner.eplb_state.add_model_calls == []
|
||||
assert runner.eplb_state.async_started is False
|
||||
|
||||
@@ -8,7 +8,6 @@ import torch.distributed as dist
|
||||
from torch.distributed import ProcessGroup
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils import is_moe_layer
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -358,17 +357,6 @@ class DeviceCommunicatorBase:
|
||||
def destroy(self):
|
||||
pass
|
||||
|
||||
def prepare_communication_buffer_for_model(self, model: torch.nn.Module) -> None:
|
||||
"""
|
||||
Prepare the communication buffer for the model.
|
||||
"""
|
||||
if not self.is_ep_communicator:
|
||||
return
|
||||
|
||||
moe_modules = [module for module in model.modules() if is_moe_layer(module)]
|
||||
for module in moe_modules:
|
||||
module.maybe_init_modular_kernel()
|
||||
|
||||
def dispatch_router_logits(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
|
||||
@@ -35,7 +35,6 @@ from vllm.distributed.eplb.eplb_communicator import create_eplb_communicator
|
||||
from vllm.distributed.parallel_state import (
|
||||
_replace_active_groups,
|
||||
get_eplb_group,
|
||||
prepare_communication_buffer_for_model,
|
||||
)
|
||||
from vllm.distributed.stateless_coordinator import StatelessGroupCoordinator
|
||||
from vllm.logger import init_logger
|
||||
@@ -485,7 +484,6 @@ class ElasticEPScalingExecutor:
|
||||
for module in moe_modules:
|
||||
if getattr(module._quant_method, "wraps_legacy_quant_method", False):
|
||||
module._replace_quant_method(module._quant_method.old_quant_method)
|
||||
prepare_communication_buffer_for_model(self.worker.model_runner.model)
|
||||
|
||||
eplb_model_state.expert_buffer = [
|
||||
torch.empty_like(w) for w in model.expert_weights[0]
|
||||
@@ -652,10 +650,6 @@ class ElasticEPScalingExecutor:
|
||||
old_num_physical_experts,
|
||||
)
|
||||
|
||||
def prepare_new_worker(self) -> None:
|
||||
with set_current_vllm_config(self.worker.vllm_config):
|
||||
prepare_communication_buffer_for_model(self.worker.model_runner.get_model())
|
||||
|
||||
def warmup_local_kernels(self) -> None:
|
||||
with set_current_vllm_config(self.worker.vllm_config):
|
||||
kernel_warmup(self.worker, process_local_only=True)
|
||||
|
||||
@@ -214,7 +214,6 @@ class ElasticEPScalingState:
|
||||
self.engine_core.available_gpu_memory_for_kv_cache = (
|
||||
ParallelConfig.sync_kv_cache_memory_size(self.new_dp_group, -1)
|
||||
)
|
||||
self._collective_rpc("elastic_ep_execute", args=("prepare_new_worker",))
|
||||
self.state = ScaleUpNewEngineState.PREPARE
|
||||
return True
|
||||
|
||||
|
||||
@@ -1234,10 +1234,6 @@ class GroupCoordinator:
|
||||
if self.mq_broadcaster is not None:
|
||||
self.mq_broadcaster = None
|
||||
|
||||
def prepare_communication_buffer_for_model(self, model: torch.nn.Module):
|
||||
if self.device_communicator is not None:
|
||||
self.device_communicator.prepare_communication_buffer_for_model(model)
|
||||
|
||||
def dispatch_router_logits(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
@@ -2041,27 +2037,6 @@ def ensure_model_parallel_initialized(
|
||||
)
|
||||
|
||||
|
||||
def prepare_communication_buffer_for_model(model: torch.nn.Module):
|
||||
"""Prepare the communication buffer for the model.
|
||||
Traditional communication libraries like NCCL are almost
|
||||
model agnostic. However, emerging new communication libraries like
|
||||
MoE all2all (DeepEP) usually allocate the communication buffer
|
||||
based on the model shape for optimal performance.
|
||||
"""
|
||||
if _TP is not None:
|
||||
_TP.prepare_communication_buffer_for_model(model)
|
||||
if _PCP is not None:
|
||||
_PCP.prepare_communication_buffer_for_model(model)
|
||||
if _PP is not None:
|
||||
_PP.prepare_communication_buffer_for_model(model)
|
||||
if _DP is not None:
|
||||
_DP.prepare_communication_buffer_for_model(model)
|
||||
if _EP is not None:
|
||||
_EP.prepare_communication_buffer_for_model(model)
|
||||
if _EPLB is not None:
|
||||
_EPLB.prepare_communication_buffer_for_model(model)
|
||||
|
||||
|
||||
def checkpoint_prepare_distributed_state() -> None:
|
||||
"""Prepare every device communicator for a process checkpoint."""
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
@@ -19,12 +19,8 @@ from vllm.model_executor.layers.fused_moe.fused_moe_modular_method import (
|
||||
FusedMoEModularMethod,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.modular_kernel import (
|
||||
FusedMoEKernel,
|
||||
FusedMoEKernelModularImpl,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.prepare_finalize import (
|
||||
MoEPrepareAndFinalizeNoDPEPModular,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
from .utils import _get_lora_aux_cuda_stream, _get_lora_device
|
||||
@@ -76,13 +72,8 @@ class FusedMoEWithLoRA(BaseLayerWithLoRA):
|
||||
assert isinstance(moe_kernel.impl, FusedMoEKernelModularImpl)
|
||||
moe_kernel.impl.shared_experts = None
|
||||
else:
|
||||
prepare_finalize = MoEPrepareAndFinalizeNoDPEPModular()
|
||||
moe_kernel = FusedMoEKernel(
|
||||
prepare_finalize,
|
||||
routed_experts.quant_method.select_gemm_impl(
|
||||
prepare_finalize, routed_experts
|
||||
),
|
||||
)
|
||||
raise RuntimeError("Old-style modular kernel not supported.")
|
||||
|
||||
assert moe_kernel.supports_lora(), (
|
||||
f"{type(moe_kernel.fused_experts).__name__} does not support LoRA. "
|
||||
"For unquantized MoE, set moe_backend='triton' or moe_backend='auto' "
|
||||
|
||||
@@ -123,20 +123,6 @@ def maybe_make_prepare_finalize(
|
||||
use_monolithic: bool = False,
|
||||
eep_stage: bool = False,
|
||||
) -> FusedMoEPrepareAndFinalize | None:
|
||||
# NOTE(rob): we are migrating each quant_method to hold the MK
|
||||
# in all cases. The allow_new_interface=False flag allow us to fall
|
||||
# back to the old method for methods that have not yet been migrated.
|
||||
#
|
||||
# In old method:
|
||||
# * maybe_init_modular_kernel() calls this function. If we are
|
||||
# using no Dp/Ep or naive all2all, we return None this function
|
||||
# returns None and no ModularKernelMethod is created. If non-naive
|
||||
# all2all is used, this returns a PrepareAndFinalize object and
|
||||
# a ModularKernelMethod is created.
|
||||
# In new method:
|
||||
# * maybe_make_prepare_finalize() is called from the oracle. We
|
||||
# always return a PrepareAndFinalize object and the quant method
|
||||
# holds the ModularKernel.
|
||||
if not moe.moe_parallel_config.use_all2all_kernels:
|
||||
if not allow_new_interface:
|
||||
return None
|
||||
|
||||
@@ -13,10 +13,6 @@ from vllm.model_executor.layers.fused_moe.config import (
|
||||
FusedMoEParallelConfig,
|
||||
FusedMoEQuantConfig,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.modular_kernel import (
|
||||
FusedMoEExpertsModular,
|
||||
FusedMoEPrepareAndFinalizeModular,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.base_config import (
|
||||
QuantizeMethodBase,
|
||||
)
|
||||
@@ -103,30 +99,6 @@ class FusedMoEMethodBase(QuantizeMethodBase):
|
||||
hidden_size, act_dtype, moe_parallel_config
|
||||
), intermediate_size_per_partition
|
||||
|
||||
def maybe_make_prepare_finalize(
|
||||
self,
|
||||
routing_tables: tuple[torch.Tensor, torch.Tensor, torch.Tensor] | None = None,
|
||||
) -> FusedMoEPrepareAndFinalizeModular | None:
|
||||
from .all2all_utils import maybe_make_prepare_finalize
|
||||
|
||||
pf = maybe_make_prepare_finalize(
|
||||
self.moe, self.moe_quant_config, routing_tables
|
||||
)
|
||||
assert pf is None or isinstance(pf, FusedMoEPrepareAndFinalizeModular)
|
||||
return pf
|
||||
|
||||
def select_gemm_impl(
|
||||
self,
|
||||
prepare_finalize: FusedMoEPrepareAndFinalizeModular,
|
||||
layer: "RoutedExperts",
|
||||
) -> FusedMoEExpertsModular:
|
||||
# based on the all2all implementation, select the appropriate
|
||||
# gemm implementation
|
||||
raise ValueError(
|
||||
f"{self.__class__.__name__} uses the new modular kernel initialization "
|
||||
"logic. This function should not be called."
|
||||
)
|
||||
|
||||
@abstractmethod
|
||||
def get_fused_moe_quant_config(
|
||||
self, layer: "RoutedExperts"
|
||||
|
||||
@@ -15,7 +15,6 @@ from vllm.model_executor.layers.fused_moe.fused_moe_method_base import (
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.modular_kernel import (
|
||||
FusedMoEKernel,
|
||||
FusedMoEPrepareAndFinalizeModular,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.runner.shared_experts import (
|
||||
SharedExperts,
|
||||
@@ -47,20 +46,6 @@ class FusedMoEModularMethod(FusedMoEMethodBase, CustomOp):
|
||||
def wraps_legacy_quant_method(self) -> bool:
|
||||
return not self.old_quant_method.supports_internal_mk
|
||||
|
||||
@staticmethod
|
||||
def make(
|
||||
routed_experts: "RoutedExperts",
|
||||
old_quant_method: FusedMoEMethodBase,
|
||||
prepare_finalize: FusedMoEPrepareAndFinalizeModular,
|
||||
) -> "FusedMoEModularMethod":
|
||||
return FusedMoEModularMethod(
|
||||
old_quant_method,
|
||||
FusedMoEKernel(
|
||||
prepare_finalize,
|
||||
old_quant_method.select_gemm_impl(prepare_finalize, routed_experts),
|
||||
),
|
||||
)
|
||||
|
||||
@property
|
||||
def skip_forward_padding(self) -> bool:
|
||||
return self.old_quant_method.skip_forward_padding
|
||||
|
||||
@@ -28,9 +28,6 @@ from vllm.model_executor.layers.fused_moe.config import (
|
||||
from vllm.model_executor.layers.fused_moe.fused_moe_method_base import (
|
||||
FusedMoEMethodBase,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.fused_moe_modular_method import (
|
||||
FusedMoEModularMethod,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.routed_experts import (
|
||||
RoutedExperts,
|
||||
)
|
||||
@@ -896,44 +893,6 @@ class MoERunner(MoERunnerInterface):
|
||||
#
|
||||
#########################################################
|
||||
|
||||
# Note: maybe_init_modular_kernel should only be called by
|
||||
# prepare_communication_buffer_for_model.
|
||||
# This is called after all weight loading and post-processing, so it
|
||||
# should be safe to swap out the quant_method.
|
||||
def maybe_init_modular_kernel(self) -> None:
|
||||
# NOTE(rob): WIP refactor. For quant methods that own the MK
|
||||
# we create the MK during process_weights_after_loading.
|
||||
if (
|
||||
self.routed_experts.quant_method.supports_internal_mk
|
||||
or self.routed_experts.quant_method.is_monolithic
|
||||
):
|
||||
return None
|
||||
|
||||
self.routed_experts._ensure_moe_quant_config_init()
|
||||
# routing_tables only needed for round-robin expert placement with
|
||||
# DeepEP all2all backend.
|
||||
routing_tables = self._expert_routing_tables()
|
||||
|
||||
if isinstance(self.routed_experts.quant_method, FusedMoEModularMethod):
|
||||
base_quant_method = self.routed_experts.quant_method.old_quant_method
|
||||
else:
|
||||
base_quant_method = self.routed_experts.quant_method
|
||||
|
||||
prepare_finalize = base_quant_method.maybe_make_prepare_finalize(
|
||||
routing_tables=routing_tables
|
||||
)
|
||||
if prepare_finalize is not None:
|
||||
logger.debug(
|
||||
"%s for %s(%s)", prepare_finalize.__class__.__name__, self, id(self)
|
||||
)
|
||||
self._replace_quant_method(
|
||||
FusedMoEModularMethod.make(
|
||||
self.routed_experts,
|
||||
base_quant_method,
|
||||
prepare_finalize,
|
||||
)
|
||||
)
|
||||
|
||||
#
|
||||
# Properties
|
||||
#
|
||||
|
||||
@@ -66,10 +66,6 @@ class MoERunnerInterface(PluggableLayer, ABC):
|
||||
#
|
||||
########################################################################
|
||||
|
||||
@abstractmethod
|
||||
def maybe_init_modular_kernel(self) -> None:
|
||||
raise NotImplementedError
|
||||
|
||||
@property
|
||||
@abstractmethod
|
||||
def layer_id(self):
|
||||
|
||||
@@ -17,10 +17,6 @@ from vllm.model_executor.layers.fused_moe.config import (
|
||||
from vllm.model_executor.layers.fused_moe.fused_moe_method_base import (
|
||||
FusedMoEMethodBase,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.modular_kernel import (
|
||||
FusedMoEExpertsModular,
|
||||
FusedMoEPrepareAndFinalizeModular,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.oracle.unquantized import (
|
||||
UnquantizedMoeBackend,
|
||||
convert_to_unquantized_kernel_format,
|
||||
@@ -56,25 +52,6 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp):
|
||||
def supports_eplb(self) -> bool:
|
||||
return True
|
||||
|
||||
def maybe_make_prepare_finalize(
|
||||
self,
|
||||
routing_tables: tuple[torch.Tensor, torch.Tensor, torch.Tensor] | None = None,
|
||||
):
|
||||
raise ValueError(
|
||||
f"{self.__class__.__name__} uses the new modular kernel initialization "
|
||||
"logic. So this function should not be called."
|
||||
)
|
||||
|
||||
def select_gemm_impl(
|
||||
self,
|
||||
prepare_finalize: FusedMoEPrepareAndFinalizeModular,
|
||||
layer: "RoutedExperts",
|
||||
) -> FusedMoEExpertsModular:
|
||||
raise ValueError(
|
||||
f"{self.__class__.__name__} uses the new modular kernel initialization "
|
||||
"logic. This function should not be called."
|
||||
)
|
||||
|
||||
def create_weights(
|
||||
self,
|
||||
layer: "RoutedExperts",
|
||||
|
||||
-8
@@ -8,7 +8,6 @@ from compressed_tensors.quantization import (
|
||||
QuantizationArgs,
|
||||
)
|
||||
|
||||
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
RoutedExperts,
|
||||
@@ -310,13 +309,6 @@ class CompressedTensorsW4A16FlydslMoEMethod(CompressedTensorsMoEMethod):
|
||||
block_shape=[0, self.group_size],
|
||||
)
|
||||
|
||||
def select_gemm_impl(
|
||||
self,
|
||||
prepare_finalize: mk.FusedMoEPrepareAndFinalizeModular,
|
||||
layer: torch.nn.Module,
|
||||
) -> mk.FusedMoEExpertsModular:
|
||||
raise NotImplementedError
|
||||
|
||||
def apply(
|
||||
self,
|
||||
layer: RoutedExperts,
|
||||
|
||||
-10
@@ -4,7 +4,6 @@
|
||||
|
||||
import torch
|
||||
|
||||
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoeWeightScaleSupported,
|
||||
@@ -242,15 +241,6 @@ class CompressedTensorsW4A4Nvfp4MoEMethod(CompressedTensorsMoEMethod):
|
||||
)
|
||||
self.moe_kernel.fused_experts.process_weights_after_loading(layer)
|
||||
|
||||
def maybe_make_prepare_finalize(
|
||||
self,
|
||||
routing_tables: tuple[torch.Tensor, torch.Tensor, torch.Tensor] | None = None,
|
||||
) -> mk.FusedMoEPrepareAndFinalizeModular | None:
|
||||
raise ValueError(
|
||||
f"{self.__class__.__name__} uses the new modular kernel initialization "
|
||||
"logic. This function should not be called."
|
||||
)
|
||||
|
||||
def get_fused_moe_quant_config(self, layer: torch.nn.Module) -> FusedMoEQuantConfig:
|
||||
return make_nvfp4_moe_quant_config(
|
||||
backend=self.nvfp4_backend,
|
||||
|
||||
-10
@@ -8,7 +8,6 @@ from compressed_tensors.quantization import (
|
||||
QuantizationStrategy,
|
||||
)
|
||||
|
||||
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
|
||||
from vllm.distributed import get_tensor_model_parallel_world_size
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
@@ -341,15 +340,6 @@ class CompressedTensorsW8A8Fp8MoEMethod(CompressedTensorsMoEMethod):
|
||||
layer=layer,
|
||||
)
|
||||
|
||||
def maybe_make_prepare_finalize(
|
||||
self,
|
||||
routing_tables: tuple[torch.Tensor, torch.Tensor, torch.Tensor] | None = None,
|
||||
) -> mk.FusedMoEPrepareAndFinalizeModular | None:
|
||||
raise ValueError(
|
||||
f"{self.__class__.__name__} uses the new modular kernel initialization "
|
||||
"logic. This function should not be called."
|
||||
)
|
||||
|
||||
def get_fused_moe_quant_config(self, layer: torch.nn.Module) -> FusedMoEQuantConfig:
|
||||
is_per_token = self.input_quant.strategy == QuantizationStrategy.TOKEN
|
||||
return make_fp8_moe_quant_config(
|
||||
|
||||
-10
@@ -7,7 +7,6 @@ from compressed_tensors.quantization import (
|
||||
QuantizationStrategy,
|
||||
)
|
||||
|
||||
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoeWeightScaleSupported,
|
||||
@@ -164,15 +163,6 @@ class CompressedTensorsW8A8Int8MoEMethod(CompressedTensorsMoEMethod):
|
||||
)
|
||||
self.moe_kernel.fused_experts.process_weights_after_loading(layer)
|
||||
|
||||
def maybe_make_prepare_finalize(
|
||||
self,
|
||||
routing_tables: tuple[torch.Tensor, torch.Tensor, torch.Tensor] | None = None,
|
||||
) -> mk.FusedMoEPrepareAndFinalizeModular | None:
|
||||
raise ValueError(
|
||||
f"{self.__class__.__name__} uses the new modular kernel initialization "
|
||||
"logic. This function should not be called."
|
||||
)
|
||||
|
||||
def get_fused_moe_quant_config(self, layer: torch.nn.Module) -> FusedMoEQuantConfig:
|
||||
return make_int8_moe_quant_config(
|
||||
int8_backend=self.int8_backend,
|
||||
|
||||
-10
@@ -3,7 +3,6 @@
|
||||
|
||||
import torch
|
||||
|
||||
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoeWeightScaleSupported,
|
||||
RoutedExperts,
|
||||
@@ -159,15 +158,6 @@ class CompressedTensorsW8A8Mxfp8MoEMethod(CompressedTensorsMoEMethod):
|
||||
layer=layer,
|
||||
)
|
||||
|
||||
def maybe_make_prepare_finalize(
|
||||
self,
|
||||
routing_tables: tuple[torch.Tensor, torch.Tensor, torch.Tensor] | None = None,
|
||||
) -> mk.FusedMoEPrepareAndFinalizeModular | None:
|
||||
raise ValueError(
|
||||
f"{self.__class__.__name__} uses the new modular kernel "
|
||||
"initialization logic. This function should not be called."
|
||||
)
|
||||
|
||||
def apply_monolithic(
|
||||
self,
|
||||
layer: RoutedExperts,
|
||||
|
||||
@@ -6,7 +6,6 @@ from typing import TYPE_CHECKING, Any
|
||||
import torch
|
||||
|
||||
import vllm.envs as envs
|
||||
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
|
||||
from vllm.config import get_current_vllm_config
|
||||
from vllm.distributed import get_tensor_model_parallel_world_size
|
||||
from vllm.logger import init_logger
|
||||
@@ -731,15 +730,6 @@ class Fp8MoEMethod(FusedMoEMethodBase):
|
||||
layer, w13, w2, w13_scale, w2_scale, w13_input_scale, w2_input_scale
|
||||
)
|
||||
|
||||
def maybe_make_prepare_finalize(
|
||||
self,
|
||||
routing_tables: tuple[torch.Tensor, torch.Tensor, torch.Tensor] | None = None,
|
||||
) -> mk.FusedMoEPrepareAndFinalizeModular | None:
|
||||
raise ValueError(
|
||||
f"{self.__class__.__name__} uses the new modular kernel initialization "
|
||||
"logic. This function should not be called."
|
||||
)
|
||||
|
||||
def get_fused_moe_quant_config(self, layer: RoutedExperts) -> FusedMoEQuantConfig:
|
||||
w1_scale = getattr(layer, f"w13_{self.weight_scale_name}")
|
||||
w2_scale = getattr(layer, f"w2_{self.weight_scale_name}")
|
||||
|
||||
@@ -8,7 +8,6 @@ import torch
|
||||
from torch.nn.parameter import Parameter
|
||||
|
||||
import vllm.envs as envs
|
||||
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
|
||||
from vllm.config import get_current_vllm_config
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.kernels.linear import (
|
||||
@@ -771,15 +770,6 @@ class ModelOptFp8MoEMethod(FusedMoEMethodBase):
|
||||
activation_key=kFp8StaticTensorSym,
|
||||
)
|
||||
|
||||
def maybe_make_prepare_finalize(
|
||||
self,
|
||||
routing_tables: tuple[torch.Tensor, torch.Tensor, torch.Tensor] | None = None,
|
||||
) -> mk.FusedMoEPrepareAndFinalizeModular | None:
|
||||
raise ValueError(
|
||||
f"{self.__class__.__name__} uses the new modular kernel initialization "
|
||||
"logic. This function should not be called."
|
||||
)
|
||||
|
||||
def create_weights(
|
||||
self,
|
||||
layer: RoutedExperts,
|
||||
@@ -1409,15 +1399,6 @@ class ModelOptNvFp4FusedMoE(FusedMoEMethodBase):
|
||||
self.nvfp4_backend
|
||||
)
|
||||
|
||||
def maybe_make_prepare_finalize(
|
||||
self,
|
||||
routing_tables: tuple[torch.Tensor, torch.Tensor, torch.Tensor] | None = None,
|
||||
) -> mk.FusedMoEPrepareAndFinalizeModular | None:
|
||||
raise ValueError(
|
||||
f"{self.__class__.__name__} uses the new modular kernel initialization "
|
||||
"logic. This function should not be called."
|
||||
)
|
||||
|
||||
def uses_weight_scale_2_pattern(self) -> bool:
|
||||
"""
|
||||
FP4 variants use 'weight_scale_2' pattern for per-tensor weight scales.
|
||||
@@ -2087,15 +2068,6 @@ class ModelOptMxFp8FusedMoE(FusedMoEMethodBase):
|
||||
):
|
||||
self._dequant_mxfp8_weights_to_bf16(layer)
|
||||
|
||||
def maybe_make_prepare_finalize(
|
||||
self,
|
||||
routing_tables: tuple[torch.Tensor, torch.Tensor, torch.Tensor] | None = None,
|
||||
) -> mk.FusedMoEPrepareAndFinalizeModular | None:
|
||||
raise ValueError(
|
||||
f"{self.__class__.__name__} uses the new modular kernel initialization "
|
||||
"logic. This function should not be called."
|
||||
)
|
||||
|
||||
def get_fused_moe_quant_config(
|
||||
self, layer: RoutedExperts
|
||||
) -> FusedMoEQuantConfig | None:
|
||||
|
||||
@@ -5,7 +5,6 @@ from abc import abstractmethod
|
||||
|
||||
import torch
|
||||
|
||||
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoEMethodBase,
|
||||
RoutedExperts,
|
||||
@@ -100,15 +99,6 @@ class OnlineMoEMethodBase(FusedMoEMethodBase):
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
pass
|
||||
|
||||
def maybe_make_prepare_finalize(
|
||||
self,
|
||||
routing_tables: tuple[torch.Tensor, torch.Tensor, torch.Tensor] | None = None,
|
||||
) -> mk.FusedMoEPrepareAndFinalizeModular | None:
|
||||
raise ValueError(
|
||||
f"{self.__class__.__name__} uses the new modular kernel "
|
||||
"initialization logic. This function should not be called."
|
||||
)
|
||||
|
||||
@property
|
||||
def supports_eplb(self) -> bool:
|
||||
return True
|
||||
|
||||
@@ -34,7 +34,6 @@ from vllm.config.compilation import CUDAGraphMode
|
||||
from vllm.distributed.parallel_state import (
|
||||
get_dcp_group,
|
||||
get_pp_group,
|
||||
prepare_communication_buffer_for_model,
|
||||
)
|
||||
from vllm.forward_context import BatchDescriptor, set_forward_context
|
||||
from vllm.logger import init_logger
|
||||
@@ -354,11 +353,6 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
time_after_load - time_before_load,
|
||||
)
|
||||
|
||||
if not load_dummy_weights:
|
||||
prepare_communication_buffer_for_model(self.model)
|
||||
if self.speculator is not None:
|
||||
prepare_communication_buffer_for_model(self.speculator.model)
|
||||
|
||||
# Initialize the components that require the model.
|
||||
self.model_state = init_model_state(
|
||||
self.vllm_config, self.model, self.encoder_cache, self.device
|
||||
|
||||
@@ -50,7 +50,6 @@ from vllm.distributed.parallel_state import (
|
||||
get_tp_group,
|
||||
graph_capture,
|
||||
is_global_first_rank,
|
||||
prepare_communication_buffer_for_model,
|
||||
)
|
||||
from vllm.forward_context import (
|
||||
BatchDescriptor,
|
||||
@@ -5387,12 +5386,7 @@ class GPUModelRunner(
|
||||
format_gib(self.model_memory_usage),
|
||||
time_after_load - time_before_load,
|
||||
)
|
||||
if not load_dummy_weights:
|
||||
prepare_communication_buffer_for_model(self.model)
|
||||
if (drafter := getattr(self, "drafter", None)) and (
|
||||
drafter_model := getattr(drafter, "model", None)
|
||||
):
|
||||
prepare_communication_buffer_for_model(drafter_model)
|
||||
|
||||
mm_config = self.model_config.multimodal_config
|
||||
self.is_multimodal_pruning_enabled = (
|
||||
supports_multimodal_pruning(self.get_model())
|
||||
|
||||
Reference in New Issue
Block a user