From acce57d8dd8bed25543f8ece834a3757208c677c Mon Sep 17 00:00:00 2001 From: Yan Ma Date: Wed, 24 Jun 2026 02:53:38 +0800 Subject: [PATCH] Deprecate old FP8 online MoE quantization class (#44514) Signed-off-by: Yan Ma --- tests/quantization/test_fp8.py | 15 +- .../model_executor/layers/quantization/fp8.py | 139 +----------------- .../model_loader/base_loader.py | 2 +- 3 files changed, 19 insertions(+), 137 deletions(-) diff --git a/tests/quantization/test_fp8.py b/tests/quantization/test_fp8.py index 499955c9f6..571e180d6c 100644 --- a/tests/quantization/test_fp8.py +++ b/tests/quantization/test_fp8.py @@ -14,6 +14,9 @@ import torch from tests.quantization.utils import is_quant_method_supported from vllm import _custom_ops as ops from vllm.config.model import ModelConfig +from vllm.model_executor.kernels.linear.scaled_mm import ( + MarlinFP8ScaledMMLinearKernel, +) from vllm.model_executor.layers.fused_moe import FusedMoE from vllm.model_executor.layers.quantization.fp8 import ( Fp8Config, @@ -46,7 +49,7 @@ MODELS = [ ) @pytest.mark.parametrize("model_id", MODELS) @pytest.mark.parametrize( - "force_marlin", [False] if current_platform.is_rocm() else [False, True] + "force_marlin", [True, False] if current_platform.is_cuda() else [False] ) @pytest.mark.parametrize( "use_rocm_aiter", [True, False] if current_platform.is_rocm() else [False] @@ -73,7 +76,7 @@ def test_model_load_and_run( ) @pytest.mark.parametrize("kv_cache_dtype", ["auto", "fp8"]) @pytest.mark.parametrize( - "force_marlin", [False] if current_platform.is_rocm() else [False, True] + "force_marlin", [True, False] if current_platform.is_cuda() else [False] ) @pytest.mark.parametrize( "use_rocm_aiter", [True, False] if current_platform.is_rocm() else [False] @@ -110,14 +113,20 @@ def test_online_quantization( assert attn._k_scale == 1.0 assert attn._v_scale == 1.0 - if current_platform.is_cuda(): + if current_platform.is_cuda() or current_platform.is_xpu(): if current_platform.supports_fp8() and not force_marlin: # For GPUs with hardware support, we keep weights in fp8 assert fc1.weight.dtype == torch.float8_e4m3fn + assert not isinstance( + fc1.quant_method.fp8_linear, MarlinFP8ScaledMMLinearKernel + ) else: # For GPUs without hardware support, we pack the fp8 weights # for weight-only quantization using Marlin kernels assert fc1.weight.dtype == torch.int32 + assert isinstance( + fc1.quant_method.fp8_linear, MarlinFP8ScaledMMLinearKernel + ) elif current_platform.is_rocm(): if current_platform.supports_fp8() and not force_marlin: # For GPUs with hardware support, we keep weights in fp8 diff --git a/vllm/model_executor/layers/quantization/fp8.py b/vllm/model_executor/layers/quantization/fp8.py index 7cdb04cfbe..869fbf7523 100644 --- a/vllm/model_executor/layers/quantization/fp8.py +++ b/vllm/model_executor/layers/quantization/fp8.py @@ -8,7 +8,6 @@ from torch.utils._python_dispatch import TorchDispatchMode import vllm.envs as envs import vllm.model_executor.layers.fused_moe.modular_kernel as mk -from vllm import _custom_ops as ops from vllm.config import get_current_vllm_config from vllm.distributed import get_tensor_model_parallel_world_size from vllm.logger import init_logger @@ -75,9 +74,6 @@ from vllm.model_executor.layers.quantization.utils.w8a8_utils import ( cutlass_fp8_supported, normalize_e4m3fn_to_e4m3fnuz, ) -from vllm.model_executor.model_loader.reload.layerwise import ( - initialize_online_processing, -) from vllm.model_executor.parameter import ( BlockQuantScaleParameter, PerTensorScaleParameter, @@ -212,10 +208,13 @@ class Fp8Config(QuantizationConfig): return Mxfp4MoEMethod(layer.moe_config) if self.is_checkpoint_fp8_serialized: - moe_quant_method = Fp8MoEMethod(self, layer) + return Fp8MoEMethod(self, layer) else: - moe_quant_method = Fp8OnlineMoEMethod(self, layer) - return moe_quant_method + from vllm.model_executor.layers.quantization.online.fp8 import ( + Fp8PerTensorOnlineMoEMethod, + ) + + return Fp8PerTensorOnlineMoEMethod(layer=layer) elif isinstance(layer, Attention): return Fp8KVCacheMethod(self) return None @@ -854,132 +853,6 @@ class Fp8MoEMethod(FusedMoEMethodBase): ) -# TODO(future PR): remove this class in favor of -# online/fp8.py::Fp8PerTensorOnlineMoEMethod -class Fp8OnlineMoEMethod(Fp8MoEMethod): - """MoE method for online FP8 quantization. - Supports loading quantized FP16/BF16 model checkpoints with dynamic - activation scaling. The weight scaling factor will be initialized after - the model weights are loaded. - - Args: - quant_config: The quantization config. - """ - - uses_meta_device: bool = True - - def __init__(self, quant_config: Fp8Config, layer: RoutedExperts): - super().__init__(quant_config, layer) - assert not quant_config.is_checkpoint_fp8_serialized - assert quant_config.activation_scheme == "dynamic" - assert quant_config.weight_block_size is None - - def create_weights( - self, - layer: RoutedExperts, - num_experts: int, - hidden_size: int, - intermediate_size_per_partition: int, - params_dtype: torch.dtype, - **extra_weight_attrs, - ): - layer.num_experts = num_experts - layer.orig_dtype = params_dtype - layer.weight_block_size = None - - # WEIGHTS - w13_weight = torch.nn.Parameter( - torch.empty( - num_experts, - 2 * intermediate_size_per_partition, - hidden_size, - device="meta", - dtype=params_dtype, - ), - requires_grad=False, - ) - layer.register_parameter("w13_weight", w13_weight) - set_weight_attrs(w13_weight, extra_weight_attrs) - - w2_weight = torch.nn.Parameter( - torch.empty( - num_experts, - hidden_size, - intermediate_size_per_partition, - device="meta", # materialized and processed during loading - dtype=params_dtype, - ), - requires_grad=False, - ) - layer.register_parameter("w2_weight", w2_weight) - set_weight_attrs(w2_weight, extra_weight_attrs) - - # BIASES (for models like GPT-OSS that have biased MoE) - if self.moe.has_bias: - w13_bias = torch.nn.Parameter( - torch.zeros( - num_experts, - 2 * intermediate_size_per_partition, - device="meta", # materialized and processed during loading - dtype=layer.orig_dtype, - ), - requires_grad=False, - ) - layer.register_parameter("w13_bias", w13_bias) - set_weight_attrs(w13_bias, extra_weight_attrs) - - w2_bias = torch.nn.Parameter( - torch.zeros( - num_experts, - hidden_size, - device="meta", # materialized and processed during loading - dtype=layer.orig_dtype, - ), - requires_grad=False, - ) - layer.register_parameter("w2_bias", w2_bias) - set_weight_attrs(w2_bias, extra_weight_attrs) - - initialize_online_processing(layer) - - def process_weights_after_loading(self, layer: RoutedExperts) -> None: - # TODO(@ksayers): inplace fp8 quant kernel, initialize scales with ones - if getattr(layer, "_already_called_process_weights_after_loading", False): - return - - fp8_dtype = current_platform.fp8_dtype() - w13 = torch.empty_like(layer.w13_weight, dtype=fp8_dtype) - w2 = torch.empty_like(layer.w2_weight, dtype=fp8_dtype) - w13_scale = torch.ones( - layer.num_experts, device=w13.device, dtype=torch.float32 - ) - w2_scale = torch.ones(layer.num_experts, device=w2.device, dtype=torch.float32) - layer.w13_input_scale = None - layer.w2_input_scale = None - - for expert in range(layer.local_num_experts): - w13[expert, :, :], w13_scale[expert] = ops.scaled_fp8_quant( - layer.w13_weight[expert, :, :] - ) - w2[expert, :, :], w2_scale[expert] = ops.scaled_fp8_quant( - layer.w2_weight[expert, :, :] - ) - - # Shuffle weights to runtime format and setup kernel. - self._setup_kernel( - layer, - w13, - w2, - w13_scale, - w2_scale, - w13_input_scale=layer.w13_input_scale, - w2_input_scale=layer.w2_input_scale, - ) - - # Prevent duplicate processing (e.g., during weight reload) - layer._already_called_process_weights_after_loading = True - - class Fp8KVCacheMethod(BaseKVCacheMethod): """ Supports loading kv-cache scaling factors from FP8 checkpoints. diff --git a/vllm/model_executor/model_loader/base_loader.py b/vllm/model_executor/model_loader/base_loader.py index 55a4dd4c28..d1f44666c5 100644 --- a/vllm/model_executor/model_loader/base_loader.py +++ b/vllm/model_executor/model_loader/base_loader.py @@ -65,7 +65,7 @@ class BaseModelLoader(ABC): # Log peak GPU memory after loading weights. This is needed # to have test coverage on peak memory for online quantization. - if current_platform.is_cuda_alike(): + if current_platform.is_cuda_alike() or current_platform.is_xpu(): peak_memory = torch.accelerator.max_memory_allocated() logger.debug_once( "Peak GPU memory after loading weights: %s GiB",