From 4cc15f2121b3421478f3186aac38de901b4a44ef Mon Sep 17 00:00:00 2001 From: siyu Date: Sat, 19 Sep 2026 17:16:39 +0800 Subject: [PATCH] [Quantization] Let ModelOpt MXFP8 layers load pre-processed weights Purpose (#57316) Signed-off-by: liusy58 Signed-off-by: Isotr0py <2037008807@qq.com> Co-authored-by: Isotr0py <2037008807@qq.com> --- tests/quantization/test_modelopt.py | 30 +++++++++++++++++++ .../kernels/linear/mxfp8/Mxfp8LinearKernel.py | 4 +++ .../kernels/linear/mxfp8/emulation.py | 2 ++ .../kernels/linear/mxfp8/flashinfer.py | 4 +++ .../kernels/linear/mxfp8/rocm_native.py | 2 ++ .../layers/quantization/modelopt.py | 19 +++++++++--- 6 files changed, 57 insertions(+), 4 deletions(-) diff --git a/tests/quantization/test_modelopt.py b/tests/quantization/test_modelopt.py index 4ca1af53a2..ae8548c370 100644 --- a/tests/quantization/test_modelopt.py +++ b/tests/quantization/test_modelopt.py @@ -144,6 +144,36 @@ def test_modelopt_mxfp8_preserves_per_row_checkpoint_scales(dist_init, monkeypat assert torch.equal(linear.weight_scale, scales) +def test_modelopt_mxfp8_pre_processed_weights_follow_kernel(monkeypatch): + """The weight cache daemon can only serve MXFP8 layers whose kernel needs + no post-load state beyond the parameters it exports.""" + from vllm.config.quantization import QuantSpec + from vllm.model_executor.kernels.linear.mxfp8 import Mxfp8LinearKernel + + method = ModelOptLinearMethod.__new__(ModelOptLinearMethod) + method.spec = QuantSpec(weight=kMxfp8Static, activation=kMxfp8Dynamic) + method.kernel = None + assert not method.supports_pre_processed_weights + + method.kernel = Mock(spec=Mxfp8LinearKernel) + method.kernel.supports_pre_processed_weights = False + assert not method.supports_pre_processed_weights + method.kernel.supports_pre_processed_weights = True + assert method.supports_pre_processed_weights + + monkeypatch.setattr( + "vllm.model_executor.layers.quantization.modelopt.is_weights_pre_processed", + lambda: True, + ) + layer = torch.nn.Module() + method.process_weights_after_loading(layer) + method.kernel.process_weights_after_loading.assert_not_called() + + method.kernel.supports_pre_processed_weights = False + with pytest.raises(RuntimeError, match="pre-processed"): + method.process_weights_after_loading(layer) + + def test_modelopt_fp8_updates_weight_dims_after_transpose(): """Humming reads weight.input_dim/output_dim. Swapping the ModelWeightParameter for a plain Parameter drops them, so the per-tensor diff --git a/vllm/model_executor/kernels/linear/mxfp8/Mxfp8LinearKernel.py b/vllm/model_executor/kernels/linear/mxfp8/Mxfp8LinearKernel.py index 3fe34029e5..fdcfeb264b 100644 --- a/vllm/model_executor/kernels/linear/mxfp8/Mxfp8LinearKernel.py +++ b/vllm/model_executor/kernels/linear/mxfp8/Mxfp8LinearKernel.py @@ -27,6 +27,10 @@ class Mxfp8LinearKernel(ABC): Marlin, emulation). """ + supports_pre_processed_weights: bool = False + """True if ``process_weights_after_loading`` only rewrites parameters, so + weights exported by the weight cache daemon can be used as-is.""" + def __init__(self, c: Mxfp8LinearLayerConfig) -> None: assert self.can_implement(c)[0] assert self.is_supported()[0] diff --git a/vllm/model_executor/kernels/linear/mxfp8/emulation.py b/vllm/model_executor/kernels/linear/mxfp8/emulation.py index 79b2fba388..da48655d37 100644 --- a/vllm/model_executor/kernels/linear/mxfp8/emulation.py +++ b/vllm/model_executor/kernels/linear/mxfp8/emulation.py @@ -16,6 +16,8 @@ from .Mxfp8LinearKernel import Mxfp8LinearKernel, Mxfp8LinearLayerConfig class EmulationMxfp8LinearKernel(Mxfp8LinearKernel): """Software emulation fallback for MXFP8 (dequant to BF16).""" + supports_pre_processed_weights = True + @classmethod def is_supported( cls, compute_capability: int | None = None diff --git a/vllm/model_executor/kernels/linear/mxfp8/flashinfer.py b/vllm/model_executor/kernels/linear/mxfp8/flashinfer.py index 02700a0ced..cc9b35d34b 100644 --- a/vllm/model_executor/kernels/linear/mxfp8/flashinfer.py +++ b/vllm/model_executor/kernels/linear/mxfp8/flashinfer.py @@ -27,6 +27,8 @@ from .Mxfp8LinearKernel import Mxfp8LinearKernel, Mxfp8LinearLayerConfig class FlashInferCutlassMxfp8LinearKernel(Mxfp8LinearKernel): """MXFP8 W8A8 GEMM via FlashInfer CUTLASS (SM100+).""" + supports_pre_processed_weights = True + @classmethod def is_supported( cls, compute_capability: int | None = None @@ -117,6 +119,8 @@ class FlashInferCutlassMxfp8LinearKernel(Mxfp8LinearKernel): class FlashInferCutedslMxfp8LinearKernel(Mxfp8LinearKernel): """MXFP8 W8A8 GEMM via FlashInfer CuTe-DSL (SM100/SM103).""" + supports_pre_processed_weights = True + @classmethod def is_supported( cls, compute_capability: int | None = None diff --git a/vllm/model_executor/kernels/linear/mxfp8/rocm_native.py b/vllm/model_executor/kernels/linear/mxfp8/rocm_native.py index 1d9789cced..c7e6537f49 100644 --- a/vllm/model_executor/kernels/linear/mxfp8/rocm_native.py +++ b/vllm/model_executor/kernels/linear/mxfp8/rocm_native.py @@ -212,6 +212,8 @@ _DOT_SCALED_K_ALIGN = 128 class RocmDotScaledMxfp8LinearKernel(Mxfp8LinearKernel): """Native CDNA4 (gfx950) MXFP8 linear via Triton ``tl.dot_scaled``.""" + supports_pre_processed_weights = True + @classmethod def is_supported( cls, compute_capability: int | None = None diff --git a/vllm/model_executor/layers/quantization/modelopt.py b/vllm/model_executor/layers/quantization/modelopt.py index c649061f33..364c52b4c8 100644 --- a/vllm/model_executor/layers/quantization/modelopt.py +++ b/vllm/model_executor/layers/quantization/modelopt.py @@ -2482,9 +2482,16 @@ class ModelOptLinearMethod(LinearMethodBase): @property def supports_pre_processed_weights(self) -> bool: # type: ignore[override] - # TODO(Isotr0py): support fp8/mxfp8 ModelOpt kernels transpose/repack. + # TODO(Isotr0py): support fp8 ModelOpt kernels transpose/repack. w = self.spec.weight - return isinstance(w, QuantKey) and w.dtype == FP4_DTYPE + return isinstance(w, QuantKey) and ( + w.dtype == FP4_DTYPE + or ( + w == kMxfp8Static + and self.kernel is not None + and self.kernel.supports_pre_processed_weights + ) + ) def create_weights( self, @@ -2524,7 +2531,13 @@ class ModelOptLinearMethod(LinearMethodBase): expose_input_quant_key(layer, self.kernel) def process_weights_after_loading(self, layer) -> None: + if self.spec.weight == kMxfp8Static and getattr(layer, "is_bmm", False): + self.kernel = init_mxfp8_linear_kernel(bmm_batch_size=layer.bmm_batch_size) if is_weights_pre_processed(): + if not self.supports_pre_processed_weights: + raise RuntimeError( + f"{type(self.kernel).__name__} cannot use pre-processed weights" + ) return self.fmt.pre_process(layer) self.wkey.process(layer, WEIGHT) @@ -2556,8 +2569,6 @@ class ModelOptLinearMethod(LinearMethodBase): persistent=False, ) layer._nvfp4_group_size_for_gather = self.ctx.group_size - if self.spec.weight == kMxfp8Static and getattr(layer, "is_bmm", False): - self.kernel = init_mxfp8_linear_kernel(bmm_batch_size=layer.bmm_batch_size) self.kernel.process_weights_after_loading(layer) def apply(self, layer, x, bias=None):