From 482c6134236d96ebab3c749cd02d245516adad1d Mon Sep 17 00:00:00 2001 From: Netanel Haber <58652339+netanel-haber@users.noreply.github.com> Date: Thu, 6 Aug 2026 00:08:55 +0300 Subject: [PATCH] [Bugfix][Humming] Preserve ModelOpt FP8 weight dimensions (#51093) Signed-off-by: Netanel Haber <58652339+netanel-haber@users.noreply.github.com> --- tests/quantization/test_modelopt.py | 23 +++++++++++++++++++ .../layers/quantization/modelopt.py | 2 ++ 2 files changed, 25 insertions(+) diff --git a/tests/quantization/test_modelopt.py b/tests/quantization/test_modelopt.py index cac49a5a80..4124085c23 100644 --- a/tests/quantization/test_modelopt.py +++ b/tests/quantization/test_modelopt.py @@ -22,6 +22,7 @@ from vllm.model_executor.kernels.linear import ( from vllm.model_executor.layers.linear import UnquantizedLinearMethod from vllm.model_executor.layers.quantization.modelopt import ( ModelOptFp8Config, + ModelOptFp8LinearMethod, ModelOptMixedPrecisionConfig, ModelOptMxFp8Config, ModelOptNvFp4Config, @@ -114,6 +115,28 @@ def test_modelopt_nvfp4_quantizes_parallel_lm_head(): assert isinstance(method, ModelOptNvFp4LinearMethod) +def test_modelopt_fp8_updates_weight_dims_after_transpose(): + layer = torch.nn.Module() + layer.register_parameter( + "weight", torch.nn.Parameter(torch.empty(3, 2), requires_grad=False) + ) + layer.register_parameter( + "weight_scale", torch.nn.Parameter(torch.ones(1), requires_grad=False) + ) + layer.register_parameter( + "input_scale", torch.nn.Parameter(torch.ones(1), requires_grad=False) + ) + + method = ModelOptFp8LinearMethod.__new__(ModelOptFp8LinearMethod) + method.fp8_linear = Mock() + method.process_weights_after_loading(layer) + + assert layer.weight.shape == (2, 3) + assert layer.weight.input_dim == 0 + assert layer.weight.output_dim == 1 + method.fp8_linear.process_weights_after_loading.assert_called_once_with(layer) + + def test_modelopt_nvfp4_leaves_excluded_parallel_lm_head_unquantized(): config = ModelOptNvFp4Config( is_checkpoint_nvfp4_serialized=True, diff --git a/vllm/model_executor/layers/quantization/modelopt.py b/vllm/model_executor/layers/quantization/modelopt.py index bac19536f4..4c33c19070 100644 --- a/vllm/model_executor/layers/quantization/modelopt.py +++ b/vllm/model_executor/layers/quantization/modelopt.py @@ -519,6 +519,8 @@ class ModelOptFp8LinearMethod(LinearMethodBase): layer.weight, layer.weight_scale, layer.logical_widths ) layer.weight = Parameter(weight.t(), requires_grad=False) + layer.weight.input_dim = 0 + layer.weight.output_dim = 1 layer.weight_scale = Parameter(max_w_scale, requires_grad=False) layer.input_scale = Parameter(layer.input_scale.max(), requires_grad=False) self.fp8_linear.process_weights_after_loading(layer)