diff --git a/tests/quantization/test_modelopt.py b/tests/quantization/test_modelopt.py index 3245023148..b43f4f8d35 100644 --- a/tests/quantization/test_modelopt.py +++ b/tests/quantization/test_modelopt.py @@ -249,10 +249,11 @@ def test_modelopt_fp8_pc_pt_checkpoint_setup(default_vllm_config, vllm_runner): assert isinstance(gate_up_proj.quant_method, ModelOptFp8PcPtLinearMethod) assert isinstance(down_proj.quant_method, ModelOptFp8PcPtLinearMethod) - assert qkv_proj.weight.dtype == torch.float8_e4m3fn - assert o_proj.weight.dtype == torch.float8_e4m3fn - assert gate_up_proj.weight.dtype == torch.float8_e4m3fn - assert down_proj.weight.dtype == torch.float8_e4m3fn + fp8_dtype = current_platform.fp8_dtype() + assert qkv_proj.weight.dtype == fp8_dtype + assert o_proj.weight.dtype == fp8_dtype + assert gate_up_proj.weight.dtype == fp8_dtype + assert down_proj.weight.dtype == fp8_dtype # Per-channel scales; activations are dynamically scaled per token. assert hasattr(qkv_proj, "weight_scale") diff --git a/vllm/model_executor/layers/quantization/modelopt.py b/vllm/model_executor/layers/quantization/modelopt.py index 4a078f1357..352bf90ea7 100644 --- a/vllm/model_executor/layers/quantization/modelopt.py +++ b/vllm/model_executor/layers/quantization/modelopt.py @@ -63,6 +63,7 @@ from vllm.model_executor.layers.quantization.utils.flashinfer_utils import ( ) from vllm.model_executor.layers.quantization.utils.fp8_utils import ( process_fp8_input_tensor_strategy_moe, + process_fp8_weight_channel_strategy, process_fp8_weight_tensor_strategy_moe, ) from vllm.model_executor.layers.quantization.utils.marlin_utils import ( @@ -604,8 +605,11 @@ class ModelOptFp8PcPtLinearMethod(LinearMethodBase): ) def process_weights_after_loading(self, layer: torch.nn.Module) -> None: - layer.weight = Parameter(layer.weight.t(), requires_grad=False) - layer.weight_scale = Parameter(layer.weight_scale.data, requires_grad=False) + weight, weight_scale, _ = process_fp8_weight_channel_strategy( + layer.weight, layer.weight_scale.data + ) + layer.weight = Parameter(weight.t(), requires_grad=False) + layer.weight_scale = Parameter(weight_scale, requires_grad=False) self.fp8_linear.process_weights_after_loading(layer) def apply(