From 68e07d59161a8d268b773c181fab17994a7c5d0a Mon Sep 17 00:00:00 2001 From: Wentao Ye <44945378+yewentao256@users.noreply.github.com> Date: Thu, 21 May 2026 04:58:09 -0400 Subject: [PATCH] [Bug] Fix ci issue `assert output_size is not None` AssertionError (#43261) Signed-off-by: yewentao256 Signed-off-by: Isotr0py Co-authored-by: Isotr0py --- .../quantization/test_cutlass_scaled_mm.py | 9 ++++++++ tests/quantization/test_fp8.py | 4 +--- .../kernels/linear/scaled_mm/cutlass.py | 21 +++++++++++++++++++ .../model_executor/layers/quantization/fp8.py | 1 + .../layers/quantization/online/fp8.py | 2 ++ 5 files changed, 34 insertions(+), 3 deletions(-) diff --git a/tests/kernels/quantization/test_cutlass_scaled_mm.py b/tests/kernels/quantization/test_cutlass_scaled_mm.py index 32eb65cfc5..a937c30fed 100644 --- a/tests/kernels/quantization/test_cutlass_scaled_mm.py +++ b/tests/kernels/quantization/test_cutlass_scaled_mm.py @@ -206,7 +206,16 @@ def test_cutlass_fp8_gemm_padded( baseline = baseline_scaled_mm(a, b, scale_a, scale_b, out_dtype, bias) + # process_weights_after_loading pad b to 16 + pad_k = (16 - k % 16) % 16 + pad_n = (16 - n % 16) % 16 + if pad_k > 0 or pad_n > 0: + b = torch.nn.functional.pad(b.t().contiguous(), (0, pad_k, 0, pad_n)).t() + if pad_n > 0 and scale_b.numel() > 1: + scale_b = torch.nn.functional.pad(scale_b, (0, pad_n), value=1.0) + kernel = object.__new__(CutlassFP8ScaledMMLinearKernel) + kernel.logical_output_size = n out = kernel.apply_scaled_mm( A=a, B=b, diff --git a/tests/quantization/test_fp8.py b/tests/quantization/test_fp8.py index 4ba0e5d3da..b93d34afbb 100644 --- a/tests/quantization/test_fp8.py +++ b/tests/quantization/test_fp8.py @@ -465,11 +465,9 @@ def test_fp8_reloading( method.process_weights_after_loading(layer) # test reloading works after loading - # assuming that no reshaping occurred - for name, shape, original_weight_loader in original_metadata: + for name, shape, _ in original_metadata: param = getattr(layer, name) weight_loader = getattr(param, "weight_loader", default_weight_loader) - assert weight_loader is original_weight_loader weight_loader(param, torch.zeros(shape)) # cannot use empty method.process_weights_after_loading(layer) diff --git a/vllm/model_executor/kernels/linear/scaled_mm/cutlass.py b/vllm/model_executor/kernels/linear/scaled_mm/cutlass.py index f268711dad..9e65edb851 100644 --- a/vllm/model_executor/kernels/linear/scaled_mm/cutlass.py +++ b/vllm/model_executor/kernels/linear/scaled_mm/cutlass.py @@ -16,6 +16,7 @@ from vllm.model_executor.layers.quantization.utils.w8a8_utils import ( CUTLASS_BLOCK_FP8_SUPPORTED, convert_to_channelwise, ) +from vllm.model_executor.utils import set_weight_attrs from vllm.platforms import current_platform from vllm.utils.torch_utils import direct_register_custom_op @@ -184,6 +185,14 @@ class CutlassFP8ScaledMMLinearKernel(FP8ScaledMMLinearKernel): pad_spec[-(2 * dim + 1)] = pad_size return torch.nn.functional.pad(x, pad_spec, value=value) + @staticmethod + def padded_weight_loader(param: torch.Tensor, loaded_weight: torch.Tensor) -> None: + if loaded_weight.shape != param.shape: + slices = tuple(slice(0, s) for s in loaded_weight.shape) + param.data[slices].copy_(loaded_weight) + else: + param.data.copy_(loaded_weight.view(param.shape)) + def process_weights_after_loading(self, layer: torch.nn.Module) -> None: weight_name, weight_scale_name, _, _ = self.layer_param_names weight = getattr(layer, weight_name) @@ -202,6 +211,12 @@ class CutlassFP8ScaledMMLinearKernel(FP8ScaledMMLinearKernel): (0, pad_k, 0, pad_n), ).t() replace_parameter(layer, weight_name, padded_weight.data) + set_weight_attrs( + getattr(layer, weight_name), + { + "weight_loader": self.padded_weight_loader, + }, + ) weight_scale = getattr(layer, weight_scale_name, None) if weight_scale is not None and pad_n > 0 and weight_scale.numel() > 1: @@ -210,6 +225,12 @@ class CutlassFP8ScaledMMLinearKernel(FP8ScaledMMLinearKernel): flat_scale, dim=0, alignment=16, value=1.0 ).view(-1, *weight_scale.shape[1:]) replace_parameter(layer, weight_scale_name, padded_scale.data) + set_weight_attrs( + getattr(layer, weight_name), + { + "weight_loader": self.padded_weight_loader, + }, + ) def apply_scaled_mm( self, diff --git a/vllm/model_executor/layers/quantization/fp8.py b/vllm/model_executor/layers/quantization/fp8.py index 0726164830..41e2e19785 100644 --- a/vllm/model_executor/layers/quantization/fp8.py +++ b/vllm/model_executor/layers/quantization/fp8.py @@ -557,6 +557,7 @@ class Fp8OnlineLinearMethod(Fp8LinearMethod): else: weight = qweight.t() replace_parameter(layer, "weight", weight.data) + self.fp8_linear.process_weights_after_loading(layer) # Prevent duplicate processing (e.g., during weight reload) layer._already_called_process_weights_after_loading = True diff --git a/vllm/model_executor/layers/quantization/online/fp8.py b/vllm/model_executor/layers/quantization/online/fp8.py index add28f1a8c..5e14eade26 100644 --- a/vllm/model_executor/layers/quantization/online/fp8.py +++ b/vllm/model_executor/layers/quantization/online/fp8.py @@ -153,6 +153,8 @@ class Fp8PerTensorOnlineLinearMethod(_Fp8OnlineLinearBase): replace_parameter(layer, "weight", qweight.t().data) replace_parameter(layer, "weight_scale", weight_scale.data) + self.fp8_linear.process_weights_after_loading(layer) + # Prevent duplicate processing (e.g., during weight reload) layer._already_called_process_weights_after_loading = True