[Bug] Fix ci issue assert output_size is not None AssertionError (#43261)

Signed-off-by: yewentao256 <zhyanwentao@126.com>
Signed-off-by: Isotr0py <Isotr0py@outlook.com>
Co-authored-by: Isotr0py <Isotr0py@outlook.com>
This commit is contained in:
Wentao Ye
2026-05-21 16:58:09 +08:00
committed by GitHub
co-authored by Isotr0py
parent ebbfb34e3e
commit 68e07d5916
5 changed files with 34 additions and 3 deletions
@@ -206,7 +206,16 @@ def test_cutlass_fp8_gemm_padded(
baseline = baseline_scaled_mm(a, b, scale_a, scale_b, out_dtype, bias)
# process_weights_after_loading pad b to 16
pad_k = (16 - k % 16) % 16
pad_n = (16 - n % 16) % 16
if pad_k > 0 or pad_n > 0:
b = torch.nn.functional.pad(b.t().contiguous(), (0, pad_k, 0, pad_n)).t()
if pad_n > 0 and scale_b.numel() > 1:
scale_b = torch.nn.functional.pad(scale_b, (0, pad_n), value=1.0)
kernel = object.__new__(CutlassFP8ScaledMMLinearKernel)
kernel.logical_output_size = n
out = kernel.apply_scaled_mm(
A=a,
B=b,
+1 -3
View File
@@ -465,11 +465,9 @@ def test_fp8_reloading(
method.process_weights_after_loading(layer)
# test reloading works after loading
# assuming that no reshaping occurred
for name, shape, original_weight_loader in original_metadata:
for name, shape, _ in original_metadata:
param = getattr(layer, name)
weight_loader = getattr(param, "weight_loader", default_weight_loader)
assert weight_loader is original_weight_loader
weight_loader(param, torch.zeros(shape)) # cannot use empty
method.process_weights_after_loading(layer)
@@ -16,6 +16,7 @@ from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
CUTLASS_BLOCK_FP8_SUPPORTED,
convert_to_channelwise,
)
from vllm.model_executor.utils import set_weight_attrs
from vllm.platforms import current_platform
from vllm.utils.torch_utils import direct_register_custom_op
@@ -184,6 +185,14 @@ class CutlassFP8ScaledMMLinearKernel(FP8ScaledMMLinearKernel):
pad_spec[-(2 * dim + 1)] = pad_size
return torch.nn.functional.pad(x, pad_spec, value=value)
@staticmethod
def padded_weight_loader(param: torch.Tensor, loaded_weight: torch.Tensor) -> None:
if loaded_weight.shape != param.shape:
slices = tuple(slice(0, s) for s in loaded_weight.shape)
param.data[slices].copy_(loaded_weight)
else:
param.data.copy_(loaded_weight.view(param.shape))
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
weight_name, weight_scale_name, _, _ = self.layer_param_names
weight = getattr(layer, weight_name)
@@ -202,6 +211,12 @@ class CutlassFP8ScaledMMLinearKernel(FP8ScaledMMLinearKernel):
(0, pad_k, 0, pad_n),
).t()
replace_parameter(layer, weight_name, padded_weight.data)
set_weight_attrs(
getattr(layer, weight_name),
{
"weight_loader": self.padded_weight_loader,
},
)
weight_scale = getattr(layer, weight_scale_name, None)
if weight_scale is not None and pad_n > 0 and weight_scale.numel() > 1:
@@ -210,6 +225,12 @@ class CutlassFP8ScaledMMLinearKernel(FP8ScaledMMLinearKernel):
flat_scale, dim=0, alignment=16, value=1.0
).view(-1, *weight_scale.shape[1:])
replace_parameter(layer, weight_scale_name, padded_scale.data)
set_weight_attrs(
getattr(layer, weight_name),
{
"weight_loader": self.padded_weight_loader,
},
)
def apply_scaled_mm(
self,
@@ -557,6 +557,7 @@ class Fp8OnlineLinearMethod(Fp8LinearMethod):
else:
weight = qweight.t()
replace_parameter(layer, "weight", weight.data)
self.fp8_linear.process_weights_after_loading(layer)
# Prevent duplicate processing (e.g., during weight reload)
layer._already_called_process_weights_after_loading = True
@@ -153,6 +153,8 @@ class Fp8PerTensorOnlineLinearMethod(_Fp8OnlineLinearBase):
replace_parameter(layer, "weight", qweight.t().data)
replace_parameter(layer, "weight_scale", weight_scale.data)
self.fp8_linear.process_weights_after_loading(layer)
# Prevent duplicate processing (e.g., during weight reload)
layer._already_called_process_weights_after_loading = True