[Bug] Fix ci issue assert output_size is not None AssertionError (#43261)
Signed-off-by: yewentao256 <zhyanwentao@126.com> Signed-off-by: Isotr0py <Isotr0py@outlook.com> Co-authored-by: Isotr0py <Isotr0py@outlook.com>
This commit is contained in:
@@ -206,7 +206,16 @@ def test_cutlass_fp8_gemm_padded(
|
||||
|
||||
baseline = baseline_scaled_mm(a, b, scale_a, scale_b, out_dtype, bias)
|
||||
|
||||
# process_weights_after_loading pad b to 16
|
||||
pad_k = (16 - k % 16) % 16
|
||||
pad_n = (16 - n % 16) % 16
|
||||
if pad_k > 0 or pad_n > 0:
|
||||
b = torch.nn.functional.pad(b.t().contiguous(), (0, pad_k, 0, pad_n)).t()
|
||||
if pad_n > 0 and scale_b.numel() > 1:
|
||||
scale_b = torch.nn.functional.pad(scale_b, (0, pad_n), value=1.0)
|
||||
|
||||
kernel = object.__new__(CutlassFP8ScaledMMLinearKernel)
|
||||
kernel.logical_output_size = n
|
||||
out = kernel.apply_scaled_mm(
|
||||
A=a,
|
||||
B=b,
|
||||
|
||||
@@ -465,11 +465,9 @@ def test_fp8_reloading(
|
||||
method.process_weights_after_loading(layer)
|
||||
|
||||
# test reloading works after loading
|
||||
# assuming that no reshaping occurred
|
||||
for name, shape, original_weight_loader in original_metadata:
|
||||
for name, shape, _ in original_metadata:
|
||||
param = getattr(layer, name)
|
||||
weight_loader = getattr(param, "weight_loader", default_weight_loader)
|
||||
assert weight_loader is original_weight_loader
|
||||
weight_loader(param, torch.zeros(shape)) # cannot use empty
|
||||
|
||||
method.process_weights_after_loading(layer)
|
||||
|
||||
@@ -16,6 +16,7 @@ from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
|
||||
CUTLASS_BLOCK_FP8_SUPPORTED,
|
||||
convert_to_channelwise,
|
||||
)
|
||||
from vllm.model_executor.utils import set_weight_attrs
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import direct_register_custom_op
|
||||
|
||||
@@ -184,6 +185,14 @@ class CutlassFP8ScaledMMLinearKernel(FP8ScaledMMLinearKernel):
|
||||
pad_spec[-(2 * dim + 1)] = pad_size
|
||||
return torch.nn.functional.pad(x, pad_spec, value=value)
|
||||
|
||||
@staticmethod
|
||||
def padded_weight_loader(param: torch.Tensor, loaded_weight: torch.Tensor) -> None:
|
||||
if loaded_weight.shape != param.shape:
|
||||
slices = tuple(slice(0, s) for s in loaded_weight.shape)
|
||||
param.data[slices].copy_(loaded_weight)
|
||||
else:
|
||||
param.data.copy_(loaded_weight.view(param.shape))
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
weight_name, weight_scale_name, _, _ = self.layer_param_names
|
||||
weight = getattr(layer, weight_name)
|
||||
@@ -202,6 +211,12 @@ class CutlassFP8ScaledMMLinearKernel(FP8ScaledMMLinearKernel):
|
||||
(0, pad_k, 0, pad_n),
|
||||
).t()
|
||||
replace_parameter(layer, weight_name, padded_weight.data)
|
||||
set_weight_attrs(
|
||||
getattr(layer, weight_name),
|
||||
{
|
||||
"weight_loader": self.padded_weight_loader,
|
||||
},
|
||||
)
|
||||
|
||||
weight_scale = getattr(layer, weight_scale_name, None)
|
||||
if weight_scale is not None and pad_n > 0 and weight_scale.numel() > 1:
|
||||
@@ -210,6 +225,12 @@ class CutlassFP8ScaledMMLinearKernel(FP8ScaledMMLinearKernel):
|
||||
flat_scale, dim=0, alignment=16, value=1.0
|
||||
).view(-1, *weight_scale.shape[1:])
|
||||
replace_parameter(layer, weight_scale_name, padded_scale.data)
|
||||
set_weight_attrs(
|
||||
getattr(layer, weight_name),
|
||||
{
|
||||
"weight_loader": self.padded_weight_loader,
|
||||
},
|
||||
)
|
||||
|
||||
def apply_scaled_mm(
|
||||
self,
|
||||
|
||||
@@ -557,6 +557,7 @@ class Fp8OnlineLinearMethod(Fp8LinearMethod):
|
||||
else:
|
||||
weight = qweight.t()
|
||||
replace_parameter(layer, "weight", weight.data)
|
||||
self.fp8_linear.process_weights_after_loading(layer)
|
||||
|
||||
# Prevent duplicate processing (e.g., during weight reload)
|
||||
layer._already_called_process_weights_after_loading = True
|
||||
|
||||
@@ -153,6 +153,8 @@ class Fp8PerTensorOnlineLinearMethod(_Fp8OnlineLinearBase):
|
||||
replace_parameter(layer, "weight", qweight.t().data)
|
||||
replace_parameter(layer, "weight_scale", weight_scale.data)
|
||||
|
||||
self.fp8_linear.process_weights_after_loading(layer)
|
||||
|
||||
# Prevent duplicate processing (e.g., during weight reload)
|
||||
layer._already_called_process_weights_after_loading = True
|
||||
|
||||
|
||||
Reference in New Issue
Block a user