[Bugfix] Canonicalize FP8 weight layout to (K, N) at the source (#44735)

Signed-off-by: mgoin <mgoin64@gmail.com>
This commit is contained in:
Michael Goin
2026-06-08 14:37:36 -06:00
committed by GitHub
parent 823a0ab754
commit 6afa25000c
3 changed files with 11 additions and 29 deletions
@@ -75,25 +75,7 @@ class MarlinFP8ScaledMMLinearKernel(FP8ScaledMMLinearKernel):
# Update layer with new values
replace_parameter(layer, "weight", weight.data)
replace_parameter(layer, "weight_scale_inv", weight_scale_inv.data)
else:
w_q, *_ = self._get_layer_params(layer)
# Compressed tensors transposes the weight to (K, N)
# for channel and tensor quant strategies.
# So we can skip the transpose if the layout is
# already (K, N).
# TODO: Remove this check once the layouts have been
# canonicalized to a standard (N, K) dimension. See issue
# #33314 for more details.
if w_q.shape != (
layer.input_size_per_partition,
layer.output_size_per_partition,
):
# transpose the weights to (K,N)
replace_parameter(
layer,
"weight",
w_q.t(),
)
# Non-block: callers must pass weight in (K, N) layout.
layer.input_scale = None
prepare_fp8_layer_for_marlin(
@@ -137,6 +137,8 @@ class CompressedTensorsW8A16Fp8(CompressedTensorsScheme):
"weight_scale",
convert_to_channelwise(layer.weight_scale, layer.logical_widths),
)
# Canonicalize to (K, N) for the kernel.
replace_parameter(layer, "weight", layer.weight.t())
self.linear_kernel.process_weights_after_loading(layer)
+8 -10
View File
@@ -384,6 +384,9 @@ class Fp8LinearMethod(LinearMethodBase):
def process_weights_after_loading(self, layer: RoutedExperts) -> None:
if self.use_marlin:
if not self.block_quant:
# Canonicalize to (K, N) for the kernel.
replace_parameter(layer, "weight", layer.weight.t())
# Only Marlin kernels support `marlin_input_dtype`; guard to avoid
# AttributeError if backend selection changes.
if hasattr(self.fp8_linear, "marlin_input_dtype"):
@@ -537,20 +540,15 @@ class Fp8OnlineLinearMethod(Fp8LinearMethod):
layer.input_scale = None
qweight, weight_scale = ops.scaled_fp8_quant(layer.weight, scale=None)
# Update layer with new values.
replace_parameter(layer, "weight", qweight.data)
# Update layer with new values. Canonicalize to (K, N) for the kernel.
replace_parameter(layer, "weight", qweight.t().data)
replace_parameter(layer, "weight_scale", weight_scale.data)
if self.use_marlin:
if self.use_marlin and hasattr(self.fp8_linear, "marlin_input_dtype"):
# Only Marlin kernels support `marlin_input_dtype`; guard to avoid
# AttributeError if backend selection changes.
if hasattr(self.fp8_linear, "marlin_input_dtype"):
self.fp8_linear.marlin_input_dtype = self.marlin_input_dtype
self.fp8_linear.process_weights_after_loading(layer)
else:
weight = qweight.t()
replace_parameter(layer, "weight", weight.data)
self.fp8_linear.process_weights_after_loading(layer)
self.fp8_linear.marlin_input_dtype = self.marlin_input_dtype
self.fp8_linear.process_weights_after_loading(layer)
# Prevent duplicate processing (e.g., during weight reload)
layer._already_called_process_weights_after_loading = True