[XPU][Bugfix] Do not transpose weight_scale_inv at load time (#47116)
Signed-off-by: Ma Jian <jian1.ma@intel.com> Co-authored-by: Kunshang Ji <kunshang.ji@intel.com>
This commit is contained in:
@@ -197,29 +197,6 @@ class XPUFp8BlockScaledMMKernel(Fp8BlockScaledMMLinearKernel):
|
||||
return False, "XPUFp8BlockScaledMM only support on XPU"
|
||||
return True, None
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module):
|
||||
super().process_weights_after_loading(layer)
|
||||
scale_attr = (
|
||||
"weight_scale_inv" if hasattr(layer, "weight_scale_inv") else "weight_scale"
|
||||
)
|
||||
scale = getattr(layer, scale_attr)
|
||||
# Models with scale_fmt=ue8m0 (e.g. DeepSeek-V4) store weight scales
|
||||
# as float8_e8m0fnu. The oneDNN fp8_gemm kernel dispatches to its
|
||||
# "block quant" path only when NEITHER scale is e8m0:
|
||||
#
|
||||
# is_block_quant = (m1_sc != e8m0) && (m2_sc != e8m0) && ...
|
||||
#
|
||||
# Since activation scales are always float32 (use_ue8m0=False on XPU,
|
||||
# DeepGEMM requires Hopper/Blackwell), an e8m0 weight scale causes
|
||||
# is_block_quant=false and falls into the wrong per-channel path,
|
||||
# producing NaN. Converting e8m0→float32 here at load time (one-time,
|
||||
# negligible overhead for small scale tensors) ensures the kernel sees
|
||||
# matching dtypes and correctly enters the block-quant path with the
|
||||
# actual group_size derived from scale tensor shapes.
|
||||
if scale.dtype == torch.float8_e8m0fnu:
|
||||
scale = scale.to(torch.float32)
|
||||
replace_parameter(layer, scale_attr, scale.data.t().contiguous())
|
||||
|
||||
def apply_block_scaled_mm(
|
||||
self,
|
||||
A: torch.Tensor,
|
||||
@@ -228,11 +205,12 @@ class XPUFp8BlockScaledMMKernel(Fp8BlockScaledMMLinearKernel):
|
||||
Bs: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
# Weight is [N, K]. Use .t() to create a [K, N] view without copying.
|
||||
# Bs is [N/128, K/128] — transpose to [K/128, N/128] for oneDNN.
|
||||
return torch.ops._xpu_C.fp8_gemm(
|
||||
A,
|
||||
B.t(),
|
||||
self.config.out_dtype,
|
||||
As,
|
||||
Bs,
|
||||
Bs.t().contiguous(),
|
||||
torch.Tensor(),
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user