[XPU] Fix double-transpose in XPUFP8ScaledMMLinearKernel for W8A8 quant method (#41689)
Signed-off-by: Libin Tang <libin.tang@intel.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
23c85343fb
commit
9946c38b7f
@@ -36,6 +36,7 @@ steps:
|
||||
python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m --block-size 64 --enforce-eager --attention-backend=TRITON_ATTN &&
|
||||
python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m --block-size 64 --enforce-eager --quantization fp8 &&
|
||||
python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m --block-size 64 --enforce-eager --kv-cache-dtype fp8 &&
|
||||
python3 examples/basic/offline_inference/generate.py --model nvidia/Llama-3.1-8B-Instruct-FP8 --block-size 64 --enforce-eager --quantization modelopt --kv-cache-dtype fp8 --attention-backend TRITON_ATTN --max-model-len 4096 &&
|
||||
python3 examples/basic/offline_inference/generate.py --model superjob/Qwen3-4B-Instruct-2507-GPTQ-Int4 --block-size 64 --enforce-eager --max-model-len 8192 &&
|
||||
python3 examples/basic/offline_inference/generate.py --model ibm-research/PowerMoE-3b --block-size 64 --enforce-eager -tp 2 &&
|
||||
python3 examples/basic/offline_inference/generate.py --model ibm-research/PowerMoE-3b --block-size 64 --enforce-eager -tp 2 --enable-expert-parallel'
|
||||
|
||||
@@ -46,7 +46,18 @@ class XPUFP8ScaledMMLinearKernel(FP8ScaledMMLinearKernel):
|
||||
self.layer_param_names = layer_param_names
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
replace_parameter(layer, "weight", layer.weight.data.t())
|
||||
# fp8_gemm_w8a16 expects weight in [in, out] layout.
|
||||
# Transpose if weight is still in [out, in] layout.
|
||||
# For square matrices, use contiguity as tie-breaker:
|
||||
# checkpoint weights are contiguous, .t() views are not.
|
||||
weight = layer.weight
|
||||
out_features, in_features = self.config.weight_shape
|
||||
|
||||
if weight.shape == (out_features, in_features) and (
|
||||
in_features != out_features or weight.is_contiguous()
|
||||
):
|
||||
replace_parameter(layer, "weight", weight.data.t())
|
||||
# else: already in [in, out] layout — no-op
|
||||
|
||||
def apply_weights(
|
||||
self,
|
||||
|
||||
Reference in New Issue
Block a user