From 2785a5e0e6eefe54c42307189f1deeb648e0a5b4 Mon Sep 17 00:00:00 2001 From: nehmathe2 Date: Tue, 16 Jun 2026 20:17:42 -0700 Subject: [PATCH] [Bugfix][ROCm] Fix FP8 per-tensor scale rank mismatch causing Inductor assertion failure (#44912) Signed-off-by: nehmathe2 Signed-off-by: Divakar Verma Signed-off-by: nehmathe Co-authored-by: TJian Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: Divakar Verma Co-authored-by: Andreas Karatzas --- vllm/model_executor/kernels/linear/scaled_mm/pytorch.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/vllm/model_executor/kernels/linear/scaled_mm/pytorch.py b/vllm/model_executor/kernels/linear/scaled_mm/pytorch.py index 9182977e95..2b6d3ed736 100644 --- a/vllm/model_executor/kernels/linear/scaled_mm/pytorch.py +++ b/vllm/model_executor/kernels/linear/scaled_mm/pytorch.py @@ -78,6 +78,12 @@ class PerTensorTorchFP8ScaledMMLinearKernel(TorchFP8ScaledMMLinearKernel): bias: torch.Tensor | None, output_shape: list, ) -> torch.Tensor: + # torch._scaled_mm under torch.compile does not support 0-D scales + if As.dim() == 0: + As = As.view(1) + if Bs.dim() == 0: + Bs = Bs.view(1) + output = torch._scaled_mm( A, B, out_dtype=out_dtype, scale_a=As, scale_b=Bs, bias=bias )