From 9354fb1ba5ca652835d0e6c88b15c3a925e1aab0 Mon Sep 17 00:00:00 2001 From: QiliangCui2023 <130511281+QiliangCui2023@users.noreply.github.com> Date: Thu, 4 Jun 2026 06:31:50 -0700 Subject: [PATCH] [Bugfix][Compile] Guard per_token_group_fp8_quant lookup on non-CUDA platforms (#44476) --- vllm/compilation/passes/fusion/matcher_utils.py | 6 ++++-- vllm/compilation/passes/fusion/rms_quant_fusion.py | 5 +++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/vllm/compilation/passes/fusion/matcher_utils.py b/vllm/compilation/passes/fusion/matcher_utils.py index 94ae2bfcb1..99b2892a77 100644 --- a/vllm/compilation/passes/fusion/matcher_utils.py +++ b/vllm/compilation/passes/fusion/matcher_utils.py @@ -36,10 +36,12 @@ QUANT_OPS: dict[QuantKey, OpOverload] = { kFp8StaticTensorSym: torch.ops._C.static_scaled_fp8_quant.default, # noqa: E501 kFp8DynamicTensorSym: torch.ops._C.dynamic_scaled_fp8_quant.default, # noqa: E501 kFp8DynamicTokenSym: torch.ops._C.dynamic_per_token_scaled_fp8_quant.default, # noqa: E501 - kFp8Dynamic128Sym: torch.ops._C.per_token_group_fp8_quant.default, # noqa: E501 - kFp8Dynamic64Sym: torch.ops._C.per_token_group_fp8_quant.default, # noqa: E501 } +if hasattr(torch.ops._C, "per_token_group_fp8_quant"): + QUANT_OPS[kFp8Dynamic128Sym] = torch.ops._C.per_token_group_fp8_quant.default # noqa: E501 + QUANT_OPS[kFp8Dynamic64Sym] = torch.ops._C.per_token_group_fp8_quant.default # noqa: E501 + if current_platform.is_cuda() and hasattr(torch.ops._C, "scaled_fp4_quant"): QUANT_OPS[kNvfp4Dynamic] = torch.ops._C.scaled_fp4_quant.out # noqa: E501 diff --git a/vllm/compilation/passes/fusion/rms_quant_fusion.py b/vllm/compilation/passes/fusion/rms_quant_fusion.py index c6a1007806..e85149a879 100644 --- a/vllm/compilation/passes/fusion/rms_quant_fusion.py +++ b/vllm/compilation/passes/fusion/rms_quant_fusion.py @@ -84,9 +84,10 @@ QUANT_OPS: dict[QuantKey, OpOverload] = { kFp8StaticTensorSym: torch.ops._C.static_scaled_fp8_quant.default, # noqa: E501 kFp8DynamicTensorSym: torch.ops._C.dynamic_scaled_fp8_quant.default, # noqa: E501 kFp8DynamicTokenSym: torch.ops._C.dynamic_per_token_scaled_fp8_quant.default, # noqa: E501 - kFp8Dynamic128Sym: torch.ops._C.per_token_group_fp8_quant.default, # noqa: E501 - kFp8Dynamic64Sym: torch.ops._C.per_token_group_fp8_quant.default, # noqa: E501 } +if hasattr(torch.ops._C, "per_token_group_fp8_quant"): + QUANT_OPS[kFp8Dynamic128Sym] = torch.ops._C.per_token_group_fp8_quant.default # noqa: E501 + QUANT_OPS[kFp8Dynamic64Sym] = torch.ops._C.per_token_group_fp8_quant.default # noqa: E501 if current_platform.is_cuda() and hasattr(torch.ops._C, "scaled_fp4_quant"): QUANT_OPS[kNvfp4Dynamic] = torch.ops._C.scaled_fp4_quant.out