diff --git a/vllm/compilation/passes/fusion/rms_quant_fusion.py b/vllm/compilation/passes/fusion/rms_quant_fusion.py index 670349a08b..6c55261e87 100644 --- a/vllm/compilation/passes/fusion/rms_quant_fusion.py +++ b/vllm/compilation/passes/fusion/rms_quant_fusion.py @@ -128,19 +128,14 @@ FUSED_OPS: dict[FusedRMSQuantKey, OpOverload] = { FusedRMSQuantKey( kFp8DynamicTokenSym, True ): torch.ops._C.rms_norm_dynamic_per_token_quant.default, # noqa: E501 - FusedRMSQuantKey( - kFp8Dynamic128Sym, False - ): torch.ops._C.rms_norm_per_block_quant.default, # noqa: E501 - FusedRMSQuantKey( - kFp8Dynamic128Sym, True - ): torch.ops._C.rms_norm_per_block_quant.default, # noqa: E501 - FusedRMSQuantKey( - kFp8Dynamic64Sym, False - ): torch.ops._C.rms_norm_per_block_quant.default, # noqa: E501 - FusedRMSQuantKey( - kFp8Dynamic64Sym, True - ): torch.ops._C.rms_norm_per_block_quant.default, # noqa: E501 } +# rms_norm_per_block_quant is CUDA-only; guard it like per_token_group_fp8_quant above. +if hasattr(torch.ops._C, "rms_norm_per_block_quant"): + _rms_norm_per_block_quant = torch.ops._C.rms_norm_per_block_quant.default + FUSED_OPS[FusedRMSQuantKey(kFp8Dynamic128Sym, False)] = _rms_norm_per_block_quant # noqa: E501 + FUSED_OPS[FusedRMSQuantKey(kFp8Dynamic128Sym, True)] = _rms_norm_per_block_quant # noqa: E501 + FUSED_OPS[FusedRMSQuantKey(kFp8Dynamic64Sym, False)] = _rms_norm_per_block_quant # noqa: E501 + FUSED_OPS[FusedRMSQuantKey(kFp8Dynamic64Sym, True)] = _rms_norm_per_block_quant # noqa: E501 class RMSNormQuantPattern: