From 75fe92a3162a68d74581ec324f04684a752e3ad2 Mon Sep 17 00:00:00 2001 From: Mohammad Miadh Angkad <176301910+mmangkad@users.noreply.github.com> Date: Mon, 13 Jul 2026 15:02:59 +0800 Subject: [PATCH] [Distributed][Perf] Enable FlashInfer MNNVL allreduce RMS quant fusion (#48064) Signed-off-by: Mohammad Miadh Angkad <176301910+mmangkad@users.noreply.github.com> --- .../passes/fusion/allreduce_rms_fusion.py | 42 +++++++++++--- .../flashinfer_all_reduce.py | 58 ++++++++++++++----- 2 files changed, 76 insertions(+), 24 deletions(-) diff --git a/vllm/compilation/passes/fusion/allreduce_rms_fusion.py b/vllm/compilation/passes/fusion/allreduce_rms_fusion.py index 906dea8101..4716e7012f 100644 --- a/vllm/compilation/passes/fusion/allreduce_rms_fusion.py +++ b/vllm/compilation/passes/fusion/allreduce_rms_fusion.py @@ -50,6 +50,20 @@ _IR_RMS_NORM_OP = torch.ops.vllm_ir.rms_norm.default _IR_FUSED_ADD_RMS_NORM_OP = torch.ops.vllm_ir.fused_add_rms_norm.default +def _view_nvfp4_scale_out_for_flashinfer( + scale_out: torch.Tensor, +) -> torch.Tensor: + """View vLLM's packed NVFP4 scale buffer as FP8 for FlashInfer.""" + return torch.ops.aten.view.dtype(scale_out, FP8_DTYPE) + + +def _view_flashinfer_nvfp4_scale_out_as_int32( + scale_out: torch.Tensor, +) -> torch.Tensor: + """View FlashInfer's NVFP4 scale buffer back as vLLM's int32 format.""" + return torch.ops.aten.view.dtype(scale_out, torch.int32) + + def _norm_input_weight_dtype_match(match: pm.Match) -> bool: """Prevent fusion when the norm input and weight dtypes differ (e.g. a Gemma fp32 weight.float()+1 gamma), covering rms_norm and fused_add_rms_norm.""" @@ -196,8 +210,8 @@ if flashinfer_comm is not None: curr_device = current_platform.get_device_capability() device_capability = curr_device.to_int() if curr_device is not None else None - # Select workspace based on pattern: quant patterns use the - # trtllm quant workspace, non-quant patterns use the primary workspace. + # Select workspace based on pattern: quant patterns use the quant + # workspace, non-quant patterns use the primary workspace. is_quant_pattern = pattern_code in ( ar_fusion_patterns.kARResidualRMSNormFP8Quant, ar_fusion_patterns.kARResidualRMSNormFP4Quant, @@ -233,9 +247,9 @@ if flashinfer_comm is not None: residual_out = allreduce_in layout_code = None - # layout_code only supported by trtllm backend - if workspace.backend == "trtllm": - # in vllm we only support swizzled layout + # vLLM quant patterns use swizzled scale-factor layout. Non-quant + # patterns ignore layout_code. + if workspace.backend in ("trtllm", "mnnvl"): layout_code = flashinfer_comm.QuantizationSFLayout.SWIZZLED_128x4 flashinfer_comm.allreduce_fusion( @@ -822,6 +836,7 @@ class AllReduceFusedRMSNormStaticQuantNVFP4Pattern(BasePattern): ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: residual = torch.zeros_like(input) result_rms = torch.empty_like(input) + output_scale_fp8 = _view_nvfp4_scale_out_for_flashinfer(output_scale) assert flashinfer_comm is not None, "FlashInfer must be enabled" allreduce = auto_functionalized( flashinfer_trtllm_fused_allreduce_norm, @@ -829,7 +844,7 @@ class AllReduceFusedRMSNormStaticQuantNVFP4Pattern(BasePattern): residual=residual, norm_out=result_rms, quant_out=quant_result, - scale_out=output_scale, + scale_out=output_scale_fp8, rms_gamma=weight, rms_eps=self.epsilon, # We don't use norm_out afterwards @@ -841,7 +856,11 @@ class AllReduceFusedRMSNormStaticQuantNVFP4Pattern(BasePattern): ) # quant_out, allreduce_output, output_scale - return allreduce[4], allreduce[1], allreduce[5] + return ( + allreduce[4], + allreduce[1], + _view_flashinfer_nvfp4_scale_out_as_int32(allreduce[5]), + ) pm.register_replacement( pattern, @@ -925,6 +944,7 @@ class AllReduceFusedAddRMSNormStaticQuantNVFP4Pattern(BasePattern): weight: torch.Tensor, input_global_scale: torch.Tensor, ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + output_scale_fp8 = _view_nvfp4_scale_out_for_flashinfer(output_scale) assert flashinfer_comm is not None, "FlashInfer must be enabled" allreduce = auto_functionalized( flashinfer_trtllm_fused_allreduce_norm, @@ -932,7 +952,7 @@ class AllReduceFusedAddRMSNormStaticQuantNVFP4Pattern(BasePattern): residual=residual, norm_out=None, quant_out=quant_result, - scale_out=output_scale, + scale_out=output_scale_fp8, rms_gamma=weight, rms_eps=self.epsilon, # We don't use norm_out afterwards @@ -943,7 +963,11 @@ class AllReduceFusedAddRMSNormStaticQuantNVFP4Pattern(BasePattern): **self.allreduce_params.get_trtllm_fused_allreduce_kwargs(), ) # quant_out, rms_norm_residual, output_scale - return allreduce[4], allreduce[2], allreduce[5] + return ( + allreduce[4], + allreduce[2], + _view_flashinfer_nvfp4_scale_out_as_int32(allreduce[5]), + ) pm.register_replacement( pattern, diff --git a/vllm/distributed/device_communicators/flashinfer_all_reduce.py b/vllm/distributed/device_communicators/flashinfer_all_reduce.py index 86a8524b68..881b265d24 100644 --- a/vllm/distributed/device_communicators/flashinfer_all_reduce.py +++ b/vllm/distributed/device_communicators/flashinfer_all_reduce.py @@ -35,7 +35,9 @@ except ImportError: # Workspace for standalone allreduce and non-quant ar+rms fusion _fi_ar_workspace = None -# Extra workspace for quant fusion patterns (only supported by trtllm backend) +# Extra workspace for quant fusion patterns. This may use either the primary +# allreduce backend or a fallback backend when the primary workspace is not +# available on the current topology. _fi_ar_quant_workspace = None @@ -189,38 +191,64 @@ def get_fi_ar_quant_workspace( """ Return the allreduce workspace for quant patterns, initializing if needed. - Always uses trtllm backend as it is the only one supporting quantization - fusion (FP8/FP4). Returns None for multi-node setups since not supported - by trtllm backend. + Backend is controlled by VLLM_FLASHINFER_ALLREDUCE_BACKEND env var, matching + non-quant fusion. With ``auto`` this prefers mnnvl and falls back to trtllm + only on single-node topologies where mnnvl multicast is unavailable. """ global _fi_ar_quant_workspace if _fi_ar_quant_workspace is not None: return _fi_ar_quant_workspace - if get_node_count() > 1: - logger.warning_once( - "Flashinfer allreduce quantization fusion is not supported for " - "multi-node allreduce. Disabling quant fusion." - ) - return None + backend, allow_trtllm_fallback = _resolve_fi_ar_backend() - # Reuse the non-quant workspace if it was already created with trtllm - if _fi_ar_workspace is not None and _fi_ar_workspace.backend == "trtllm": + if get_node_count() > 1 and backend == "trtllm": + raise ValueError( + "Flashinfer allreduce quantization fusion is not supported for " + "multi-node allreduce with 'trtllm' backend. Please use 'mnnvl' " + "backend instead." + ) + + # Reuse the non-quant workspace if it was already created with the same + # backend. + if _fi_ar_workspace is not None and _fi_ar_workspace.backend == backend: + _fi_ar_quant_workspace = _fi_ar_workspace + return _fi_ar_quant_workspace + + if ( + _fi_ar_workspace is not None + and _fi_ar_workspace.backend == "trtllm" + and allow_trtllm_fallback + and backend != "trtllm" + ): _fi_ar_quant_workspace = _fi_ar_workspace return _fi_ar_quant_workspace _fi_ar_quant_workspace = _create_workspace( - "trtllm", world_size, rank, max_token_num, hidden_dim, dtype, group + backend, world_size, rank, max_token_num, hidden_dim, dtype, group ) + if _fi_ar_quant_workspace is None and allow_trtllm_fallback and backend != "trtllm": + logger.warning_once( + "FlashInfer mnnvl allreduce quantization fusion workspace unavailable " + "(likely no NVSwitch multicast support); falling back to trtllm " + "backend for single node." + ) + backend = "trtllm" + if _fi_ar_workspace is not None and _fi_ar_workspace.backend == backend: + _fi_ar_quant_workspace = _fi_ar_workspace + else: + _fi_ar_quant_workspace = _create_workspace( + backend, world_size, rank, max_token_num, hidden_dim, dtype, group + ) + if _fi_ar_quant_workspace is not None: logger.info_once( "Initialized FlashInfer Allreduce norm quantization " - "fusion workspace with backend=trtllm" + f"fusion workspace with backend={backend}" ) else: logger.warning_once( "Failed to initialize FlashInfer Allreduce norm quantization " - "fusion workspace with backend=trtllm" + f"fusion workspace with backend={backend}" ) return _fi_ar_quant_workspace