[Distributed][Perf] Enable FlashInfer MNNVL allreduce RMS quant fusion (#48064)

Signed-off-by: Mohammad Miadh Angkad <176301910+mmangkad@users.noreply.github.com>
This commit is contained in:
Mohammad Miadh Angkad
2026-07-13 15:02:59 +08:00
committed by GitHub
parent b7b58d1eba
commit 75fe92a316
2 changed files with 76 additions and 24 deletions
@@ -50,6 +50,20 @@ _IR_RMS_NORM_OP = torch.ops.vllm_ir.rms_norm.default
_IR_FUSED_ADD_RMS_NORM_OP = torch.ops.vllm_ir.fused_add_rms_norm.default
def _view_nvfp4_scale_out_for_flashinfer(
scale_out: torch.Tensor,
) -> torch.Tensor:
"""View vLLM's packed NVFP4 scale buffer as FP8 for FlashInfer."""
return torch.ops.aten.view.dtype(scale_out, FP8_DTYPE)
def _view_flashinfer_nvfp4_scale_out_as_int32(
scale_out: torch.Tensor,
) -> torch.Tensor:
"""View FlashInfer's NVFP4 scale buffer back as vLLM's int32 format."""
return torch.ops.aten.view.dtype(scale_out, torch.int32)
def _norm_input_weight_dtype_match(match: pm.Match) -> bool:
"""Prevent fusion when the norm input and weight dtypes differ (e.g. a Gemma
fp32 weight.float()+1 gamma), covering rms_norm and fused_add_rms_norm."""
@@ -196,8 +210,8 @@ if flashinfer_comm is not None:
curr_device = current_platform.get_device_capability()
device_capability = curr_device.to_int() if curr_device is not None else None
# Select workspace based on pattern: quant patterns use the
# trtllm quant workspace, non-quant patterns use the primary workspace.
# Select workspace based on pattern: quant patterns use the quant
# workspace, non-quant patterns use the primary workspace.
is_quant_pattern = pattern_code in (
ar_fusion_patterns.kARResidualRMSNormFP8Quant,
ar_fusion_patterns.kARResidualRMSNormFP4Quant,
@@ -233,9 +247,9 @@ if flashinfer_comm is not None:
residual_out = allreduce_in
layout_code = None
# layout_code only supported by trtllm backend
if workspace.backend == "trtllm":
# in vllm we only support swizzled layout
# vLLM quant patterns use swizzled scale-factor layout. Non-quant
# patterns ignore layout_code.
if workspace.backend in ("trtllm", "mnnvl"):
layout_code = flashinfer_comm.QuantizationSFLayout.SWIZZLED_128x4
flashinfer_comm.allreduce_fusion(
@@ -822,6 +836,7 @@ class AllReduceFusedRMSNormStaticQuantNVFP4Pattern(BasePattern):
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
residual = torch.zeros_like(input)
result_rms = torch.empty_like(input)
output_scale_fp8 = _view_nvfp4_scale_out_for_flashinfer(output_scale)
assert flashinfer_comm is not None, "FlashInfer must be enabled"
allreduce = auto_functionalized(
flashinfer_trtllm_fused_allreduce_norm,
@@ -829,7 +844,7 @@ class AllReduceFusedRMSNormStaticQuantNVFP4Pattern(BasePattern):
residual=residual,
norm_out=result_rms,
quant_out=quant_result,
scale_out=output_scale,
scale_out=output_scale_fp8,
rms_gamma=weight,
rms_eps=self.epsilon,
# We don't use norm_out afterwards
@@ -841,7 +856,11 @@ class AllReduceFusedRMSNormStaticQuantNVFP4Pattern(BasePattern):
)
# quant_out, allreduce_output, output_scale
return allreduce[4], allreduce[1], allreduce[5]
return (
allreduce[4],
allreduce[1],
_view_flashinfer_nvfp4_scale_out_as_int32(allreduce[5]),
)
pm.register_replacement(
pattern,
@@ -925,6 +944,7 @@ class AllReduceFusedAddRMSNormStaticQuantNVFP4Pattern(BasePattern):
weight: torch.Tensor,
input_global_scale: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
output_scale_fp8 = _view_nvfp4_scale_out_for_flashinfer(output_scale)
assert flashinfer_comm is not None, "FlashInfer must be enabled"
allreduce = auto_functionalized(
flashinfer_trtllm_fused_allreduce_norm,
@@ -932,7 +952,7 @@ class AllReduceFusedAddRMSNormStaticQuantNVFP4Pattern(BasePattern):
residual=residual,
norm_out=None,
quant_out=quant_result,
scale_out=output_scale,
scale_out=output_scale_fp8,
rms_gamma=weight,
rms_eps=self.epsilon,
# We don't use norm_out afterwards
@@ -943,7 +963,11 @@ class AllReduceFusedAddRMSNormStaticQuantNVFP4Pattern(BasePattern):
**self.allreduce_params.get_trtllm_fused_allreduce_kwargs(),
)
# quant_out, rms_norm_residual, output_scale
return allreduce[4], allreduce[2], allreduce[5]
return (
allreduce[4],
allreduce[2],
_view_flashinfer_nvfp4_scale_out_as_int32(allreduce[5]),
)
pm.register_replacement(
pattern,
@@ -35,7 +35,9 @@ except ImportError:
# Workspace for standalone allreduce and non-quant ar+rms fusion
_fi_ar_workspace = None
# Extra workspace for quant fusion patterns (only supported by trtllm backend)
# Extra workspace for quant fusion patterns. This may use either the primary
# allreduce backend or a fallback backend when the primary workspace is not
# available on the current topology.
_fi_ar_quant_workspace = None
@@ -189,38 +191,64 @@ def get_fi_ar_quant_workspace(
"""
Return the allreduce workspace for quant patterns, initializing if needed.
Always uses trtllm backend as it is the only one supporting quantization
fusion (FP8/FP4). Returns None for multi-node setups since not supported
by trtllm backend.
Backend is controlled by VLLM_FLASHINFER_ALLREDUCE_BACKEND env var, matching
non-quant fusion. With ``auto`` this prefers mnnvl and falls back to trtllm
only on single-node topologies where mnnvl multicast is unavailable.
"""
global _fi_ar_quant_workspace
if _fi_ar_quant_workspace is not None:
return _fi_ar_quant_workspace
if get_node_count() > 1:
logger.warning_once(
"Flashinfer allreduce quantization fusion is not supported for "
"multi-node allreduce. Disabling quant fusion."
)
return None
backend, allow_trtllm_fallback = _resolve_fi_ar_backend()
# Reuse the non-quant workspace if it was already created with trtllm
if _fi_ar_workspace is not None and _fi_ar_workspace.backend == "trtllm":
if get_node_count() > 1 and backend == "trtllm":
raise ValueError(
"Flashinfer allreduce quantization fusion is not supported for "
"multi-node allreduce with 'trtllm' backend. Please use 'mnnvl' "
"backend instead."
)
# Reuse the non-quant workspace if it was already created with the same
# backend.
if _fi_ar_workspace is not None and _fi_ar_workspace.backend == backend:
_fi_ar_quant_workspace = _fi_ar_workspace
return _fi_ar_quant_workspace
if (
_fi_ar_workspace is not None
and _fi_ar_workspace.backend == "trtllm"
and allow_trtllm_fallback
and backend != "trtllm"
):
_fi_ar_quant_workspace = _fi_ar_workspace
return _fi_ar_quant_workspace
_fi_ar_quant_workspace = _create_workspace(
"trtllm", world_size, rank, max_token_num, hidden_dim, dtype, group
backend, world_size, rank, max_token_num, hidden_dim, dtype, group
)
if _fi_ar_quant_workspace is None and allow_trtllm_fallback and backend != "trtllm":
logger.warning_once(
"FlashInfer mnnvl allreduce quantization fusion workspace unavailable "
"(likely no NVSwitch multicast support); falling back to trtllm "
"backend for single node."
)
backend = "trtllm"
if _fi_ar_workspace is not None and _fi_ar_workspace.backend == backend:
_fi_ar_quant_workspace = _fi_ar_workspace
else:
_fi_ar_quant_workspace = _create_workspace(
backend, world_size, rank, max_token_num, hidden_dim, dtype, group
)
if _fi_ar_quant_workspace is not None:
logger.info_once(
"Initialized FlashInfer Allreduce norm quantization "
"fusion workspace with backend=trtllm"
f"fusion workspace with backend={backend}"
)
else:
logger.warning_once(
"Failed to initialize FlashInfer Allreduce norm quantization "
"fusion workspace with backend=trtllm"
f"fusion workspace with backend={backend}"
)
return _fi_ar_quant_workspace