[Distributed][Perf] Enable FlashInfer MNNVL allreduce RMS quant fusion (#48064)
Signed-off-by: Mohammad Miadh Angkad <176301910+mmangkad@users.noreply.github.com>
This commit is contained in:
@@ -50,6 +50,20 @@ _IR_RMS_NORM_OP = torch.ops.vllm_ir.rms_norm.default
|
||||
_IR_FUSED_ADD_RMS_NORM_OP = torch.ops.vllm_ir.fused_add_rms_norm.default
|
||||
|
||||
|
||||
def _view_nvfp4_scale_out_for_flashinfer(
|
||||
scale_out: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
"""View vLLM's packed NVFP4 scale buffer as FP8 for FlashInfer."""
|
||||
return torch.ops.aten.view.dtype(scale_out, FP8_DTYPE)
|
||||
|
||||
|
||||
def _view_flashinfer_nvfp4_scale_out_as_int32(
|
||||
scale_out: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
"""View FlashInfer's NVFP4 scale buffer back as vLLM's int32 format."""
|
||||
return torch.ops.aten.view.dtype(scale_out, torch.int32)
|
||||
|
||||
|
||||
def _norm_input_weight_dtype_match(match: pm.Match) -> bool:
|
||||
"""Prevent fusion when the norm input and weight dtypes differ (e.g. a Gemma
|
||||
fp32 weight.float()+1 gamma), covering rms_norm and fused_add_rms_norm."""
|
||||
@@ -196,8 +210,8 @@ if flashinfer_comm is not None:
|
||||
curr_device = current_platform.get_device_capability()
|
||||
device_capability = curr_device.to_int() if curr_device is not None else None
|
||||
|
||||
# Select workspace based on pattern: quant patterns use the
|
||||
# trtllm quant workspace, non-quant patterns use the primary workspace.
|
||||
# Select workspace based on pattern: quant patterns use the quant
|
||||
# workspace, non-quant patterns use the primary workspace.
|
||||
is_quant_pattern = pattern_code in (
|
||||
ar_fusion_patterns.kARResidualRMSNormFP8Quant,
|
||||
ar_fusion_patterns.kARResidualRMSNormFP4Quant,
|
||||
@@ -233,9 +247,9 @@ if flashinfer_comm is not None:
|
||||
residual_out = allreduce_in
|
||||
|
||||
layout_code = None
|
||||
# layout_code only supported by trtllm backend
|
||||
if workspace.backend == "trtllm":
|
||||
# in vllm we only support swizzled layout
|
||||
# vLLM quant patterns use swizzled scale-factor layout. Non-quant
|
||||
# patterns ignore layout_code.
|
||||
if workspace.backend in ("trtllm", "mnnvl"):
|
||||
layout_code = flashinfer_comm.QuantizationSFLayout.SWIZZLED_128x4
|
||||
|
||||
flashinfer_comm.allreduce_fusion(
|
||||
@@ -822,6 +836,7 @@ class AllReduceFusedRMSNormStaticQuantNVFP4Pattern(BasePattern):
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
residual = torch.zeros_like(input)
|
||||
result_rms = torch.empty_like(input)
|
||||
output_scale_fp8 = _view_nvfp4_scale_out_for_flashinfer(output_scale)
|
||||
assert flashinfer_comm is not None, "FlashInfer must be enabled"
|
||||
allreduce = auto_functionalized(
|
||||
flashinfer_trtllm_fused_allreduce_norm,
|
||||
@@ -829,7 +844,7 @@ class AllReduceFusedRMSNormStaticQuantNVFP4Pattern(BasePattern):
|
||||
residual=residual,
|
||||
norm_out=result_rms,
|
||||
quant_out=quant_result,
|
||||
scale_out=output_scale,
|
||||
scale_out=output_scale_fp8,
|
||||
rms_gamma=weight,
|
||||
rms_eps=self.epsilon,
|
||||
# We don't use norm_out afterwards
|
||||
@@ -841,7 +856,11 @@ class AllReduceFusedRMSNormStaticQuantNVFP4Pattern(BasePattern):
|
||||
)
|
||||
|
||||
# quant_out, allreduce_output, output_scale
|
||||
return allreduce[4], allreduce[1], allreduce[5]
|
||||
return (
|
||||
allreduce[4],
|
||||
allreduce[1],
|
||||
_view_flashinfer_nvfp4_scale_out_as_int32(allreduce[5]),
|
||||
)
|
||||
|
||||
pm.register_replacement(
|
||||
pattern,
|
||||
@@ -925,6 +944,7 @@ class AllReduceFusedAddRMSNormStaticQuantNVFP4Pattern(BasePattern):
|
||||
weight: torch.Tensor,
|
||||
input_global_scale: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
output_scale_fp8 = _view_nvfp4_scale_out_for_flashinfer(output_scale)
|
||||
assert flashinfer_comm is not None, "FlashInfer must be enabled"
|
||||
allreduce = auto_functionalized(
|
||||
flashinfer_trtllm_fused_allreduce_norm,
|
||||
@@ -932,7 +952,7 @@ class AllReduceFusedAddRMSNormStaticQuantNVFP4Pattern(BasePattern):
|
||||
residual=residual,
|
||||
norm_out=None,
|
||||
quant_out=quant_result,
|
||||
scale_out=output_scale,
|
||||
scale_out=output_scale_fp8,
|
||||
rms_gamma=weight,
|
||||
rms_eps=self.epsilon,
|
||||
# We don't use norm_out afterwards
|
||||
@@ -943,7 +963,11 @@ class AllReduceFusedAddRMSNormStaticQuantNVFP4Pattern(BasePattern):
|
||||
**self.allreduce_params.get_trtllm_fused_allreduce_kwargs(),
|
||||
)
|
||||
# quant_out, rms_norm_residual, output_scale
|
||||
return allreduce[4], allreduce[2], allreduce[5]
|
||||
return (
|
||||
allreduce[4],
|
||||
allreduce[2],
|
||||
_view_flashinfer_nvfp4_scale_out_as_int32(allreduce[5]),
|
||||
)
|
||||
|
||||
pm.register_replacement(
|
||||
pattern,
|
||||
|
||||
@@ -35,7 +35,9 @@ except ImportError:
|
||||
|
||||
# Workspace for standalone allreduce and non-quant ar+rms fusion
|
||||
_fi_ar_workspace = None
|
||||
# Extra workspace for quant fusion patterns (only supported by trtllm backend)
|
||||
# Extra workspace for quant fusion patterns. This may use either the primary
|
||||
# allreduce backend or a fallback backend when the primary workspace is not
|
||||
# available on the current topology.
|
||||
_fi_ar_quant_workspace = None
|
||||
|
||||
|
||||
@@ -189,38 +191,64 @@ def get_fi_ar_quant_workspace(
|
||||
"""
|
||||
Return the allreduce workspace for quant patterns, initializing if needed.
|
||||
|
||||
Always uses trtllm backend as it is the only one supporting quantization
|
||||
fusion (FP8/FP4). Returns None for multi-node setups since not supported
|
||||
by trtllm backend.
|
||||
Backend is controlled by VLLM_FLASHINFER_ALLREDUCE_BACKEND env var, matching
|
||||
non-quant fusion. With ``auto`` this prefers mnnvl and falls back to trtllm
|
||||
only on single-node topologies where mnnvl multicast is unavailable.
|
||||
"""
|
||||
global _fi_ar_quant_workspace
|
||||
if _fi_ar_quant_workspace is not None:
|
||||
return _fi_ar_quant_workspace
|
||||
|
||||
if get_node_count() > 1:
|
||||
logger.warning_once(
|
||||
"Flashinfer allreduce quantization fusion is not supported for "
|
||||
"multi-node allreduce. Disabling quant fusion."
|
||||
)
|
||||
return None
|
||||
backend, allow_trtllm_fallback = _resolve_fi_ar_backend()
|
||||
|
||||
# Reuse the non-quant workspace if it was already created with trtllm
|
||||
if _fi_ar_workspace is not None and _fi_ar_workspace.backend == "trtllm":
|
||||
if get_node_count() > 1 and backend == "trtllm":
|
||||
raise ValueError(
|
||||
"Flashinfer allreduce quantization fusion is not supported for "
|
||||
"multi-node allreduce with 'trtllm' backend. Please use 'mnnvl' "
|
||||
"backend instead."
|
||||
)
|
||||
|
||||
# Reuse the non-quant workspace if it was already created with the same
|
||||
# backend.
|
||||
if _fi_ar_workspace is not None and _fi_ar_workspace.backend == backend:
|
||||
_fi_ar_quant_workspace = _fi_ar_workspace
|
||||
return _fi_ar_quant_workspace
|
||||
|
||||
if (
|
||||
_fi_ar_workspace is not None
|
||||
and _fi_ar_workspace.backend == "trtllm"
|
||||
and allow_trtllm_fallback
|
||||
and backend != "trtllm"
|
||||
):
|
||||
_fi_ar_quant_workspace = _fi_ar_workspace
|
||||
return _fi_ar_quant_workspace
|
||||
|
||||
_fi_ar_quant_workspace = _create_workspace(
|
||||
"trtllm", world_size, rank, max_token_num, hidden_dim, dtype, group
|
||||
backend, world_size, rank, max_token_num, hidden_dim, dtype, group
|
||||
)
|
||||
if _fi_ar_quant_workspace is None and allow_trtllm_fallback and backend != "trtllm":
|
||||
logger.warning_once(
|
||||
"FlashInfer mnnvl allreduce quantization fusion workspace unavailable "
|
||||
"(likely no NVSwitch multicast support); falling back to trtllm "
|
||||
"backend for single node."
|
||||
)
|
||||
backend = "trtllm"
|
||||
if _fi_ar_workspace is not None and _fi_ar_workspace.backend == backend:
|
||||
_fi_ar_quant_workspace = _fi_ar_workspace
|
||||
else:
|
||||
_fi_ar_quant_workspace = _create_workspace(
|
||||
backend, world_size, rank, max_token_num, hidden_dim, dtype, group
|
||||
)
|
||||
|
||||
if _fi_ar_quant_workspace is not None:
|
||||
logger.info_once(
|
||||
"Initialized FlashInfer Allreduce norm quantization "
|
||||
"fusion workspace with backend=trtllm"
|
||||
f"fusion workspace with backend={backend}"
|
||||
)
|
||||
else:
|
||||
logger.warning_once(
|
||||
"Failed to initialize FlashInfer Allreduce norm quantization "
|
||||
"fusion workspace with backend=trtllm"
|
||||
f"fusion workspace with backend={backend}"
|
||||
)
|
||||
|
||||
return _fi_ar_quant_workspace
|
||||
|
||||
Reference in New Issue
Block a user