[Kernel] FlashInfer: switch allreduce fusion to unified API (#33985)
Signed-off-by: Mohammad Miadh Angkad <176301910+mmangkad@users.noreply.github.com>
This commit is contained in:
@@ -1,5 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import contextlib
|
||||
from importlib.util import find_spec
|
||||
from types import ModuleType
|
||||
|
||||
@@ -36,7 +37,9 @@ if find_spec("flashinfer"):
|
||||
try:
|
||||
import flashinfer.comm as _flashinfer_comm
|
||||
|
||||
if hasattr(_flashinfer_comm, "trtllm_allreduce_fusion"):
|
||||
if hasattr(_flashinfer_comm, "allreduce_fusion") and hasattr(
|
||||
_flashinfer_comm, "create_allreduce_fusion_workspace"
|
||||
):
|
||||
flashinfer_comm = _flashinfer_comm
|
||||
except ImportError:
|
||||
pass
|
||||
@@ -79,7 +82,7 @@ _FI_ALLREDUCE_ONE_SHOT_MAX_SIZES_MB: dict[int, dict[int, float]] = {
|
||||
|
||||
|
||||
if flashinfer_comm is not None:
|
||||
_FI_WORKSPACE_TENSOR = None
|
||||
_FI_WORKSPACE = None
|
||||
MiB = 1024 * 1024
|
||||
|
||||
def call_trtllm_fused_allreduce_norm(
|
||||
@@ -87,10 +90,8 @@ if flashinfer_comm is not None:
|
||||
residual: torch.Tensor,
|
||||
rms_gamma: torch.Tensor,
|
||||
rms_eps: float,
|
||||
world_rank: int,
|
||||
world_size: int,
|
||||
launch_with_pdl: bool,
|
||||
trigger_completion_at_end: bool,
|
||||
fp32_acc: bool,
|
||||
max_token_num: int,
|
||||
pattern_code: int,
|
||||
@@ -121,7 +122,7 @@ if flashinfer_comm is not None:
|
||||
max_one_shot_size is None or current_tensor_size <= max_one_shot_size * MiB
|
||||
)
|
||||
|
||||
assert _FI_WORKSPACE_TENSOR is not None, (
|
||||
assert _FI_WORKSPACE is not None, (
|
||||
"Flashinfer must be enabled when using flashinfer"
|
||||
)
|
||||
if norm_out is None:
|
||||
@@ -134,24 +135,18 @@ if flashinfer_comm is not None:
|
||||
residual_out = allreduce_in
|
||||
# For the sizes that are smaller than the max size,
|
||||
# we only use flashinfer one shot allreduce
|
||||
flashinfer_comm.trtllm_allreduce_fusion(
|
||||
allreduce_in=allreduce_in,
|
||||
token_num=allreduce_in.shape[0],
|
||||
flashinfer_comm.allreduce_fusion(
|
||||
input=allreduce_in,
|
||||
workspace=_FI_WORKSPACE,
|
||||
pattern=pattern_code,
|
||||
residual_in=residual,
|
||||
residual_out=residual_out,
|
||||
norm_out=norm_out,
|
||||
rms_gamma=rms_gamma,
|
||||
rms_eps=rms_eps,
|
||||
world_rank=world_rank,
|
||||
world_size=world_size,
|
||||
hidden_dim=allreduce_in.shape[-1],
|
||||
workspace_ptrs=_FI_WORKSPACE_TENSOR,
|
||||
launch_with_pdl=launch_with_pdl,
|
||||
use_oneshot=use_oneshot,
|
||||
trigger_completion_at_end=trigger_completion_at_end,
|
||||
fp32_acc=fp32_acc,
|
||||
pattern_code=pattern_code,
|
||||
allreduce_out=None,
|
||||
quant_out=quant_out,
|
||||
scale_out=scale_out,
|
||||
# in vllm we only support swizzled layout
|
||||
@@ -164,10 +159,8 @@ if flashinfer_comm is not None:
|
||||
residual: torch.Tensor,
|
||||
rms_gamma: torch.Tensor,
|
||||
rms_eps: float,
|
||||
world_rank: int,
|
||||
world_size: int,
|
||||
launch_with_pdl: bool,
|
||||
trigger_completion_at_end: bool,
|
||||
fp32_acc: bool,
|
||||
max_token_num: int,
|
||||
pattern_code: int,
|
||||
@@ -200,25 +193,18 @@ class FlashInferFusedAllReduceParams:
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
rank: int,
|
||||
world_size: int,
|
||||
use_fp32_lamport: bool = False,
|
||||
max_token_num: int = 1024,
|
||||
) -> None:
|
||||
self.rank = rank
|
||||
self.world_size = world_size
|
||||
self.use_fp32_lamport = use_fp32_lamport
|
||||
self.trigger_completion_at_end = True
|
||||
self.launch_with_pdl = True
|
||||
self.fp32_acc = True
|
||||
self.max_token_num = max_token_num
|
||||
|
||||
def get_trtllm_fused_allreduce_kwargs(self) -> dict[str, bool | int]:
|
||||
return {
|
||||
"world_rank": self.rank,
|
||||
"world_size": self.world_size,
|
||||
"launch_with_pdl": self.launch_with_pdl,
|
||||
"trigger_completion_at_end": self.trigger_completion_at_end,
|
||||
"fp32_acc": self.fp32_acc,
|
||||
"max_token_num": self.max_token_num,
|
||||
}
|
||||
@@ -712,7 +698,6 @@ class AllReduceFusionPass(VllmPatternMatcherPass):
|
||||
self.hidden_dim = config.model_config.get_hidden_size()
|
||||
self.group = get_tp_group().device_group
|
||||
rank = get_tensor_model_parallel_rank()
|
||||
use_fp32_lamport = self.model_dtype == torch.float32
|
||||
if flashinfer_comm is None:
|
||||
logger.warning(
|
||||
"Flashinfer is not installed or comm module not found, "
|
||||
@@ -730,7 +715,7 @@ class AllReduceFusionPass(VllmPatternMatcherPass):
|
||||
self.tp_size,
|
||||
)
|
||||
return
|
||||
element_size = 4 if use_fp32_lamport else 2
|
||||
element_size = torch.tensor([], dtype=self.model_dtype).element_size()
|
||||
self.max_token_num = max_size // (self.hidden_dim * element_size)
|
||||
# take the min to save workspace size and we'll never use more
|
||||
# than max_num_batched_tokens anyways
|
||||
@@ -744,23 +729,19 @@ class AllReduceFusionPass(VllmPatternMatcherPass):
|
||||
scope="global",
|
||||
)
|
||||
|
||||
self.ipc_handles, workspace_tensor = (
|
||||
flashinfer_comm.trtllm_create_ipc_workspace_for_all_reduce_fusion(
|
||||
tp_rank=rank,
|
||||
tp_size=self.tp_size,
|
||||
max_token_num=self.max_token_num,
|
||||
hidden_dim=self.hidden_dim,
|
||||
group=self.group,
|
||||
use_fp32_lamport=use_fp32_lamport,
|
||||
)
|
||||
self.workspace = flashinfer_comm.create_allreduce_fusion_workspace(
|
||||
backend="trtllm",
|
||||
world_size=self.tp_size,
|
||||
rank=rank,
|
||||
max_token_num=self.max_token_num,
|
||||
hidden_dim=self.hidden_dim,
|
||||
dtype=self.model_dtype,
|
||||
)
|
||||
|
||||
global _FI_WORKSPACE_TENSOR
|
||||
_FI_WORKSPACE_TENSOR = workspace_tensor
|
||||
global _FI_WORKSPACE
|
||||
_FI_WORKSPACE = self.workspace
|
||||
self.allreduce_params = FlashInferFusedAllReduceParams(
|
||||
rank=rank,
|
||||
world_size=self.tp_size,
|
||||
use_fp32_lamport=use_fp32_lamport,
|
||||
max_token_num=self.max_token_num,
|
||||
)
|
||||
|
||||
@@ -832,7 +813,6 @@ class AllReduceFusionPass(VllmPatternMatcherPass):
|
||||
def __del__(self) -> None:
|
||||
if getattr(self, "disabled", True):
|
||||
return
|
||||
if flashinfer_comm is not None:
|
||||
flashinfer_comm.trtllm_destroy_ipc_workspace_for_all_reduce(
|
||||
self.ipc_handles, self.group
|
||||
)
|
||||
if getattr(self, "workspace", None) is not None:
|
||||
with contextlib.suppress(Exception):
|
||||
self.workspace.destroy()
|
||||
|
||||
Reference in New Issue
Block a user