diff --git a/vllm/compilation/passes/fusion/allreduce_rms_fusion.py b/vllm/compilation/passes/fusion/allreduce_rms_fusion.py index 83423c02a6..5406f611e8 100644 --- a/vllm/compilation/passes/fusion/allreduce_rms_fusion.py +++ b/vllm/compilation/passes/fusion/allreduce_rms_fusion.py @@ -44,6 +44,10 @@ from .matcher_utils import MatcherQuantFP8 FP8_DTYPE = current_platform.fp8_dtype() + +# The empirical value for small batch +PDL_ADVANCE_LAUNCH_TOKENS = 16 + logger = init_logger(__name__) flashinfer_comm: ModuleType | None = None @@ -204,6 +208,7 @@ if flashinfer_comm is not None: layout_code=layout_code, use_oneshot=use_oneshot, fp32_acc=fp32_acc, + trigger_completion_at_end=num_tokens > PDL_ADVANCE_LAUNCH_TOKENS, ) def call_trtllm_fused_allreduce_norm_fake( diff --git a/vllm/distributed/device_communicators/flashinfer_all_reduce.py b/vllm/distributed/device_communicators/flashinfer_all_reduce.py index a65789a283..2594c0cf16 100644 --- a/vllm/distributed/device_communicators/flashinfer_all_reduce.py +++ b/vllm/distributed/device_communicators/flashinfer_all_reduce.py @@ -19,6 +19,8 @@ from vllm.platforms import current_platform logger = init_logger(__name__) +# The empirical value for small batch +PDL_ADVANCE_LAUNCH_TOKENS = 16 fi_ar_available = False try: @@ -312,7 +314,7 @@ class FlashInferAllReduce: return self._ensure_workspace(hidden_dim, input_tensor.dtype) def all_reduce(self, input_tensor: torch.Tensor) -> torch.Tensor: - _, hidden_dim = input_tensor.shape + num_tokens, hidden_dim = input_tensor.shape workspace = get_fi_ar_workspace( world_size=self.world_size, rank=self.rank, @@ -325,6 +327,8 @@ class FlashInferAllReduce: input=input_tensor, workspace=workspace, pattern=flashinfer_comm.AllReduceFusionPattern.kAllReduce, + launch_with_pdl=True, + trigger_completion_at_end=num_tokens > PDL_ADVANCE_LAUNCH_TOKENS, ) def destroy(self):