[Minor] Bigger overlap for FI AR (#43103)

Signed-off-by: Jee Jee Li <jeejeelee@inferact.ai>
This commit is contained in:
Jee Jee Li
2026-05-20 21:20:57 -07:00
committed by GitHub
parent 5d041cc1fe
commit ee05e8137e
2 changed files with 10 additions and 1 deletions
@@ -44,6 +44,10 @@ from .matcher_utils import MatcherQuantFP8
FP8_DTYPE = current_platform.fp8_dtype()
# The empirical value for small batch
PDL_ADVANCE_LAUNCH_TOKENS = 16
logger = init_logger(__name__)
flashinfer_comm: ModuleType | None = None
@@ -204,6 +208,7 @@ if flashinfer_comm is not None:
layout_code=layout_code,
use_oneshot=use_oneshot,
fp32_acc=fp32_acc,
trigger_completion_at_end=num_tokens > PDL_ADVANCE_LAUNCH_TOKENS,
)
def call_trtllm_fused_allreduce_norm_fake(
@@ -19,6 +19,8 @@ from vllm.platforms import current_platform
logger = init_logger(__name__)
# The empirical value for small batch
PDL_ADVANCE_LAUNCH_TOKENS = 16
fi_ar_available = False
try:
@@ -312,7 +314,7 @@ class FlashInferAllReduce:
return self._ensure_workspace(hidden_dim, input_tensor.dtype)
def all_reduce(self, input_tensor: torch.Tensor) -> torch.Tensor:
_, hidden_dim = input_tensor.shape
num_tokens, hidden_dim = input_tensor.shape
workspace = get_fi_ar_workspace(
world_size=self.world_size,
rank=self.rank,
@@ -325,6 +327,8 @@ class FlashInferAllReduce:
input=input_tensor,
workspace=workspace,
pattern=flashinfer_comm.AllReduceFusionPattern.kAllReduce,
launch_with_pdl=True,
trigger_completion_at_end=num_tokens > PDL_ADVANCE_LAUNCH_TOKENS,
)
def destroy(self):