[Minor] Bigger overlap for FI AR (#43103)
Signed-off-by: Jee Jee Li <jeejeelee@inferact.ai>
This commit is contained in:
@@ -44,6 +44,10 @@ from .matcher_utils import MatcherQuantFP8
|
||||
|
||||
FP8_DTYPE = current_platform.fp8_dtype()
|
||||
|
||||
|
||||
# The empirical value for small batch
|
||||
PDL_ADVANCE_LAUNCH_TOKENS = 16
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
flashinfer_comm: ModuleType | None = None
|
||||
@@ -204,6 +208,7 @@ if flashinfer_comm is not None:
|
||||
layout_code=layout_code,
|
||||
use_oneshot=use_oneshot,
|
||||
fp32_acc=fp32_acc,
|
||||
trigger_completion_at_end=num_tokens > PDL_ADVANCE_LAUNCH_TOKENS,
|
||||
)
|
||||
|
||||
def call_trtllm_fused_allreduce_norm_fake(
|
||||
|
||||
@@ -19,6 +19,8 @@ from vllm.platforms import current_platform
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
# The empirical value for small batch
|
||||
PDL_ADVANCE_LAUNCH_TOKENS = 16
|
||||
|
||||
fi_ar_available = False
|
||||
try:
|
||||
@@ -312,7 +314,7 @@ class FlashInferAllReduce:
|
||||
return self._ensure_workspace(hidden_dim, input_tensor.dtype)
|
||||
|
||||
def all_reduce(self, input_tensor: torch.Tensor) -> torch.Tensor:
|
||||
_, hidden_dim = input_tensor.shape
|
||||
num_tokens, hidden_dim = input_tensor.shape
|
||||
workspace = get_fi_ar_workspace(
|
||||
world_size=self.world_size,
|
||||
rank=self.rank,
|
||||
@@ -325,6 +327,8 @@ class FlashInferAllReduce:
|
||||
input=input_tensor,
|
||||
workspace=workspace,
|
||||
pattern=flashinfer_comm.AllReduceFusionPattern.kAllReduce,
|
||||
launch_with_pdl=True,
|
||||
trigger_completion_at_end=num_tokens > PDL_ADVANCE_LAUNCH_TOKENS,
|
||||
)
|
||||
|
||||
def destroy(self):
|
||||
|
||||
Reference in New Issue
Block a user