[Minor] Bigger overlap for FI AR (#43103)
Signed-off-by: Jee Jee Li <jeejeelee@inferact.ai>
This commit is contained in:
@@ -44,6 +44,10 @@ from .matcher_utils import MatcherQuantFP8
|
||||
|
||||
FP8_DTYPE = current_platform.fp8_dtype()
|
||||
|
||||
|
||||
# The empirical value for small batch
|
||||
PDL_ADVANCE_LAUNCH_TOKENS = 16
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
flashinfer_comm: ModuleType | None = None
|
||||
@@ -204,6 +208,7 @@ if flashinfer_comm is not None:
|
||||
layout_code=layout_code,
|
||||
use_oneshot=use_oneshot,
|
||||
fp32_acc=fp32_acc,
|
||||
trigger_completion_at_end=num_tokens > PDL_ADVANCE_LAUNCH_TOKENS,
|
||||
)
|
||||
|
||||
def call_trtllm_fused_allreduce_norm_fake(
|
||||
|
||||
Reference in New Issue
Block a user