[compile] Add FlashInfer FP8 async TP fusion and preserve allreduce fusion ordering #27893 (#39505)

Signed-off-by: baonudesifeizhai <baonudesifeizhai@gmail.com>
Signed-off-by: baonudesifeizhai <85092850+baonudesifeizhai@users.noreply.github.com>
Signed-off-by: roG0d <baonudesifeizhai@gmail.com>
This commit is contained in:
baonudesifeizhai
2026-05-01 05:08:34 +00:00
committed by GitHub
parent 947138b6c2
commit c3868bbbe4
6 changed files with 397 additions and 30 deletions
@@ -31,6 +31,7 @@ logger = init_logger(__name__)
# Only apply sequence parallelism for models with hidden_size >= threshold
SP_MIN_HIDDEN_SIZE: dict[int, int] = {
90: 8192, # H100: only for models with hidden_size >= 8192
100: 8192, # Blackwell family: only for models with hidden_size >= 8192
}
# Min size per GPU per device capability for sequence parallelism
@@ -38,6 +39,8 @@ SP_MIN_HIDDEN_SIZE: dict[int, int] = {
# This ensures the threshold scales appropriately with tensor parallelism
SP_MIN_PER_GPU_SIZE_MB: dict[int, float] = {
90: 8, # 8MB per GPU for H100
# Use a more conservative threshold on Blackwell so TP8 starts later.
100: 32,
}
@@ -67,7 +70,12 @@ def get_sequence_parallelism_threshold(
capability = current_platform.get_device_capability()
if capability is None:
return None
device_capability = capability.to_int()
# Collapse Blackwell variants (sm100/sm103/...) into one policy bucket.
if current_platform.is_device_capability_family(100):
device_capability = 100
else:
device_capability = capability.to_int()
# Check if device has configured thresholds
min_hidden_size = SP_MIN_HIDDEN_SIZE.get(device_capability)