[Kernel] Porting the TRTLLM minimax_allreduce_rms kernels (#37045)
Signed-off-by: Jee Jee Li <pandaleefree@gmail.com>
This commit is contained in:
@@ -134,6 +134,8 @@ class PassConfig:
|
||||
"""Enable async TP."""
|
||||
fuse_allreduce_rms: bool = None # type: ignore[assignment]
|
||||
"""Enable flashinfer allreduce fusion."""
|
||||
fuse_minimax_qk_norm: bool = None # type: ignore[assignment]
|
||||
"""Enable fused allreduce+RMSNorm for MiniMax QK norm."""
|
||||
enable_qk_norm_rope_fusion: bool = False
|
||||
"""Enable fused Q/K RMSNorm + RoPE pass."""
|
||||
|
||||
|
||||
@@ -1627,6 +1627,22 @@ class VllmConfig:
|
||||
compile_range_end,
|
||||
)
|
||||
|
||||
if compilation_config.pass_config.fuse_minimax_qk_norm:
|
||||
from vllm.compilation.passes.fusion.minimax_qk_norm_fusion import (
|
||||
MAX_TOKEN_NUM,
|
||||
)
|
||||
|
||||
max_token_num = min(
|
||||
MAX_TOKEN_NUM, self.scheduler_config.max_num_batched_tokens
|
||||
)
|
||||
if compile_range_end is not None and max_token_num < compile_range_end:
|
||||
computed_compile_ranges_endpoints.append(max_token_num)
|
||||
else:
|
||||
logger.debug(
|
||||
"Max num batched tokens below MiniMax QK norm fusion threshold, "
|
||||
"MiniMax QK norm fusion enabled for all num_tokens."
|
||||
)
|
||||
|
||||
if compilation_config.compile_ranges_endpoints is not None:
|
||||
for x in compilation_config.compile_ranges_endpoints:
|
||||
assert isinstance(x, int)
|
||||
|
||||
Reference in New Issue
Block a user