[kernel] Integrate FlashInfer BF16 CuTeDSL Low Latency GEMM (#50572)

Signed-off-by: jiahanc <173873397+jiahanc@users.noreply.github.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: Jee Jee Li <pandaleefree@gmail.com>
This commit is contained in:
Jiahan Chang (Cyrus)
2026-08-27 22:29:12 +00:00
committed by GitHub
co-authored by OpenAI Codex Jee Jee Li
parent f79a2f5582
commit 7c877062ac
7 changed files with 349 additions and 23 deletions
+26 -11
View File
@@ -247,6 +247,31 @@ def _flashinfer_autotune_skip_ops(runner: "GPUModelRunner") -> set[str] | None:
return None
_FLASHINFER_BF16_AUTOTUNE_MAX_TOKENS = 32
def _flashinfer_autotune_token_counts(runner: "GPUModelRunner") -> tuple[int, ...]:
max_tokens = runner.scheduler_config.max_num_batched_tokens
linear_backend = runner.vllm_config.kernel_config.linear_backend
if (
linear_backend == "flashinfer_cutedsl"
and max_tokens > _FLASHINFER_BF16_AUTOTUNE_MAX_TOKENS
):
return max_tokens, _FLASHINFER_BF16_AUTOTUNE_MAX_TOKENS
return (max_tokens,)
def _run_flashinfer_autotune_dummy_runs(runner: "GPUModelRunner") -> None:
for num_tokens in _flashinfer_autotune_token_counts(runner):
logger.info("Running FlashInfer autotune with %d tokens.", num_tokens)
runner._dummy_run(
num_tokens=num_tokens,
skip_eplb=True,
is_profile=True,
randomize_inputs=True,
)
def flashinfer_autotune(runner: "GPUModelRunner") -> None:
"""
Autotune FlashInfer operations.
@@ -284,20 +309,10 @@ def flashinfer_autotune(runner: "GPUModelRunner") -> None:
logger.info_once("Using FlashInfer autotune cache file: %s", cache_path)
# We skip EPLB here since we don't want to record dummy metrics.
# When autotuning with number of tokens m, flashinfer will autotune
# operations for all number of tokens up to m, so we only need to
# run with the max number of tokens.
# Randomize inputs to avoid every token pick the same experts,
# which lead to some EP ranks receiving no tokens and skipping their
# MoE kernel entirely, and cause hang due to all-reduce collective
# during synchronized autotuning.
dummy_run_kwargs = dict(
num_tokens=runner.scheduler_config.max_num_batched_tokens,
skip_eplb=True,
is_profile=True,
randomize_inputs=True,
)
# Read cached autotune results and broadcast to all ranks.
cached_results: bytes | None = None
if is_leader and cache_path.exists():
@@ -316,7 +331,7 @@ def flashinfer_autotune(runner: "GPUModelRunner") -> None:
torch.inference_mode(),
fi_utils.autotune(tune_mode=True, **autotune_kwargs),
):
runner._dummy_run(**dummy_run_kwargs)
_run_flashinfer_autotune_dummy_runs(runner)
finally:
set_autotune_process_group(None)