[Kernel] Remove WAR for LL CuTeDSL BF16 GEMM (#60434)

Signed-off-by: jiahanc <173873397+jiahanc@users.noreply.github.com>
This commit is contained in:
Jiahan Chang (Cyrus)
2026-10-08 05:38:07 +00:00
committed by GitHub
parent 81198e97ba
commit 816173ca9d
5 changed files with 16 additions and 93 deletions
+1 -4
View File
@@ -128,16 +128,13 @@ def _can_use_flashinfer_cutedsl_bf16(
if (
k <= 0
or n <= 0
or x.numel() == 0
or weight.shape[1] != k
or k % 128 != 0
or x.data_ptr() % 32 != 0
or weight.data_ptr() % 32 != 0
):
return False
m = x.numel() // k
if not 1 <= m <= 32:
return False
return bias is None or (
bias.is_cuda
and bias.device == x.device
+2 -48
View File
@@ -316,9 +316,6 @@ def _flashinfer_autotune_skip_ops(runner: "GPUModelRunner") -> set[str] | None:
return None
_FLASHINFER_BF16_AUTOTUNE_MAX_TOKENS = 32
def _flashinfer_deferred_moe_token_counts(
runner: "GPUModelRunner",
) -> tuple[int, ...]:
@@ -339,19 +336,10 @@ def _flashinfer_deferred_moe_token_counts(
return tuple(dict.fromkeys(token_counts))
def _flashinfer_autotune_token_counts(
runner: "GPUModelRunner", *, include_bf16: bool = True
) -> tuple[int, ...]:
def _flashinfer_autotune_token_counts(runner: "GPUModelRunner") -> tuple[int, ...]:
max_tokens = runner.scheduler_config.max_num_batched_tokens
# Tune the widest bucket set first so bounded passes reuse its configs.
token_counts = [max_tokens]
linear_backend = runner.vllm_config.kernel_config.linear_backend
if (
include_bf16
and linear_backend == "flashinfer_cutedsl"
and max_tokens > _FLASHINFER_BF16_AUTOTUNE_MAX_TOKENS
):
token_counts.append(_FLASHINFER_BF16_AUTOTUNE_MAX_TOKENS)
token_counts.extend(_flashinfer_deferred_moe_token_counts(runner))
return tuple(dict.fromkeys(token_counts))
@@ -362,7 +350,7 @@ def _run_flashinfer_autotune_dummy_runs(
import vllm.utils.flashinfer as fi_utils
dummy_run_kwargs = {"skip_attn": True} if skip_attn else {}
for num_tokens in _flashinfer_autotune_token_counts(runner, include_bf16=False):
for num_tokens in _flashinfer_autotune_token_counts(runner):
# The drafter may use more tokens than this pass in the same dummy run.
# Include its max M when building the autotune buckets.
max_tuning_tokens = num_tokens
@@ -398,36 +386,6 @@ def _run_flashinfer_autotune_dummy_runs(
)
def _run_flashinfer_bf16_autotune_dummy_run(
runner: "GPUModelRunner",
*,
skip_ops: set[str] | None = None,
skip_attn: bool = False,
) -> None:
import vllm.utils.flashinfer as fi_utils
if (
runner.vllm_config.kernel_config.linear_backend != "flashinfer_cutedsl"
or runner.scheduler_config.max_num_batched_tokens
<= _FLASHINFER_BF16_AUTOTUNE_MAX_TOKENS
):
return
num_tokens = _FLASHINFER_BF16_AUTOTUNE_MAX_TOKENS
tuning_buckets = fi_utils.flashinfer_get_hybrid_num_tokens_buckets(num_tokens)
logger.info("Running FlashInfer BF16-only autotune with %d tokens.", num_tokens)
# Other ops execute from their existing cache or fallback, without the
# BF16 bucket cap. No full-model autotune context may enclose this pass.
with fi_utils.autotune_bf16_only(tuning_buckets, skip_ops=skip_ops):
runner._dummy_run(
num_tokens=num_tokens,
skip_eplb=True,
is_profile=True,
randomize_inputs=True,
**({"skip_attn": True} if skip_attn else {}),
)
def _autotune_cache_fingerprint(path: Path) -> tuple[str, int] | None:
"""Identify a saved autotune file by its FlashInfer metadata and size."""
try:
@@ -534,10 +492,6 @@ def flashinfer_autotune(runner: "GPUModelRunner") -> None:
_run_flashinfer_autotune_dummy_runs(runner, skip_attn=hisparse_enabled)
replayssm_autotune_warmup(runner)
_autotune_kimi_k3_kda_qkvg(runner.get_model())
with torch.inference_mode():
_run_flashinfer_bf16_autotune_dummy_run(
runner, skip_ops=skip_ops, skip_attn=hisparse_enabled
)
finally:
set_autotune_process_group(None)