[Kernel] Remove WAR for LL CuTeDSL BF16 GEMM (#60434)
Signed-off-by: jiahanc <173873397+jiahanc@users.noreply.github.com>
This commit is contained in:
@@ -25,6 +25,8 @@ def require_flashinfer_bf16_cutedsl() -> None:
|
||||
(8, True, False),
|
||||
(16, False, True),
|
||||
(32, True, True),
|
||||
(64, True, True),
|
||||
(512, False, False),
|
||||
],
|
||||
)
|
||||
def test_flashinfer_bf16_cutedsl_correctness(
|
||||
|
||||
@@ -39,12 +39,11 @@ def _make_moe(*, max_deferred_tokens: int = 128, enabled: bool = True):
|
||||
return moe
|
||||
|
||||
|
||||
def _make_runner(modules, *, max_tokens: int = 8192, linear_backend: str = "auto"):
|
||||
def _make_runner(modules, *, max_tokens: int = 8192):
|
||||
"""Create a runner carrying only the state used by FlashInfer warmup."""
|
||||
return SimpleNamespace(
|
||||
scheduler_config=SimpleNamespace(max_num_batched_tokens=max_tokens),
|
||||
vllm_config=SimpleNamespace(
|
||||
kernel_config=SimpleNamespace(linear_backend=linear_backend),
|
||||
attention_config=SimpleNamespace(hisparse_config=None),
|
||||
parallel_config=SimpleNamespace(data_parallel_rank=0),
|
||||
),
|
||||
@@ -63,21 +62,19 @@ def test_flashinfer_autotune_token_counts_include_deferred_moe_limits():
|
||||
_make_moe(max_deferred_tokens=128),
|
||||
_make_moe(max_deferred_tokens=64, enabled=False),
|
||||
_make_moe(max_deferred_tokens=-1),
|
||||
],
|
||||
linear_backend="flashinfer_cutedsl",
|
||||
]
|
||||
)
|
||||
|
||||
with patch("vllm.model_executor.layers.fused_moe.MoERunner", _FakeMoERunner):
|
||||
token_counts = _flashinfer_autotune_token_counts(runner)
|
||||
|
||||
assert token_counts == (8192, 32, 128)
|
||||
assert token_counts == (8192, 128)
|
||||
|
||||
|
||||
def test_flashinfer_autotune_token_counts_are_bounded_and_deduplicated():
|
||||
runner = _make_runner(
|
||||
[_make_moe(max_deferred_tokens=4096)],
|
||||
max_tokens=32,
|
||||
linear_backend="flashinfer_cutedsl",
|
||||
)
|
||||
|
||||
with patch("vllm.model_executor.layers.fused_moe.MoERunner", _FakeMoERunner):
|
||||
|
||||
@@ -128,16 +128,13 @@ def _can_use_flashinfer_cutedsl_bf16(
|
||||
if (
|
||||
k <= 0
|
||||
or n <= 0
|
||||
or x.numel() == 0
|
||||
or weight.shape[1] != k
|
||||
or k % 128 != 0
|
||||
or x.data_ptr() % 32 != 0
|
||||
or weight.data_ptr() % 32 != 0
|
||||
):
|
||||
return False
|
||||
|
||||
m = x.numel() // k
|
||||
if not 1 <= m <= 32:
|
||||
return False
|
||||
return bias is None or (
|
||||
bias.is_cuda
|
||||
and bias.device == x.device
|
||||
|
||||
@@ -316,9 +316,6 @@ def _flashinfer_autotune_skip_ops(runner: "GPUModelRunner") -> set[str] | None:
|
||||
return None
|
||||
|
||||
|
||||
_FLASHINFER_BF16_AUTOTUNE_MAX_TOKENS = 32
|
||||
|
||||
|
||||
def _flashinfer_deferred_moe_token_counts(
|
||||
runner: "GPUModelRunner",
|
||||
) -> tuple[int, ...]:
|
||||
@@ -339,19 +336,10 @@ def _flashinfer_deferred_moe_token_counts(
|
||||
return tuple(dict.fromkeys(token_counts))
|
||||
|
||||
|
||||
def _flashinfer_autotune_token_counts(
|
||||
runner: "GPUModelRunner", *, include_bf16: bool = True
|
||||
) -> tuple[int, ...]:
|
||||
def _flashinfer_autotune_token_counts(runner: "GPUModelRunner") -> tuple[int, ...]:
|
||||
max_tokens = runner.scheduler_config.max_num_batched_tokens
|
||||
# Tune the widest bucket set first so bounded passes reuse its configs.
|
||||
token_counts = [max_tokens]
|
||||
linear_backend = runner.vllm_config.kernel_config.linear_backend
|
||||
if (
|
||||
include_bf16
|
||||
and linear_backend == "flashinfer_cutedsl"
|
||||
and max_tokens > _FLASHINFER_BF16_AUTOTUNE_MAX_TOKENS
|
||||
):
|
||||
token_counts.append(_FLASHINFER_BF16_AUTOTUNE_MAX_TOKENS)
|
||||
token_counts.extend(_flashinfer_deferred_moe_token_counts(runner))
|
||||
return tuple(dict.fromkeys(token_counts))
|
||||
|
||||
@@ -362,7 +350,7 @@ def _run_flashinfer_autotune_dummy_runs(
|
||||
import vllm.utils.flashinfer as fi_utils
|
||||
|
||||
dummy_run_kwargs = {"skip_attn": True} if skip_attn else {}
|
||||
for num_tokens in _flashinfer_autotune_token_counts(runner, include_bf16=False):
|
||||
for num_tokens in _flashinfer_autotune_token_counts(runner):
|
||||
# The drafter may use more tokens than this pass in the same dummy run.
|
||||
# Include its max M when building the autotune buckets.
|
||||
max_tuning_tokens = num_tokens
|
||||
@@ -398,36 +386,6 @@ def _run_flashinfer_autotune_dummy_runs(
|
||||
)
|
||||
|
||||
|
||||
def _run_flashinfer_bf16_autotune_dummy_run(
|
||||
runner: "GPUModelRunner",
|
||||
*,
|
||||
skip_ops: set[str] | None = None,
|
||||
skip_attn: bool = False,
|
||||
) -> None:
|
||||
import vllm.utils.flashinfer as fi_utils
|
||||
|
||||
if (
|
||||
runner.vllm_config.kernel_config.linear_backend != "flashinfer_cutedsl"
|
||||
or runner.scheduler_config.max_num_batched_tokens
|
||||
<= _FLASHINFER_BF16_AUTOTUNE_MAX_TOKENS
|
||||
):
|
||||
return
|
||||
|
||||
num_tokens = _FLASHINFER_BF16_AUTOTUNE_MAX_TOKENS
|
||||
tuning_buckets = fi_utils.flashinfer_get_hybrid_num_tokens_buckets(num_tokens)
|
||||
logger.info("Running FlashInfer BF16-only autotune with %d tokens.", num_tokens)
|
||||
# Other ops execute from their existing cache or fallback, without the
|
||||
# BF16 bucket cap. No full-model autotune context may enclose this pass.
|
||||
with fi_utils.autotune_bf16_only(tuning_buckets, skip_ops=skip_ops):
|
||||
runner._dummy_run(
|
||||
num_tokens=num_tokens,
|
||||
skip_eplb=True,
|
||||
is_profile=True,
|
||||
randomize_inputs=True,
|
||||
**({"skip_attn": True} if skip_attn else {}),
|
||||
)
|
||||
|
||||
|
||||
def _autotune_cache_fingerprint(path: Path) -> tuple[str, int] | None:
|
||||
"""Identify a saved autotune file by its FlashInfer metadata and size."""
|
||||
try:
|
||||
@@ -534,10 +492,6 @@ def flashinfer_autotune(runner: "GPUModelRunner") -> None:
|
||||
_run_flashinfer_autotune_dummy_runs(runner, skip_attn=hisparse_enabled)
|
||||
replayssm_autotune_warmup(runner)
|
||||
_autotune_kimi_k3_kda_qkvg(runner.get_model())
|
||||
with torch.inference_mode():
|
||||
_run_flashinfer_bf16_autotune_dummy_run(
|
||||
runner, skip_ops=skip_ops, skip_attn=hisparse_enabled
|
||||
)
|
||||
finally:
|
||||
set_autotune_process_group(None)
|
||||
|
||||
|
||||
@@ -11,8 +11,7 @@ import importlib
|
||||
import importlib.util
|
||||
import os
|
||||
import shutil
|
||||
from collections.abc import Callable, Iterator
|
||||
from contextvars import ContextVar
|
||||
from collections.abc import Callable
|
||||
from typing import Any, NoReturn
|
||||
|
||||
import requests
|
||||
@@ -27,24 +26,6 @@ from vllm.utils.torch_utils import PIN_MEMORY
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
_bf16_autotune_buckets: ContextVar[tuple[int, ...] | None] = ContextVar(
|
||||
"flashinfer_bf16_autotune_buckets", default=None
|
||||
)
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def autotune_bf16_only(
|
||||
tuning_buckets: tuple[int, ...], *, skip_ops: set[str] | None = None
|
||||
) -> Iterator[None]:
|
||||
"""Tune BF16 calls with bounded buckets, outside full-model autotuning."""
|
||||
token = _bf16_autotune_buckets.set(tuning_buckets)
|
||||
try:
|
||||
with autotune(tune_mode=False, skip_ops=skip_ops):
|
||||
yield
|
||||
finally:
|
||||
_bf16_autotune_buckets.reset(token)
|
||||
|
||||
|
||||
def flashinfer_bf16_mm(
|
||||
a: torch.Tensor,
|
||||
b: torch.Tensor,
|
||||
@@ -54,21 +35,14 @@ def flashinfer_bf16_mm(
|
||||
) -> torch.Tensor:
|
||||
from flashinfer import mm_bf16
|
||||
|
||||
tuning_buckets = _bf16_autotune_buckets.get()
|
||||
tuning = (
|
||||
autotune(tune_mode=True, tuning_buckets=tuning_buckets)
|
||||
if tuning_buckets is not None
|
||||
else contextlib.nullcontext()
|
||||
return mm_bf16(
|
||||
a,
|
||||
b,
|
||||
bias=bias,
|
||||
pdl=pdl,
|
||||
out_dtype=torch.bfloat16,
|
||||
backend=backend,
|
||||
)
|
||||
with tuning:
|
||||
return mm_bf16(
|
||||
a,
|
||||
b,
|
||||
bias=bias,
|
||||
pdl=pdl,
|
||||
out_dtype=torch.bfloat16,
|
||||
backend=backend,
|
||||
)
|
||||
|
||||
|
||||
# This is the storage path for the cubins, it can be replaced
|
||||
@@ -1295,7 +1269,6 @@ def is_flashinfer_cudnn_fp8_prefill_attn_supported() -> bool:
|
||||
__all__ = [
|
||||
"has_flashinfer",
|
||||
"flashinfer_bf16_mm",
|
||||
"autotune_bf16_only",
|
||||
"has_flashinfer_bf16_gemm",
|
||||
"is_flashinfer_bf16_gemm_supported",
|
||||
"is_flashinfer_cutedsl_bf16_gemm_supported",
|
||||
|
||||
Reference in New Issue
Block a user