[Perf] Re-enable flashinfer autotune by default and cleanup (#42857)

Signed-off-by: wzhao18 <wzhao18.sz@gmail.com>
This commit is contained in:
Wei Zhao
2026-05-18 09:12:27 -07:00
committed by GitHub
parent b12745e4f3
commit 8c296de63b
5 changed files with 111 additions and 79 deletions
+2 -6
View File
@@ -223,9 +223,7 @@ OPTIMIZATION_LEVEL_01 = {
"use_inductor_graph_partition": False,
},
"kernel_config": {
# Disabled for now due to correctness issues:
# https://github.com/flashinfer-ai/flashinfer/issues/3197
"enable_flashinfer_autotune": False,
"enable_flashinfer_autotune": True,
},
}
OPTIMIZATION_LEVEL_02 = {
@@ -246,9 +244,7 @@ OPTIMIZATION_LEVEL_02 = {
"use_inductor_graph_partition": False,
},
"kernel_config": {
# Disabled for now due to correctness issues:
# https://github.com/flashinfer-ai/flashinfer/issues/3197
"enable_flashinfer_autotune": False,
"enable_flashinfer_autotune": True,
},
}
OPTIMIZATION_LEVEL_03 = {
@@ -149,24 +149,21 @@ class FlashInferCuteDSLExperts(mk.FusedMoEExpertsModular):
# The functional API expects x_sf with trailing dim: (M, K//16, 1).
x_sf = a1q_scale.unsqueeze(-1)
from vllm.utils.flashinfer import _is_fi_autotuning, autotune
with autotune(_is_fi_autotuning):
flashinfer_cute_dsl_fused_moe_nvfp4(
x=hidden_states,
x_sf=x_sf,
token_selected_experts=topk_ids.to(torch.int32),
token_final_scales=topk_weights.float(),
w1_weight=w1,
w1_weight_sf=self.w1_scale,
w1_alpha=self.g1_alphas,
fc2_input_scale=self.a2_gscale,
w2_weight=w2,
w2_weight_sf=self.w2_scale,
w2_alpha=self.g2_alphas,
num_experts=self.global_num_experts,
top_k=self.topk,
num_local_experts=self.local_num_experts,
local_expert_offset=self.local_expert_offset,
moe_output=output,
)
flashinfer_cute_dsl_fused_moe_nvfp4(
x=hidden_states,
x_sf=x_sf,
token_selected_experts=topk_ids.to(torch.int32),
token_final_scales=topk_weights.float(),
w1_weight=w1,
w1_weight_sf=self.w1_scale,
w1_alpha=self.g1_alphas,
fc2_input_scale=self.a2_gscale,
w2_weight=w2,
w2_weight_sf=self.w2_scale,
w2_alpha=self.g2_alphas,
num_experts=self.global_num_experts,
top_k=self.topk,
num_local_experts=self.local_num_experts,
local_expert_offset=self.local_expert_offset,
moe_output=output,
)
@@ -194,39 +194,36 @@ class TrtLlmMxfp4ExpertsMonolithic(
device=hidden_states.device,
)
from vllm.utils.flashinfer import _is_fi_autotuning, autotune
with autotune(_is_fi_autotuning):
trtllm_fp4_block_scale_moe(
routing_logits=router_logits.to(torch.bfloat16),
routing_bias=None,
hidden_states=x_quant,
hidden_states_scale=x_scale,
gemm1_weights=w1,
gemm1_weights_scale=self.w1_scale,
gemm1_bias=self.w1_bias,
gemm1_alpha=self.gemm1_alpha,
gemm1_beta=self.gemm1_beta,
gemm1_clamp_limit=self.gemm1_clamp_limit,
gemm2_weights=w2,
gemm2_weights_scale=self.w2_scale,
gemm2_bias=self.w2_bias,
output1_scale_scalar=None,
output1_scale_gate_scalar=None,
output2_scale_scalar=None,
num_experts=global_num_experts,
top_k=self.topk,
n_group=None,
topk_group=None,
intermediate_size=self.intermediate_size_per_partition,
local_expert_offset=self.ep_rank * self.local_num_experts,
local_num_experts=self.local_num_experts,
routed_scaling_factor=None,
routing_method_type=self.routing_method_type,
do_finalize=True,
tune_max_num_tokens=max(self.max_capture_size, 1),
output=output,
)
trtllm_fp4_block_scale_moe(
routing_logits=router_logits.to(torch.bfloat16),
routing_bias=None,
hidden_states=x_quant,
hidden_states_scale=x_scale,
gemm1_weights=w1,
gemm1_weights_scale=self.w1_scale,
gemm1_bias=self.w1_bias,
gemm1_alpha=self.gemm1_alpha,
gemm1_beta=self.gemm1_beta,
gemm1_clamp_limit=self.gemm1_clamp_limit,
gemm2_weights=w2,
gemm2_weights_scale=self.w2_scale,
gemm2_bias=self.w2_bias,
output1_scale_scalar=None,
output1_scale_gate_scalar=None,
output2_scale_scalar=None,
num_experts=global_num_experts,
top_k=self.topk,
n_group=None,
topk_group=None,
intermediate_size=self.intermediate_size_per_partition,
local_expert_offset=self.ep_rank * self.local_num_experts,
local_num_experts=self.local_num_experts,
routed_scaling_factor=None,
routing_method_type=self.routing_method_type,
do_finalize=True,
tune_max_num_tokens=max(self.max_capture_size, 1),
output=output,
)
return output
@@ -350,9 +347,6 @@ class TrtLlmMxfp4ExpertsModular(TrtLlmMxfp4ExpertsBase, mk.FusedMoEExpertsModula
from flashinfer import trtllm_fp4_block_scale_routed_moe
from vllm.utils.flashinfer import _is_fi_autotuning, autotune
with autotune(_is_fi_autotuning):
trtllm_fp4_block_scale_routed_moe(**kwargs)
trtllm_fp4_block_scale_routed_moe(**kwargs)
return output
+60 -14
View File
@@ -87,23 +87,69 @@ def flashinfer_autotune(runner: "GPUModelRunner") -> None:
future calls to FlashInfer will use the best implementation.
Without autotuning, FlashInfer will rely on heuristics, which may
be significantly slower.
Tuning is performed only on rank 0. The resulting cache is broadcast
to every rank so all ranks dispatch the same kernel tactic.
"""
import os
import tempfile
import vllm.utils.flashinfer as fi_utils
from vllm.distributed.parallel_state import get_world_group
with torch.inference_mode(), fi_utils.autotune():
# Certain FlashInfer kernels (e.g. nvfp4 routed moe) are
# incompatible with autotuning. This state is used to skip
# those kernels during the autotuning process.
fi_utils._is_fi_autotuning = True
world = get_world_group()
is_leader = world.rank_in_group == 0
# We skip EPLB here since we don't want to record dummy metrics
# When autotuning with number of tokens m, flashinfer will autotune
# operations for all number of tokens up to m.
# So we only need to run with the max number of tokens.
runner._dummy_run(
runner.scheduler_config.max_num_batched_tokens,
skip_eplb=True,
is_profile=True,
cache_dir = tempfile.mkdtemp(prefix="vllm_flashinfer_autotune_")
cache_path = os.path.join(cache_dir, "autotune_cache.json")
# We skip EPLB here since we don't want to record dummy metrics.
# When autotuning with number of tokens m, flashinfer will autotune
# operations for all number of tokens up to m, so we only need to
# run with the max number of tokens.
dummy_run_kwargs = dict(
num_tokens=runner.scheduler_config.max_num_batched_tokens,
skip_eplb=True,
is_profile=True,
)
with torch.inference_mode():
if is_leader:
with fi_utils.autotune(tune_mode=True, cache=cache_path):
runner._dummy_run(**dummy_run_kwargs)
else:
runner._dummy_run(**dummy_run_kwargs)
# Broadcast autotune cache from rank 0 to all other ranks so every
# rank loads the same set of chosen tactics.
tune_results: bytes | None = None
if is_leader and os.path.exists(cache_path):
with open(cache_path, "rb") as f:
tune_results = f.read()
tune_results = world.broadcast_object(tune_results, src=0)
if tune_results is None:
logger.warning(
"No FlashInfer autotune cache entries found."
"Falling back to default tactics."
)
else:
if not is_leader:
with open(cache_path, "wb") as f:
f.write(tune_results)
from flashinfer.autotuner import AutoTuner
AutoTuner.get().load_configs(cache_path)
logger.info(
"FlashInfer autotune cache loaded on rank %d from %s.",
world.rank_in_group,
cache_path,
)
fi_utils._is_fi_autotuning = False
try:
if os.path.exists(cache_path):
os.unlink(cache_path)
os.rmdir(cache_dir)
except OSError:
pass
-1
View File
@@ -144,7 +144,6 @@ autotune = _lazy_import_wrapper(
"autotune",
fallback_fn=lambda *args, **kwargs: contextlib.nullcontext(),
)
_is_fi_autotuning: bool = False
@functools.cache