[Perf] Re-enable flashinfer autotune by default and cleanup (#42857)
Signed-off-by: wzhao18 <wzhao18.sz@gmail.com>
This commit is contained in:
+2
-6
@@ -223,9 +223,7 @@ OPTIMIZATION_LEVEL_01 = {
|
||||
"use_inductor_graph_partition": False,
|
||||
},
|
||||
"kernel_config": {
|
||||
# Disabled for now due to correctness issues:
|
||||
# https://github.com/flashinfer-ai/flashinfer/issues/3197
|
||||
"enable_flashinfer_autotune": False,
|
||||
"enable_flashinfer_autotune": True,
|
||||
},
|
||||
}
|
||||
OPTIMIZATION_LEVEL_02 = {
|
||||
@@ -246,9 +244,7 @@ OPTIMIZATION_LEVEL_02 = {
|
||||
"use_inductor_graph_partition": False,
|
||||
},
|
||||
"kernel_config": {
|
||||
# Disabled for now due to correctness issues:
|
||||
# https://github.com/flashinfer-ai/flashinfer/issues/3197
|
||||
"enable_flashinfer_autotune": False,
|
||||
"enable_flashinfer_autotune": True,
|
||||
},
|
||||
}
|
||||
OPTIMIZATION_LEVEL_03 = {
|
||||
|
||||
@@ -149,24 +149,21 @@ class FlashInferCuteDSLExperts(mk.FusedMoEExpertsModular):
|
||||
# The functional API expects x_sf with trailing dim: (M, K//16, 1).
|
||||
x_sf = a1q_scale.unsqueeze(-1)
|
||||
|
||||
from vllm.utils.flashinfer import _is_fi_autotuning, autotune
|
||||
|
||||
with autotune(_is_fi_autotuning):
|
||||
flashinfer_cute_dsl_fused_moe_nvfp4(
|
||||
x=hidden_states,
|
||||
x_sf=x_sf,
|
||||
token_selected_experts=topk_ids.to(torch.int32),
|
||||
token_final_scales=topk_weights.float(),
|
||||
w1_weight=w1,
|
||||
w1_weight_sf=self.w1_scale,
|
||||
w1_alpha=self.g1_alphas,
|
||||
fc2_input_scale=self.a2_gscale,
|
||||
w2_weight=w2,
|
||||
w2_weight_sf=self.w2_scale,
|
||||
w2_alpha=self.g2_alphas,
|
||||
num_experts=self.global_num_experts,
|
||||
top_k=self.topk,
|
||||
num_local_experts=self.local_num_experts,
|
||||
local_expert_offset=self.local_expert_offset,
|
||||
moe_output=output,
|
||||
)
|
||||
flashinfer_cute_dsl_fused_moe_nvfp4(
|
||||
x=hidden_states,
|
||||
x_sf=x_sf,
|
||||
token_selected_experts=topk_ids.to(torch.int32),
|
||||
token_final_scales=topk_weights.float(),
|
||||
w1_weight=w1,
|
||||
w1_weight_sf=self.w1_scale,
|
||||
w1_alpha=self.g1_alphas,
|
||||
fc2_input_scale=self.a2_gscale,
|
||||
w2_weight=w2,
|
||||
w2_weight_sf=self.w2_scale,
|
||||
w2_alpha=self.g2_alphas,
|
||||
num_experts=self.global_num_experts,
|
||||
top_k=self.topk,
|
||||
num_local_experts=self.local_num_experts,
|
||||
local_expert_offset=self.local_expert_offset,
|
||||
moe_output=output,
|
||||
)
|
||||
|
||||
@@ -194,39 +194,36 @@ class TrtLlmMxfp4ExpertsMonolithic(
|
||||
device=hidden_states.device,
|
||||
)
|
||||
|
||||
from vllm.utils.flashinfer import _is_fi_autotuning, autotune
|
||||
|
||||
with autotune(_is_fi_autotuning):
|
||||
trtllm_fp4_block_scale_moe(
|
||||
routing_logits=router_logits.to(torch.bfloat16),
|
||||
routing_bias=None,
|
||||
hidden_states=x_quant,
|
||||
hidden_states_scale=x_scale,
|
||||
gemm1_weights=w1,
|
||||
gemm1_weights_scale=self.w1_scale,
|
||||
gemm1_bias=self.w1_bias,
|
||||
gemm1_alpha=self.gemm1_alpha,
|
||||
gemm1_beta=self.gemm1_beta,
|
||||
gemm1_clamp_limit=self.gemm1_clamp_limit,
|
||||
gemm2_weights=w2,
|
||||
gemm2_weights_scale=self.w2_scale,
|
||||
gemm2_bias=self.w2_bias,
|
||||
output1_scale_scalar=None,
|
||||
output1_scale_gate_scalar=None,
|
||||
output2_scale_scalar=None,
|
||||
num_experts=global_num_experts,
|
||||
top_k=self.topk,
|
||||
n_group=None,
|
||||
topk_group=None,
|
||||
intermediate_size=self.intermediate_size_per_partition,
|
||||
local_expert_offset=self.ep_rank * self.local_num_experts,
|
||||
local_num_experts=self.local_num_experts,
|
||||
routed_scaling_factor=None,
|
||||
routing_method_type=self.routing_method_type,
|
||||
do_finalize=True,
|
||||
tune_max_num_tokens=max(self.max_capture_size, 1),
|
||||
output=output,
|
||||
)
|
||||
trtllm_fp4_block_scale_moe(
|
||||
routing_logits=router_logits.to(torch.bfloat16),
|
||||
routing_bias=None,
|
||||
hidden_states=x_quant,
|
||||
hidden_states_scale=x_scale,
|
||||
gemm1_weights=w1,
|
||||
gemm1_weights_scale=self.w1_scale,
|
||||
gemm1_bias=self.w1_bias,
|
||||
gemm1_alpha=self.gemm1_alpha,
|
||||
gemm1_beta=self.gemm1_beta,
|
||||
gemm1_clamp_limit=self.gemm1_clamp_limit,
|
||||
gemm2_weights=w2,
|
||||
gemm2_weights_scale=self.w2_scale,
|
||||
gemm2_bias=self.w2_bias,
|
||||
output1_scale_scalar=None,
|
||||
output1_scale_gate_scalar=None,
|
||||
output2_scale_scalar=None,
|
||||
num_experts=global_num_experts,
|
||||
top_k=self.topk,
|
||||
n_group=None,
|
||||
topk_group=None,
|
||||
intermediate_size=self.intermediate_size_per_partition,
|
||||
local_expert_offset=self.ep_rank * self.local_num_experts,
|
||||
local_num_experts=self.local_num_experts,
|
||||
routed_scaling_factor=None,
|
||||
routing_method_type=self.routing_method_type,
|
||||
do_finalize=True,
|
||||
tune_max_num_tokens=max(self.max_capture_size, 1),
|
||||
output=output,
|
||||
)
|
||||
|
||||
return output
|
||||
|
||||
@@ -350,9 +347,6 @@ class TrtLlmMxfp4ExpertsModular(TrtLlmMxfp4ExpertsBase, mk.FusedMoEExpertsModula
|
||||
|
||||
from flashinfer import trtllm_fp4_block_scale_routed_moe
|
||||
|
||||
from vllm.utils.flashinfer import _is_fi_autotuning, autotune
|
||||
|
||||
with autotune(_is_fi_autotuning):
|
||||
trtllm_fp4_block_scale_routed_moe(**kwargs)
|
||||
trtllm_fp4_block_scale_routed_moe(**kwargs)
|
||||
|
||||
return output
|
||||
|
||||
@@ -87,23 +87,69 @@ def flashinfer_autotune(runner: "GPUModelRunner") -> None:
|
||||
future calls to FlashInfer will use the best implementation.
|
||||
Without autotuning, FlashInfer will rely on heuristics, which may
|
||||
be significantly slower.
|
||||
|
||||
Tuning is performed only on rank 0. The resulting cache is broadcast
|
||||
to every rank so all ranks dispatch the same kernel tactic.
|
||||
"""
|
||||
import os
|
||||
import tempfile
|
||||
|
||||
import vllm.utils.flashinfer as fi_utils
|
||||
from vllm.distributed.parallel_state import get_world_group
|
||||
|
||||
with torch.inference_mode(), fi_utils.autotune():
|
||||
# Certain FlashInfer kernels (e.g. nvfp4 routed moe) are
|
||||
# incompatible with autotuning. This state is used to skip
|
||||
# those kernels during the autotuning process.
|
||||
fi_utils._is_fi_autotuning = True
|
||||
world = get_world_group()
|
||||
is_leader = world.rank_in_group == 0
|
||||
|
||||
# We skip EPLB here since we don't want to record dummy metrics
|
||||
# When autotuning with number of tokens m, flashinfer will autotune
|
||||
# operations for all number of tokens up to m.
|
||||
# So we only need to run with the max number of tokens.
|
||||
runner._dummy_run(
|
||||
runner.scheduler_config.max_num_batched_tokens,
|
||||
skip_eplb=True,
|
||||
is_profile=True,
|
||||
cache_dir = tempfile.mkdtemp(prefix="vllm_flashinfer_autotune_")
|
||||
cache_path = os.path.join(cache_dir, "autotune_cache.json")
|
||||
|
||||
# We skip EPLB here since we don't want to record dummy metrics.
|
||||
# When autotuning with number of tokens m, flashinfer will autotune
|
||||
# operations for all number of tokens up to m, so we only need to
|
||||
# run with the max number of tokens.
|
||||
dummy_run_kwargs = dict(
|
||||
num_tokens=runner.scheduler_config.max_num_batched_tokens,
|
||||
skip_eplb=True,
|
||||
is_profile=True,
|
||||
)
|
||||
|
||||
with torch.inference_mode():
|
||||
if is_leader:
|
||||
with fi_utils.autotune(tune_mode=True, cache=cache_path):
|
||||
runner._dummy_run(**dummy_run_kwargs)
|
||||
else:
|
||||
runner._dummy_run(**dummy_run_kwargs)
|
||||
|
||||
# Broadcast autotune cache from rank 0 to all other ranks so every
|
||||
# rank loads the same set of chosen tactics.
|
||||
tune_results: bytes | None = None
|
||||
if is_leader and os.path.exists(cache_path):
|
||||
with open(cache_path, "rb") as f:
|
||||
tune_results = f.read()
|
||||
|
||||
tune_results = world.broadcast_object(tune_results, src=0)
|
||||
|
||||
if tune_results is None:
|
||||
logger.warning(
|
||||
"No FlashInfer autotune cache entries found."
|
||||
"Falling back to default tactics."
|
||||
)
|
||||
else:
|
||||
if not is_leader:
|
||||
with open(cache_path, "wb") as f:
|
||||
f.write(tune_results)
|
||||
from flashinfer.autotuner import AutoTuner
|
||||
|
||||
AutoTuner.get().load_configs(cache_path)
|
||||
logger.info(
|
||||
"FlashInfer autotune cache loaded on rank %d from %s.",
|
||||
world.rank_in_group,
|
||||
cache_path,
|
||||
)
|
||||
|
||||
fi_utils._is_fi_autotuning = False
|
||||
try:
|
||||
if os.path.exists(cache_path):
|
||||
os.unlink(cache_path)
|
||||
os.rmdir(cache_dir)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
@@ -144,7 +144,6 @@ autotune = _lazy_import_wrapper(
|
||||
"autotune",
|
||||
fallback_fn=lambda *args, **kwargs: contextlib.nullcontext(),
|
||||
)
|
||||
_is_fi_autotuning: bool = False
|
||||
|
||||
|
||||
@functools.cache
|
||||
|
||||
Reference in New Issue
Block a user