diff --git a/vllm/config/vllm.py b/vllm/config/vllm.py index 5414b9d04f..a2753218f5 100644 --- a/vllm/config/vllm.py +++ b/vllm/config/vllm.py @@ -223,9 +223,7 @@ OPTIMIZATION_LEVEL_01 = { "use_inductor_graph_partition": False, }, "kernel_config": { - # Disabled for now due to correctness issues: - # https://github.com/flashinfer-ai/flashinfer/issues/3197 - "enable_flashinfer_autotune": False, + "enable_flashinfer_autotune": True, }, } OPTIMIZATION_LEVEL_02 = { @@ -246,9 +244,7 @@ OPTIMIZATION_LEVEL_02 = { "use_inductor_graph_partition": False, }, "kernel_config": { - # Disabled for now due to correctness issues: - # https://github.com/flashinfer-ai/flashinfer/issues/3197 - "enable_flashinfer_autotune": False, + "enable_flashinfer_autotune": True, }, } OPTIMIZATION_LEVEL_03 = { diff --git a/vllm/model_executor/layers/fused_moe/experts/flashinfer_cutedsl_moe.py b/vllm/model_executor/layers/fused_moe/experts/flashinfer_cutedsl_moe.py index 5ce58220b0..2310982792 100644 --- a/vllm/model_executor/layers/fused_moe/experts/flashinfer_cutedsl_moe.py +++ b/vllm/model_executor/layers/fused_moe/experts/flashinfer_cutedsl_moe.py @@ -149,24 +149,21 @@ class FlashInferCuteDSLExperts(mk.FusedMoEExpertsModular): # The functional API expects x_sf with trailing dim: (M, K//16, 1). x_sf = a1q_scale.unsqueeze(-1) - from vllm.utils.flashinfer import _is_fi_autotuning, autotune - - with autotune(_is_fi_autotuning): - flashinfer_cute_dsl_fused_moe_nvfp4( - x=hidden_states, - x_sf=x_sf, - token_selected_experts=topk_ids.to(torch.int32), - token_final_scales=topk_weights.float(), - w1_weight=w1, - w1_weight_sf=self.w1_scale, - w1_alpha=self.g1_alphas, - fc2_input_scale=self.a2_gscale, - w2_weight=w2, - w2_weight_sf=self.w2_scale, - w2_alpha=self.g2_alphas, - num_experts=self.global_num_experts, - top_k=self.topk, - num_local_experts=self.local_num_experts, - local_expert_offset=self.local_expert_offset, - moe_output=output, - ) + flashinfer_cute_dsl_fused_moe_nvfp4( + x=hidden_states, + x_sf=x_sf, + token_selected_experts=topk_ids.to(torch.int32), + token_final_scales=topk_weights.float(), + w1_weight=w1, + w1_weight_sf=self.w1_scale, + w1_alpha=self.g1_alphas, + fc2_input_scale=self.a2_gscale, + w2_weight=w2, + w2_weight_sf=self.w2_scale, + w2_alpha=self.g2_alphas, + num_experts=self.global_num_experts, + top_k=self.topk, + num_local_experts=self.local_num_experts, + local_expert_offset=self.local_expert_offset, + moe_output=output, + ) diff --git a/vllm/model_executor/layers/fused_moe/experts/trtllm_mxfp4_moe.py b/vllm/model_executor/layers/fused_moe/experts/trtllm_mxfp4_moe.py index 7308ed195c..64c163d73e 100644 --- a/vllm/model_executor/layers/fused_moe/experts/trtllm_mxfp4_moe.py +++ b/vllm/model_executor/layers/fused_moe/experts/trtllm_mxfp4_moe.py @@ -194,39 +194,36 @@ class TrtLlmMxfp4ExpertsMonolithic( device=hidden_states.device, ) - from vllm.utils.flashinfer import _is_fi_autotuning, autotune - - with autotune(_is_fi_autotuning): - trtllm_fp4_block_scale_moe( - routing_logits=router_logits.to(torch.bfloat16), - routing_bias=None, - hidden_states=x_quant, - hidden_states_scale=x_scale, - gemm1_weights=w1, - gemm1_weights_scale=self.w1_scale, - gemm1_bias=self.w1_bias, - gemm1_alpha=self.gemm1_alpha, - gemm1_beta=self.gemm1_beta, - gemm1_clamp_limit=self.gemm1_clamp_limit, - gemm2_weights=w2, - gemm2_weights_scale=self.w2_scale, - gemm2_bias=self.w2_bias, - output1_scale_scalar=None, - output1_scale_gate_scalar=None, - output2_scale_scalar=None, - num_experts=global_num_experts, - top_k=self.topk, - n_group=None, - topk_group=None, - intermediate_size=self.intermediate_size_per_partition, - local_expert_offset=self.ep_rank * self.local_num_experts, - local_num_experts=self.local_num_experts, - routed_scaling_factor=None, - routing_method_type=self.routing_method_type, - do_finalize=True, - tune_max_num_tokens=max(self.max_capture_size, 1), - output=output, - ) + trtllm_fp4_block_scale_moe( + routing_logits=router_logits.to(torch.bfloat16), + routing_bias=None, + hidden_states=x_quant, + hidden_states_scale=x_scale, + gemm1_weights=w1, + gemm1_weights_scale=self.w1_scale, + gemm1_bias=self.w1_bias, + gemm1_alpha=self.gemm1_alpha, + gemm1_beta=self.gemm1_beta, + gemm1_clamp_limit=self.gemm1_clamp_limit, + gemm2_weights=w2, + gemm2_weights_scale=self.w2_scale, + gemm2_bias=self.w2_bias, + output1_scale_scalar=None, + output1_scale_gate_scalar=None, + output2_scale_scalar=None, + num_experts=global_num_experts, + top_k=self.topk, + n_group=None, + topk_group=None, + intermediate_size=self.intermediate_size_per_partition, + local_expert_offset=self.ep_rank * self.local_num_experts, + local_num_experts=self.local_num_experts, + routed_scaling_factor=None, + routing_method_type=self.routing_method_type, + do_finalize=True, + tune_max_num_tokens=max(self.max_capture_size, 1), + output=output, + ) return output @@ -350,9 +347,6 @@ class TrtLlmMxfp4ExpertsModular(TrtLlmMxfp4ExpertsBase, mk.FusedMoEExpertsModula from flashinfer import trtllm_fp4_block_scale_routed_moe - from vllm.utils.flashinfer import _is_fi_autotuning, autotune - - with autotune(_is_fi_autotuning): - trtllm_fp4_block_scale_routed_moe(**kwargs) + trtllm_fp4_block_scale_routed_moe(**kwargs) return output diff --git a/vllm/model_executor/warmup/kernel_warmup.py b/vllm/model_executor/warmup/kernel_warmup.py index 70abd8a6c5..cb5917d3a1 100644 --- a/vllm/model_executor/warmup/kernel_warmup.py +++ b/vllm/model_executor/warmup/kernel_warmup.py @@ -87,23 +87,69 @@ def flashinfer_autotune(runner: "GPUModelRunner") -> None: future calls to FlashInfer will use the best implementation. Without autotuning, FlashInfer will rely on heuristics, which may be significantly slower. + + Tuning is performed only on rank 0. The resulting cache is broadcast + to every rank so all ranks dispatch the same kernel tactic. """ + import os + import tempfile + import vllm.utils.flashinfer as fi_utils + from vllm.distributed.parallel_state import get_world_group - with torch.inference_mode(), fi_utils.autotune(): - # Certain FlashInfer kernels (e.g. nvfp4 routed moe) are - # incompatible with autotuning. This state is used to skip - # those kernels during the autotuning process. - fi_utils._is_fi_autotuning = True + world = get_world_group() + is_leader = world.rank_in_group == 0 - # We skip EPLB here since we don't want to record dummy metrics - # When autotuning with number of tokens m, flashinfer will autotune - # operations for all number of tokens up to m. - # So we only need to run with the max number of tokens. - runner._dummy_run( - runner.scheduler_config.max_num_batched_tokens, - skip_eplb=True, - is_profile=True, + cache_dir = tempfile.mkdtemp(prefix="vllm_flashinfer_autotune_") + cache_path = os.path.join(cache_dir, "autotune_cache.json") + + # We skip EPLB here since we don't want to record dummy metrics. + # When autotuning with number of tokens m, flashinfer will autotune + # operations for all number of tokens up to m, so we only need to + # run with the max number of tokens. + dummy_run_kwargs = dict( + num_tokens=runner.scheduler_config.max_num_batched_tokens, + skip_eplb=True, + is_profile=True, + ) + + with torch.inference_mode(): + if is_leader: + with fi_utils.autotune(tune_mode=True, cache=cache_path): + runner._dummy_run(**dummy_run_kwargs) + else: + runner._dummy_run(**dummy_run_kwargs) + + # Broadcast autotune cache from rank 0 to all other ranks so every + # rank loads the same set of chosen tactics. + tune_results: bytes | None = None + if is_leader and os.path.exists(cache_path): + with open(cache_path, "rb") as f: + tune_results = f.read() + + tune_results = world.broadcast_object(tune_results, src=0) + + if tune_results is None: + logger.warning( + "No FlashInfer autotune cache entries found." + "Falling back to default tactics." + ) + else: + if not is_leader: + with open(cache_path, "wb") as f: + f.write(tune_results) + from flashinfer.autotuner import AutoTuner + + AutoTuner.get().load_configs(cache_path) + logger.info( + "FlashInfer autotune cache loaded on rank %d from %s.", + world.rank_in_group, + cache_path, ) - fi_utils._is_fi_autotuning = False + try: + if os.path.exists(cache_path): + os.unlink(cache_path) + os.rmdir(cache_dir) + except OSError: + pass diff --git a/vllm/utils/flashinfer.py b/vllm/utils/flashinfer.py index 54d8645dc1..287de921c1 100644 --- a/vllm/utils/flashinfer.py +++ b/vllm/utils/flashinfer.py @@ -144,7 +144,6 @@ autotune = _lazy_import_wrapper( "autotune", fallback_fn=lambda *args, **kwargs: contextlib.nullcontext(), ) -_is_fi_autotuning: bool = False @functools.cache