[FlashInfer] Migrate autotune warmup to flashinfer.autotune_v2 managed cache (#55671)
Signed-off-by: Yang Xu <yanxu@nvidia.com> Signed-off-by: Zihua Wu <13583761+lucifer1004@users.noreply.github.com> Signed-off-by: YangXu <yanxu@nvidia.com> Co-authored-by: Zihua Wu <13583761+lucifer1004@users.noreply.github.com> Co-authored-by: Kimi Code <noreply@moonshot.cn> Co-authored-by: Codex <noreply@openai.com> Co-authored-by: Misha Goin <mgoin64@gmail.com> Co-authored-by: vllm-agent <claw@inferact.ai>
This commit is contained in:
co-authored by
Zihua Wu
Kimi Code
Codex
Misha Goin
vllm-agent
parent
b3a7e2ff6e
commit
71c20f73f2
@@ -23,6 +23,8 @@ from vllm.model_executor.warmup.cutedsl_warmup import cutedsl_warmup
|
||||
from vllm.model_executor.warmup.deep_gemm_warmup import deep_gemm_warmup
|
||||
from vllm.model_executor.warmup.flashinfer_autotune_cache import (
|
||||
resolve_flashinfer_autotune_file,
|
||||
resolve_flashinfer_autotune_v2_root,
|
||||
use_flashinfer_autotune_v2,
|
||||
)
|
||||
from vllm.model_executor.warmup.flashinfer_sparse_mla_warmup import (
|
||||
autotune_hisparse_flashinfer_attention,
|
||||
@@ -419,7 +421,7 @@ def flashinfer_autotune(runner: "GPUModelRunner", *, skip_attn: bool = False) ->
|
||||
otherwise the world group tunes together. Per-tactic timings are
|
||||
averaged over the tuning group so all its ranks select the same tactic.
|
||||
|
||||
Results are persisted per rank: FlashInfer keys MoE entries by tp/ep rank
|
||||
Legacy results are persisted per rank: FlashInfer keys MoE entries by tp/ep rank
|
||||
(``MoERunner.get_cache_key_extras``), so one rank's file only hits on that
|
||||
rank. A rank with a cache hit skips the per-tactic reduce the others block
|
||||
in, so ranks keep loaded configs only if every rank in the tuning group
|
||||
@@ -450,38 +452,45 @@ def flashinfer_autotune(runner: "GPUModelRunner", *, skip_attn: bool = False) ->
|
||||
)
|
||||
autotune_kwargs["skip_ops"] = skip_ops
|
||||
|
||||
cache_path = resolve_flashinfer_autotune_file(runner)
|
||||
# The world group only spans DP ranks when vLLM folds DP into it.
|
||||
dp_rank = runner.vllm_config.parallel_config.data_parallel_rank
|
||||
cache_path = cache_path.with_name(
|
||||
f"{cache_path.stem}_dp{dp_rank}_rank{world.rank_in_group}{cache_path.suffix}"
|
||||
)
|
||||
if is_leader:
|
||||
logger.info_once("Using FlashInfer autotune cache file: %s", cache_path)
|
||||
use_autotune_v2 = use_flashinfer_autotune_v2(runner)
|
||||
if use_autotune_v2:
|
||||
from flashinfer import autotune_v2, autotune_v2_reload
|
||||
|
||||
# We skip EPLB here since we don't want to record dummy metrics.
|
||||
# Randomize inputs to avoid every token pick the same experts,
|
||||
# which lead to some EP ranks receiving no tokens and skipping their
|
||||
# MoE kernel entirely, and cause hang due to all-reduce collective
|
||||
# during synchronized autotuning.
|
||||
if _all_ranks_have_matching_cache(cache_path, tune_group):
|
||||
loaded = tuner.load_configs(str(cache_path))
|
||||
if tune_group.world_size > 1:
|
||||
loaded_by_rank: list[bool | None] = [None] * tune_group.world_size
|
||||
torch.distributed.all_gather_object(
|
||||
loaded_by_rank, loaded, group=tune_group.cpu_group
|
||||
cache_root = resolve_flashinfer_autotune_v2_root()
|
||||
if is_leader:
|
||||
logger.info_once(
|
||||
"Using FlashInfer managed autotune cache (root=%s)",
|
||||
cache_root if cache_root is not None else "flashinfer default",
|
||||
)
|
||||
loaded = all(loaded_by_rank)
|
||||
if not loaded:
|
||||
tuner.clear_cache()
|
||||
autotune_context = autotune_v2(
|
||||
mode="tune", cache_root=cache_root, **autotune_kwargs
|
||||
)
|
||||
else:
|
||||
cache_path = resolve_flashinfer_autotune_file(runner)
|
||||
# The world group only spans DP ranks when vLLM folds DP into it.
|
||||
dp_rank = runner.vllm_config.parallel_config.data_parallel_rank
|
||||
cache_path = cache_path.with_name(
|
||||
f"{cache_path.stem}_dp{dp_rank}_rank{world.rank_in_group}{cache_path.suffix}"
|
||||
)
|
||||
if is_leader:
|
||||
logger.info_once("Using FlashInfer autotune cache file: %s", cache_path)
|
||||
|
||||
if _all_ranks_have_matching_cache(cache_path, tune_group):
|
||||
loaded = tuner.load_configs(str(cache_path))
|
||||
if tune_group.world_size > 1:
|
||||
loaded_by_rank: list[bool | None] = [None] * tune_group.world_size
|
||||
torch.distributed.all_gather_object(
|
||||
loaded_by_rank, loaded, group=tune_group.cpu_group
|
||||
)
|
||||
loaded = all(loaded_by_rank)
|
||||
if not loaded:
|
||||
tuner.clear_cache()
|
||||
autotune_context = fi_utils.autotune(tune_mode=True, **autotune_kwargs)
|
||||
|
||||
group = tune_group.cpu_group if tune_group.world_size > 1 else None
|
||||
set_autotune_process_group(group)
|
||||
try:
|
||||
with (
|
||||
torch.inference_mode(),
|
||||
fi_utils.autotune(tune_mode=True, **autotune_kwargs),
|
||||
):
|
||||
with torch.inference_mode(), autotune_context:
|
||||
hisparse_enabled = (
|
||||
runner.vllm_config.attention_config.hisparse_config is not None
|
||||
)
|
||||
@@ -498,7 +507,10 @@ def flashinfer_autotune(runner: "GPUModelRunner", *, skip_attn: bool = False) ->
|
||||
|
||||
if world.world_size > 1:
|
||||
world.barrier()
|
||||
# Skip the rewrite when nothing was tuned this start (every entry came from
|
||||
# the file). FlashInfer gates its own autotune(cache=...) save the same way.
|
||||
if tuner._dirty:
|
||||
if use_autotune_v2:
|
||||
if world.world_size > 1:
|
||||
# All ranks serve the shared store's final state after publishing.
|
||||
autotune_v2_reload()
|
||||
world.barrier()
|
||||
elif tuner._dirty:
|
||||
tuner.save_configs(str(cache_path))
|
||||
|
||||
Reference in New Issue
Block a user