[Bugfix] Persist FlashInfer autotune cache per rank to fix the rank-0-only cache-hit deadlock (#57635)
Signed-off-by: Madhav Jivrajani <madhav.jiv@gmail.com> Signed-off-by: NickLucche <nicolo.lucchesi@mistral.ai> Co-authored-by: NickLucche <nicolo.lucchesi@mistral.ai> Co-authored-by: Wei Zhao <51183510+wzhao18@users.noreply.github.com>
This commit is contained in:
co-authored by
NickLucche
Wei Zhao
parent
02b83919aa
commit
3403e0f176
@@ -5,8 +5,10 @@ This is useful specifically for JIT'ed kernels as we don't want JIT'ing to
|
||||
happen during model execution.
|
||||
"""
|
||||
|
||||
import json
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import torch
|
||||
@@ -18,7 +20,6 @@ from vllm.model_executor.warmup.cutedsl_warmup import cutedsl_warmup
|
||||
from vllm.model_executor.warmup.deep_gemm_warmup import deep_gemm_warmup
|
||||
from vllm.model_executor.warmup.flashinfer_autotune_cache import (
|
||||
resolve_flashinfer_autotune_file,
|
||||
write_flashinfer_autotune_cache,
|
||||
)
|
||||
from vllm.model_executor.warmup.flashinfer_sparse_mla_warmup import (
|
||||
autotune_hisparse_flashinfer_attention,
|
||||
@@ -408,6 +409,26 @@ def _run_flashinfer_bf16_autotune_dummy_run(
|
||||
)
|
||||
|
||||
|
||||
def _autotune_cache_fingerprint(path: Path) -> tuple[str, int] | None:
|
||||
"""Identify a saved autotune file by its FlashInfer metadata and size."""
|
||||
try:
|
||||
configs = json.loads(path.read_text())
|
||||
metadata = configs.pop("_metadata", None)
|
||||
except (OSError, ValueError, AttributeError):
|
||||
return None
|
||||
return json.dumps(metadata, sort_keys=True), len(configs)
|
||||
|
||||
|
||||
def _all_ranks_have_matching_cache(path: Path, group) -> bool:
|
||||
"""True iff every rank in ``group`` has its own, mutually consistent file."""
|
||||
fingerprint = _autotune_cache_fingerprint(path)
|
||||
if group.world_size == 1:
|
||||
return fingerprint is not None
|
||||
gathered: list[tuple[str, int] | None] = [None] * group.world_size
|
||||
torch.distributed.all_gather_object(gathered, fingerprint, group=group.cpu_group)
|
||||
return fingerprint is not None and all(f == fingerprint for f in gathered)
|
||||
|
||||
|
||||
def flashinfer_autotune(runner: "GPUModelRunner") -> None:
|
||||
"""Autotune FlashInfer operations.
|
||||
FlashInfer have many implementations for the same operation,
|
||||
@@ -421,6 +442,12 @@ def flashinfer_autotune(runner: "GPUModelRunner") -> None:
|
||||
so each stage's TP group tunes separately with its own cache file;
|
||||
otherwise the world group tunes together. Per-tactic timings are
|
||||
averaged over the tuning group so all its ranks select the same tactic.
|
||||
|
||||
Results are persisted per rank: FlashInfer keys MoE entries by tp/ep rank
|
||||
(``MoERunner.get_cache_key_extras``), so one rank's file only hits on that
|
||||
rank. A rank with a cache hit skips the per-tactic reduce the others block
|
||||
in, so ranks keep loaded configs only if every rank in the tuning group
|
||||
has a matching file and successfully loads it.
|
||||
"""
|
||||
from flashinfer.autotuner import AutoTuner, set_autotune_process_group
|
||||
|
||||
@@ -447,11 +474,11 @@ def flashinfer_autotune(runner: "GPUModelRunner") -> None:
|
||||
autotune_kwargs["skip_ops"] = skip_ops
|
||||
|
||||
cache_path = resolve_flashinfer_autotune_file(runner)
|
||||
if pp_size > 1:
|
||||
ranks = "-".join(str(rank) for rank in tune_group.ranks)
|
||||
cache_path = cache_path.with_name(
|
||||
f"{cache_path.stem}_tp_{ranks}{cache_path.suffix}"
|
||||
)
|
||||
# The world group only spans DP ranks when vLLM folds DP into it.
|
||||
dp_rank = runner.vllm_config.parallel_config.data_parallel_rank
|
||||
cache_path = cache_path.with_name(
|
||||
f"{cache_path.stem}_dp{dp_rank}_rank{world.rank_in_group}{cache_path.suffix}"
|
||||
)
|
||||
if is_leader:
|
||||
logger.info_once("Using FlashInfer autotune cache file: %s", cache_path)
|
||||
|
||||
@@ -460,16 +487,16 @@ def flashinfer_autotune(runner: "GPUModelRunner") -> None:
|
||||
# which lead to some EP ranks receiving no tokens and skipping their
|
||||
# MoE kernel entirely, and cause hang due to all-reduce collective
|
||||
# during synchronized autotuning.
|
||||
# Read cached autotune results and broadcast within the tuning group.
|
||||
cached_results: bytes | None = None
|
||||
if is_leader and cache_path.exists():
|
||||
with open(cache_path, "rb") as f:
|
||||
cached_results = f.read()
|
||||
cached_results = tune_group.broadcast_object(cached_results, src=0)
|
||||
if cached_results is not None:
|
||||
write_flashinfer_autotune_cache(cache_path, cached_results)
|
||||
tune_group.barrier()
|
||||
tuner.load_configs(str(cache_path))
|
||||
if _all_ranks_have_matching_cache(cache_path, tune_group):
|
||||
loaded = tuner.load_configs(str(cache_path))
|
||||
if tune_group.world_size > 1:
|
||||
loaded_by_rank: list[bool | None] = [None] * tune_group.world_size
|
||||
torch.distributed.all_gather_object(
|
||||
loaded_by_rank, loaded, group=tune_group.cpu_group
|
||||
)
|
||||
loaded = all(loaded_by_rank)
|
||||
if not loaded:
|
||||
tuner.clear_cache()
|
||||
|
||||
group = tune_group.cpu_group if tune_group.world_size > 1 else None
|
||||
set_autotune_process_group(group)
|
||||
@@ -497,5 +524,7 @@ def flashinfer_autotune(runner: "GPUModelRunner") -> None:
|
||||
|
||||
if world.world_size > 1:
|
||||
world.barrier()
|
||||
if is_leader:
|
||||
# Skip the rewrite when nothing was tuned this start (every entry came from
|
||||
# the file). FlashInfer gates its own autotune(cache=...) save the same way.
|
||||
if tuner._dirty:
|
||||
tuner.save_configs(str(cache_path))
|
||||
|
||||
Reference in New Issue
Block a user