[Bugfix] Persist FlashInfer autotune cache per rank to fix the rank-0-only cache-hit deadlock (#57635)
Signed-off-by: Madhav Jivrajani <madhav.jiv@gmail.com> Signed-off-by: NickLucche <nicolo.lucchesi@mistral.ai> Co-authored-by: NickLucche <nicolo.lucchesi@mistral.ai> Co-authored-by: Wei Zhao <51183510+wzhao18@users.noreply.github.com>
This commit is contained in:
co-authored by
NickLucche
Wei Zhao
parent
02b83919aa
commit
3403e0f176
@@ -46,6 +46,7 @@ def _make_runner(modules, *, max_tokens: int = 8192, linear_backend: str = "auto
|
|||||||
vllm_config=SimpleNamespace(
|
vllm_config=SimpleNamespace(
|
||||||
kernel_config=SimpleNamespace(linear_backend=linear_backend),
|
kernel_config=SimpleNamespace(linear_backend=linear_backend),
|
||||||
attention_config=SimpleNamespace(hisparse_config=None),
|
attention_config=SimpleNamespace(hisparse_config=None),
|
||||||
|
parallel_config=SimpleNamespace(data_parallel_rank=0),
|
||||||
),
|
),
|
||||||
get_model=Mock(
|
get_model=Mock(
|
||||||
return_value=SimpleNamespace(modules=Mock(return_value=modules))
|
return_value=SimpleNamespace(modules=Mock(return_value=modules))
|
||||||
@@ -146,12 +147,20 @@ class _AutotuneGroup:
|
|||||||
def record(self, operation):
|
def record(self, operation):
|
||||||
self.run.collectives[self.ranks][self.run.rank].append(operation)
|
self.run.collectives[self.ranks][self.run.rank].append(operation)
|
||||||
|
|
||||||
def broadcast_object(self, obj, src=0):
|
def all_gather_object(self, out, obj):
|
||||||
assert src == 0
|
# Ranks run sequentially: snapshot every rank's file before any saves.
|
||||||
self.record(("broadcast", src))
|
self.record(("all_gather",))
|
||||||
if self.rank_in_group == src:
|
if isinstance(obj, bool):
|
||||||
self.run.broadcasts[self.ranks] = obj
|
out[:] = [rank not in self.run.rejected_load_ranks for rank in self.ranks]
|
||||||
return self.run.broadcasts[self.ranks]
|
assert out[self.rank_in_group] == obj
|
||||||
|
return
|
||||||
|
out[:] = self.run.gathered.setdefault(
|
||||||
|
self.ranks,
|
||||||
|
[
|
||||||
|
warmup._autotune_cache_fingerprint(self.run.cache_path(rank))
|
||||||
|
for rank in self.ranks
|
||||||
|
],
|
||||||
|
)
|
||||||
|
|
||||||
def barrier(self):
|
def barrier(self):
|
||||||
self.record(("barrier",))
|
self.record(("barrier",))
|
||||||
@@ -162,10 +171,18 @@ class _AutotuneTuner:
|
|||||||
self.run = run
|
self.run = run
|
||||||
self.cache = {}
|
self.cache = {}
|
||||||
self.loaded = None
|
self.loaded = None
|
||||||
|
self._dirty = False
|
||||||
|
|
||||||
def load_configs(self, path):
|
def load_configs(self, path):
|
||||||
|
if self.run.rank in self.run.rejected_load_ranks:
|
||||||
|
return False
|
||||||
self.loaded = json.loads(Path(path).read_text())
|
self.loaded = json.loads(Path(path).read_text())
|
||||||
self.cache.update(self.loaded)
|
self.cache.update(self.loaded)
|
||||||
|
return True
|
||||||
|
|
||||||
|
def clear_cache(self):
|
||||||
|
self.cache.clear()
|
||||||
|
self._dirty = False
|
||||||
|
|
||||||
def save_configs(self, path):
|
def save_configs(self, path):
|
||||||
self.run.saves.append((self.run.rank, Path(path), dict(self.cache)))
|
self.run.saves.append((self.run.rank, Path(path), dict(self.cache)))
|
||||||
@@ -182,21 +199,27 @@ class _AutotuneTuner:
|
|||||||
if group is not None:
|
if group is not None:
|
||||||
group.record(("all_reduce", operation, tactic))
|
group.record(("all_reduce", operation, tactic))
|
||||||
self.cache[operation] = self.run.rank // self.run.tp
|
self.cache[operation] = self.run.rank // self.run.tp
|
||||||
|
self._dirty = True
|
||||||
|
|
||||||
|
|
||||||
class _AutotuneRun:
|
class _AutotuneRun:
|
||||||
def __init__(self, pp, tp):
|
def __init__(self, pp, tp, cache_dir):
|
||||||
self.pp, self.tp = pp, tp
|
self.pp, self.tp = pp, tp
|
||||||
|
self.cache_dir = cache_dir
|
||||||
self.rank = 0
|
self.rank = 0
|
||||||
self.tuning_group = None
|
self.tuning_group = None
|
||||||
self.collectives: dict[tuple[int, ...], dict[int, list[tuple[Any, ...]]]] = (
|
self.collectives: dict[tuple[int, ...], dict[int, list[tuple[Any, ...]]]] = (
|
||||||
defaultdict(lambda: defaultdict(list))
|
defaultdict(lambda: defaultdict(list))
|
||||||
)
|
)
|
||||||
self.broadcasts = {}
|
self.gathered = {}
|
||||||
|
self.rejected_load_ranks: set[int] = set()
|
||||||
self.tuners = {}
|
self.tuners = {}
|
||||||
self.saves = []
|
self.saves = []
|
||||||
self.profile_groups = defaultdict(list)
|
self.profile_groups = defaultdict(list)
|
||||||
|
|
||||||
|
def cache_path(self, rank):
|
||||||
|
return self.cache_dir / f"autotune_configs_dp0_rank{rank}.json"
|
||||||
|
|
||||||
def world(self):
|
def world(self):
|
||||||
return _AutotuneGroup(self, range(self.pp * self.tp))
|
return _AutotuneGroup(self, range(self.pp * self.tp))
|
||||||
|
|
||||||
@@ -235,7 +258,7 @@ def autotune_run(monkeypatch, tmp_path):
|
|||||||
from vllm.distributed import parallel_state
|
from vllm.distributed import parallel_state
|
||||||
|
|
||||||
def make_run(*, pp=2, tp=4):
|
def make_run(*, pp=2, tp=4):
|
||||||
run = _AutotuneRun(pp, tp)
|
run = _AutotuneRun(pp, tp, tmp_path)
|
||||||
autotuner = ModuleType("flashinfer.autotuner")
|
autotuner = ModuleType("flashinfer.autotuner")
|
||||||
monkeypatch.setattr(
|
monkeypatch.setattr(
|
||||||
autotuner,
|
autotuner,
|
||||||
@@ -251,6 +274,10 @@ def autotune_run(monkeypatch, tmp_path):
|
|||||||
monkeypatch.setattr(parallel_state, "get_tp_group", run.tensor_group)
|
monkeypatch.setattr(parallel_state, "get_tp_group", run.tensor_group)
|
||||||
monkeypatch.setattr(parallel_state, "get_pp_group", run.pipeline_group)
|
monkeypatch.setattr(parallel_state, "get_pp_group", run.pipeline_group)
|
||||||
monkeypatch.setattr(fi_utils, "autotune", lambda **kwargs: nullcontext())
|
monkeypatch.setattr(fi_utils, "autotune", lambda **kwargs: nullcontext())
|
||||||
|
monkeypatch.setattr(
|
||||||
|
"torch.distributed.all_gather_object",
|
||||||
|
lambda out, obj, group: group.all_gather_object(out, obj),
|
||||||
|
)
|
||||||
monkeypatch.setattr(
|
monkeypatch.setattr(
|
||||||
warmup,
|
warmup,
|
||||||
"resolve_flashinfer_autotune_file",
|
"resolve_flashinfer_autotune_file",
|
||||||
@@ -286,19 +313,18 @@ def test_pp_stage_cache_roundtrip_isolated_and_asymmetric_hits_safe(
|
|||||||
legacy = tmp_path / "autotune_configs.json"
|
legacy = tmp_path / "autotune_configs.json"
|
||||||
legacy.write_text('{"legacy_world_cache": 99}')
|
legacy.write_text('{"legacy_world_cache": 99}')
|
||||||
cold = autotune_run().execute()
|
cold = autotune_run().execute()
|
||||||
assert [rank for rank, _, _ in cold.saves] == [0, 4]
|
|
||||||
paths = [path for _, path, _ in cold.saves]
|
|
||||||
assert len(set(paths)) == 2 and legacy not in paths
|
|
||||||
assert json.loads(legacy.read_text()) == {"legacy_world_cache": 99}
|
|
||||||
assert cold.saves[0][2] == {"shared_gemm": 0, "pp0_extra_gemm": 0}
|
|
||||||
assert cold.saves[1][2] == {"shared_gemm": 1}
|
|
||||||
cold.assert_collectives_match()
|
cold.assert_collectives_match()
|
||||||
|
assert [rank for rank, _, _ in cold.saves] == list(range(8))
|
||||||
|
assert [path for _, path, _ in cold.saves] == [cold.cache_path(r) for r in range(8)]
|
||||||
|
assert json.loads(legacy.read_text()) == {"legacy_world_cache": 99}
|
||||||
|
stage_caches = [{"shared_gemm": 0, "pp0_extra_gemm": 0}, {"shared_gemm": 1}]
|
||||||
|
assert all(cache == stage_caches[r // 4] for r, _, cache in cold.saves)
|
||||||
warm = autotune_run().execute()
|
warm = autotune_run().execute()
|
||||||
warm.assert_collectives_match()
|
warm.assert_collectives_match()
|
||||||
assert not warm.profile_groups
|
assert not warm.profile_groups and not warm.saves
|
||||||
for rank, tuner in warm.tuners.items():
|
for rank, tuner in warm.tuners.items():
|
||||||
assert tuner.loaded == cold.saves[rank // 4][2]
|
assert tuner.loaded == stage_caches[rank // 4]
|
||||||
paths[1].unlink()
|
cold.cache_path(5).unlink()
|
||||||
mixed = autotune_run().execute()
|
mixed = autotune_run().execute()
|
||||||
mixed.assert_collectives_match()
|
mixed.assert_collectives_match()
|
||||||
assert set(mixed.profile_groups) == {4, 5, 6, 7}
|
assert set(mixed.profile_groups) == {4, 5, 6, 7}
|
||||||
@@ -306,10 +332,33 @@ def test_pp_stage_cache_roundtrip_isolated_and_asymmetric_hits_safe(
|
|||||||
assert all(mixed.tuners[rank].loaded is None for rank in range(4, 8))
|
assert all(mixed.tuners[rank].loaded is None for rank in range(4, 8))
|
||||||
|
|
||||||
|
|
||||||
def test_pp1_retains_world_synchronization_and_existing_cache_name(autotune_run):
|
def test_pp1_tunes_world_group_and_saves_per_rank(autotune_run):
|
||||||
run = autotune_run(pp=1, tp=4).execute()
|
run = autotune_run(pp=1, tp=4).execute()
|
||||||
run.assert_collectives_match()
|
run.assert_collectives_match()
|
||||||
assert [(rank, path.name) for rank, path, _ in run.saves] == [
|
assert [(rank, path) for rank, path, _ in run.saves] == [
|
||||||
(0, "autotune_configs.json")
|
(rank, run.cache_path(rank)) for rank in range(4)
|
||||||
]
|
]
|
||||||
assert all(groups == [(0, 1, 2, 3)] for groups in run.profile_groups.values())
|
assert all(groups == [(0, 1, 2, 3)] for groups in run.profile_groups.values())
|
||||||
|
|
||||||
|
|
||||||
|
def test_mismatched_rank_caches_are_ignored_by_every_rank(autotune_run):
|
||||||
|
"""One rank loading while others tune deadlocks the per-tactic reduce."""
|
||||||
|
cold = autotune_run(pp=1, tp=4).execute()
|
||||||
|
stale = cold.cache_path(2)
|
||||||
|
stale.write_text(json.dumps({**json.loads(stale.read_text()), "old_gemm": 0}))
|
||||||
|
rerun = autotune_run(pp=1, tp=4).execute()
|
||||||
|
rerun.assert_collectives_match()
|
||||||
|
assert all(tuner.loaded is None for tuner in rerun.tuners.values())
|
||||||
|
assert set(rerun.profile_groups) == {0, 1, 2, 3}
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("pp, tp", [(1, 1), (1, 4), (2, 4)])
|
||||||
|
def test_rejected_cache_retunes_every_rank_in_its_tuning_group(autotune_run, pp, tp):
|
||||||
|
"""Matching files can still be incompatible with one rank's runtime."""
|
||||||
|
autotune_run(pp=pp, tp=tp).execute()
|
||||||
|
rerun = autotune_run(pp=pp, tp=tp)
|
||||||
|
rerun.rejected_load_ranks = {tp - 1}
|
||||||
|
rerun.execute()
|
||||||
|
rerun.assert_collectives_match()
|
||||||
|
assert set(rerun.profile_groups) == set(range(tp))
|
||||||
|
assert {rank for rank, _, _ in rerun.saves} == set(range(tp))
|
||||||
|
|||||||
@@ -5,8 +5,10 @@ This is useful specifically for JIT'ed kernels as we don't want JIT'ing to
|
|||||||
happen during model execution.
|
happen during model execution.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
import json
|
||||||
import sys
|
import sys
|
||||||
import time
|
import time
|
||||||
|
from pathlib import Path
|
||||||
from typing import TYPE_CHECKING
|
from typing import TYPE_CHECKING
|
||||||
|
|
||||||
import torch
|
import torch
|
||||||
@@ -18,7 +20,6 @@ from vllm.model_executor.warmup.cutedsl_warmup import cutedsl_warmup
|
|||||||
from vllm.model_executor.warmup.deep_gemm_warmup import deep_gemm_warmup
|
from vllm.model_executor.warmup.deep_gemm_warmup import deep_gemm_warmup
|
||||||
from vllm.model_executor.warmup.flashinfer_autotune_cache import (
|
from vllm.model_executor.warmup.flashinfer_autotune_cache import (
|
||||||
resolve_flashinfer_autotune_file,
|
resolve_flashinfer_autotune_file,
|
||||||
write_flashinfer_autotune_cache,
|
|
||||||
)
|
)
|
||||||
from vllm.model_executor.warmup.flashinfer_sparse_mla_warmup import (
|
from vllm.model_executor.warmup.flashinfer_sparse_mla_warmup import (
|
||||||
autotune_hisparse_flashinfer_attention,
|
autotune_hisparse_flashinfer_attention,
|
||||||
@@ -408,6 +409,26 @@ def _run_flashinfer_bf16_autotune_dummy_run(
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _autotune_cache_fingerprint(path: Path) -> tuple[str, int] | None:
|
||||||
|
"""Identify a saved autotune file by its FlashInfer metadata and size."""
|
||||||
|
try:
|
||||||
|
configs = json.loads(path.read_text())
|
||||||
|
metadata = configs.pop("_metadata", None)
|
||||||
|
except (OSError, ValueError, AttributeError):
|
||||||
|
return None
|
||||||
|
return json.dumps(metadata, sort_keys=True), len(configs)
|
||||||
|
|
||||||
|
|
||||||
|
def _all_ranks_have_matching_cache(path: Path, group) -> bool:
|
||||||
|
"""True iff every rank in ``group`` has its own, mutually consistent file."""
|
||||||
|
fingerprint = _autotune_cache_fingerprint(path)
|
||||||
|
if group.world_size == 1:
|
||||||
|
return fingerprint is not None
|
||||||
|
gathered: list[tuple[str, int] | None] = [None] * group.world_size
|
||||||
|
torch.distributed.all_gather_object(gathered, fingerprint, group=group.cpu_group)
|
||||||
|
return fingerprint is not None and all(f == fingerprint for f in gathered)
|
||||||
|
|
||||||
|
|
||||||
def flashinfer_autotune(runner: "GPUModelRunner") -> None:
|
def flashinfer_autotune(runner: "GPUModelRunner") -> None:
|
||||||
"""Autotune FlashInfer operations.
|
"""Autotune FlashInfer operations.
|
||||||
FlashInfer have many implementations for the same operation,
|
FlashInfer have many implementations for the same operation,
|
||||||
@@ -421,6 +442,12 @@ def flashinfer_autotune(runner: "GPUModelRunner") -> None:
|
|||||||
so each stage's TP group tunes separately with its own cache file;
|
so each stage's TP group tunes separately with its own cache file;
|
||||||
otherwise the world group tunes together. Per-tactic timings are
|
otherwise the world group tunes together. Per-tactic timings are
|
||||||
averaged over the tuning group so all its ranks select the same tactic.
|
averaged over the tuning group so all its ranks select the same tactic.
|
||||||
|
|
||||||
|
Results are persisted per rank: FlashInfer keys MoE entries by tp/ep rank
|
||||||
|
(``MoERunner.get_cache_key_extras``), so one rank's file only hits on that
|
||||||
|
rank. A rank with a cache hit skips the per-tactic reduce the others block
|
||||||
|
in, so ranks keep loaded configs only if every rank in the tuning group
|
||||||
|
has a matching file and successfully loads it.
|
||||||
"""
|
"""
|
||||||
from flashinfer.autotuner import AutoTuner, set_autotune_process_group
|
from flashinfer.autotuner import AutoTuner, set_autotune_process_group
|
||||||
|
|
||||||
@@ -447,11 +474,11 @@ def flashinfer_autotune(runner: "GPUModelRunner") -> None:
|
|||||||
autotune_kwargs["skip_ops"] = skip_ops
|
autotune_kwargs["skip_ops"] = skip_ops
|
||||||
|
|
||||||
cache_path = resolve_flashinfer_autotune_file(runner)
|
cache_path = resolve_flashinfer_autotune_file(runner)
|
||||||
if pp_size > 1:
|
# The world group only spans DP ranks when vLLM folds DP into it.
|
||||||
ranks = "-".join(str(rank) for rank in tune_group.ranks)
|
dp_rank = runner.vllm_config.parallel_config.data_parallel_rank
|
||||||
cache_path = cache_path.with_name(
|
cache_path = cache_path.with_name(
|
||||||
f"{cache_path.stem}_tp_{ranks}{cache_path.suffix}"
|
f"{cache_path.stem}_dp{dp_rank}_rank{world.rank_in_group}{cache_path.suffix}"
|
||||||
)
|
)
|
||||||
if is_leader:
|
if is_leader:
|
||||||
logger.info_once("Using FlashInfer autotune cache file: %s", cache_path)
|
logger.info_once("Using FlashInfer autotune cache file: %s", cache_path)
|
||||||
|
|
||||||
@@ -460,16 +487,16 @@ def flashinfer_autotune(runner: "GPUModelRunner") -> None:
|
|||||||
# which lead to some EP ranks receiving no tokens and skipping their
|
# which lead to some EP ranks receiving no tokens and skipping their
|
||||||
# MoE kernel entirely, and cause hang due to all-reduce collective
|
# MoE kernel entirely, and cause hang due to all-reduce collective
|
||||||
# during synchronized autotuning.
|
# during synchronized autotuning.
|
||||||
# Read cached autotune results and broadcast within the tuning group.
|
if _all_ranks_have_matching_cache(cache_path, tune_group):
|
||||||
cached_results: bytes | None = None
|
loaded = tuner.load_configs(str(cache_path))
|
||||||
if is_leader and cache_path.exists():
|
if tune_group.world_size > 1:
|
||||||
with open(cache_path, "rb") as f:
|
loaded_by_rank: list[bool | None] = [None] * tune_group.world_size
|
||||||
cached_results = f.read()
|
torch.distributed.all_gather_object(
|
||||||
cached_results = tune_group.broadcast_object(cached_results, src=0)
|
loaded_by_rank, loaded, group=tune_group.cpu_group
|
||||||
if cached_results is not None:
|
)
|
||||||
write_flashinfer_autotune_cache(cache_path, cached_results)
|
loaded = all(loaded_by_rank)
|
||||||
tune_group.barrier()
|
if not loaded:
|
||||||
tuner.load_configs(str(cache_path))
|
tuner.clear_cache()
|
||||||
|
|
||||||
group = tune_group.cpu_group if tune_group.world_size > 1 else None
|
group = tune_group.cpu_group if tune_group.world_size > 1 else None
|
||||||
set_autotune_process_group(group)
|
set_autotune_process_group(group)
|
||||||
@@ -497,5 +524,7 @@ def flashinfer_autotune(runner: "GPUModelRunner") -> None:
|
|||||||
|
|
||||||
if world.world_size > 1:
|
if world.world_size > 1:
|
||||||
world.barrier()
|
world.barrier()
|
||||||
if is_leader:
|
# Skip the rewrite when nothing was tuned this start (every entry came from
|
||||||
|
# the file). FlashInfer gates its own autotune(cache=...) save the same way.
|
||||||
|
if tuner._dirty:
|
||||||
tuner.save_configs(str(cache_path))
|
tuner.save_configs(str(cache_path))
|
||||||
|
|||||||
Reference in New Issue
Block a user