[Bugfix][PP][Spec Decode] MiniMax-M3 EAGLE3 aux-state relay at PP > 1 and per-stage FlashInfer autotune (#57197)
Signed-off-by: venkywonka <23023424+venkywonka@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -1,16 +1,23 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import json
|
||||
import sys
|
||||
from collections import defaultdict
|
||||
from contextlib import nullcontext
|
||||
from inspect import signature
|
||||
from types import SimpleNamespace
|
||||
from pathlib import Path
|
||||
from types import ModuleType, SimpleNamespace
|
||||
from typing import Any
|
||||
from unittest.mock import Mock, call, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.model_executor.warmup import kernel_warmup as warmup
|
||||
from vllm.model_executor.warmup.kernel_warmup import (
|
||||
_flashinfer_autotune_token_counts,
|
||||
_run_flashinfer_autotune_dummy_runs,
|
||||
flashinfer_autotune,
|
||||
)
|
||||
|
||||
pytestmark = pytest.mark.cpu_test
|
||||
@@ -37,7 +44,8 @@ def _make_runner(modules, *, max_tokens: int = 8192, linear_backend: str = "auto
|
||||
return SimpleNamespace(
|
||||
scheduler_config=SimpleNamespace(max_num_batched_tokens=max_tokens),
|
||||
vllm_config=SimpleNamespace(
|
||||
kernel_config=SimpleNamespace(linear_backend=linear_backend)
|
||||
kernel_config=SimpleNamespace(linear_backend=linear_backend),
|
||||
attention_config=SimpleNamespace(hisparse_config=None),
|
||||
),
|
||||
get_model=Mock(
|
||||
return_value=SimpleNamespace(modules=Mock(return_value=modules))
|
||||
@@ -125,3 +133,183 @@ def test_flashinfer_autotune_uses_token_buckets_for_each_dummy_run(skip_attn):
|
||||
**({"skip_attn": True} if skip_attn else {}),
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
class _AutotuneGroup:
|
||||
def __init__(self, run, ranks):
|
||||
self.run = run
|
||||
self.ranks = tuple(ranks)
|
||||
self.world_size = len(self.ranks)
|
||||
self.rank_in_group = self.ranks.index(run.rank)
|
||||
self.cpu_group = self
|
||||
|
||||
def record(self, operation):
|
||||
self.run.collectives[self.ranks][self.run.rank].append(operation)
|
||||
|
||||
def broadcast_object(self, obj, src=0):
|
||||
assert src == 0
|
||||
self.record(("broadcast", src))
|
||||
if self.rank_in_group == src:
|
||||
self.run.broadcasts[self.ranks] = obj
|
||||
return self.run.broadcasts[self.ranks]
|
||||
|
||||
def barrier(self):
|
||||
self.record(("barrier",))
|
||||
|
||||
|
||||
class _AutotuneTuner:
|
||||
def __init__(self, run):
|
||||
self.run = run
|
||||
self.cache = {}
|
||||
self.loaded = None
|
||||
|
||||
def load_configs(self, path):
|
||||
self.loaded = json.loads(Path(path).read_text())
|
||||
self.cache.update(self.loaded)
|
||||
|
||||
def save_configs(self, path):
|
||||
self.run.saves.append((self.run.rank, Path(path), dict(self.cache)))
|
||||
Path(path).write_text(json.dumps(self.cache))
|
||||
|
||||
def profile(self, operation):
|
||||
if operation in self.cache:
|
||||
return
|
||||
group = self.run.tuning_group
|
||||
self.run.profile_groups[self.run.rank].append(
|
||||
None if group is None else group.ranks
|
||||
)
|
||||
for tactic in range(2):
|
||||
if group is not None:
|
||||
group.record(("all_reduce", operation, tactic))
|
||||
self.cache[operation] = self.run.rank // self.run.tp
|
||||
|
||||
|
||||
class _AutotuneRun:
|
||||
def __init__(self, pp, tp):
|
||||
self.pp, self.tp = pp, tp
|
||||
self.rank = 0
|
||||
self.tuning_group = None
|
||||
self.collectives: dict[tuple[int, ...], dict[int, list[tuple[Any, ...]]]] = (
|
||||
defaultdict(lambda: defaultdict(list))
|
||||
)
|
||||
self.broadcasts = {}
|
||||
self.tuners = {}
|
||||
self.saves = []
|
||||
self.profile_groups = defaultdict(list)
|
||||
|
||||
def world(self):
|
||||
return _AutotuneGroup(self, range(self.pp * self.tp))
|
||||
|
||||
def tensor_group(self):
|
||||
start = self.rank // self.tp * self.tp
|
||||
return _AutotuneGroup(self, range(start, start + self.tp))
|
||||
|
||||
def pipeline_group(self):
|
||||
return SimpleNamespace(world_size=self.pp)
|
||||
|
||||
def set_group(self, group):
|
||||
self.tuning_group = group
|
||||
|
||||
def dummy_runs(self, runner, *, skip_attn=False):
|
||||
self.tuners[self.rank].profile("shared_gemm")
|
||||
if self.pp > 1 and self.rank // self.tp == 0:
|
||||
self.tuners[self.rank].profile("pp0_extra_gemm")
|
||||
|
||||
def execute(self):
|
||||
for rank in range(self.pp * self.tp):
|
||||
self.rank = rank
|
||||
self.tuners[rank] = _AutotuneTuner(self)
|
||||
flashinfer_autotune(_make_runner([]))
|
||||
return self
|
||||
|
||||
def assert_collectives_match(self):
|
||||
for ranks, traces in self.collectives.items():
|
||||
assert set(traces) == set(ranks)
|
||||
expected = traces[ranks[0]]
|
||||
assert all(trace == expected for trace in traces.values()), dict(traces)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def autotune_run(monkeypatch, tmp_path):
|
||||
import vllm.utils.flashinfer as fi_utils
|
||||
from vllm.distributed import parallel_state
|
||||
|
||||
def make_run(*, pp=2, tp=4):
|
||||
run = _AutotuneRun(pp, tp)
|
||||
autotuner = ModuleType("flashinfer.autotuner")
|
||||
monkeypatch.setattr(
|
||||
autotuner,
|
||||
"AutoTuner",
|
||||
SimpleNamespace(get=lambda: run.tuners[run.rank]),
|
||||
raising=False,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
autotuner, "set_autotune_process_group", run.set_group, raising=False
|
||||
)
|
||||
monkeypatch.setitem(sys.modules, "flashinfer.autotuner", autotuner)
|
||||
monkeypatch.setattr(parallel_state, "get_world_group", run.world)
|
||||
monkeypatch.setattr(parallel_state, "get_tp_group", run.tensor_group)
|
||||
monkeypatch.setattr(parallel_state, "get_pp_group", run.pipeline_group)
|
||||
monkeypatch.setattr(fi_utils, "autotune", lambda **kwargs: nullcontext())
|
||||
monkeypatch.setattr(
|
||||
warmup,
|
||||
"resolve_flashinfer_autotune_file",
|
||||
lambda runner: tmp_path / "autotune_configs.json",
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
warmup, "_flashinfer_autotune_skip_ops", lambda runner: None
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
warmup, "_run_flashinfer_autotune_dummy_runs", run.dummy_runs
|
||||
)
|
||||
monkeypatch.setattr(warmup, "replayssm_autotune_warmup", lambda runner: None)
|
||||
monkeypatch.setattr(warmup, "_autotune_kimi_k3_kda_qkvg", lambda model: None)
|
||||
return run
|
||||
|
||||
return make_run
|
||||
|
||||
|
||||
@pytest.mark.parametrize("tp", [1, 4])
|
||||
def test_heterogeneous_pp_stages_have_compatible_collectives(autotune_run, tp):
|
||||
run = autotune_run(tp=tp).execute()
|
||||
run.assert_collectives_match()
|
||||
for rank, groups in run.profile_groups.items():
|
||||
expected = tuple(range(rank // tp * tp, (rank // tp + 1) * tp))
|
||||
assert groups and all(
|
||||
group == (expected if tp > 1 else None) for group in groups
|
||||
)
|
||||
|
||||
|
||||
def test_pp_stage_cache_roundtrip_isolated_and_asymmetric_hits_safe(
|
||||
autotune_run, tmp_path
|
||||
):
|
||||
legacy = tmp_path / "autotune_configs.json"
|
||||
legacy.write_text('{"legacy_world_cache": 99}')
|
||||
cold = autotune_run().execute()
|
||||
assert [rank for rank, _, _ in cold.saves] == [0, 4]
|
||||
paths = [path for _, path, _ in cold.saves]
|
||||
assert len(set(paths)) == 2 and legacy not in paths
|
||||
assert json.loads(legacy.read_text()) == {"legacy_world_cache": 99}
|
||||
assert cold.saves[0][2] == {"shared_gemm": 0, "pp0_extra_gemm": 0}
|
||||
assert cold.saves[1][2] == {"shared_gemm": 1}
|
||||
cold.assert_collectives_match()
|
||||
warm = autotune_run().execute()
|
||||
warm.assert_collectives_match()
|
||||
assert not warm.profile_groups
|
||||
for rank, tuner in warm.tuners.items():
|
||||
assert tuner.loaded == cold.saves[rank // 4][2]
|
||||
paths[1].unlink()
|
||||
mixed = autotune_run().execute()
|
||||
mixed.assert_collectives_match()
|
||||
assert set(mixed.profile_groups) == {4, 5, 6, 7}
|
||||
assert all(mixed.tuners[rank].loaded is not None for rank in range(4))
|
||||
assert all(mixed.tuners[rank].loaded is None for rank in range(4, 8))
|
||||
|
||||
|
||||
def test_pp1_retains_world_synchronization_and_existing_cache_name(autotune_run):
|
||||
run = autotune_run(pp=1, tp=4).execute()
|
||||
run.assert_collectives_match()
|
||||
assert [(rank, path.name) for rank, path, _ in run.saves] == [
|
||||
(0, "autotune_configs.json")
|
||||
]
|
||||
assert all(groups == [(0, 1, 2, 3)] for groups in run.profile_groups.values())
|
||||
|
||||
Reference in New Issue
Block a user