[Bugfix][PP][Spec Decode] MiniMax-M3 EAGLE3 aux-state relay at PP > 1 and per-stage FlashInfer autotune (#57197)

Signed-off-by: venkywonka <23023424+venkywonka@users.noreply.github.com>
Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Venky
2026-10-01 01:14:24 +04:00
committed by GitHub
co-authored by Claude Opus 5.5
parent 73c7cae4d7
commit 42f0c17ea7
4 changed files with 382 additions and 15 deletions
@@ -1,16 +1,23 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import json
import sys
from collections import defaultdict
from contextlib import nullcontext
from inspect import signature
from types import SimpleNamespace
from pathlib import Path
from types import ModuleType, SimpleNamespace
from typing import Any
from unittest.mock import Mock, call, patch
import pytest
from vllm.model_executor.warmup import kernel_warmup as warmup
from vllm.model_executor.warmup.kernel_warmup import (
_flashinfer_autotune_token_counts,
_run_flashinfer_autotune_dummy_runs,
flashinfer_autotune,
)
pytestmark = pytest.mark.cpu_test
@@ -37,7 +44,8 @@ def _make_runner(modules, *, max_tokens: int = 8192, linear_backend: str = "auto
return SimpleNamespace(
scheduler_config=SimpleNamespace(max_num_batched_tokens=max_tokens),
vllm_config=SimpleNamespace(
kernel_config=SimpleNamespace(linear_backend=linear_backend)
kernel_config=SimpleNamespace(linear_backend=linear_backend),
attention_config=SimpleNamespace(hisparse_config=None),
),
get_model=Mock(
return_value=SimpleNamespace(modules=Mock(return_value=modules))
@@ -125,3 +133,183 @@ def test_flashinfer_autotune_uses_token_buckets_for_each_dummy_run(skip_attn):
**({"skip_attn": True} if skip_attn else {}),
),
]
class _AutotuneGroup:
def __init__(self, run, ranks):
self.run = run
self.ranks = tuple(ranks)
self.world_size = len(self.ranks)
self.rank_in_group = self.ranks.index(run.rank)
self.cpu_group = self
def record(self, operation):
self.run.collectives[self.ranks][self.run.rank].append(operation)
def broadcast_object(self, obj, src=0):
assert src == 0
self.record(("broadcast", src))
if self.rank_in_group == src:
self.run.broadcasts[self.ranks] = obj
return self.run.broadcasts[self.ranks]
def barrier(self):
self.record(("barrier",))
class _AutotuneTuner:
def __init__(self, run):
self.run = run
self.cache = {}
self.loaded = None
def load_configs(self, path):
self.loaded = json.loads(Path(path).read_text())
self.cache.update(self.loaded)
def save_configs(self, path):
self.run.saves.append((self.run.rank, Path(path), dict(self.cache)))
Path(path).write_text(json.dumps(self.cache))
def profile(self, operation):
if operation in self.cache:
return
group = self.run.tuning_group
self.run.profile_groups[self.run.rank].append(
None if group is None else group.ranks
)
for tactic in range(2):
if group is not None:
group.record(("all_reduce", operation, tactic))
self.cache[operation] = self.run.rank // self.run.tp
class _AutotuneRun:
def __init__(self, pp, tp):
self.pp, self.tp = pp, tp
self.rank = 0
self.tuning_group = None
self.collectives: dict[tuple[int, ...], dict[int, list[tuple[Any, ...]]]] = (
defaultdict(lambda: defaultdict(list))
)
self.broadcasts = {}
self.tuners = {}
self.saves = []
self.profile_groups = defaultdict(list)
def world(self):
return _AutotuneGroup(self, range(self.pp * self.tp))
def tensor_group(self):
start = self.rank // self.tp * self.tp
return _AutotuneGroup(self, range(start, start + self.tp))
def pipeline_group(self):
return SimpleNamespace(world_size=self.pp)
def set_group(self, group):
self.tuning_group = group
def dummy_runs(self, runner, *, skip_attn=False):
self.tuners[self.rank].profile("shared_gemm")
if self.pp > 1 and self.rank // self.tp == 0:
self.tuners[self.rank].profile("pp0_extra_gemm")
def execute(self):
for rank in range(self.pp * self.tp):
self.rank = rank
self.tuners[rank] = _AutotuneTuner(self)
flashinfer_autotune(_make_runner([]))
return self
def assert_collectives_match(self):
for ranks, traces in self.collectives.items():
assert set(traces) == set(ranks)
expected = traces[ranks[0]]
assert all(trace == expected for trace in traces.values()), dict(traces)
@pytest.fixture
def autotune_run(monkeypatch, tmp_path):
import vllm.utils.flashinfer as fi_utils
from vllm.distributed import parallel_state
def make_run(*, pp=2, tp=4):
run = _AutotuneRun(pp, tp)
autotuner = ModuleType("flashinfer.autotuner")
monkeypatch.setattr(
autotuner,
"AutoTuner",
SimpleNamespace(get=lambda: run.tuners[run.rank]),
raising=False,
)
monkeypatch.setattr(
autotuner, "set_autotune_process_group", run.set_group, raising=False
)
monkeypatch.setitem(sys.modules, "flashinfer.autotuner", autotuner)
monkeypatch.setattr(parallel_state, "get_world_group", run.world)
monkeypatch.setattr(parallel_state, "get_tp_group", run.tensor_group)
monkeypatch.setattr(parallel_state, "get_pp_group", run.pipeline_group)
monkeypatch.setattr(fi_utils, "autotune", lambda **kwargs: nullcontext())
monkeypatch.setattr(
warmup,
"resolve_flashinfer_autotune_file",
lambda runner: tmp_path / "autotune_configs.json",
)
monkeypatch.setattr(
warmup, "_flashinfer_autotune_skip_ops", lambda runner: None
)
monkeypatch.setattr(
warmup, "_run_flashinfer_autotune_dummy_runs", run.dummy_runs
)
monkeypatch.setattr(warmup, "replayssm_autotune_warmup", lambda runner: None)
monkeypatch.setattr(warmup, "_autotune_kimi_k3_kda_qkvg", lambda model: None)
return run
return make_run
@pytest.mark.parametrize("tp", [1, 4])
def test_heterogeneous_pp_stages_have_compatible_collectives(autotune_run, tp):
run = autotune_run(tp=tp).execute()
run.assert_collectives_match()
for rank, groups in run.profile_groups.items():
expected = tuple(range(rank // tp * tp, (rank // tp + 1) * tp))
assert groups and all(
group == (expected if tp > 1 else None) for group in groups
)
def test_pp_stage_cache_roundtrip_isolated_and_asymmetric_hits_safe(
autotune_run, tmp_path
):
legacy = tmp_path / "autotune_configs.json"
legacy.write_text('{"legacy_world_cache": 99}')
cold = autotune_run().execute()
assert [rank for rank, _, _ in cold.saves] == [0, 4]
paths = [path for _, path, _ in cold.saves]
assert len(set(paths)) == 2 and legacy not in paths
assert json.loads(legacy.read_text()) == {"legacy_world_cache": 99}
assert cold.saves[0][2] == {"shared_gemm": 0, "pp0_extra_gemm": 0}
assert cold.saves[1][2] == {"shared_gemm": 1}
cold.assert_collectives_match()
warm = autotune_run().execute()
warm.assert_collectives_match()
assert not warm.profile_groups
for rank, tuner in warm.tuners.items():
assert tuner.loaded == cold.saves[rank // 4][2]
paths[1].unlink()
mixed = autotune_run().execute()
mixed.assert_collectives_match()
assert set(mixed.profile_groups) == {4, 5, 6, 7}
assert all(mixed.tuners[rank].loaded is not None for rank in range(4))
assert all(mixed.tuners[rank].loaded is None for rank in range(4, 8))
def test_pp1_retains_world_synchronization_and_existing_cache_name(autotune_run):
run = autotune_run(pp=1, tp=4).execute()
run.assert_collectives_match()
assert [(rank, path.name) for rank, path, _ in run.saves] == [
(0, "autotune_configs.json")
]
assert all(groups == [(0, 1, 2, 3)] for groups in run.profile_groups.values())
@@ -3,10 +3,15 @@
from types import SimpleNamespace
import pytest
import torch
from vllm.distributed import parallel_state
from vllm.model_executor.models.interfaces import EagleModelMixin
from vllm.model_executor.models.mimo import MiMoModel
from vllm.models.minimax_m3.nvidia import model as minimax_m3
from vllm.sequence import IntermediateTensors
from vllm.v1.worker.gpu.spec_decode.eagle.eagle3_utils import (
reserve_aux_intermediate_tensor_slots,
verify_supports_aux_hidden_states_over_pp,
)
@@ -24,3 +29,150 @@ def test_mimo_does_not_inherit_aux_hidden_state_pp_support():
assert not inner.supports_aux_hidden_states_over_pp
with pytest.raises(ValueError, match="does not support eagle3"):
verify_supports_aux_hidden_states_over_pp(target, "eagle3")
class _PPAuxLayer(torch.nn.Module):
"""Aliases and mutates residual in place like MiniMaxM3DecoderLayer."""
ffn_all_reduce_deferred = False
def __init__(self, index):
super().__init__()
self.index = index
def forward(self, positions, hidden_states, residual):
total = hidden_states if residual is None else hidden_states + residual
residual = hidden_states if residual is None else residual
residual.copy_(total + self.index + 1)
return torch.full_like(hidden_states, 10 * (self.index + 1)), residual
class _PPAuxNorm(torch.nn.Module):
"""Clobbers its inputs so aux taps that alias them would fail."""
def forward(self, hidden_states, residual):
output = hidden_states + residual
hidden_states.fill_(-101)
residual.fill_(-103)
return output, residual
@pytest.fixture
def minimax_pp_stage(monkeypatch):
"""Use the real MiniMax constructor/forward and aux helpers on CPU tensors."""
group = SimpleNamespace(world_size=2, is_first_rank=True, is_last_rank=False)
bounds = [0, 3]
monkeypatch.setattr(minimax_m3, "get_pp_group", lambda: group)
monkeypatch.setattr(parallel_state, "get_pp_group", lambda: group)
monkeypatch.setattr(parallel_state, "model_parallel_is_initialized", lambda: True)
monkeypatch.setattr(
minimax_m3,
"VocabParallelEmbedding",
lambda *args, **kwargs: torch.nn.Identity(),
)
monkeypatch.setattr(
minimax_m3, "MiniMAXGemmaRMSNorm", lambda *args, **kwargs: _PPAuxNorm()
)
monkeypatch.setattr(
minimax_m3,
"MiniMaxM3DecoderLayer",
lambda **kwargs: _PPAuxLayer(int(kwargs["prefix"].rsplit(".", 1)[1])),
)
def make_layers(num_layers, build, prefix):
return (
*bounds,
torch.nn.ModuleList(
build(f"{prefix}.{index}")
if bounds[0] <= index < bounds[1]
else torch.nn.Identity()
for index in range(num_layers)
),
)
monkeypatch.setattr(minimax_m3, "make_layers", make_layers)
def make_stage(start, end, taps, world_size=2):
bounds[:] = [start, end]
group.world_size = world_size
group.is_first_rank = start == 0
group.is_last_rank = end == 6
config = SimpleNamespace(
vocab_size=16, hidden_size=6144, num_hidden_layers=6, rms_norm_eps=1e-6
)
model = minimax_m3.MiniMaxM3Model(
vllm_config=SimpleNamespace(
model_config=SimpleNamespace(hf_text_config=config), quant_config=None
)
)
model._set_aux_hidden_state_layers(taps)
target = SimpleNamespace(
model=model,
make_empty_intermediate_tensors=model.make_empty_intermediate_tensors,
)
reserve_aux_intermediate_tensor_slots(target)
return model, target
return make_stage
# Input 7; _PPAuxLayer i adds 11 * (i + 1) to hidden + residual.
def _expected_aux_value(tap):
return 7 + 11 * tap * (tap + 1) // 2
def _pp_aux_state(value):
return torch.full((2, 6144), float(value), device="cpu")
@pytest.mark.parametrize("taps", [(0, 3, 6), (0, 1, 3), (4, 5, 6)])
def test_minimax_pp2_preserves_ordered_global_aux_taps(minimax_pp_stage, taps):
first, _ = minimax_pp_stage(0, 3, taps)
sent = first(None, None, None, _pp_aux_state(7))
last, target = minimax_pp_stage(3, 6, taps)
reserved = target.make_empty_intermediate_tensors(2, torch.float32, "cpu")
assert set(sent.tensors) == set(reserved.tensors)
result = last(None, None, sent)
assert isinstance(result, tuple)
output, aux = result
torch.testing.assert_close(output, _pp_aux_state(_expected_aux_value(6)))
for hidden, tap in zip(aux, taps, strict=True):
torch.testing.assert_close(hidden, _pp_aux_state(_expected_aux_value(tap)))
def test_minimax_missing_upstream_aux_slot_is_not_silently_dropped(minimax_pp_stage):
model, _ = minimax_pp_stage(3, 6, (0, 3, 6))
incoming = IntermediateTensors(
{
"hidden_states": _pp_aux_state(30),
"residual": _pp_aux_state(43),
"aux_hidden_states_0": _pp_aux_state(7),
}
)
with pytest.raises(RuntimeError, match="Missing aux_hidden_states_1"):
model(None, None, incoming)
@pytest.mark.parametrize("taps", [(), (0, 3, 6)])
def test_minimax_pp1_output_and_aux_taps(minimax_pp_stage, taps):
model, _ = minimax_pp_stage(0, 6, taps, world_size=1)
output = model(None, None, None, _pp_aux_state(7))
if taps:
output, aux = output
for hidden, tap in zip(aux, taps, strict=True):
torch.testing.assert_close(hidden, _pp_aux_state(_expected_aux_value(tap)))
torch.testing.assert_close(output, _pp_aux_state(_expected_aux_value(6)))
def test_minimax_pp2_without_aux_keeps_hidden_and_residual_transport(minimax_pp_stage):
first, _ = minimax_pp_stage(0, 3, ())
sent = first(None, None, None, _pp_aux_state(7))
assert set(sent.tensors) == {"hidden_states", "residual"}
torch.testing.assert_close(sent["hidden_states"], _pp_aux_state(30))
torch.testing.assert_close(sent["residual"], _pp_aux_state(43))
last, target = minimax_pp_stage(3, 6, ())
assert set(
target.make_empty_intermediate_tensors(12, torch.bfloat16, "cpu").tensors
) == {"hidden_states", "residual"}
output = last(None, None, sent)
torch.testing.assert_close(output, _pp_aux_state(_expected_aux_value(6)))
+21 -9
View File
@@ -410,17 +410,24 @@ def flashinfer_autotune(runner: "GPUModelRunner") -> None:
Without autotuning, FlashInfer will rely on heuristics, which may
be significantly slower.
Every rank profiles the same tactics. When distributed, per-tactic
timings are averaged over the world CPU group so all ranks select the
same tactic.
With PP > 1, stages run different layers and may profile different ops,
so each stage's TP group tunes separately with its own cache file;
otherwise the world group tunes together. Per-tactic timings are
averaged over the tuning group so all its ranks select the same tactic.
"""
from flashinfer.autotuner import AutoTuner, set_autotune_process_group
import vllm.utils.flashinfer as fi_utils
from vllm.distributed.parallel_state import get_world_group
from vllm.distributed.parallel_state import (
get_pp_group,
get_tp_group,
get_world_group,
)
world = get_world_group()
is_leader = world.rank_in_group == 0
pp_size = get_pp_group().world_size
tune_group = get_tp_group() if pp_size > 1 else world
is_leader = tune_group.rank_in_group == 0
tuner = AutoTuner.get()
autotune_kwargs: dict = {}
@@ -433,6 +440,11 @@ def flashinfer_autotune(runner: "GPUModelRunner") -> None:
autotune_kwargs["skip_ops"] = skip_ops
cache_path = resolve_flashinfer_autotune_file(runner)
if pp_size > 1:
ranks = "-".join(str(rank) for rank in tune_group.ranks)
cache_path = cache_path.with_name(
f"{cache_path.stem}_tp_{ranks}{cache_path.suffix}"
)
if is_leader:
logger.info_once("Using FlashInfer autotune cache file: %s", cache_path)
@@ -441,18 +453,18 @@ def flashinfer_autotune(runner: "GPUModelRunner") -> None:
# which lead to some EP ranks receiving no tokens and skipping their
# MoE kernel entirely, and cause hang due to all-reduce collective
# during synchronized autotuning.
# Read cached autotune results and broadcast to all ranks.
# Read cached autotune results and broadcast within the tuning group.
cached_results: bytes | None = None
if is_leader and cache_path.exists():
with open(cache_path, "rb") as f:
cached_results = f.read()
cached_results = world.broadcast_object(cached_results, src=0)
cached_results = tune_group.broadcast_object(cached_results, src=0)
if cached_results is not None:
write_flashinfer_autotune_cache(cache_path, cached_results)
world.barrier()
tune_group.barrier()
tuner.load_configs(str(cache_path))
group = world.cpu_group if world.world_size > 1 else None
group = tune_group.cpu_group if tune_group.world_size > 1 else None
set_autotune_process_group(group)
try:
with (
+19 -4
View File
@@ -794,6 +794,7 @@ class MiniMaxM3DecoderLayer(nn.Module):
class MiniMaxM3Model(nn.Module, EagleModelMixin):
fall_back_to_pt_during_load = False
supports_aux_hidden_states_over_pp = True
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
super().__init__()
@@ -886,9 +887,18 @@ class MiniMaxM3Model(nn.Module, EagleModelMixin):
hidden_states = intermediate_tensors["hidden_states"]
residual = intermediate_tensors["residual"]
# EAGLE3 is not yet compatible with pipeline parallel
aux_hidden_states = self._maybe_add_hidden_state([], 0, hidden_states, residual)
for idx, layer in enumerate(self.layers[self.start_layer : self.end_layer]):
remote_aux = self.collect_remote_aux_hidden_states(intermediate_tensors)
aux_hidden_states: list[torch.Tensor] = []
if get_pp_group().is_first_rank:
self._maybe_add_hidden_state(
aux_hidden_states, self.start_layer, hidden_states, residual
)
# Preserve the entry tap before in-place residual updates.
if aux_hidden_states:
aux_hidden_states[0] = aux_hidden_states[0].clone()
for idx, layer in enumerate(
self.layers[self.start_layer : self.end_layer], start=self.start_layer
):
hidden_states, residual = layer(positions, hidden_states, residual)
self._maybe_add_hidden_state(
aux_hidden_states, idx + 1, hidden_states, residual
@@ -896,7 +906,11 @@ class MiniMaxM3Model(nn.Module, EagleModelMixin):
if not get_pp_group().is_last_rank:
return IntermediateTensors(
{"hidden_states": hidden_states, "residual": residual}
{
"hidden_states": hidden_states,
"residual": residual,
**self.pack_local_aux_hidden_states(aux_hidden_states),
}
)
if self.fuse_final_norm_allreduce:
@@ -906,6 +920,7 @@ class MiniMaxM3Model(nn.Module, EagleModelMixin):
else:
hidden_states, _ = self.norm(hidden_states, residual)
aux_hidden_states = remote_aux + aux_hidden_states
if len(aux_hidden_states) > 0:
return hidden_states, aux_hidden_states
return hidden_states