[Bugfix][PP][Spec Decode] MiniMax-M3 EAGLE3 aux-state relay at PP > 1 and per-stage FlashInfer autotune (#57197)

Signed-off-by: venkywonka <23023424+venkywonka@users.noreply.github.com>
Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Venky
2026-10-01 01:14:24 +04:00
committed by GitHub
co-authored by Claude Opus 5.5
parent 73c7cae4d7
commit 42f0c17ea7
4 changed files with 382 additions and 15 deletions
@@ -1,16 +1,23 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import json
import sys
from collections import defaultdict
from contextlib import nullcontext
from inspect import signature
from types import SimpleNamespace
from pathlib import Path
from types import ModuleType, SimpleNamespace
from typing import Any
from unittest.mock import Mock, call, patch
import pytest
from vllm.model_executor.warmup import kernel_warmup as warmup
from vllm.model_executor.warmup.kernel_warmup import (
_flashinfer_autotune_token_counts,
_run_flashinfer_autotune_dummy_runs,
flashinfer_autotune,
)
pytestmark = pytest.mark.cpu_test
@@ -37,7 +44,8 @@ def _make_runner(modules, *, max_tokens: int = 8192, linear_backend: str = "auto
return SimpleNamespace(
scheduler_config=SimpleNamespace(max_num_batched_tokens=max_tokens),
vllm_config=SimpleNamespace(
kernel_config=SimpleNamespace(linear_backend=linear_backend)
kernel_config=SimpleNamespace(linear_backend=linear_backend),
attention_config=SimpleNamespace(hisparse_config=None),
),
get_model=Mock(
return_value=SimpleNamespace(modules=Mock(return_value=modules))
@@ -125,3 +133,183 @@ def test_flashinfer_autotune_uses_token_buckets_for_each_dummy_run(skip_attn):
**({"skip_attn": True} if skip_attn else {}),
),
]
class _AutotuneGroup:
def __init__(self, run, ranks):
self.run = run
self.ranks = tuple(ranks)
self.world_size = len(self.ranks)
self.rank_in_group = self.ranks.index(run.rank)
self.cpu_group = self
def record(self, operation):
self.run.collectives[self.ranks][self.run.rank].append(operation)
def broadcast_object(self, obj, src=0):
assert src == 0
self.record(("broadcast", src))
if self.rank_in_group == src:
self.run.broadcasts[self.ranks] = obj
return self.run.broadcasts[self.ranks]
def barrier(self):
self.record(("barrier",))
class _AutotuneTuner:
def __init__(self, run):
self.run = run
self.cache = {}
self.loaded = None
def load_configs(self, path):
self.loaded = json.loads(Path(path).read_text())
self.cache.update(self.loaded)
def save_configs(self, path):
self.run.saves.append((self.run.rank, Path(path), dict(self.cache)))
Path(path).write_text(json.dumps(self.cache))
def profile(self, operation):
if operation in self.cache:
return
group = self.run.tuning_group
self.run.profile_groups[self.run.rank].append(
None if group is None else group.ranks
)
for tactic in range(2):
if group is not None:
group.record(("all_reduce", operation, tactic))
self.cache[operation] = self.run.rank // self.run.tp
class _AutotuneRun:
def __init__(self, pp, tp):
self.pp, self.tp = pp, tp
self.rank = 0
self.tuning_group = None
self.collectives: dict[tuple[int, ...], dict[int, list[tuple[Any, ...]]]] = (
defaultdict(lambda: defaultdict(list))
)
self.broadcasts = {}
self.tuners = {}
self.saves = []
self.profile_groups = defaultdict(list)
def world(self):
return _AutotuneGroup(self, range(self.pp * self.tp))
def tensor_group(self):
start = self.rank // self.tp * self.tp
return _AutotuneGroup(self, range(start, start + self.tp))
def pipeline_group(self):
return SimpleNamespace(world_size=self.pp)
def set_group(self, group):
self.tuning_group = group
def dummy_runs(self, runner, *, skip_attn=False):
self.tuners[self.rank].profile("shared_gemm")
if self.pp > 1 and self.rank // self.tp == 0:
self.tuners[self.rank].profile("pp0_extra_gemm")
def execute(self):
for rank in range(self.pp * self.tp):
self.rank = rank
self.tuners[rank] = _AutotuneTuner(self)
flashinfer_autotune(_make_runner([]))
return self
def assert_collectives_match(self):
for ranks, traces in self.collectives.items():
assert set(traces) == set(ranks)
expected = traces[ranks[0]]
assert all(trace == expected for trace in traces.values()), dict(traces)
@pytest.fixture
def autotune_run(monkeypatch, tmp_path):
import vllm.utils.flashinfer as fi_utils
from vllm.distributed import parallel_state
def make_run(*, pp=2, tp=4):
run = _AutotuneRun(pp, tp)
autotuner = ModuleType("flashinfer.autotuner")
monkeypatch.setattr(
autotuner,
"AutoTuner",
SimpleNamespace(get=lambda: run.tuners[run.rank]),
raising=False,
)
monkeypatch.setattr(
autotuner, "set_autotune_process_group", run.set_group, raising=False
)
monkeypatch.setitem(sys.modules, "flashinfer.autotuner", autotuner)
monkeypatch.setattr(parallel_state, "get_world_group", run.world)
monkeypatch.setattr(parallel_state, "get_tp_group", run.tensor_group)
monkeypatch.setattr(parallel_state, "get_pp_group", run.pipeline_group)
monkeypatch.setattr(fi_utils, "autotune", lambda **kwargs: nullcontext())
monkeypatch.setattr(
warmup,
"resolve_flashinfer_autotune_file",
lambda runner: tmp_path / "autotune_configs.json",
)
monkeypatch.setattr(
warmup, "_flashinfer_autotune_skip_ops", lambda runner: None
)
monkeypatch.setattr(
warmup, "_run_flashinfer_autotune_dummy_runs", run.dummy_runs
)
monkeypatch.setattr(warmup, "replayssm_autotune_warmup", lambda runner: None)
monkeypatch.setattr(warmup, "_autotune_kimi_k3_kda_qkvg", lambda model: None)
return run
return make_run
@pytest.mark.parametrize("tp", [1, 4])
def test_heterogeneous_pp_stages_have_compatible_collectives(autotune_run, tp):
run = autotune_run(tp=tp).execute()
run.assert_collectives_match()
for rank, groups in run.profile_groups.items():
expected = tuple(range(rank // tp * tp, (rank // tp + 1) * tp))
assert groups and all(
group == (expected if tp > 1 else None) for group in groups
)
def test_pp_stage_cache_roundtrip_isolated_and_asymmetric_hits_safe(
autotune_run, tmp_path
):
legacy = tmp_path / "autotune_configs.json"
legacy.write_text('{"legacy_world_cache": 99}')
cold = autotune_run().execute()
assert [rank for rank, _, _ in cold.saves] == [0, 4]
paths = [path for _, path, _ in cold.saves]
assert len(set(paths)) == 2 and legacy not in paths
assert json.loads(legacy.read_text()) == {"legacy_world_cache": 99}
assert cold.saves[0][2] == {"shared_gemm": 0, "pp0_extra_gemm": 0}
assert cold.saves[1][2] == {"shared_gemm": 1}
cold.assert_collectives_match()
warm = autotune_run().execute()
warm.assert_collectives_match()
assert not warm.profile_groups
for rank, tuner in warm.tuners.items():
assert tuner.loaded == cold.saves[rank // 4][2]
paths[1].unlink()
mixed = autotune_run().execute()
mixed.assert_collectives_match()
assert set(mixed.profile_groups) == {4, 5, 6, 7}
assert all(mixed.tuners[rank].loaded is not None for rank in range(4))
assert all(mixed.tuners[rank].loaded is None for rank in range(4, 8))
def test_pp1_retains_world_synchronization_and_existing_cache_name(autotune_run):
run = autotune_run(pp=1, tp=4).execute()
run.assert_collectives_match()
assert [(rank, path.name) for rank, path, _ in run.saves] == [
(0, "autotune_configs.json")
]
assert all(groups == [(0, 1, 2, 3)] for groups in run.profile_groups.values())
@@ -3,10 +3,15 @@
from types import SimpleNamespace
import pytest
import torch
from vllm.distributed import parallel_state
from vllm.model_executor.models.interfaces import EagleModelMixin
from vllm.model_executor.models.mimo import MiMoModel
from vllm.models.minimax_m3.nvidia import model as minimax_m3
from vllm.sequence import IntermediateTensors
from vllm.v1.worker.gpu.spec_decode.eagle.eagle3_utils import (
reserve_aux_intermediate_tensor_slots,
verify_supports_aux_hidden_states_over_pp,
)
@@ -24,3 +29,150 @@ def test_mimo_does_not_inherit_aux_hidden_state_pp_support():
assert not inner.supports_aux_hidden_states_over_pp
with pytest.raises(ValueError, match="does not support eagle3"):
verify_supports_aux_hidden_states_over_pp(target, "eagle3")
class _PPAuxLayer(torch.nn.Module):
"""Aliases and mutates residual in place like MiniMaxM3DecoderLayer."""
ffn_all_reduce_deferred = False
def __init__(self, index):
super().__init__()
self.index = index
def forward(self, positions, hidden_states, residual):
total = hidden_states if residual is None else hidden_states + residual
residual = hidden_states if residual is None else residual
residual.copy_(total + self.index + 1)
return torch.full_like(hidden_states, 10 * (self.index + 1)), residual
class _PPAuxNorm(torch.nn.Module):
"""Clobbers its inputs so aux taps that alias them would fail."""
def forward(self, hidden_states, residual):
output = hidden_states + residual
hidden_states.fill_(-101)
residual.fill_(-103)
return output, residual
@pytest.fixture
def minimax_pp_stage(monkeypatch):
"""Use the real MiniMax constructor/forward and aux helpers on CPU tensors."""
group = SimpleNamespace(world_size=2, is_first_rank=True, is_last_rank=False)
bounds = [0, 3]
monkeypatch.setattr(minimax_m3, "get_pp_group", lambda: group)
monkeypatch.setattr(parallel_state, "get_pp_group", lambda: group)
monkeypatch.setattr(parallel_state, "model_parallel_is_initialized", lambda: True)
monkeypatch.setattr(
minimax_m3,
"VocabParallelEmbedding",
lambda *args, **kwargs: torch.nn.Identity(),
)
monkeypatch.setattr(
minimax_m3, "MiniMAXGemmaRMSNorm", lambda *args, **kwargs: _PPAuxNorm()
)
monkeypatch.setattr(
minimax_m3,
"MiniMaxM3DecoderLayer",
lambda **kwargs: _PPAuxLayer(int(kwargs["prefix"].rsplit(".", 1)[1])),
)
def make_layers(num_layers, build, prefix):
return (
*bounds,
torch.nn.ModuleList(
build(f"{prefix}.{index}")
if bounds[0] <= index < bounds[1]
else torch.nn.Identity()
for index in range(num_layers)
),
)
monkeypatch.setattr(minimax_m3, "make_layers", make_layers)
def make_stage(start, end, taps, world_size=2):
bounds[:] = [start, end]
group.world_size = world_size
group.is_first_rank = start == 0
group.is_last_rank = end == 6
config = SimpleNamespace(
vocab_size=16, hidden_size=6144, num_hidden_layers=6, rms_norm_eps=1e-6
)
model = minimax_m3.MiniMaxM3Model(
vllm_config=SimpleNamespace(
model_config=SimpleNamespace(hf_text_config=config), quant_config=None
)
)
model._set_aux_hidden_state_layers(taps)
target = SimpleNamespace(
model=model,
make_empty_intermediate_tensors=model.make_empty_intermediate_tensors,
)
reserve_aux_intermediate_tensor_slots(target)
return model, target
return make_stage
# Input 7; _PPAuxLayer i adds 11 * (i + 1) to hidden + residual.
def _expected_aux_value(tap):
return 7 + 11 * tap * (tap + 1) // 2
def _pp_aux_state(value):
return torch.full((2, 6144), float(value), device="cpu")
@pytest.mark.parametrize("taps", [(0, 3, 6), (0, 1, 3), (4, 5, 6)])
def test_minimax_pp2_preserves_ordered_global_aux_taps(minimax_pp_stage, taps):
first, _ = minimax_pp_stage(0, 3, taps)
sent = first(None, None, None, _pp_aux_state(7))
last, target = minimax_pp_stage(3, 6, taps)
reserved = target.make_empty_intermediate_tensors(2, torch.float32, "cpu")
assert set(sent.tensors) == set(reserved.tensors)
result = last(None, None, sent)
assert isinstance(result, tuple)
output, aux = result
torch.testing.assert_close(output, _pp_aux_state(_expected_aux_value(6)))
for hidden, tap in zip(aux, taps, strict=True):
torch.testing.assert_close(hidden, _pp_aux_state(_expected_aux_value(tap)))
def test_minimax_missing_upstream_aux_slot_is_not_silently_dropped(minimax_pp_stage):
model, _ = minimax_pp_stage(3, 6, (0, 3, 6))
incoming = IntermediateTensors(
{
"hidden_states": _pp_aux_state(30),
"residual": _pp_aux_state(43),
"aux_hidden_states_0": _pp_aux_state(7),
}
)
with pytest.raises(RuntimeError, match="Missing aux_hidden_states_1"):
model(None, None, incoming)
@pytest.mark.parametrize("taps", [(), (0, 3, 6)])
def test_minimax_pp1_output_and_aux_taps(minimax_pp_stage, taps):
model, _ = minimax_pp_stage(0, 6, taps, world_size=1)
output = model(None, None, None, _pp_aux_state(7))
if taps:
output, aux = output
for hidden, tap in zip(aux, taps, strict=True):
torch.testing.assert_close(hidden, _pp_aux_state(_expected_aux_value(tap)))
torch.testing.assert_close(output, _pp_aux_state(_expected_aux_value(6)))
def test_minimax_pp2_without_aux_keeps_hidden_and_residual_transport(minimax_pp_stage):
first, _ = minimax_pp_stage(0, 3, ())
sent = first(None, None, None, _pp_aux_state(7))
assert set(sent.tensors) == {"hidden_states", "residual"}
torch.testing.assert_close(sent["hidden_states"], _pp_aux_state(30))
torch.testing.assert_close(sent["residual"], _pp_aux_state(43))
last, target = minimax_pp_stage(3, 6, ())
assert set(
target.make_empty_intermediate_tensors(12, torch.bfloat16, "cpu").tensors
) == {"hidden_states", "residual"}
output = last(None, None, sent)
torch.testing.assert_close(output, _pp_aux_state(_expected_aux_value(6)))