diff --git a/tests/model_executor/test_flashinfer_autotune_warmup.py b/tests/model_executor/test_flashinfer_autotune_warmup.py index dd162629b8..22bb54d71f 100644 --- a/tests/model_executor/test_flashinfer_autotune_warmup.py +++ b/tests/model_executor/test_flashinfer_autotune_warmup.py @@ -1,16 +1,23 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import json +import sys +from collections import defaultdict +from contextlib import nullcontext from inspect import signature -from types import SimpleNamespace +from pathlib import Path +from types import ModuleType, SimpleNamespace from typing import Any from unittest.mock import Mock, call, patch import pytest +from vllm.model_executor.warmup import kernel_warmup as warmup from vllm.model_executor.warmup.kernel_warmup import ( _flashinfer_autotune_token_counts, _run_flashinfer_autotune_dummy_runs, + flashinfer_autotune, ) pytestmark = pytest.mark.cpu_test @@ -37,7 +44,8 @@ def _make_runner(modules, *, max_tokens: int = 8192, linear_backend: str = "auto return SimpleNamespace( scheduler_config=SimpleNamespace(max_num_batched_tokens=max_tokens), vllm_config=SimpleNamespace( - kernel_config=SimpleNamespace(linear_backend=linear_backend) + kernel_config=SimpleNamespace(linear_backend=linear_backend), + attention_config=SimpleNamespace(hisparse_config=None), ), get_model=Mock( return_value=SimpleNamespace(modules=Mock(return_value=modules)) @@ -125,3 +133,183 @@ def test_flashinfer_autotune_uses_token_buckets_for_each_dummy_run(skip_attn): **({"skip_attn": True} if skip_attn else {}), ), ] + + +class _AutotuneGroup: + def __init__(self, run, ranks): + self.run = run + self.ranks = tuple(ranks) + self.world_size = len(self.ranks) + self.rank_in_group = self.ranks.index(run.rank) + self.cpu_group = self + + def record(self, operation): + self.run.collectives[self.ranks][self.run.rank].append(operation) + + def broadcast_object(self, obj, src=0): + assert src == 0 + self.record(("broadcast", src)) + if self.rank_in_group == src: + self.run.broadcasts[self.ranks] = obj + return self.run.broadcasts[self.ranks] + + def barrier(self): + self.record(("barrier",)) + + +class _AutotuneTuner: + def __init__(self, run): + self.run = run + self.cache = {} + self.loaded = None + + def load_configs(self, path): + self.loaded = json.loads(Path(path).read_text()) + self.cache.update(self.loaded) + + def save_configs(self, path): + self.run.saves.append((self.run.rank, Path(path), dict(self.cache))) + Path(path).write_text(json.dumps(self.cache)) + + def profile(self, operation): + if operation in self.cache: + return + group = self.run.tuning_group + self.run.profile_groups[self.run.rank].append( + None if group is None else group.ranks + ) + for tactic in range(2): + if group is not None: + group.record(("all_reduce", operation, tactic)) + self.cache[operation] = self.run.rank // self.run.tp + + +class _AutotuneRun: + def __init__(self, pp, tp): + self.pp, self.tp = pp, tp + self.rank = 0 + self.tuning_group = None + self.collectives: dict[tuple[int, ...], dict[int, list[tuple[Any, ...]]]] = ( + defaultdict(lambda: defaultdict(list)) + ) + self.broadcasts = {} + self.tuners = {} + self.saves = [] + self.profile_groups = defaultdict(list) + + def world(self): + return _AutotuneGroup(self, range(self.pp * self.tp)) + + def tensor_group(self): + start = self.rank // self.tp * self.tp + return _AutotuneGroup(self, range(start, start + self.tp)) + + def pipeline_group(self): + return SimpleNamespace(world_size=self.pp) + + def set_group(self, group): + self.tuning_group = group + + def dummy_runs(self, runner, *, skip_attn=False): + self.tuners[self.rank].profile("shared_gemm") + if self.pp > 1 and self.rank // self.tp == 0: + self.tuners[self.rank].profile("pp0_extra_gemm") + + def execute(self): + for rank in range(self.pp * self.tp): + self.rank = rank + self.tuners[rank] = _AutotuneTuner(self) + flashinfer_autotune(_make_runner([])) + return self + + def assert_collectives_match(self): + for ranks, traces in self.collectives.items(): + assert set(traces) == set(ranks) + expected = traces[ranks[0]] + assert all(trace == expected for trace in traces.values()), dict(traces) + + +@pytest.fixture +def autotune_run(monkeypatch, tmp_path): + import vllm.utils.flashinfer as fi_utils + from vllm.distributed import parallel_state + + def make_run(*, pp=2, tp=4): + run = _AutotuneRun(pp, tp) + autotuner = ModuleType("flashinfer.autotuner") + monkeypatch.setattr( + autotuner, + "AutoTuner", + SimpleNamespace(get=lambda: run.tuners[run.rank]), + raising=False, + ) + monkeypatch.setattr( + autotuner, "set_autotune_process_group", run.set_group, raising=False + ) + monkeypatch.setitem(sys.modules, "flashinfer.autotuner", autotuner) + monkeypatch.setattr(parallel_state, "get_world_group", run.world) + monkeypatch.setattr(parallel_state, "get_tp_group", run.tensor_group) + monkeypatch.setattr(parallel_state, "get_pp_group", run.pipeline_group) + monkeypatch.setattr(fi_utils, "autotune", lambda **kwargs: nullcontext()) + monkeypatch.setattr( + warmup, + "resolve_flashinfer_autotune_file", + lambda runner: tmp_path / "autotune_configs.json", + ) + monkeypatch.setattr( + warmup, "_flashinfer_autotune_skip_ops", lambda runner: None + ) + monkeypatch.setattr( + warmup, "_run_flashinfer_autotune_dummy_runs", run.dummy_runs + ) + monkeypatch.setattr(warmup, "replayssm_autotune_warmup", lambda runner: None) + monkeypatch.setattr(warmup, "_autotune_kimi_k3_kda_qkvg", lambda model: None) + return run + + return make_run + + +@pytest.mark.parametrize("tp", [1, 4]) +def test_heterogeneous_pp_stages_have_compatible_collectives(autotune_run, tp): + run = autotune_run(tp=tp).execute() + run.assert_collectives_match() + for rank, groups in run.profile_groups.items(): + expected = tuple(range(rank // tp * tp, (rank // tp + 1) * tp)) + assert groups and all( + group == (expected if tp > 1 else None) for group in groups + ) + + +def test_pp_stage_cache_roundtrip_isolated_and_asymmetric_hits_safe( + autotune_run, tmp_path +): + legacy = tmp_path / "autotune_configs.json" + legacy.write_text('{"legacy_world_cache": 99}') + cold = autotune_run().execute() + assert [rank for rank, _, _ in cold.saves] == [0, 4] + paths = [path for _, path, _ in cold.saves] + assert len(set(paths)) == 2 and legacy not in paths + assert json.loads(legacy.read_text()) == {"legacy_world_cache": 99} + assert cold.saves[0][2] == {"shared_gemm": 0, "pp0_extra_gemm": 0} + assert cold.saves[1][2] == {"shared_gemm": 1} + cold.assert_collectives_match() + warm = autotune_run().execute() + warm.assert_collectives_match() + assert not warm.profile_groups + for rank, tuner in warm.tuners.items(): + assert tuner.loaded == cold.saves[rank // 4][2] + paths[1].unlink() + mixed = autotune_run().execute() + mixed.assert_collectives_match() + assert set(mixed.profile_groups) == {4, 5, 6, 7} + assert all(mixed.tuners[rank].loaded is not None for rank in range(4)) + assert all(mixed.tuners[rank].loaded is None for rank in range(4, 8)) + + +def test_pp1_retains_world_synchronization_and_existing_cache_name(autotune_run): + run = autotune_run(pp=1, tp=4).execute() + run.assert_collectives_match() + assert [(rank, path.name) for rank, path, _ in run.saves] == [ + (0, "autotune_configs.json") + ] + assert all(groups == [(0, 1, 2, 3)] for groups in run.profile_groups.values()) diff --git a/tests/v1/worker/test_eagle3_aux_hidden_states_pp.py b/tests/v1/worker/test_eagle3_aux_hidden_states_pp.py index b22da39c91..309a81b36c 100644 --- a/tests/v1/worker/test_eagle3_aux_hidden_states_pp.py +++ b/tests/v1/worker/test_eagle3_aux_hidden_states_pp.py @@ -3,10 +3,15 @@ from types import SimpleNamespace import pytest +import torch +from vllm.distributed import parallel_state from vllm.model_executor.models.interfaces import EagleModelMixin from vllm.model_executor.models.mimo import MiMoModel +from vllm.models.minimax_m3.nvidia import model as minimax_m3 +from vllm.sequence import IntermediateTensors from vllm.v1.worker.gpu.spec_decode.eagle.eagle3_utils import ( + reserve_aux_intermediate_tensor_slots, verify_supports_aux_hidden_states_over_pp, ) @@ -24,3 +29,150 @@ def test_mimo_does_not_inherit_aux_hidden_state_pp_support(): assert not inner.supports_aux_hidden_states_over_pp with pytest.raises(ValueError, match="does not support eagle3"): verify_supports_aux_hidden_states_over_pp(target, "eagle3") + + +class _PPAuxLayer(torch.nn.Module): + """Aliases and mutates residual in place like MiniMaxM3DecoderLayer.""" + + ffn_all_reduce_deferred = False + + def __init__(self, index): + super().__init__() + self.index = index + + def forward(self, positions, hidden_states, residual): + total = hidden_states if residual is None else hidden_states + residual + residual = hidden_states if residual is None else residual + residual.copy_(total + self.index + 1) + return torch.full_like(hidden_states, 10 * (self.index + 1)), residual + + +class _PPAuxNorm(torch.nn.Module): + """Clobbers its inputs so aux taps that alias them would fail.""" + + def forward(self, hidden_states, residual): + output = hidden_states + residual + hidden_states.fill_(-101) + residual.fill_(-103) + return output, residual + + +@pytest.fixture +def minimax_pp_stage(monkeypatch): + """Use the real MiniMax constructor/forward and aux helpers on CPU tensors.""" + group = SimpleNamespace(world_size=2, is_first_rank=True, is_last_rank=False) + bounds = [0, 3] + monkeypatch.setattr(minimax_m3, "get_pp_group", lambda: group) + monkeypatch.setattr(parallel_state, "get_pp_group", lambda: group) + monkeypatch.setattr(parallel_state, "model_parallel_is_initialized", lambda: True) + monkeypatch.setattr( + minimax_m3, + "VocabParallelEmbedding", + lambda *args, **kwargs: torch.nn.Identity(), + ) + monkeypatch.setattr( + minimax_m3, "MiniMAXGemmaRMSNorm", lambda *args, **kwargs: _PPAuxNorm() + ) + monkeypatch.setattr( + minimax_m3, + "MiniMaxM3DecoderLayer", + lambda **kwargs: _PPAuxLayer(int(kwargs["prefix"].rsplit(".", 1)[1])), + ) + + def make_layers(num_layers, build, prefix): + return ( + *bounds, + torch.nn.ModuleList( + build(f"{prefix}.{index}") + if bounds[0] <= index < bounds[1] + else torch.nn.Identity() + for index in range(num_layers) + ), + ) + + monkeypatch.setattr(minimax_m3, "make_layers", make_layers) + + def make_stage(start, end, taps, world_size=2): + bounds[:] = [start, end] + group.world_size = world_size + group.is_first_rank = start == 0 + group.is_last_rank = end == 6 + config = SimpleNamespace( + vocab_size=16, hidden_size=6144, num_hidden_layers=6, rms_norm_eps=1e-6 + ) + model = minimax_m3.MiniMaxM3Model( + vllm_config=SimpleNamespace( + model_config=SimpleNamespace(hf_text_config=config), quant_config=None + ) + ) + model._set_aux_hidden_state_layers(taps) + target = SimpleNamespace( + model=model, + make_empty_intermediate_tensors=model.make_empty_intermediate_tensors, + ) + reserve_aux_intermediate_tensor_slots(target) + return model, target + + return make_stage + + +# Input 7; _PPAuxLayer i adds 11 * (i + 1) to hidden + residual. +def _expected_aux_value(tap): + return 7 + 11 * tap * (tap + 1) // 2 + + +def _pp_aux_state(value): + return torch.full((2, 6144), float(value), device="cpu") + + +@pytest.mark.parametrize("taps", [(0, 3, 6), (0, 1, 3), (4, 5, 6)]) +def test_minimax_pp2_preserves_ordered_global_aux_taps(minimax_pp_stage, taps): + first, _ = minimax_pp_stage(0, 3, taps) + sent = first(None, None, None, _pp_aux_state(7)) + last, target = minimax_pp_stage(3, 6, taps) + reserved = target.make_empty_intermediate_tensors(2, torch.float32, "cpu") + assert set(sent.tensors) == set(reserved.tensors) + result = last(None, None, sent) + assert isinstance(result, tuple) + output, aux = result + torch.testing.assert_close(output, _pp_aux_state(_expected_aux_value(6))) + for hidden, tap in zip(aux, taps, strict=True): + torch.testing.assert_close(hidden, _pp_aux_state(_expected_aux_value(tap))) + + +def test_minimax_missing_upstream_aux_slot_is_not_silently_dropped(minimax_pp_stage): + model, _ = minimax_pp_stage(3, 6, (0, 3, 6)) + incoming = IntermediateTensors( + { + "hidden_states": _pp_aux_state(30), + "residual": _pp_aux_state(43), + "aux_hidden_states_0": _pp_aux_state(7), + } + ) + with pytest.raises(RuntimeError, match="Missing aux_hidden_states_1"): + model(None, None, incoming) + + +@pytest.mark.parametrize("taps", [(), (0, 3, 6)]) +def test_minimax_pp1_output_and_aux_taps(minimax_pp_stage, taps): + model, _ = minimax_pp_stage(0, 6, taps, world_size=1) + output = model(None, None, None, _pp_aux_state(7)) + if taps: + output, aux = output + for hidden, tap in zip(aux, taps, strict=True): + torch.testing.assert_close(hidden, _pp_aux_state(_expected_aux_value(tap))) + torch.testing.assert_close(output, _pp_aux_state(_expected_aux_value(6))) + + +def test_minimax_pp2_without_aux_keeps_hidden_and_residual_transport(minimax_pp_stage): + first, _ = minimax_pp_stage(0, 3, ()) + sent = first(None, None, None, _pp_aux_state(7)) + assert set(sent.tensors) == {"hidden_states", "residual"} + torch.testing.assert_close(sent["hidden_states"], _pp_aux_state(30)) + torch.testing.assert_close(sent["residual"], _pp_aux_state(43)) + last, target = minimax_pp_stage(3, 6, ()) + assert set( + target.make_empty_intermediate_tensors(12, torch.bfloat16, "cpu").tensors + ) == {"hidden_states", "residual"} + output = last(None, None, sent) + torch.testing.assert_close(output, _pp_aux_state(_expected_aux_value(6))) diff --git a/vllm/model_executor/warmup/kernel_warmup.py b/vllm/model_executor/warmup/kernel_warmup.py index 33fe7002aa..228ce73803 100644 --- a/vllm/model_executor/warmup/kernel_warmup.py +++ b/vllm/model_executor/warmup/kernel_warmup.py @@ -410,17 +410,24 @@ def flashinfer_autotune(runner: "GPUModelRunner") -> None: Without autotuning, FlashInfer will rely on heuristics, which may be significantly slower. - Every rank profiles the same tactics. When distributed, per-tactic - timings are averaged over the world CPU group so all ranks select the - same tactic. + With PP > 1, stages run different layers and may profile different ops, + so each stage's TP group tunes separately with its own cache file; + otherwise the world group tunes together. Per-tactic timings are + averaged over the tuning group so all its ranks select the same tactic. """ from flashinfer.autotuner import AutoTuner, set_autotune_process_group import vllm.utils.flashinfer as fi_utils - from vllm.distributed.parallel_state import get_world_group + from vllm.distributed.parallel_state import ( + get_pp_group, + get_tp_group, + get_world_group, + ) world = get_world_group() - is_leader = world.rank_in_group == 0 + pp_size = get_pp_group().world_size + tune_group = get_tp_group() if pp_size > 1 else world + is_leader = tune_group.rank_in_group == 0 tuner = AutoTuner.get() autotune_kwargs: dict = {} @@ -433,6 +440,11 @@ def flashinfer_autotune(runner: "GPUModelRunner") -> None: autotune_kwargs["skip_ops"] = skip_ops cache_path = resolve_flashinfer_autotune_file(runner) + if pp_size > 1: + ranks = "-".join(str(rank) for rank in tune_group.ranks) + cache_path = cache_path.with_name( + f"{cache_path.stem}_tp_{ranks}{cache_path.suffix}" + ) if is_leader: logger.info_once("Using FlashInfer autotune cache file: %s", cache_path) @@ -441,18 +453,18 @@ def flashinfer_autotune(runner: "GPUModelRunner") -> None: # which lead to some EP ranks receiving no tokens and skipping their # MoE kernel entirely, and cause hang due to all-reduce collective # during synchronized autotuning. - # Read cached autotune results and broadcast to all ranks. + # Read cached autotune results and broadcast within the tuning group. cached_results: bytes | None = None if is_leader and cache_path.exists(): with open(cache_path, "rb") as f: cached_results = f.read() - cached_results = world.broadcast_object(cached_results, src=0) + cached_results = tune_group.broadcast_object(cached_results, src=0) if cached_results is not None: write_flashinfer_autotune_cache(cache_path, cached_results) - world.barrier() + tune_group.barrier() tuner.load_configs(str(cache_path)) - group = world.cpu_group if world.world_size > 1 else None + group = tune_group.cpu_group if tune_group.world_size > 1 else None set_autotune_process_group(group) try: with ( diff --git a/vllm/models/minimax_m3/nvidia/model.py b/vllm/models/minimax_m3/nvidia/model.py index d1c0a01c40..1c0138946f 100644 --- a/vllm/models/minimax_m3/nvidia/model.py +++ b/vllm/models/minimax_m3/nvidia/model.py @@ -794,6 +794,7 @@ class MiniMaxM3DecoderLayer(nn.Module): class MiniMaxM3Model(nn.Module, EagleModelMixin): fall_back_to_pt_during_load = False + supports_aux_hidden_states_over_pp = True def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): super().__init__() @@ -886,9 +887,18 @@ class MiniMaxM3Model(nn.Module, EagleModelMixin): hidden_states = intermediate_tensors["hidden_states"] residual = intermediate_tensors["residual"] - # EAGLE3 is not yet compatible with pipeline parallel - aux_hidden_states = self._maybe_add_hidden_state([], 0, hidden_states, residual) - for idx, layer in enumerate(self.layers[self.start_layer : self.end_layer]): + remote_aux = self.collect_remote_aux_hidden_states(intermediate_tensors) + aux_hidden_states: list[torch.Tensor] = [] + if get_pp_group().is_first_rank: + self._maybe_add_hidden_state( + aux_hidden_states, self.start_layer, hidden_states, residual + ) + # Preserve the entry tap before in-place residual updates. + if aux_hidden_states: + aux_hidden_states[0] = aux_hidden_states[0].clone() + for idx, layer in enumerate( + self.layers[self.start_layer : self.end_layer], start=self.start_layer + ): hidden_states, residual = layer(positions, hidden_states, residual) self._maybe_add_hidden_state( aux_hidden_states, idx + 1, hidden_states, residual @@ -896,7 +906,11 @@ class MiniMaxM3Model(nn.Module, EagleModelMixin): if not get_pp_group().is_last_rank: return IntermediateTensors( - {"hidden_states": hidden_states, "residual": residual} + { + "hidden_states": hidden_states, + "residual": residual, + **self.pack_local_aux_hidden_states(aux_hidden_states), + } ) if self.fuse_final_norm_allreduce: @@ -906,6 +920,7 @@ class MiniMaxM3Model(nn.Module, EagleModelMixin): else: hidden_states, _ = self.norm(hidden_states, residual) + aux_hidden_states = remote_aux + aux_hidden_states if len(aux_hidden_states) > 0: return hidden_states, aux_hidden_states return hidden_states