[Bugfix][PP][Spec Decode] MiniMax-M3 EAGLE3 aux-state relay at PP > 1 and per-stage FlashInfer autotune (#57197)
Signed-off-by: venkywonka <23023424+venkywonka@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -1,16 +1,23 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import json
|
||||
import sys
|
||||
from collections import defaultdict
|
||||
from contextlib import nullcontext
|
||||
from inspect import signature
|
||||
from types import SimpleNamespace
|
||||
from pathlib import Path
|
||||
from types import ModuleType, SimpleNamespace
|
||||
from typing import Any
|
||||
from unittest.mock import Mock, call, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.model_executor.warmup import kernel_warmup as warmup
|
||||
from vllm.model_executor.warmup.kernel_warmup import (
|
||||
_flashinfer_autotune_token_counts,
|
||||
_run_flashinfer_autotune_dummy_runs,
|
||||
flashinfer_autotune,
|
||||
)
|
||||
|
||||
pytestmark = pytest.mark.cpu_test
|
||||
@@ -37,7 +44,8 @@ def _make_runner(modules, *, max_tokens: int = 8192, linear_backend: str = "auto
|
||||
return SimpleNamespace(
|
||||
scheduler_config=SimpleNamespace(max_num_batched_tokens=max_tokens),
|
||||
vllm_config=SimpleNamespace(
|
||||
kernel_config=SimpleNamespace(linear_backend=linear_backend)
|
||||
kernel_config=SimpleNamespace(linear_backend=linear_backend),
|
||||
attention_config=SimpleNamespace(hisparse_config=None),
|
||||
),
|
||||
get_model=Mock(
|
||||
return_value=SimpleNamespace(modules=Mock(return_value=modules))
|
||||
@@ -125,3 +133,183 @@ def test_flashinfer_autotune_uses_token_buckets_for_each_dummy_run(skip_attn):
|
||||
**({"skip_attn": True} if skip_attn else {}),
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
class _AutotuneGroup:
|
||||
def __init__(self, run, ranks):
|
||||
self.run = run
|
||||
self.ranks = tuple(ranks)
|
||||
self.world_size = len(self.ranks)
|
||||
self.rank_in_group = self.ranks.index(run.rank)
|
||||
self.cpu_group = self
|
||||
|
||||
def record(self, operation):
|
||||
self.run.collectives[self.ranks][self.run.rank].append(operation)
|
||||
|
||||
def broadcast_object(self, obj, src=0):
|
||||
assert src == 0
|
||||
self.record(("broadcast", src))
|
||||
if self.rank_in_group == src:
|
||||
self.run.broadcasts[self.ranks] = obj
|
||||
return self.run.broadcasts[self.ranks]
|
||||
|
||||
def barrier(self):
|
||||
self.record(("barrier",))
|
||||
|
||||
|
||||
class _AutotuneTuner:
|
||||
def __init__(self, run):
|
||||
self.run = run
|
||||
self.cache = {}
|
||||
self.loaded = None
|
||||
|
||||
def load_configs(self, path):
|
||||
self.loaded = json.loads(Path(path).read_text())
|
||||
self.cache.update(self.loaded)
|
||||
|
||||
def save_configs(self, path):
|
||||
self.run.saves.append((self.run.rank, Path(path), dict(self.cache)))
|
||||
Path(path).write_text(json.dumps(self.cache))
|
||||
|
||||
def profile(self, operation):
|
||||
if operation in self.cache:
|
||||
return
|
||||
group = self.run.tuning_group
|
||||
self.run.profile_groups[self.run.rank].append(
|
||||
None if group is None else group.ranks
|
||||
)
|
||||
for tactic in range(2):
|
||||
if group is not None:
|
||||
group.record(("all_reduce", operation, tactic))
|
||||
self.cache[operation] = self.run.rank // self.run.tp
|
||||
|
||||
|
||||
class _AutotuneRun:
|
||||
def __init__(self, pp, tp):
|
||||
self.pp, self.tp = pp, tp
|
||||
self.rank = 0
|
||||
self.tuning_group = None
|
||||
self.collectives: dict[tuple[int, ...], dict[int, list[tuple[Any, ...]]]] = (
|
||||
defaultdict(lambda: defaultdict(list))
|
||||
)
|
||||
self.broadcasts = {}
|
||||
self.tuners = {}
|
||||
self.saves = []
|
||||
self.profile_groups = defaultdict(list)
|
||||
|
||||
def world(self):
|
||||
return _AutotuneGroup(self, range(self.pp * self.tp))
|
||||
|
||||
def tensor_group(self):
|
||||
start = self.rank // self.tp * self.tp
|
||||
return _AutotuneGroup(self, range(start, start + self.tp))
|
||||
|
||||
def pipeline_group(self):
|
||||
return SimpleNamespace(world_size=self.pp)
|
||||
|
||||
def set_group(self, group):
|
||||
self.tuning_group = group
|
||||
|
||||
def dummy_runs(self, runner, *, skip_attn=False):
|
||||
self.tuners[self.rank].profile("shared_gemm")
|
||||
if self.pp > 1 and self.rank // self.tp == 0:
|
||||
self.tuners[self.rank].profile("pp0_extra_gemm")
|
||||
|
||||
def execute(self):
|
||||
for rank in range(self.pp * self.tp):
|
||||
self.rank = rank
|
||||
self.tuners[rank] = _AutotuneTuner(self)
|
||||
flashinfer_autotune(_make_runner([]))
|
||||
return self
|
||||
|
||||
def assert_collectives_match(self):
|
||||
for ranks, traces in self.collectives.items():
|
||||
assert set(traces) == set(ranks)
|
||||
expected = traces[ranks[0]]
|
||||
assert all(trace == expected for trace in traces.values()), dict(traces)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def autotune_run(monkeypatch, tmp_path):
|
||||
import vllm.utils.flashinfer as fi_utils
|
||||
from vllm.distributed import parallel_state
|
||||
|
||||
def make_run(*, pp=2, tp=4):
|
||||
run = _AutotuneRun(pp, tp)
|
||||
autotuner = ModuleType("flashinfer.autotuner")
|
||||
monkeypatch.setattr(
|
||||
autotuner,
|
||||
"AutoTuner",
|
||||
SimpleNamespace(get=lambda: run.tuners[run.rank]),
|
||||
raising=False,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
autotuner, "set_autotune_process_group", run.set_group, raising=False
|
||||
)
|
||||
monkeypatch.setitem(sys.modules, "flashinfer.autotuner", autotuner)
|
||||
monkeypatch.setattr(parallel_state, "get_world_group", run.world)
|
||||
monkeypatch.setattr(parallel_state, "get_tp_group", run.tensor_group)
|
||||
monkeypatch.setattr(parallel_state, "get_pp_group", run.pipeline_group)
|
||||
monkeypatch.setattr(fi_utils, "autotune", lambda **kwargs: nullcontext())
|
||||
monkeypatch.setattr(
|
||||
warmup,
|
||||
"resolve_flashinfer_autotune_file",
|
||||
lambda runner: tmp_path / "autotune_configs.json",
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
warmup, "_flashinfer_autotune_skip_ops", lambda runner: None
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
warmup, "_run_flashinfer_autotune_dummy_runs", run.dummy_runs
|
||||
)
|
||||
monkeypatch.setattr(warmup, "replayssm_autotune_warmup", lambda runner: None)
|
||||
monkeypatch.setattr(warmup, "_autotune_kimi_k3_kda_qkvg", lambda model: None)
|
||||
return run
|
||||
|
||||
return make_run
|
||||
|
||||
|
||||
@pytest.mark.parametrize("tp", [1, 4])
|
||||
def test_heterogeneous_pp_stages_have_compatible_collectives(autotune_run, tp):
|
||||
run = autotune_run(tp=tp).execute()
|
||||
run.assert_collectives_match()
|
||||
for rank, groups in run.profile_groups.items():
|
||||
expected = tuple(range(rank // tp * tp, (rank // tp + 1) * tp))
|
||||
assert groups and all(
|
||||
group == (expected if tp > 1 else None) for group in groups
|
||||
)
|
||||
|
||||
|
||||
def test_pp_stage_cache_roundtrip_isolated_and_asymmetric_hits_safe(
|
||||
autotune_run, tmp_path
|
||||
):
|
||||
legacy = tmp_path / "autotune_configs.json"
|
||||
legacy.write_text('{"legacy_world_cache": 99}')
|
||||
cold = autotune_run().execute()
|
||||
assert [rank for rank, _, _ in cold.saves] == [0, 4]
|
||||
paths = [path for _, path, _ in cold.saves]
|
||||
assert len(set(paths)) == 2 and legacy not in paths
|
||||
assert json.loads(legacy.read_text()) == {"legacy_world_cache": 99}
|
||||
assert cold.saves[0][2] == {"shared_gemm": 0, "pp0_extra_gemm": 0}
|
||||
assert cold.saves[1][2] == {"shared_gemm": 1}
|
||||
cold.assert_collectives_match()
|
||||
warm = autotune_run().execute()
|
||||
warm.assert_collectives_match()
|
||||
assert not warm.profile_groups
|
||||
for rank, tuner in warm.tuners.items():
|
||||
assert tuner.loaded == cold.saves[rank // 4][2]
|
||||
paths[1].unlink()
|
||||
mixed = autotune_run().execute()
|
||||
mixed.assert_collectives_match()
|
||||
assert set(mixed.profile_groups) == {4, 5, 6, 7}
|
||||
assert all(mixed.tuners[rank].loaded is not None for rank in range(4))
|
||||
assert all(mixed.tuners[rank].loaded is None for rank in range(4, 8))
|
||||
|
||||
|
||||
def test_pp1_retains_world_synchronization_and_existing_cache_name(autotune_run):
|
||||
run = autotune_run(pp=1, tp=4).execute()
|
||||
run.assert_collectives_match()
|
||||
assert [(rank, path.name) for rank, path, _ in run.saves] == [
|
||||
(0, "autotune_configs.json")
|
||||
]
|
||||
assert all(groups == [(0, 1, 2, 3)] for groups in run.profile_groups.values())
|
||||
|
||||
@@ -3,10 +3,15 @@
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.distributed import parallel_state
|
||||
from vllm.model_executor.models.interfaces import EagleModelMixin
|
||||
from vllm.model_executor.models.mimo import MiMoModel
|
||||
from vllm.models.minimax_m3.nvidia import model as minimax_m3
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.v1.worker.gpu.spec_decode.eagle.eagle3_utils import (
|
||||
reserve_aux_intermediate_tensor_slots,
|
||||
verify_supports_aux_hidden_states_over_pp,
|
||||
)
|
||||
|
||||
@@ -24,3 +29,150 @@ def test_mimo_does_not_inherit_aux_hidden_state_pp_support():
|
||||
assert not inner.supports_aux_hidden_states_over_pp
|
||||
with pytest.raises(ValueError, match="does not support eagle3"):
|
||||
verify_supports_aux_hidden_states_over_pp(target, "eagle3")
|
||||
|
||||
|
||||
class _PPAuxLayer(torch.nn.Module):
|
||||
"""Aliases and mutates residual in place like MiniMaxM3DecoderLayer."""
|
||||
|
||||
ffn_all_reduce_deferred = False
|
||||
|
||||
def __init__(self, index):
|
||||
super().__init__()
|
||||
self.index = index
|
||||
|
||||
def forward(self, positions, hidden_states, residual):
|
||||
total = hidden_states if residual is None else hidden_states + residual
|
||||
residual = hidden_states if residual is None else residual
|
||||
residual.copy_(total + self.index + 1)
|
||||
return torch.full_like(hidden_states, 10 * (self.index + 1)), residual
|
||||
|
||||
|
||||
class _PPAuxNorm(torch.nn.Module):
|
||||
"""Clobbers its inputs so aux taps that alias them would fail."""
|
||||
|
||||
def forward(self, hidden_states, residual):
|
||||
output = hidden_states + residual
|
||||
hidden_states.fill_(-101)
|
||||
residual.fill_(-103)
|
||||
return output, residual
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def minimax_pp_stage(monkeypatch):
|
||||
"""Use the real MiniMax constructor/forward and aux helpers on CPU tensors."""
|
||||
group = SimpleNamespace(world_size=2, is_first_rank=True, is_last_rank=False)
|
||||
bounds = [0, 3]
|
||||
monkeypatch.setattr(minimax_m3, "get_pp_group", lambda: group)
|
||||
monkeypatch.setattr(parallel_state, "get_pp_group", lambda: group)
|
||||
monkeypatch.setattr(parallel_state, "model_parallel_is_initialized", lambda: True)
|
||||
monkeypatch.setattr(
|
||||
minimax_m3,
|
||||
"VocabParallelEmbedding",
|
||||
lambda *args, **kwargs: torch.nn.Identity(),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
minimax_m3, "MiniMAXGemmaRMSNorm", lambda *args, **kwargs: _PPAuxNorm()
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
minimax_m3,
|
||||
"MiniMaxM3DecoderLayer",
|
||||
lambda **kwargs: _PPAuxLayer(int(kwargs["prefix"].rsplit(".", 1)[1])),
|
||||
)
|
||||
|
||||
def make_layers(num_layers, build, prefix):
|
||||
return (
|
||||
*bounds,
|
||||
torch.nn.ModuleList(
|
||||
build(f"{prefix}.{index}")
|
||||
if bounds[0] <= index < bounds[1]
|
||||
else torch.nn.Identity()
|
||||
for index in range(num_layers)
|
||||
),
|
||||
)
|
||||
|
||||
monkeypatch.setattr(minimax_m3, "make_layers", make_layers)
|
||||
|
||||
def make_stage(start, end, taps, world_size=2):
|
||||
bounds[:] = [start, end]
|
||||
group.world_size = world_size
|
||||
group.is_first_rank = start == 0
|
||||
group.is_last_rank = end == 6
|
||||
config = SimpleNamespace(
|
||||
vocab_size=16, hidden_size=6144, num_hidden_layers=6, rms_norm_eps=1e-6
|
||||
)
|
||||
model = minimax_m3.MiniMaxM3Model(
|
||||
vllm_config=SimpleNamespace(
|
||||
model_config=SimpleNamespace(hf_text_config=config), quant_config=None
|
||||
)
|
||||
)
|
||||
model._set_aux_hidden_state_layers(taps)
|
||||
target = SimpleNamespace(
|
||||
model=model,
|
||||
make_empty_intermediate_tensors=model.make_empty_intermediate_tensors,
|
||||
)
|
||||
reserve_aux_intermediate_tensor_slots(target)
|
||||
return model, target
|
||||
|
||||
return make_stage
|
||||
|
||||
|
||||
# Input 7; _PPAuxLayer i adds 11 * (i + 1) to hidden + residual.
|
||||
def _expected_aux_value(tap):
|
||||
return 7 + 11 * tap * (tap + 1) // 2
|
||||
|
||||
|
||||
def _pp_aux_state(value):
|
||||
return torch.full((2, 6144), float(value), device="cpu")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("taps", [(0, 3, 6), (0, 1, 3), (4, 5, 6)])
|
||||
def test_minimax_pp2_preserves_ordered_global_aux_taps(minimax_pp_stage, taps):
|
||||
first, _ = minimax_pp_stage(0, 3, taps)
|
||||
sent = first(None, None, None, _pp_aux_state(7))
|
||||
last, target = minimax_pp_stage(3, 6, taps)
|
||||
reserved = target.make_empty_intermediate_tensors(2, torch.float32, "cpu")
|
||||
assert set(sent.tensors) == set(reserved.tensors)
|
||||
result = last(None, None, sent)
|
||||
assert isinstance(result, tuple)
|
||||
output, aux = result
|
||||
torch.testing.assert_close(output, _pp_aux_state(_expected_aux_value(6)))
|
||||
for hidden, tap in zip(aux, taps, strict=True):
|
||||
torch.testing.assert_close(hidden, _pp_aux_state(_expected_aux_value(tap)))
|
||||
|
||||
|
||||
def test_minimax_missing_upstream_aux_slot_is_not_silently_dropped(minimax_pp_stage):
|
||||
model, _ = minimax_pp_stage(3, 6, (0, 3, 6))
|
||||
incoming = IntermediateTensors(
|
||||
{
|
||||
"hidden_states": _pp_aux_state(30),
|
||||
"residual": _pp_aux_state(43),
|
||||
"aux_hidden_states_0": _pp_aux_state(7),
|
||||
}
|
||||
)
|
||||
with pytest.raises(RuntimeError, match="Missing aux_hidden_states_1"):
|
||||
model(None, None, incoming)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("taps", [(), (0, 3, 6)])
|
||||
def test_minimax_pp1_output_and_aux_taps(minimax_pp_stage, taps):
|
||||
model, _ = minimax_pp_stage(0, 6, taps, world_size=1)
|
||||
output = model(None, None, None, _pp_aux_state(7))
|
||||
if taps:
|
||||
output, aux = output
|
||||
for hidden, tap in zip(aux, taps, strict=True):
|
||||
torch.testing.assert_close(hidden, _pp_aux_state(_expected_aux_value(tap)))
|
||||
torch.testing.assert_close(output, _pp_aux_state(_expected_aux_value(6)))
|
||||
|
||||
|
||||
def test_minimax_pp2_without_aux_keeps_hidden_and_residual_transport(minimax_pp_stage):
|
||||
first, _ = minimax_pp_stage(0, 3, ())
|
||||
sent = first(None, None, None, _pp_aux_state(7))
|
||||
assert set(sent.tensors) == {"hidden_states", "residual"}
|
||||
torch.testing.assert_close(sent["hidden_states"], _pp_aux_state(30))
|
||||
torch.testing.assert_close(sent["residual"], _pp_aux_state(43))
|
||||
last, target = minimax_pp_stage(3, 6, ())
|
||||
assert set(
|
||||
target.make_empty_intermediate_tensors(12, torch.bfloat16, "cpu").tensors
|
||||
) == {"hidden_states", "residual"}
|
||||
output = last(None, None, sent)
|
||||
torch.testing.assert_close(output, _pp_aux_state(_expected_aux_value(6)))
|
||||
|
||||
@@ -410,17 +410,24 @@ def flashinfer_autotune(runner: "GPUModelRunner") -> None:
|
||||
Without autotuning, FlashInfer will rely on heuristics, which may
|
||||
be significantly slower.
|
||||
|
||||
Every rank profiles the same tactics. When distributed, per-tactic
|
||||
timings are averaged over the world CPU group so all ranks select the
|
||||
same tactic.
|
||||
With PP > 1, stages run different layers and may profile different ops,
|
||||
so each stage's TP group tunes separately with its own cache file;
|
||||
otherwise the world group tunes together. Per-tactic timings are
|
||||
averaged over the tuning group so all its ranks select the same tactic.
|
||||
"""
|
||||
from flashinfer.autotuner import AutoTuner, set_autotune_process_group
|
||||
|
||||
import vllm.utils.flashinfer as fi_utils
|
||||
from vllm.distributed.parallel_state import get_world_group
|
||||
from vllm.distributed.parallel_state import (
|
||||
get_pp_group,
|
||||
get_tp_group,
|
||||
get_world_group,
|
||||
)
|
||||
|
||||
world = get_world_group()
|
||||
is_leader = world.rank_in_group == 0
|
||||
pp_size = get_pp_group().world_size
|
||||
tune_group = get_tp_group() if pp_size > 1 else world
|
||||
is_leader = tune_group.rank_in_group == 0
|
||||
tuner = AutoTuner.get()
|
||||
|
||||
autotune_kwargs: dict = {}
|
||||
@@ -433,6 +440,11 @@ def flashinfer_autotune(runner: "GPUModelRunner") -> None:
|
||||
autotune_kwargs["skip_ops"] = skip_ops
|
||||
|
||||
cache_path = resolve_flashinfer_autotune_file(runner)
|
||||
if pp_size > 1:
|
||||
ranks = "-".join(str(rank) for rank in tune_group.ranks)
|
||||
cache_path = cache_path.with_name(
|
||||
f"{cache_path.stem}_tp_{ranks}{cache_path.suffix}"
|
||||
)
|
||||
if is_leader:
|
||||
logger.info_once("Using FlashInfer autotune cache file: %s", cache_path)
|
||||
|
||||
@@ -441,18 +453,18 @@ def flashinfer_autotune(runner: "GPUModelRunner") -> None:
|
||||
# which lead to some EP ranks receiving no tokens and skipping their
|
||||
# MoE kernel entirely, and cause hang due to all-reduce collective
|
||||
# during synchronized autotuning.
|
||||
# Read cached autotune results and broadcast to all ranks.
|
||||
# Read cached autotune results and broadcast within the tuning group.
|
||||
cached_results: bytes | None = None
|
||||
if is_leader and cache_path.exists():
|
||||
with open(cache_path, "rb") as f:
|
||||
cached_results = f.read()
|
||||
cached_results = world.broadcast_object(cached_results, src=0)
|
||||
cached_results = tune_group.broadcast_object(cached_results, src=0)
|
||||
if cached_results is not None:
|
||||
write_flashinfer_autotune_cache(cache_path, cached_results)
|
||||
world.barrier()
|
||||
tune_group.barrier()
|
||||
tuner.load_configs(str(cache_path))
|
||||
|
||||
group = world.cpu_group if world.world_size > 1 else None
|
||||
group = tune_group.cpu_group if tune_group.world_size > 1 else None
|
||||
set_autotune_process_group(group)
|
||||
try:
|
||||
with (
|
||||
|
||||
@@ -794,6 +794,7 @@ class MiniMaxM3DecoderLayer(nn.Module):
|
||||
|
||||
class MiniMaxM3Model(nn.Module, EagleModelMixin):
|
||||
fall_back_to_pt_during_load = False
|
||||
supports_aux_hidden_states_over_pp = True
|
||||
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
super().__init__()
|
||||
@@ -886,9 +887,18 @@ class MiniMaxM3Model(nn.Module, EagleModelMixin):
|
||||
hidden_states = intermediate_tensors["hidden_states"]
|
||||
residual = intermediate_tensors["residual"]
|
||||
|
||||
# EAGLE3 is not yet compatible with pipeline parallel
|
||||
aux_hidden_states = self._maybe_add_hidden_state([], 0, hidden_states, residual)
|
||||
for idx, layer in enumerate(self.layers[self.start_layer : self.end_layer]):
|
||||
remote_aux = self.collect_remote_aux_hidden_states(intermediate_tensors)
|
||||
aux_hidden_states: list[torch.Tensor] = []
|
||||
if get_pp_group().is_first_rank:
|
||||
self._maybe_add_hidden_state(
|
||||
aux_hidden_states, self.start_layer, hidden_states, residual
|
||||
)
|
||||
# Preserve the entry tap before in-place residual updates.
|
||||
if aux_hidden_states:
|
||||
aux_hidden_states[0] = aux_hidden_states[0].clone()
|
||||
for idx, layer in enumerate(
|
||||
self.layers[self.start_layer : self.end_layer], start=self.start_layer
|
||||
):
|
||||
hidden_states, residual = layer(positions, hidden_states, residual)
|
||||
self._maybe_add_hidden_state(
|
||||
aux_hidden_states, idx + 1, hidden_states, residual
|
||||
@@ -896,7 +906,11 @@ class MiniMaxM3Model(nn.Module, EagleModelMixin):
|
||||
|
||||
if not get_pp_group().is_last_rank:
|
||||
return IntermediateTensors(
|
||||
{"hidden_states": hidden_states, "residual": residual}
|
||||
{
|
||||
"hidden_states": hidden_states,
|
||||
"residual": residual,
|
||||
**self.pack_local_aux_hidden_states(aux_hidden_states),
|
||||
}
|
||||
)
|
||||
|
||||
if self.fuse_final_norm_allreduce:
|
||||
@@ -906,6 +920,7 @@ class MiniMaxM3Model(nn.Module, EagleModelMixin):
|
||||
else:
|
||||
hidden_states, _ = self.norm(hidden_states, residual)
|
||||
|
||||
aux_hidden_states = remote_aux + aux_hidden_states
|
||||
if len(aux_hidden_states) > 0:
|
||||
return hidden_states, aux_hidden_states
|
||||
return hidden_states
|
||||
|
||||
Reference in New Issue
Block a user