From ba1892907940c6baa50f4b385ae4f21de5452c4d Mon Sep 17 00:00:00 2001 From: aoshen02 Date: Wed, 22 Jul 2026 09:56:39 +0800 Subject: [PATCH] [Bugfix][SpecDecode] Scope MTP completeness checks outside bucketed updates (#49178) Signed-off-by: aoshen02 Co-authored-by: aoshen02 --- .../model_loader/test_mtp_validation.py | 19 +++++++ .../distributed/weight_transfer/ipc_engine.py | 7 ++- .../weight_transfer/nccl_engine.py | 57 ++++++++++--------- .../model_loader/mtp_validation.py | 26 +++++++++ vllm/model_executor/models/bailing_moe_mtp.py | 8 ++- vllm/model_executor/models/deepseek_mtp.py | 5 +- vllm/model_executor/models/step3p5_mtp.py | 5 +- vllm/models/deepseek_v32/nvidia/mtp.py | 5 +- vllm/models/deepseek_v4/amd/mtp.py | 5 +- vllm/models/deepseek_v4/nvidia/mtp.py | 5 +- vllm/models/deepseek_v4/xpu/mtp.py | 5 +- vllm/models/inkling/nvidia/mtp.py | 5 +- vllm/models/minimax_m3/amd/mtp.py | 8 ++- vllm/models/minimax_m3/nvidia/mtp.py | 8 ++- 14 files changed, 131 insertions(+), 37 deletions(-) create mode 100644 tests/model_executor/model_loader/test_mtp_validation.py create mode 100644 vllm/model_executor/model_loader/mtp_validation.py diff --git a/tests/model_executor/model_loader/test_mtp_validation.py b/tests/model_executor/model_loader/test_mtp_validation.py new file mode 100644 index 0000000000..ecccaa0cd7 --- /dev/null +++ b/tests/model_executor/model_loader/test_mtp_validation.py @@ -0,0 +1,19 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import pytest + +from vllm.model_executor.model_loader.mtp_validation import ( + disable_mtp_completeness_check, + is_mtp_completeness_check_enabled, +) + + +def test_disable_mtp_completeness_check_is_scoped(): + assert is_mtp_completeness_check_enabled() + + with pytest.raises(RuntimeError), disable_mtp_completeness_check(): + assert not is_mtp_completeness_check_enabled() + raise RuntimeError + + assert is_mtp_completeness_check_enabled() diff --git a/vllm/distributed/weight_transfer/ipc_engine.py b/vllm/distributed/weight_transfer/ipc_engine.py index f1b6070893..b191e8c50d 100644 --- a/vllm/distributed/weight_transfer/ipc_engine.py +++ b/vllm/distributed/weight_transfer/ipc_engine.py @@ -274,7 +274,12 @@ class IPCWeightTransferEngine( weight = rebuild_cuda_tensor(*list_args) weights.append((name, weight)) - self.model.load_weights(weights) + from vllm.model_executor.model_loader.mtp_validation import ( + disable_mtp_completeness_check, + ) + + with disable_mtp_completeness_check(): + self.model.load_weights(weights) def shutdown(self) -> None: pass diff --git a/vllm/distributed/weight_transfer/nccl_engine.py b/vllm/distributed/weight_transfer/nccl_engine.py index 5838a8ba8a..0ba44f3486 100644 --- a/vllm/distributed/weight_transfer/nccl_engine.py +++ b/vllm/distributed/weight_transfer/nccl_engine.py @@ -168,36 +168,41 @@ class NCCLWeightTransferEngine( "Call init_transfer_engine() first." ) - if update_info.packed: - # Build iterator of (name, (shape, dtype)) from update_info - def state_dict_info_iterator(): + from vllm.model_executor.model_loader.mtp_validation import ( + disable_mtp_completeness_check, + ) + + with disable_mtp_completeness_check(): + if update_info.packed: + # Build iterator of (name, (shape, dtype)) from update_info + def state_dict_info_iterator(): + for name, dtype_name, shape in zip( + update_info.names, update_info.dtype_names, update_info.shapes + ): + dtype = getattr(torch, dtype_name) + yield (name, (shape, dtype)) + + packed_nccl_broadcast_consumer( + iterator=state_dict_info_iterator(), + group=self.model_update_group, + src=0, + post_unpack_func=self.model.load_weights, + buffer_size_bytes=update_info.packed_buffer_size_bytes, + num_buffers=update_info.packed_num_buffers, + device=self.device, + ) + else: + # Use simple one-by-one broadcasting for name, dtype_name, shape in zip( update_info.names, update_info.dtype_names, update_info.shapes ): dtype = getattr(torch, dtype_name) - yield (name, (shape, dtype)) - - packed_nccl_broadcast_consumer( - iterator=state_dict_info_iterator(), - group=self.model_update_group, - src=0, - post_unpack_func=self.model.load_weights, - buffer_size_bytes=update_info.packed_buffer_size_bytes, - num_buffers=update_info.packed_num_buffers, - device=self.device, - ) - else: - # Use simple one-by-one broadcasting - for name, dtype_name, shape in zip( - update_info.names, update_info.dtype_names, update_info.shapes - ): - dtype = getattr(torch, dtype_name) - weight = torch.empty(shape, dtype=dtype, device=self.device) - self.model_update_group.broadcast( - weight, src=0, stream=torch.cuda.current_stream() - ) - self.model.load_weights([(name, weight)]) - del weight + weight = torch.empty(shape, dtype=dtype, device=self.device) + self.model_update_group.broadcast( + weight, src=0, stream=torch.cuda.current_stream() + ) + self.model.load_weights([(name, weight)]) + del weight def shutdown(self) -> None: if self.model_update_group is not None: diff --git a/vllm/model_executor/model_loader/mtp_validation.py b/vllm/model_executor/model_loader/mtp_validation.py new file mode 100644 index 0000000000..3f20756abc --- /dev/null +++ b/vllm/model_executor/model_loader/mtp_validation.py @@ -0,0 +1,26 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Scoped controls for MTP checkpoint completeness validation.""" + +from collections.abc import Iterator +from contextlib import contextmanager +from contextvars import ContextVar + +_mtp_completeness_check_enabled: ContextVar[bool] = ContextVar( + "mtp_completeness_check_enabled", default=True +) + + +def is_mtp_completeness_check_enabled() -> bool: + """Return whether MTP completeness validation is enabled in this scope.""" + return _mtp_completeness_check_enabled.get() + + +@contextmanager +def disable_mtp_completeness_check() -> Iterator[None]: + """Temporarily disable MTP completeness validation for one weight load.""" + token = _mtp_completeness_check_enabled.set(False) + try: + yield + finally: + _mtp_completeness_check_enabled.reset(token) diff --git a/vllm/model_executor/models/bailing_moe_mtp.py b/vllm/model_executor/models/bailing_moe_mtp.py index da6b1ddb8b..53679edf89 100644 --- a/vllm/model_executor/models/bailing_moe_mtp.py +++ b/vllm/model_executor/models/bailing_moe_mtp.py @@ -20,6 +20,9 @@ from vllm.model_executor.layers.vocab_parallel_embedding import ( ParallelLMHead, VocabParallelEmbedding, ) +from vllm.model_executor.model_loader.mtp_validation import ( + is_mtp_completeness_check_enabled, +) from vllm.model_executor.model_loader.weight_utils import ( default_weight_loader, maybe_remap_kv_scale_name, @@ -370,7 +373,10 @@ class BailingMoeV25MTPModel(nn.Module): self.model.mtp_start_layer_idx, self.model.mtp_start_layer_idx + self.model.num_mtp_layers, ): - if layer_idx not in loaded_mtp_layers: + if ( + layer_idx not in loaded_mtp_layers + and is_mtp_completeness_check_enabled() + ): raise ValueError( f"Bailing MTP speculative decoding layer {layer_idx} " "weights are missing from checkpoint. Use a checkpoint " diff --git a/vllm/model_executor/models/deepseek_mtp.py b/vllm/model_executor/models/deepseek_mtp.py index 3a0c21fe7d..65f860a4a3 100644 --- a/vllm/model_executor/models/deepseek_mtp.py +++ b/vllm/model_executor/models/deepseek_mtp.py @@ -21,6 +21,9 @@ from vllm.model_executor.layers.vocab_parallel_embedding import ( ParallelLMHead, VocabParallelEmbedding, ) +from vllm.model_executor.model_loader.mtp_validation import ( + is_mtp_completeness_check_enabled, +) from vllm.model_executor.model_loader.weight_utils import ( default_weight_loader, maybe_remap_kv_scale_name, @@ -516,7 +519,7 @@ class DeepSeekMTP(nn.Module, DeepseekV2MixtureOfExperts): self.model.mtp_start_layer_idx, self.model.mtp_start_layer_idx + self.model.num_mtp_layers, ): - if layer_idx not in loaded_layers: + if layer_idx not in loaded_layers and is_mtp_completeness_check_enabled(): raise ValueError( f"MTP speculative decoding layer {layer_idx} weights " f"missing from checkpoint. The checkpoint may have " diff --git a/vllm/model_executor/models/step3p5_mtp.py b/vllm/model_executor/models/step3p5_mtp.py index b533a9111d..7e95274771 100644 --- a/vllm/model_executor/models/step3p5_mtp.py +++ b/vllm/model_executor/models/step3p5_mtp.py @@ -15,6 +15,9 @@ from vllm.model_executor.layers.vocab_parallel_embedding import ( ParallelLMHead, VocabParallelEmbedding, ) +from vllm.model_executor.model_loader.mtp_validation import ( + is_mtp_completeness_check_enabled, +) from vllm.model_executor.model_loader.weight_utils import default_weight_loader from vllm.sequence import IntermediateTensors @@ -283,7 +286,7 @@ class Step3p5MTP(nn.Module): and getattr(param, "requires_grad", False) is False } params_need_to_load -= optional_params - if params_need_to_load != loaded_params: + if params_need_to_load != loaded_params and is_mtp_completeness_check_enabled(): missing_params = list(params_need_to_load - loaded_params) param_name_example = missing_params[0] raise RuntimeError( diff --git a/vllm/models/deepseek_v32/nvidia/mtp.py b/vllm/models/deepseek_v32/nvidia/mtp.py index d3d8e5aae9..c8f2fcc5ff 100644 --- a/vllm/models/deepseek_v32/nvidia/mtp.py +++ b/vllm/models/deepseek_v32/nvidia/mtp.py @@ -17,6 +17,9 @@ from vllm.model_executor.layers.logits_processor import LogitsProcessor from vllm.model_executor.layers.vocab_parallel_embedding import ( VocabParallelEmbedding, ) +from vllm.model_executor.model_loader.mtp_validation import ( + is_mtp_completeness_check_enabled, +) from vllm.model_executor.model_loader.weight_utils import ( default_weight_loader, maybe_remap_kv_scale_name, @@ -416,7 +419,7 @@ class DeepseekV32MTP(nn.Module, DeepseekV2MixtureOfExperts): self.model.mtp_start_layer_idx, self.model.mtp_start_layer_idx + self.model.num_mtp_layers, ): - if layer_idx not in loaded_layers: + if layer_idx not in loaded_layers and is_mtp_completeness_check_enabled(): raise ValueError( f"MTP speculative decoding layer {layer_idx} weights " f"missing from checkpoint." diff --git a/vllm/models/deepseek_v4/amd/mtp.py b/vllm/models/deepseek_v4/amd/mtp.py index f5ef4bb067..a12b401f0d 100644 --- a/vllm/models/deepseek_v4/amd/mtp.py +++ b/vllm/models/deepseek_v4/amd/mtp.py @@ -32,6 +32,9 @@ from vllm.model_executor.layers.mhc import HCHeadOp from vllm.model_executor.layers.vocab_parallel_embedding import ( VocabParallelEmbedding, ) +from vllm.model_executor.model_loader.mtp_validation import ( + is_mtp_completeness_check_enabled, +) from vllm.model_executor.model_loader.weight_utils import default_weight_loader from vllm.model_executor.models.deepseek_mtp import SharedHead from vllm.model_executor.models.deepseek_v2 import get_spec_layer_idx_from_weight_name @@ -461,7 +464,7 @@ class DeepSeekV4MTP(nn.Module): self.model.mtp_start_layer_idx, self.model.mtp_start_layer_idx + self.model.num_mtp_layers, ): - if layer_idx not in loaded_layers: + if layer_idx not in loaded_layers and is_mtp_completeness_check_enabled(): raise ValueError( f"MTP speculative decoding layer {layer_idx} weights " f"missing from checkpoint. The checkpoint may have " diff --git a/vllm/models/deepseek_v4/nvidia/mtp.py b/vllm/models/deepseek_v4/nvidia/mtp.py index 64715deae9..8aa3d2c9a2 100644 --- a/vllm/models/deepseek_v4/nvidia/mtp.py +++ b/vllm/models/deepseek_v4/nvidia/mtp.py @@ -37,6 +37,9 @@ from vllm.model_executor.layers.logits_processor import LogitsProcessor from vllm.model_executor.layers.vocab_parallel_embedding import ( VocabParallelEmbedding, ) +from vllm.model_executor.model_loader.mtp_validation import ( + is_mtp_completeness_check_enabled, +) from vllm.model_executor.model_loader.weight_utils import default_weight_loader from vllm.model_executor.models.deepseek_mtp import SharedHead from vllm.model_executor.models.deepseek_v2 import get_spec_layer_idx_from_weight_name @@ -467,7 +470,7 @@ class DeepSeekV4MTP(nn.Module): self.model.mtp_start_layer_idx, self.model.mtp_start_layer_idx + self.model.num_mtp_layers, ): - if layer_idx not in loaded_layers: + if layer_idx not in loaded_layers and is_mtp_completeness_check_enabled(): raise ValueError( f"MTP speculative decoding layer {layer_idx} weights " f"missing from checkpoint. The checkpoint may have " diff --git a/vllm/models/deepseek_v4/xpu/mtp.py b/vllm/models/deepseek_v4/xpu/mtp.py index 8baca78b8b..35687ccf16 100644 --- a/vllm/models/deepseek_v4/xpu/mtp.py +++ b/vllm/models/deepseek_v4/xpu/mtp.py @@ -32,6 +32,9 @@ from vllm.model_executor.layers.mhc import HCHeadOp from vllm.model_executor.layers.vocab_parallel_embedding import ( VocabParallelEmbedding, ) +from vllm.model_executor.model_loader.mtp_validation import ( + is_mtp_completeness_check_enabled, +) from vllm.model_executor.model_loader.weight_utils import default_weight_loader from vllm.model_executor.models.deepseek_mtp import SharedHead from vllm.model_executor.models.deepseek_v2 import get_spec_layer_idx_from_weight_name @@ -469,7 +472,7 @@ class DeepSeekV4MTP(nn.Module): self.model.mtp_start_layer_idx, self.model.mtp_start_layer_idx + self.model.num_mtp_layers, ): - if layer_idx not in loaded_layers: + if layer_idx not in loaded_layers and is_mtp_completeness_check_enabled(): raise ValueError( f"MTP speculative decoding layer {layer_idx} weights " f"missing from checkpoint. The checkpoint may have " diff --git a/vllm/models/inkling/nvidia/mtp.py b/vllm/models/inkling/nvidia/mtp.py index a2559d4cf0..34cde020ed 100644 --- a/vllm/models/inkling/nvidia/mtp.py +++ b/vllm/models/inkling/nvidia/mtp.py @@ -25,6 +25,9 @@ from vllm.config import VllmConfig from vllm.model_executor.layers.linear import ReplicatedLinear from vllm.model_executor.layers.logits_processor import LogitsProcessor from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead +from vllm.model_executor.model_loader.mtp_validation import ( + is_mtp_completeness_check_enabled, +) from vllm.model_executor.model_loader.weight_utils import default_weight_loader from vllm.model_executor.models.utils import maybe_prefix from vllm.sequence import IntermediateTensors @@ -396,7 +399,7 @@ def _load_inkling_mtp_weights( for name in params if name.startswith("model.layers.") or name.startswith("model.chain_norm.") } - if missing := sorted(required - loaded): + if (missing := sorted(required - loaded)) and is_mtp_completeness_check_enabled(): raise ValueError( "Inkling MTP checkpoint is missing required parameters: " + ", ".join(missing) diff --git a/vllm/models/minimax_m3/amd/mtp.py b/vllm/models/minimax_m3/amd/mtp.py index f62face1d2..cfb26d7948 100644 --- a/vllm/models/minimax_m3/amd/mtp.py +++ b/vllm/models/minimax_m3/amd/mtp.py @@ -35,6 +35,9 @@ from vllm.model_executor.layers.vocab_parallel_embedding import ( ParallelLMHead, VocabParallelEmbedding, ) +from vllm.model_executor.model_loader.mtp_validation import ( + is_mtp_completeness_check_enabled, +) from vllm.model_executor.model_loader.weight_utils import ( default_weight_loader, maybe_remap_kv_scale_name, @@ -322,7 +325,10 @@ class MiniMaxM3MTP(nn.Module): # Validate that weights were loaded for each MTP layer. for layer_idx in range(self.model.num_mtp_layers): - if layer_idx not in loaded_mtp_layers: + if ( + layer_idx not in loaded_mtp_layers + and is_mtp_completeness_check_enabled() + ): raise ValueError( f"Failed to load MTP layer {layer_idx} weights from checkpoint." ) diff --git a/vllm/models/minimax_m3/nvidia/mtp.py b/vllm/models/minimax_m3/nvidia/mtp.py index e2c7f8821d..832c872a10 100644 --- a/vllm/models/minimax_m3/nvidia/mtp.py +++ b/vllm/models/minimax_m3/nvidia/mtp.py @@ -19,6 +19,9 @@ from vllm.model_executor.layers.vocab_parallel_embedding import ( ParallelLMHead, VocabParallelEmbedding, ) +from vllm.model_executor.model_loader.mtp_validation import ( + is_mtp_completeness_check_enabled, +) from vllm.model_executor.model_loader.weight_utils import ( default_weight_loader, maybe_remap_kv_scale_name, @@ -304,7 +307,10 @@ class MiniMaxM3MTP(nn.Module): # Validate that weights were loaded for each MTP layer. for layer_idx in range(self.model.num_mtp_layers): - if layer_idx not in loaded_mtp_layers: + if ( + layer_idx not in loaded_mtp_layers + and is_mtp_completeness_check_enabled() + ): raise ValueError( f"Failed to load MTP layer {layer_idx} weights from checkpoint." )