From dca96bf97b350204937a72c4d386b79b425f3989 Mon Sep 17 00:00:00 2001 From: Canlin Guo Date: Sat, 12 Sep 2026 14:22:30 +0800 Subject: [PATCH] [Bugfix][Spec Decode] Avoid fastsafetensors deadlock for PP draft models (#54416) Signed-off-by: Canlin Co-authored-by: Codex --- .../test_draft_attention_backend_override.py | 9 ++++++++- .../test_draft_moe_backend_override.py | 9 ++++++++- vllm/v1/worker/gpu/spec_decode/dflash/utils.py | 2 ++ vllm/v1/worker/gpu/spec_decode/dspark/utils.py | 2 ++ vllm/v1/worker/gpu/spec_decode/eagle/utils.py | 4 ++++ vllm/v1/worker/gpu/spec_decode/utils.py | 18 ++++++++++++++++++ 6 files changed, 42 insertions(+), 2 deletions(-) diff --git a/tests/v1/spec_decode/test_draft_attention_backend_override.py b/tests/v1/spec_decode/test_draft_attention_backend_override.py index 5984d33769..410afe8d61 100644 --- a/tests/v1/spec_decode/test_draft_attention_backend_override.py +++ b/tests/v1/spec_decode/test_draft_attention_backend_override.py @@ -6,11 +6,13 @@ assert on the config ``load_eagle_model`` hands to ``get_model``. """ -from dataclasses import dataclass +from dataclasses import dataclass, field +from types import SimpleNamespace from unittest.mock import patch import pytest +from vllm.config import LoadConfig from vllm.v1.worker.gpu.spec_decode.eagle.utils import load_eagle_model @@ -43,6 +45,7 @@ class _VllmConfig: kernel_config: _KernelConfig cache_config: _CacheConfig speculative_config: _SpeculativeConfig + load_config: LoadConfig = field(default_factory=LoadConfig) def _config(target_backend: str, draft_backend: str | None) -> _VllmConfig: @@ -65,6 +68,10 @@ def _capture_draft_config(cfg): with ( patch("vllm.v1.worker.gpu.spec_decode.eagle.utils.get_model", _fake_get_model), + patch( + "vllm.v1.worker.gpu.spec_decode.utils.get_pp_group", + return_value=SimpleNamespace(world_size=1), + ), pytest.raises(_Captured) as exc, ): load_eagle_model(object(), cfg) diff --git a/tests/v1/spec_decode/test_draft_moe_backend_override.py b/tests/v1/spec_decode/test_draft_moe_backend_override.py index d4acd9e4be..5a4522e5b3 100644 --- a/tests/v1/spec_decode/test_draft_moe_backend_override.py +++ b/tests/v1/spec_decode/test_draft_moe_backend_override.py @@ -8,11 +8,13 @@ target is typically unquantized, and quantized-only backends reject it, so the server fails to start rather than falling back. """ -from dataclasses import dataclass +from dataclasses import dataclass, field +from types import SimpleNamespace from unittest.mock import patch import pytest +from vllm.config import LoadConfig from vllm.v1.worker.gpu.spec_decode.eagle.utils import load_eagle_model @@ -39,6 +41,7 @@ class _VllmConfig: kernel_config: _KernelConfig cache_config: _CacheConfig speculative_config: _SpeculativeConfig + load_config: LoadConfig = field(default_factory=LoadConfig) def _config(target_moe: str, draft_moe: str | None) -> _VllmConfig: @@ -62,6 +65,10 @@ def _capture_draft_config(cfg): with ( patch("vllm.v1.worker.gpu.spec_decode.eagle.utils.get_model", _fake_get_model), + patch( + "vllm.v1.worker.gpu.spec_decode.utils.get_pp_group", + return_value=SimpleNamespace(world_size=1), + ), pytest.raises(_Captured) as exc, ): load_eagle_model(object(), cfg) diff --git a/vllm/v1/worker/gpu/spec_decode/dflash/utils.py b/vllm/v1/worker/gpu/spec_decode/dflash/utils.py index 6226dea1c2..7323b29312 100644 --- a/vllm/v1/worker/gpu/spec_decode/dflash/utils.py +++ b/vllm/v1/worker/gpu/spec_decode/dflash/utils.py @@ -9,6 +9,7 @@ from vllm.v1.worker.gpu.spec_decode.eagle.utils import ( get_target_lm_head, maybe_share_target_embed, ) +from vllm.v1.worker.gpu.spec_decode.utils import get_pp_safe_draft_load_config def load_dflash_model(target_model: nn.Module, vllm_config: VllmConfig) -> nn.Module: @@ -37,6 +38,7 @@ def load_dflash_model(target_model: nn.Module, vllm_config: VllmConfig) -> nn.Mo if speculative_config.kv_cache_dtype is not None else vllm_config.cache_config ), + load_config=get_pp_safe_draft_load_config(vllm_config.load_config), ) with set_model_tag("dflash_head"): dflash_model = get_model( diff --git a/vllm/v1/worker/gpu/spec_decode/dspark/utils.py b/vllm/v1/worker/gpu/spec_decode/dspark/utils.py index 5b13530ef7..db1e17a9f2 100644 --- a/vllm/v1/worker/gpu/spec_decode/dspark/utils.py +++ b/vllm/v1/worker/gpu/spec_decode/dspark/utils.py @@ -6,6 +6,7 @@ import torch.nn as nn from vllm.config import ModelConfig, VllmConfig, replace from vllm.logger import init_logger from vllm.v1.attention.backends.registry import AttentionBackendEnum +from vllm.v1.worker.gpu.spec_decode.utils import get_pp_safe_draft_load_config logger = init_logger(__name__) @@ -73,6 +74,7 @@ def load_dspark_model(target_model: nn.Module, vllm_config: VllmConfig) -> nn.Mo if speculative_config.kv_cache_dtype is not None else vllm_config.cache_config ), + load_config=get_pp_safe_draft_load_config(vllm_config.load_config), ) # VllmConfig post-init restores the target's quant config because the target # config is retained for DSpark's target-layer metadata, so we must override it. diff --git a/vllm/v1/worker/gpu/spec_decode/eagle/utils.py b/vllm/v1/worker/gpu/spec_decode/eagle/utils.py index a85f21a3cd..b93a79e455 100644 --- a/vllm/v1/worker/gpu/spec_decode/eagle/utils.py +++ b/vllm/v1/worker/gpu/spec_decode/eagle/utils.py @@ -8,6 +8,7 @@ from vllm.distributed.parallel_state import get_pp_group from vllm.lora.layers.base import BaseLayerWithLoRA from vllm.model_executor.model_loader import get_model from vllm.model_executor.models.utils import PPMissingLayer +from vllm.v1.worker.gpu.spec_decode.utils import get_pp_safe_draft_load_config def _should_share(eagle: nn.Module, flag: str, draft, target) -> bool: @@ -103,6 +104,9 @@ def load_eagle_model(target_model: nn.Module, vllm_config: VllmConfig) -> nn.Mod backend=speculative_config.attention_backend, ), ) + draft_load_config = get_pp_safe_draft_load_config(vllm_config.load_config) + if draft_load_config is not vllm_config.load_config: + vllm_config = replace(vllm_config, load_config=draft_load_config) with set_model_tag("eagle_head"): eagle_model = get_model( vllm_config=vllm_config, model_config=draft_model_config diff --git a/vllm/v1/worker/gpu/spec_decode/utils.py b/vllm/v1/worker/gpu/spec_decode/utils.py index e25672e953..f73acb2bda 100644 --- a/vllm/v1/worker/gpu/spec_decode/utils.py +++ b/vllm/v1/worker/gpu/spec_decode/utils.py @@ -3,10 +3,28 @@ import numpy as np import torch +from vllm.config import LoadConfig, replace +from vllm.distributed import get_pp_group +from vllm.logger import init_logger from vllm.v1.outputs import DraftTokenIds from vllm.v1.worker.gpu.async_utils import async_copy_to_np from vllm.v1.worker.gpu.input_batch import InputBatch +logger = init_logger(__name__) + + +def get_pp_safe_draft_load_config(load_config: LoadConfig) -> LoadConfig: + """Avoid collectives that include PP ranks without a draft model.""" + if get_pp_group().world_size > 1 and load_config.load_format == "fastsafetensors": + logger.warning_once( + "fastsafetensors cannot load a draft model instantiated on only " + "one pipeline stage; falling back to the standard safetensors " + "loader for the draft model. The target model still uses " + "fastsafetensors." + ) + return replace(load_config, load_format="auto") + return load_config + class DraftTokensHandler: def __init__(self, device: torch.device | None = None):