[Bugfix][Spec Decode] Avoid fastsafetensors deadlock for PP draft models (#54416)

Signed-off-by: Canlin <canlinguosdu@gmail.com>
Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
Canlin Guo
2026-09-12 14:22:30 +08:00
committed by GitHub
co-authored by Codex
parent eed1f3d0c6
commit dca96bf97b
6 changed files with 42 additions and 2 deletions
@@ -6,11 +6,13 @@
assert on the config ``load_eagle_model`` hands to ``get_model``.
"""
from dataclasses import dataclass
from dataclasses import dataclass, field
from types import SimpleNamespace
from unittest.mock import patch
import pytest
from vllm.config import LoadConfig
from vllm.v1.worker.gpu.spec_decode.eagle.utils import load_eagle_model
@@ -43,6 +45,7 @@ class _VllmConfig:
kernel_config: _KernelConfig
cache_config: _CacheConfig
speculative_config: _SpeculativeConfig
load_config: LoadConfig = field(default_factory=LoadConfig)
def _config(target_backend: str, draft_backend: str | None) -> _VllmConfig:
@@ -65,6 +68,10 @@ def _capture_draft_config(cfg):
with (
patch("vllm.v1.worker.gpu.spec_decode.eagle.utils.get_model", _fake_get_model),
patch(
"vllm.v1.worker.gpu.spec_decode.utils.get_pp_group",
return_value=SimpleNamespace(world_size=1),
),
pytest.raises(_Captured) as exc,
):
load_eagle_model(object(), cfg)
@@ -8,11 +8,13 @@ target is typically unquantized, and quantized-only backends reject it, so the
server fails to start rather than falling back.
"""
from dataclasses import dataclass
from dataclasses import dataclass, field
from types import SimpleNamespace
from unittest.mock import patch
import pytest
from vllm.config import LoadConfig
from vllm.v1.worker.gpu.spec_decode.eagle.utils import load_eagle_model
@@ -39,6 +41,7 @@ class _VllmConfig:
kernel_config: _KernelConfig
cache_config: _CacheConfig
speculative_config: _SpeculativeConfig
load_config: LoadConfig = field(default_factory=LoadConfig)
def _config(target_moe: str, draft_moe: str | None) -> _VllmConfig:
@@ -62,6 +65,10 @@ def _capture_draft_config(cfg):
with (
patch("vllm.v1.worker.gpu.spec_decode.eagle.utils.get_model", _fake_get_model),
patch(
"vllm.v1.worker.gpu.spec_decode.utils.get_pp_group",
return_value=SimpleNamespace(world_size=1),
),
pytest.raises(_Captured) as exc,
):
load_eagle_model(object(), cfg)
@@ -9,6 +9,7 @@ from vllm.v1.worker.gpu.spec_decode.eagle.utils import (
get_target_lm_head,
maybe_share_target_embed,
)
from vllm.v1.worker.gpu.spec_decode.utils import get_pp_safe_draft_load_config
def load_dflash_model(target_model: nn.Module, vllm_config: VllmConfig) -> nn.Module:
@@ -37,6 +38,7 @@ def load_dflash_model(target_model: nn.Module, vllm_config: VllmConfig) -> nn.Mo
if speculative_config.kv_cache_dtype is not None
else vllm_config.cache_config
),
load_config=get_pp_safe_draft_load_config(vllm_config.load_config),
)
with set_model_tag("dflash_head"):
dflash_model = get_model(
@@ -6,6 +6,7 @@ import torch.nn as nn
from vllm.config import ModelConfig, VllmConfig, replace
from vllm.logger import init_logger
from vllm.v1.attention.backends.registry import AttentionBackendEnum
from vllm.v1.worker.gpu.spec_decode.utils import get_pp_safe_draft_load_config
logger = init_logger(__name__)
@@ -73,6 +74,7 @@ def load_dspark_model(target_model: nn.Module, vllm_config: VllmConfig) -> nn.Mo
if speculative_config.kv_cache_dtype is not None
else vllm_config.cache_config
),
load_config=get_pp_safe_draft_load_config(vllm_config.load_config),
)
# VllmConfig post-init restores the target's quant config because the target
# config is retained for DSpark's target-layer metadata, so we must override it.
@@ -8,6 +8,7 @@ from vllm.distributed.parallel_state import get_pp_group
from vllm.lora.layers.base import BaseLayerWithLoRA
from vllm.model_executor.model_loader import get_model
from vllm.model_executor.models.utils import PPMissingLayer
from vllm.v1.worker.gpu.spec_decode.utils import get_pp_safe_draft_load_config
def _should_share(eagle: nn.Module, flag: str, draft, target) -> bool:
@@ -103,6 +104,9 @@ def load_eagle_model(target_model: nn.Module, vllm_config: VllmConfig) -> nn.Mod
backend=speculative_config.attention_backend,
),
)
draft_load_config = get_pp_safe_draft_load_config(vllm_config.load_config)
if draft_load_config is not vllm_config.load_config:
vllm_config = replace(vllm_config, load_config=draft_load_config)
with set_model_tag("eagle_head"):
eagle_model = get_model(
vllm_config=vllm_config, model_config=draft_model_config
+18
View File
@@ -3,10 +3,28 @@
import numpy as np
import torch
from vllm.config import LoadConfig, replace
from vllm.distributed import get_pp_group
from vllm.logger import init_logger
from vllm.v1.outputs import DraftTokenIds
from vllm.v1.worker.gpu.async_utils import async_copy_to_np
from vllm.v1.worker.gpu.input_batch import InputBatch
logger = init_logger(__name__)
def get_pp_safe_draft_load_config(load_config: LoadConfig) -> LoadConfig:
"""Avoid collectives that include PP ranks without a draft model."""
if get_pp_group().world_size > 1 and load_config.load_format == "fastsafetensors":
logger.warning_once(
"fastsafetensors cannot load a draft model instantiated on only "
"one pipeline stage; falling back to the standard safetensors "
"loader for the draft model. The target model still uses "
"fastsafetensors."
)
return replace(load_config, load_format="auto")
return load_config
class DraftTokensHandler:
def __init__(self, device: torch.device | None = None):