[Bugfix][Spec Decode] Avoid fastsafetensors deadlock for PP draft models (#54416)
Signed-off-by: Canlin <canlinguosdu@gmail.com> Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
@@ -6,11 +6,13 @@
|
||||
assert on the config ``load_eagle_model`` hands to ``get_model``.
|
||||
"""
|
||||
|
||||
from dataclasses import dataclass
|
||||
from dataclasses import dataclass, field
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.config import LoadConfig
|
||||
from vllm.v1.worker.gpu.spec_decode.eagle.utils import load_eagle_model
|
||||
|
||||
|
||||
@@ -43,6 +45,7 @@ class _VllmConfig:
|
||||
kernel_config: _KernelConfig
|
||||
cache_config: _CacheConfig
|
||||
speculative_config: _SpeculativeConfig
|
||||
load_config: LoadConfig = field(default_factory=LoadConfig)
|
||||
|
||||
|
||||
def _config(target_backend: str, draft_backend: str | None) -> _VllmConfig:
|
||||
@@ -65,6 +68,10 @@ def _capture_draft_config(cfg):
|
||||
|
||||
with (
|
||||
patch("vllm.v1.worker.gpu.spec_decode.eagle.utils.get_model", _fake_get_model),
|
||||
patch(
|
||||
"vllm.v1.worker.gpu.spec_decode.utils.get_pp_group",
|
||||
return_value=SimpleNamespace(world_size=1),
|
||||
),
|
||||
pytest.raises(_Captured) as exc,
|
||||
):
|
||||
load_eagle_model(object(), cfg)
|
||||
|
||||
@@ -8,11 +8,13 @@ target is typically unquantized, and quantized-only backends reject it, so the
|
||||
server fails to start rather than falling back.
|
||||
"""
|
||||
|
||||
from dataclasses import dataclass
|
||||
from dataclasses import dataclass, field
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.config import LoadConfig
|
||||
from vllm.v1.worker.gpu.spec_decode.eagle.utils import load_eagle_model
|
||||
|
||||
|
||||
@@ -39,6 +41,7 @@ class _VllmConfig:
|
||||
kernel_config: _KernelConfig
|
||||
cache_config: _CacheConfig
|
||||
speculative_config: _SpeculativeConfig
|
||||
load_config: LoadConfig = field(default_factory=LoadConfig)
|
||||
|
||||
|
||||
def _config(target_moe: str, draft_moe: str | None) -> _VllmConfig:
|
||||
@@ -62,6 +65,10 @@ def _capture_draft_config(cfg):
|
||||
|
||||
with (
|
||||
patch("vllm.v1.worker.gpu.spec_decode.eagle.utils.get_model", _fake_get_model),
|
||||
patch(
|
||||
"vllm.v1.worker.gpu.spec_decode.utils.get_pp_group",
|
||||
return_value=SimpleNamespace(world_size=1),
|
||||
),
|
||||
pytest.raises(_Captured) as exc,
|
||||
):
|
||||
load_eagle_model(object(), cfg)
|
||||
|
||||
@@ -9,6 +9,7 @@ from vllm.v1.worker.gpu.spec_decode.eagle.utils import (
|
||||
get_target_lm_head,
|
||||
maybe_share_target_embed,
|
||||
)
|
||||
from vllm.v1.worker.gpu.spec_decode.utils import get_pp_safe_draft_load_config
|
||||
|
||||
|
||||
def load_dflash_model(target_model: nn.Module, vllm_config: VllmConfig) -> nn.Module:
|
||||
@@ -37,6 +38,7 @@ def load_dflash_model(target_model: nn.Module, vllm_config: VllmConfig) -> nn.Mo
|
||||
if speculative_config.kv_cache_dtype is not None
|
||||
else vllm_config.cache_config
|
||||
),
|
||||
load_config=get_pp_safe_draft_load_config(vllm_config.load_config),
|
||||
)
|
||||
with set_model_tag("dflash_head"):
|
||||
dflash_model = get_model(
|
||||
|
||||
@@ -6,6 +6,7 @@ import torch.nn as nn
|
||||
from vllm.config import ModelConfig, VllmConfig, replace
|
||||
from vllm.logger import init_logger
|
||||
from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
||||
from vllm.v1.worker.gpu.spec_decode.utils import get_pp_safe_draft_load_config
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -73,6 +74,7 @@ def load_dspark_model(target_model: nn.Module, vllm_config: VllmConfig) -> nn.Mo
|
||||
if speculative_config.kv_cache_dtype is not None
|
||||
else vllm_config.cache_config
|
||||
),
|
||||
load_config=get_pp_safe_draft_load_config(vllm_config.load_config),
|
||||
)
|
||||
# VllmConfig post-init restores the target's quant config because the target
|
||||
# config is retained for DSpark's target-layer metadata, so we must override it.
|
||||
|
||||
@@ -8,6 +8,7 @@ from vllm.distributed.parallel_state import get_pp_group
|
||||
from vllm.lora.layers.base import BaseLayerWithLoRA
|
||||
from vllm.model_executor.model_loader import get_model
|
||||
from vllm.model_executor.models.utils import PPMissingLayer
|
||||
from vllm.v1.worker.gpu.spec_decode.utils import get_pp_safe_draft_load_config
|
||||
|
||||
|
||||
def _should_share(eagle: nn.Module, flag: str, draft, target) -> bool:
|
||||
@@ -103,6 +104,9 @@ def load_eagle_model(target_model: nn.Module, vllm_config: VllmConfig) -> nn.Mod
|
||||
backend=speculative_config.attention_backend,
|
||||
),
|
||||
)
|
||||
draft_load_config = get_pp_safe_draft_load_config(vllm_config.load_config)
|
||||
if draft_load_config is not vllm_config.load_config:
|
||||
vllm_config = replace(vllm_config, load_config=draft_load_config)
|
||||
with set_model_tag("eagle_head"):
|
||||
eagle_model = get_model(
|
||||
vllm_config=vllm_config, model_config=draft_model_config
|
||||
|
||||
@@ -3,10 +3,28 @@
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
from vllm.config import LoadConfig, replace
|
||||
from vllm.distributed import get_pp_group
|
||||
from vllm.logger import init_logger
|
||||
from vllm.v1.outputs import DraftTokenIds
|
||||
from vllm.v1.worker.gpu.async_utils import async_copy_to_np
|
||||
from vllm.v1.worker.gpu.input_batch import InputBatch
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def get_pp_safe_draft_load_config(load_config: LoadConfig) -> LoadConfig:
|
||||
"""Avoid collectives that include PP ranks without a draft model."""
|
||||
if get_pp_group().world_size > 1 and load_config.load_format == "fastsafetensors":
|
||||
logger.warning_once(
|
||||
"fastsafetensors cannot load a draft model instantiated on only "
|
||||
"one pipeline stage; falling back to the standard safetensors "
|
||||
"loader for the draft model. The target model still uses "
|
||||
"fastsafetensors."
|
||||
)
|
||||
return replace(load_config, load_format="auto")
|
||||
return load_config
|
||||
|
||||
|
||||
class DraftTokensHandler:
|
||||
def __init__(self, device: torch.device | None = None):
|
||||
|
||||
Reference in New Issue
Block a user