[Bugfix][Spec Decode] Avoid fastsafetensors deadlock for PP draft models (#54416)

Signed-off-by: Canlin <canlinguosdu@gmail.com>
Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
Canlin Guo
2026-09-12 14:22:30 +08:00
committed by GitHub
co-authored by Codex
parent eed1f3d0c6
commit dca96bf97b
6 changed files with 42 additions and 2 deletions
+18
View File
@@ -3,10 +3,28 @@
import numpy as np
import torch
from vllm.config import LoadConfig, replace
from vllm.distributed import get_pp_group
from vllm.logger import init_logger
from vllm.v1.outputs import DraftTokenIds
from vllm.v1.worker.gpu.async_utils import async_copy_to_np
from vllm.v1.worker.gpu.input_batch import InputBatch
logger = init_logger(__name__)
def get_pp_safe_draft_load_config(load_config: LoadConfig) -> LoadConfig:
"""Avoid collectives that include PP ranks without a draft model."""
if get_pp_group().world_size > 1 and load_config.load_format == "fastsafetensors":
logger.warning_once(
"fastsafetensors cannot load a draft model instantiated on only "
"one pipeline stage; falling back to the standard safetensors "
"loader for the draft model. The target model still uses "
"fastsafetensors."
)
return replace(load_config, load_format="auto")
return load_config
class DraftTokensHandler:
def __init__(self, device: torch.device | None = None):