[Fast Start] Support fp4 (#55465)
Signed-off-by: Isotr0py <Isotr0py@outlook.com> Co-authored-by: Isotr0py <Isotr0py@outlook.com>
This commit is contained in:
@@ -7,34 +7,33 @@ warm restarts (weights mapped from the daemon via CUDA IPC) must both serve
|
||||
identical outputs.
|
||||
"""
|
||||
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import threading
|
||||
import time
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm import SamplingParams
|
||||
from vllm.assets.image import ImageAsset
|
||||
from vllm.model_executor.model_loader.weight_cache.protocol import get_socket_path
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
MODEL = "Qwen/Qwen3.5-0.8B"
|
||||
PROMPTS = [
|
||||
"Hello, my name is",
|
||||
"The capital of France is",
|
||||
]
|
||||
DAEMON_TIMEOUT_S = 600
|
||||
|
||||
|
||||
class WeightCacheDaemon:
|
||||
"""Context manager running the real weight cache daemon as a subprocess."""
|
||||
|
||||
_READY_MARKER = "Weight cache daemon READY"
|
||||
|
||||
def __init__(self, model: str, tp_size: int):
|
||||
def __init__(self, model: str, tp_size: int, extra_args: list[str] | None = None):
|
||||
# Short base path: Unix socket paths are limited to ~107 characters.
|
||||
self.socket_dir = tempfile.mkdtemp(prefix="vllm_ipc_")
|
||||
self.tp_size = tp_size
|
||||
self._cmd = [
|
||||
sys.executable,
|
||||
"-m",
|
||||
@@ -46,6 +45,7 @@ class WeightCacheDaemon:
|
||||
"--weight-cache-socket-dir",
|
||||
self.socket_dir,
|
||||
"--enforce-eager",
|
||||
*(extra_args or []),
|
||||
]
|
||||
self._proc: subprocess.Popen | None = None
|
||||
self._lines: list[str] = []
|
||||
@@ -75,6 +75,13 @@ class WeightCacheDaemon:
|
||||
|
||||
def _wait_ready(self, timeout_s: float) -> None:
|
||||
assert self._proc is not None
|
||||
# Each rank binds its socket only once the model is fully cached.
|
||||
# Poll for the socket files rather than a log line: model loading can
|
||||
# pull in JIT compilers that swap the process's stderr and swallow
|
||||
# everything logged afterwards, making log-based readiness flaky.
|
||||
expected = [
|
||||
get_socket_path(gpu_id, self.socket_dir) for gpu_id in range(self.tp_size)
|
||||
]
|
||||
deadline = time.monotonic() + timeout_s
|
||||
while time.monotonic() < deadline:
|
||||
if self._proc.poll() is not None:
|
||||
@@ -82,7 +89,7 @@ class WeightCacheDaemon:
|
||||
f"Weight cache daemon exited with {self._proc.returncode}:\n"
|
||||
f"{self._logs()}"
|
||||
)
|
||||
if self._READY_MARKER in self._logs():
|
||||
if all(os.path.exists(path) for path in expected):
|
||||
return
|
||||
time.sleep(1.0)
|
||||
raise TimeoutError(
|
||||
@@ -100,24 +107,76 @@ class WeightCacheDaemon:
|
||||
shutil.rmtree(self.socket_dir, ignore_errors=True)
|
||||
|
||||
|
||||
def generate(vllm_runner, socket_dir: str | None, fallback: bool, **kwargs):
|
||||
@dataclass
|
||||
class ModelCase:
|
||||
model: str
|
||||
prompts: list[str]
|
||||
images: list | None = None
|
||||
llm_kwargs: dict[str, Any] = field(default_factory=dict)
|
||||
daemon_args: list[str] = field(default_factory=list)
|
||||
|
||||
|
||||
def generate(
|
||||
vllm_runner,
|
||||
case: ModelCase,
|
||||
socket_dir: str | None,
|
||||
fallback: bool,
|
||||
):
|
||||
extra_config = (
|
||||
{} if socket_dir is None else {"socket_dir": socket_dir, "fallback": fallback}
|
||||
)
|
||||
with vllm_runner(
|
||||
MODEL,
|
||||
case.model,
|
||||
load_format="auto" if socket_dir is None else "ipc_cache",
|
||||
model_loader_extra_config=extra_config,
|
||||
# Greedy outputs are compared across engine restarts; cap the batch
|
||||
# size so scheduling cannot change the results.
|
||||
max_num_seqs=1,
|
||||
**kwargs,
|
||||
**case.llm_kwargs,
|
||||
) as llm:
|
||||
sampling_params = SamplingParams(temperature=0, max_tokens=16, ignore_eos=True)
|
||||
return llm.generate(PROMPTS, sampling_params)
|
||||
return llm.generate(case.prompts, sampling_params, images=case.images)
|
||||
|
||||
|
||||
def test_ipc_cache_cold_start_and_warm_restart(vllm_runner):
|
||||
# Both hybrid-recurrent models need chunked prefill for their mamba-style
|
||||
# cache mode.
|
||||
QWEN_CASE = ModelCase(
|
||||
model="Qwen/Qwen3.5-0.8B",
|
||||
prompts=[
|
||||
"Hello, my name is",
|
||||
"The capital of France is",
|
||||
],
|
||||
llm_kwargs=dict(
|
||||
gpu_memory_utilization=0.3,
|
||||
enforce_eager=True,
|
||||
enable_chunked_prefill=True,
|
||||
),
|
||||
)
|
||||
|
||||
# Kimi K3 adds the mxfp4-pack MoE path (post-load kernel setup runs in
|
||||
# pre-processed mode) and a vision tower, exercised with an image request.
|
||||
K3_CASE = ModelCase(
|
||||
model="tiny-random/kimi-k3",
|
||||
prompts=[
|
||||
"The capital of France is",
|
||||
"def quicksort(arr):",
|
||||
"<|kimi_image_placeholder|>What is shown in the image?",
|
||||
],
|
||||
images=[None, None, ImageAsset("stop_sign").pil_image],
|
||||
llm_kwargs=dict(
|
||||
gpu_memory_utilization=0.3,
|
||||
enforce_eager=True,
|
||||
trust_remote_code=True,
|
||||
enable_chunked_prefill=True,
|
||||
# The image placeholder expands to >1k tokens for the stop_sign asset.
|
||||
max_model_len=4096,
|
||||
),
|
||||
daemon_args=["--trust-remote-code"],
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("case", [QWEN_CASE, K3_CASE], ids=["qwen3.5", "kimi-k3"])
|
||||
def test_ipc_cache_cold_start_and_warm_restart(vllm_runner, case: ModelCase):
|
||||
"""Cold start falls back to disk; warm restarts load weights via CUDA IPC.
|
||||
|
||||
All runs must produce outputs identical to a default-loader baseline. The
|
||||
@@ -126,32 +185,26 @@ def test_ipc_cache_cold_start_and_warm_restart(vllm_runner):
|
||||
"""
|
||||
if not current_platform.is_cuda_alike():
|
||||
pytest.skip("Weight cache IPC sharing requires CUDA or ROCm")
|
||||
llm_kwargs = dict(
|
||||
gpu_memory_utilization=0.3,
|
||||
enforce_eager=True,
|
||||
# Qwen3.5 is a hybrid mamba model; its mamba cache mode requires
|
||||
# chunked prefill.
|
||||
enable_chunked_prefill=True,
|
||||
)
|
||||
if case is K3_CASE and not current_platform.is_device_capability_family(100):
|
||||
pytest.skip("Kimi K3 IPC weight cache requires an SM100 MXFP4 backend")
|
||||
|
||||
# Baseline: plain disk loading with the default loader.
|
||||
baseline_outputs = generate(vllm_runner, None, fallback=True, **llm_kwargs)
|
||||
baseline_outputs = generate(
|
||||
vllm_runner,
|
||||
case,
|
||||
None,
|
||||
fallback=True,
|
||||
)
|
||||
assert all(text for _, texts in baseline_outputs for text in texts)
|
||||
|
||||
# Cold start: no daemon is serving, so the loader falls back to disk.
|
||||
with tempfile.TemporaryDirectory(prefix="vllm_ipc_empty_") as empty_socket_dir:
|
||||
cold_outputs = generate(
|
||||
vllm_runner, empty_socket_dir, fallback=True, **llm_kwargs
|
||||
)
|
||||
cold_outputs = generate(vllm_runner, case, empty_socket_dir, fallback=True)
|
||||
|
||||
with WeightCacheDaemon(MODEL, tp_size=1) as daemon:
|
||||
warm_outputs = generate(
|
||||
vllm_runner, daemon.socket_dir, fallback=False, **llm_kwargs
|
||||
)
|
||||
with WeightCacheDaemon(case.model, tp_size=1, extra_args=case.daemon_args) as d:
|
||||
warm_outputs = generate(vllm_runner, case, d.socket_dir, fallback=False)
|
||||
# Warm restart: a second engine lifetime against the same daemon.
|
||||
restart_outputs = generate(
|
||||
vllm_runner, daemon.socket_dir, fallback=False, **llm_kwargs
|
||||
)
|
||||
restart_outputs = generate(vllm_runner, case, d.socket_dir, fallback=False)
|
||||
|
||||
assert cold_outputs == baseline_outputs
|
||||
assert warm_outputs == baseline_outputs
|
||||
|
||||
Reference in New Issue
Block a user