[Mini-vLLM] Cut dead subsystems: models, entrypoints, backends, utils, tests

- Drop dead model modules, keeping files live imports require
- Remove anthropic, cohere, codegen entrypoints and batch STT modes
- Delete unused attention backends, MoE experts, and NPU/XPU util leaves
- Strip dead third_party trees and ROCm leaves with no CUDA path
- Remove mooncake EC connectors: dependency missing since branch base
- Re-audit verified by full import sweep and Qwen3.5 smoke test

Co-authored-by: Hinata <hinata@idihore.id>
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-10-19 00:20:48 +07:00
co-authored by Hinata
parent 6a95f62e2f
commit 81074e94a3
940 changed files with 11 additions and 380533 deletions
@@ -1,28 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
from vllm import SamplingParams
from vllm.platforms import current_platform
test_model = "openai-community/gpt2"
prompts = [
"Hello, my name is",
"The president of the United States is",
"The capital of France is",
"The future of AI is",
]
# Create a sampling params object.
sampling_params = SamplingParams(temperature=0.8, top_p=0.95, seed=0)
@pytest.mark.skipif(
not current_platform.is_cuda_alike(),
reason="fastsafetensors requires NVIDIA/AMD GPUs",
)
def test_model_loader_download_files(vllm_runner):
with vllm_runner(test_model, load_format="fastsafetensors") as llm:
deserialized_outputs = llm.generate(prompts, sampling_params)
assert deserialized_outputs
@@ -1,49 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import glob
import tempfile
import huggingface_hub.constants
import pytest
import torch
from vllm.model_executor.model_loader.weight_utils import (
download_weights_from_hf,
fastsafetensors_weights_iterator,
safetensors_weights_iterator,
)
from vllm.platforms import current_platform
@pytest.mark.skipif(
not current_platform.is_cuda_alike(),
reason="fastsafetensors requires NVIDIA/AMD GPUs",
)
@pytest.mark.parametrize("queue_size", [0, 1])
def test_fastsafetensors_model_loader(monkeypatch, queue_size):
monkeypatch.setenv("VLLM_FASTSAFETENSORS_QUEUE_SIZE", str(queue_size))
with tempfile.TemporaryDirectory() as tmpdir:
huggingface_hub.constants.HF_HUB_OFFLINE = False
download_weights_from_hf(
"openai-community/gpt2", allow_patterns=["*.safetensors"], cache_dir=tmpdir
)
safetensors = glob.glob(f"{tmpdir}/**/*.safetensors", recursive=True)
assert len(safetensors) > 0
fastsafetensors_tensors = {}
hf_safetensors_tensors = {}
for name, tensor in fastsafetensors_weights_iterator(safetensors, True):
fastsafetensors_tensors[name] = tensor
for name, tensor in safetensors_weights_iterator(safetensors, True):
hf_safetensors_tensors[name] = tensor
assert len(fastsafetensors_tensors) == len(hf_safetensors_tensors)
for name, fastsafetensors_tensor in fastsafetensors_tensors.items():
fastsafetensors_tensor = fastsafetensors_tensor.to("cpu")
assert fastsafetensors_tensor.dtype == hf_safetensors_tensors[name].dtype
assert fastsafetensors_tensor.shape == hf_safetensors_tensors[name].shape
assert torch.all(fastsafetensors_tensor.eq(hf_safetensors_tensors[name]))
@@ -1,28 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
from vllm import SamplingParams
from vllm.platforms import current_platform
test_model = "openai-community/gpt2"
prompts = [
"Hello, my name is",
"The president of the United States is",
"The capital of France is",
"The future of AI is",
]
# Create a sampling params object.
sampling_params = SamplingParams(temperature=0.8, top_p=0.95, seed=0)
@pytest.mark.skipif(
not current_platform.is_cuda(),
reason="InstantTensor requires NVIDIA GPUs",
)
def test_model_loader_download_files(vllm_runner):
with vllm_runner(test_model, load_format="instanttensor") as llm:
deserialized_outputs = llm.generate(prompts, sampling_params)
assert deserialized_outputs
@@ -1,104 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import glob
import sys
from types import SimpleNamespace
from unittest.mock import MagicMock
import pytest
import torch
from safetensors.torch import save_file
from vllm.model_executor.model_loader.weight_utils import (
download_weights_from_hf,
instanttensor_weights_iterator,
safetensors_weights_iterator,
)
from vllm.platforms import current_platform
@pytest.mark.parametrize(
("weight_name", "local_ids", "collective"),
[
("mlp.experts.gate_proj.weight", {0}, True),
("mlp.experts.0.gate_proj.weight", {0}, False),
("mlp.experts.0.gate_proj.weight", set(), False),
],
)
def test_instanttensor_ep_collective_loading(
monkeypatch, tmp_path, weight_name, local_ids, collective
):
"""Keep fused checkpoints collective and expert decisions rank-independent."""
from vllm.model_executor.model_loader import weight_utils
path = tmp_path / "model.safetensors"
save_file({weight_name: torch.ones(1)}, str(path))
native_open = MagicMock()
reader = native_open.return_value.__enter__.return_value
reader.total_tensor_size = 0
reader.tensors.return_value = ()
monkeypatch.setitem(
sys.modules, "instanttensor", SimpleNamespace(safe_open=native_open)
)
group = SimpleNamespace(world_size=2, device_group=object())
monkeypatch.setattr(weight_utils, "get_world_group", lambda: group)
monkeypatch.setattr(current_platform, "is_cuda", lambda: True)
monkeypatch.setattr(current_platform, "current_device", lambda: 0)
list(instanttensor_weights_iterator([str(path)], False, local_expert_ids=local_ids))
kwargs = native_open.call_args.kwargs
assert kwargs["process_group"] is (group.device_group if collective else None)
@pytest.mark.skipif(
not current_platform.is_cuda(),
reason="InstantTensor requires NVIDIA GPUs",
)
def test_instanttensor_ep_filter_cuda(tmp_path):
weights = {
"mlp.experts.0.gate_proj.weight": torch.ones(2),
"mlp.experts.1.gate_proj.weight": torch.ones(2),
"mlp.experts.1.gate_proj.weight_scale": torch.ones(1),
"model.layers[0].attn.weight": torch.ones(2),
}
path = tmp_path / "model.safetensors"
save_file(weights, str(path))
loaded = dict(
instanttensor_weights_iterator([str(path)], False, local_expert_ids={0})
)
del weights["mlp.experts.1.gate_proj.weight"]
assert loaded.keys() == weights.keys()
for name, tensor in loaded.items():
torch.testing.assert_close(tensor.cpu(), weights[name], rtol=0, atol=0)
@pytest.mark.skipif(
not current_platform.is_cuda(),
reason="InstantTensor requires NVIDIA GPUs",
)
def test_instanttensor_model_loader():
model_dir = download_weights_from_hf(
"openai-community/gpt2", cache_dir=None, allow_patterns=["*.safetensors"]
)
safetensors = glob.glob(f"{model_dir}/*.safetensors")
assert len(safetensors) > 0
instanttensor_tensors = {}
hf_safetensors_tensors = {}
for name, tensor in instanttensor_weights_iterator(safetensors, True):
instanttensor_tensors[name] = tensor.to("cpu")
for name, tensor in safetensors_weights_iterator(safetensors, True):
hf_safetensors_tensors[name] = tensor
assert len(instanttensor_tensors) == len(hf_safetensors_tensors)
for name, instanttensor_tensor in instanttensor_tensors.items():
assert instanttensor_tensor.dtype == hf_safetensors_tensors[name].dtype
assert instanttensor_tensor.shape == hf_safetensors_tensors[name].shape
assert torch.all(instanttensor_tensor.eq(hf_safetensors_tensors[name]))
if __name__ == "__main__":
test_instanttensor_model_loader()
@@ -1,721 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import json
import os
import types
from datetime import timedelta
from pathlib import Path
from unittest.mock import patch
import pytest
import torch
from safetensors.torch import load_file, save_file
from tests.utils import create_new_process_for_each_test
from vllm import SamplingParams
from vllm.config.load import LoadConfig
from vllm.model_executor.model_loader import get_model_loader
from vllm.model_executor.model_loader import runai_streamer_loader as rsl
from vllm.utils.mem_constants import GiB_bytes
load_format = "runai_streamer"
test_model = "openai-community/gpt2"
# TODO(amacaskill): Replace with a GKE owned GCS bucket.
test_gcs_model = "gs://vertex-model-garden-public-us/codegemma/codegemma-2b/"
prompts = [
"Hello, my name is",
"The president of the United States is",
"The capital of France is",
"The future of AI is",
]
# Create a sampling params object.
sampling_params = SamplingParams(temperature=0.8, top_p=0.95, seed=0)
def get_runai_model_loader():
load_config = LoadConfig(load_format=load_format)
return get_model_loader(load_config)
def test_get_model_loader_with_runai_flag():
model_loader = get_runai_model_loader()
assert model_loader.__class__.__name__ == "RunaiModelStreamerLoader"
def test_checkpoint_selection_hook_is_optional_for_model_detection():
from vllm.model_executor.models.interfaces import SupportsMultiModal
from vllm.model_executor.models.interfaces_base import (
VllmModel,
is_text_generation_model,
)
class ModelWithoutHook(torch.nn.Module):
def __init__(self, vllm_config, prefix=""):
super().__init__()
def embed_input_ids(self, input_ids):
return input_ids
def forward(self, input_ids, positions):
return input_ids
def compute_logits(self, hidden_states):
return hidden_states
assert is_text_generation_model(ModelWithoutHook)
assert is_text_generation_model(ModelWithoutHook(None))
class ModelWithDefaultRule(ModelWithoutHook, VllmModel):
pass
assert not ModelWithDefaultRule(None).is_unused_checkpoint_weight("any.weight")
# Structural models do not inherit the protocol's optional default hook.
for names in ([], ["language_model"]):
model = torch.nn.Module()
model.language_model = ModelWithoutHook(None)
model._language_model_names = names
assert SupportsMultiModal.get_language_model(model) is model.language_model
def test_runai_model_loader_download_files(vllm_runner):
with vllm_runner(test_model, load_format=load_format) as llm:
deserialized_outputs = llm.generate(prompts, sampling_params)
assert deserialized_outputs
# In-process MTP compilation must not retain GPU resources in the pytest worker.
@create_new_process_for_each_test("spawn")
def test_runai_mtp_real_checkpoint(vllm_runner, tmp_path, monkeypatch):
"""Skip main-only shards without changing a real MTP model's generation."""
from vllm.model_executor.models.qwen3_5_mtp import Qwen3_5MTP
from vllm.platforms import current_platform
from vllm.transformers_utils.repo_utils import hf_api
from vllm.v1.metrics.reader import Counter
if not current_platform.is_cuda_alike():
pytest.skip("MTP serving requires CUDA or ROCm")
checkpoint = tmp_path / "checkpoint"
hf_api().snapshot_download(
repo_id="Qwen/Qwen3.5-0.8B-Base", local_dir=str(checkpoint)
)
original_files = list(checkpoint.glob("*.safetensors"))
tensors = {}
for path in original_files:
tensors.update(load_file(path))
# Reshard real, unmodified weights so selection cannot pass vacuously on
# the published single mixed file. This predicate needs no model state.
selector = Qwen3_5MTP.__new__(Qwen3_5MTP)
main: dict[str, torch.Tensor] = {}
draft: dict[str, torch.Tensor] = {}
for name, tensor in tensors.items():
(main if selector.is_unused_checkpoint_weight(name) else draft)[name] = tensor
assert main and draft
files = [checkpoint / "main.safetensors", checkpoint / "draft.safetensors"]
for path, contents in zip(files, (main, draft)):
save_file(contents, path, metadata={"format": "pt"})
for path in original_files:
path.unlink()
(checkpoint / "model.safetensors.index.json").write_text(
json.dumps(
{
"weight_map": {
name: path.name
for path, data in zip(files, (main, draft))
for name in data
}
}
)
)
del tensors, main, draft
monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0")
opened = []
real_stream = rsl.runai_safetensors_weights_iterator
def stream(paths, *args):
opened.append(list(paths))
yield from real_stream(paths, *args)
monkeypatch.setattr(rsl, "runai_safetensors_weights_iterator", stream)
results = []
for load_format in ("auto", "runai_streamer"):
with vllm_runner(
str(checkpoint),
load_format=load_format,
block_size=None,
enable_chunked_prefill=True,
disable_log_stats=False,
limit_mm_per_prompt={"image": 0, "video": 0},
speculative_config={"method": "mtp", "num_speculative_tokens": 1},
) as runner:
outputs = runner.generate_greedy(prompts, max_tokens=32)
results.append(outputs)
accepted = next(
m.value
for m in runner.llm.get_metrics()
if isinstance(m, Counter)
and m.name == "vllm:spec_decode_num_accepted_tokens"
)
assert accepted > 0
assert len(opened) == 2
assert set(opened[0]) == {str(path) for path in files}
assert opened[1] == [str(files[1])]
matches = sum(a == b for a, b in zip(*results))
assert matches >= 3, "RunAI MTP generation diverged from the Default loader"
@pytest.mark.skip(
reason="Temporarily disabled due to GCS access issues. "
"TODO: Re-enable this test once the underlying issue is resolved."
)
def test_runai_model_loader_download_files_gcs(
vllm_runner, monkeypatch: pytest.MonkeyPatch
):
monkeypatch.setenv("GOOGLE_CLOUD_PROJECT", "fake-project")
monkeypatch.setenv("RUNAI_STREAMER_GCS_USE_ANONYMOUS_CREDENTIALS", "true")
monkeypatch.setenv(
"CLOUD_STORAGE_EMULATOR_ENDPOINT", "https://storage.googleapis.com"
)
with vllm_runner(test_gcs_model, load_format=load_format) as llm:
deserialized_outputs = llm.generate(prompts, sampling_params)
assert deserialized_outputs
def test_runai_passes_revision_by_name():
# revision must reach download_safetensors_index_file_from_hf as the
# ``revision`` keyword, not the positional ``subfolder`` slot.
fake_self = types.SimpleNamespace(
load_config=types.SimpleNamespace(download_dir="/cache", ignore_patterns=[])
)
with (
patch.object(rsl, "is_runai_obj_uri", return_value=False),
patch.object(rsl, "download_weights_from_hf", return_value="/folder"),
patch.object(
rsl, "list_safetensors", return_value=["/folder/model.safetensors"]
),
patch.object(rsl, "download_safetensors_index_file_from_hf") as mock_idx,
):
rsl.RunaiModelStreamerLoader._prepare_weights(fake_self, "org/model", "myrev")
mock_idx.assert_called_once()
assert mock_idx.call_args.kwargs.get("revision") == "myrev"
assert "myrev" not in mock_idx.call_args.args
def _runai_loader(extra):
return rsl.RunaiModelStreamerLoader(
LoadConfig(load_format="runai_streamer", model_loader_extra_config=extra)
)
@pytest.mark.parametrize(
"extra, match",
[
({"typo_key": 1}, "Unexpected extra config"),
({"distributed": "yes"}, "distributed must be a bool"),
({"concurrency": "16"}, "concurrency must be a positive integer"),
({"concurrency": -1}, "concurrency must be a positive integer"),
],
)
def test_runai_rejects_invalid_extra_config(extra, match):
# The loader used to silently drop unknown keys / wrong types / negatives.
with pytest.raises(ValueError, match=match):
_runai_loader(extra)
def test_runai_accepts_valid_extra_config():
with patch.dict(os.environ, {}, clear=False):
os.environ.pop("RUNAI_STREAMER_CONCURRENCY", None)
os.environ.pop("RUNAI_STREAMER_MEMORY_LIMIT", None)
loader = _runai_loader(
{"distributed": True, "concurrency": 16, "memory_limit": 1024}
)
assert loader._is_distributed is True
assert os.environ["RUNAI_STREAMER_CONCURRENCY"] == "16"
assert os.environ["RUNAI_STREAMER_MEMORY_LIMIT"] == "1024"
def test_runai_invalid_extra_config_leaves_environ_untouched():
# A later invalid key must not leave an earlier valid key applied to
# os.environ (all values are validated before any global mutation).
with patch.dict(os.environ, {}, clear=False):
os.environ.pop("RUNAI_STREAMER_CONCURRENCY", None)
with pytest.raises(ValueError, match="memory_limit must be an integer >= -1"):
_runai_loader({"concurrency": 16, "memory_limit": -5})
assert "RUNAI_STREAMER_CONCURRENCY" not in os.environ
@pytest.mark.parametrize(
"model_config,expected_source",
[
# Object storage: model_weights holds the URI, model the pulled config dir.
(
types.SimpleNamespace(
model="/tmp/pulled-config-files",
model_weights="s3://bucket/weights",
revision="myrev",
),
"s3://bucket/weights",
),
# HF repo or local path: model_weights is empty, model is the source.
(
types.SimpleNamespace(
model="org/model", model_weights="", revision="myrev"
),
"org/model",
),
],
ids=["object_storage", "hf_repo"],
)
def test_runai_get_all_weights_resolves_source(model_config, expected_source):
fake_self = types.SimpleNamespace(
_get_weights_iterator=lambda path, revision, is_unused_weight=None: iter(
[(f"{path}@{revision}", None)]
)
)
weights = list(
rsl.RunaiModelStreamerLoader.get_all_weights(
fake_self, model_config, model=None
)
)
assert weights == [(f"{expected_source}@myrev", None)]
@pytest.mark.parametrize("selection", ["draft", "unadapted", "reject_all"])
def test_runai_selects_shards_before_streaming(tmp_path, monkeypatch, selection):
"""Unused shards must not reach RunAI; mixed shards retain their tensors."""
contents = [
{"main.weight": torch.tensor([1.0])},
{"main.bias": torch.tensor([2.0]), "draft.weight": torch.tensor([3.0])},
{"draft.bias": torch.tensor([4.0])},
]
files = []
for i, tensors in enumerate(contents):
path = str(tmp_path / f"part-{i}.safetensors")
save_file(tensors, path)
files.append(path)
monkeypatch.setattr(rsl, "list_safetensors", lambda path: files)
opened = []
real_stream = rsl.runai_safetensors_weights_iterator
def stream(paths, *args):
opened.extend(paths)
yield from real_stream(paths, *args)
monkeypatch.setattr(rsl, "runai_safetensors_weights_iterator", stream)
model = torch.nn.Module()
if selection == "draft":
model.is_unused_checkpoint_weight = lambda name: not name.startswith("draft.")
elif selection == "reject_all":
model.is_unused_checkpoint_weight = lambda name: True
config = types.SimpleNamespace(model=str(tmp_path), model_weights="", revision=None)
loaded = dict(_runai_loader({}).get_all_weights(config, model))
expected = files[1:] if selection == "draft" else files
assert opened == expected
assert set(loaded) == {
name
for tensors in (contents[1:] if selection == "draft" else contents)
for name in tensors
}
for name, tensor in {**contents[1], **contents[2]}.items():
torch.testing.assert_close(loaded[name], tensor)
def _check_distributed_selection(rank, rendezvous, scenario):
torch.distributed.init_process_group(
"gloo",
init_method=rendezvous,
rank=rank,
world_size=2,
timeout=timedelta(seconds=30),
)
root = "s3://bucket/weights"
if scenario == "different_roots":
root += str(rank)
files = [root + "/main.safetensors", root + "/draft.safetensors"]
def pull_files(source, destination, **kwargs):
if scenario == "metadata_failure" and rank == 1:
raise OSError("Index temporarily unavailable")
(Path(destination) / "model.safetensors.index.json").write_text(
json.dumps(
{
"weight_map": {
"main.weight": "main.safetensors",
"draft.weight": "draft.safetensors",
}
}
)
)
try:
with pytest.MonkeyPatch.context() as monkeypatch:
monkeypatch.setattr(rsl, "list_safetensors", lambda path: files)
monkeypatch.setattr(rsl, "runai_pull_files", pull_files)
monkeypatch.setattr(
rsl,
"_get_remote_checkpoint_keys",
lambda paths: [
["main.weight"]
if path.endswith("main.safetensors")
else ["draft.weight"]
for path in paths
],
)
monkeypatch.setattr(
rsl,
"get_world_group",
lambda: types.SimpleNamespace(
world_size=2, cpu_group=torch.distributed.group.WORLD
),
raising=False,
)
monkeypatch.setattr(
rsl,
"get_pp_group",
lambda: types.SimpleNamespace(
world_size=2 if scenario == "pipeline_draft" else 1
),
raising=False,
)
if scenario == "pipeline_draft" and rank == 0:
# Earlier PP stages never invoke the draft loader.
torch.distributed.barrier()
return
# Exercise real selection and Gloo; observe the distributed stream input.
monkeypatch.setattr(
rsl,
"runai_safetensors_weights_iterator",
lambda paths, *args: iter((path, None) for path in paths),
)
model = torch.nn.Module()
if not (scenario == "missing_hook" and rank == 1):
needed = (
"main." if scenario == "different_roles" and rank == 1 else "draft."
)
model.is_unused_checkpoint_weight = lambda name: (
not name.startswith(needed)
)
config = types.SimpleNamespace(model=root, model_weights="", revision=None)
selected = [
path
for path, _ in _runai_loader({"distributed": True}).get_all_weights(
config, model
)
]
expected = files[1:] if scenario == "healthy" else files
assert selected == expected
if scenario == "pipeline_draft":
torch.distributed.barrier()
finally:
torch.distributed.destroy_process_group()
@pytest.mark.parametrize(
"scenario",
[
"healthy",
"metadata_failure",
"different_roles",
"missing_hook",
"different_roots",
"pipeline_draft",
],
)
def test_distributed_runai_preserves_every_ranks_required_shards(tmp_path, scenario):
"""All ranks must stream the same union even if one cannot filter safely."""
torch.multiprocessing.spawn(
_check_distributed_selection,
args=((tmp_path / "rendezvous").as_uri(), scenario),
nprocs=2,
)
@pytest.mark.parametrize("scheme", ["s3", "gs", "az"])
@pytest.mark.parametrize(
"index_kind",
[
"valid",
"missing",
"malformed",
"unavailable",
"unadapted",
"missing_shard",
"stale",
"header_unavailable",
],
)
def test_runai_remote_selection_uses_actual_weight_source(
tmp_path, monkeypatch, scheme, index_kind
):
root = f"{scheme}://bucket/weights"
files = [
root + "/main/part.safetensors",
root + "/draft/part.safetensors",
root + "/mixed.safetensors",
root + "/extra.safetensors",
]
monkeypatch.setattr(rsl, "list_safetensors", lambda path: files)
pulled = []
def pull_files(source, destination, allow_pattern, ignore_pattern=None):
pulled.append((source.rstrip("/"), allow_pattern))
if index_kind == "unavailable":
# Storage SDKs do not necessarily derive errors from OSError.
raise Exception("Object metadata unavailable")
index = Path(destination) / "model.safetensors.index.json"
if index_kind in ("valid", "missing_shard", "stale", "header_unavailable"):
index.write_text(
json.dumps(
{
"weight_map": {
"main.weight": "main/part.safetensors",
"draft.weight": (
"absent.safetensors"
if index_kind == "missing_shard"
else "draft/part.safetensors"
),
"main.bias": "mixed.safetensors",
"draft.bias": "mixed.safetensors",
}
}
)
)
elif index_kind == "malformed":
index.write_text("not JSON")
monkeypatch.setattr(rsl, "runai_pull_files", pull_files, raising=False)
headers_checked = []
def checkpoint_keys(paths):
headers_checked.extend(paths)
if index_kind == "header_unavailable":
raise OSError("Header temporarily unavailable")
return [
["main.weight", "draft.unindexed_weight"]
if index_kind == "stale"
else ["main.weight"]
for _ in paths
]
monkeypatch.setattr(
rsl, "_get_remote_checkpoint_keys", checkpoint_keys, raising=False
)
opened = []
def stream(paths, *args):
opened.extend(paths)
return iter(())
monkeypatch.setattr(rsl, "runai_safetensors_weights_iterator", stream)
model = torch.nn.Module()
if index_kind != "unadapted":
model.is_unused_checkpoint_weight = lambda name: not name.startswith("draft.")
# The config directory is not the checkpoint: using its index could drop
# tensors from an independent model_weights source.
(tmp_path / "model.safetensors.index.json").write_text("not the weights index")
config = types.SimpleNamespace(
model=str(tmp_path), model_weights=root, revision=None
)
list(_runai_loader({}).get_all_weights(config, model))
assert opened == (files[1:] if index_kind == "valid" else files)
assert headers_checked == (
files[:1] if index_kind in ("valid", "stale", "header_unavailable") else []
)
assert pulled == (
[]
if index_kind == "unadapted"
else [
(
root,
[
"weights/model.safetensors.index.json",
"model.safetensors.index.json",
],
)
]
)
@pytest.mark.parametrize(
"weight_map",
[None, {}, [], {"draft.weight": None}, {"draft.weight": "../part.safetensors"}],
)
def test_runai_invalid_weight_map_keeps_original_files(monkeypatch, weight_map):
files = ["s3://bucket/model/part.safetensors"]
monkeypatch.setattr(rsl, "list_safetensors", lambda path: files)
def pull_files(source, directory, **kwargs):
(Path(directory) / "model.safetensors.index.json").write_text(
json.dumps({"weight_map": weight_map})
)
monkeypatch.setattr(rsl, "runai_pull_files", pull_files)
monkeypatch.setattr(
rsl, "runai_safetensors_weights_iterator", lambda paths, *args: iter(paths)
)
model = torch.nn.Module()
model.is_unused_checkpoint_weight = lambda name: True
config = types.SimpleNamespace(
model="s3://bucket/model", model_weights="", revision=None
)
assert list(_runai_loader({}).get_all_weights(config, model)) == files
def test_runai_model_rule_error_is_not_hidden_by_metadata_fallback(monkeypatch):
files = ["s3://bucket/model/part.safetensors"]
monkeypatch.setattr(rsl, "list_safetensors", lambda path: files)
def pull_files(source, directory, **kwargs):
(Path(directory) / "model.safetensors.index.json").write_text(
json.dumps({"weight_map": {"draft.weight": "part.safetensors"}})
)
monkeypatch.setattr(rsl, "runai_pull_files", pull_files)
model = torch.nn.Module()
def invalid_rule(name):
raise ValueError("Invalid model rule")
model.is_unused_checkpoint_weight = invalid_rule
config = types.SimpleNamespace(
model="s3://bucket/model", model_weights="", revision=None
)
with pytest.raises(ValueError, match="Invalid model rule"):
list(_runai_loader({}).get_all_weights(config, model))
def test_runai_header_reader_uses_actual_tensor_names(tmp_path):
"""The RunAI header reader sees names even when an index omits them."""
path = tmp_path / "mixed.safetensors"
save_file(
{
"main.weight": torch.tensor([1.0]),
"draft.unindexed_weight": torch.tensor([2.0]),
},
path,
metadata={"format": "pt"},
)
(names,) = rsl._get_remote_checkpoint_keys([str(path)])
assert set(names) == {"main.weight", "draft.unindexed_weight"}
@pytest.mark.parametrize("stale", [False, True])
def test_runai_stale_index_keeps_actual_draft_weights(tmp_path, monkeypatch, stale):
"""Verify excluded headers before dropping a shard with real draft weights."""
root = "s3://bucket/checkpoint"
main_path = tmp_path / "main.safetensors"
draft_path = tmp_path / "draft.safetensors"
main_tensors = {"main.weight": torch.tensor([1.0])}
if stale:
main_tensors["draft.unindexed_weight"] = torch.tensor([2.0])
save_file(main_tensors, main_path)
save_file({"draft.weight": torch.tensor([3.0])}, draft_path)
paths = {
root + "/main.safetensors": str(main_path),
root + "/draft.safetensors": str(draft_path),
}
monkeypatch.setattr(rsl, "list_safetensors", lambda path: list(paths))
def pull_files(source, directory, **kwargs):
(Path(directory) / "model.safetensors.index.json").write_text(
json.dumps(
{
"weight_map": {
"main.weight": "main.safetensors",
"draft.weight": "draft.safetensors",
}
}
)
)
monkeypatch.setattr(rsl, "runai_pull_files", pull_files)
read_headers = rsl._get_remote_checkpoint_keys
monkeypatch.setattr(
rsl,
"_get_remote_checkpoint_keys",
lambda files: read_headers([paths[path] for path in files]),
)
stream = rsl.runai_safetensors_weights_iterator
def stream_local(files, *args):
for name, tensor in stream([paths[path] for path in files], *args):
yield name, tensor.clone()
monkeypatch.setattr(rsl, "runai_safetensors_weights_iterator", stream_local)
model = torch.nn.Module()
model.is_unused_checkpoint_weight = lambda name: not name.startswith("draft.")
config = types.SimpleNamespace(model=root, model_weights="", revision=None)
loaded = dict(_runai_loader({}).get_all_weights(config, model))
assert set(loaded) == (
{"main.weight", "draft.unindexed_weight", "draft.weight"}
if stale
else {"draft.weight"}
)
torch.testing.assert_close(loaded["draft.weight"], torch.tensor([3.0]))
if stale:
torch.testing.assert_close(
loaded["draft.unindexed_weight"], torch.tensor([2.0])
)
@pytest.mark.parametrize(
"base_model,mul_model,add_model",
[
(
"Qwen/Qwen3-0.6B",
"inference-optimization/Qwen3-0.6B-debug-multiply",
"inference-optimization/Qwen3-0.6B-debug-add",
),
],
)
def test_runai_deep_sleep_reload_weights(base_model, mul_model, add_model, vllm_runner):
free, total = torch.accelerator.get_memory_info()
used_bytes_baseline = total - free
def sleep_and_reload(llm, path):
# Level 2 discards the parameter memory, so after wake_up the weights
# can only come from the reload: a tensor the streamer fails to
# deliver changes the output instead of keeping a stale valid value.
llm.get_llm().sleep(level=2)
free, total = torch.accelerator.get_memory_info()
assert total - free - used_bytes_baseline < 3 * GiB_bytes
llm.get_llm().wake_up(tags=["weights"])
llm.collective_rpc("reload_weights", kwargs={"weights_path": path})
free, total = torch.accelerator.get_memory_info()
assert total - free - used_bytes_baseline < 4 * GiB_bytes
llm.get_llm().wake_up(tags=["kv_cache"])
with vllm_runner(
model_name=base_model,
load_format=load_format,
enable_sleep_mode=True,
enable_prefix_caching=False,
max_model_len=16,
max_num_seqs=1,
) as llm:
base_output = llm.generate_greedy(["3 4 ="], max_tokens=4)
sleep_and_reload(llm, mul_model)
mul_perp = llm.generate_prompt_perplexity(["3 4 = 12"], mask=["3 4 ="])[0]
add_perp = llm.generate_prompt_perplexity(["3 4 = 7"], mask=["3 4 ="])[0]
assert mul_perp < add_perp
sleep_and_reload(llm, add_model)
mul_perp = llm.generate_prompt_perplexity(["3 4 = 12"], mask=["3 4 ="])[0]
add_perp = llm.generate_prompt_perplexity(["3 4 = 7"], mask=["3 4 ="])[0]
assert add_perp < mul_perp
# Round trip to the original weights over discarded memory must
# reproduce the pre-sleep output.
sleep_and_reload(llm, base_model)
assert llm.generate_greedy(["3 4 ="], max_tokens=4) == base_output
@@ -1,617 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import asyncio
import gc
import json
import os
import pathlib
import subprocess
import sys
from typing import Any
from unittest.mock import Mock
import pytest
import torch
import vllm.model_executor.model_loader.tensorizer
from tests.utils import VLLM_PATH, RemoteOpenAIServer
from vllm import SamplingParams
from vllm.engine.arg_utils import EngineArgs
from vllm.model_executor.model_loader.tensorizer import (
TensorizerConfig,
TensorSerializer,
is_vllm_tensorized,
open_stream,
tensorize_vllm_model,
)
from vllm.model_executor.model_loader.tensorizer_loader import (
BLACKLISTED_TENSORIZER_ARGS,
)
from vllm.utils.import_utils import PlaceholderModule
from .conftest import DummyExecutor, assert_from_collective_rpc
try:
import tensorizer
from tensorizer import EncryptionParams
except ImportError:
tensorizer = PlaceholderModule("tensorizer")
EncryptionParams = tensorizer.placeholder_attr("EncryptionParams")
class TensorizerCaughtError(Exception):
pass
EXAMPLES_PATH = VLLM_PATH / "examples"
pytest_plugins = ("pytest_asyncio",)
prompts = [
"Hello, my name is",
"The president of the United States is",
"The capital of France is",
"The future of AI is",
]
# Create a sampling params object.
sampling_params = SamplingParams(temperature=0.8, top_p=0.95, seed=0)
def patch_init_and_catch_error(self, obj, method_name, expected_error: type[Exception]):
original = getattr(obj, method_name, None)
if original is None:
raise ValueError("Method '{}' not found.".format(method_name))
def wrapper(*args, **kwargs):
try:
return original(*args, **kwargs)
except expected_error as err:
raise TensorizerCaughtError from err
setattr(obj, method_name, wrapper)
self.load_model()
def assert_specific_tensorizer_error_is_raised(
executor,
obj: Any,
method_name: str,
expected_error: type[Exception],
):
with pytest.raises(TensorizerCaughtError):
executor.collective_rpc(
patch_init_and_catch_error,
args=(
obj,
method_name,
expected_error,
),
)
def is_curl_installed():
try:
subprocess.check_call(["curl", "--version"])
return True
except (subprocess.CalledProcessError, FileNotFoundError):
return False
def write_keyfile(keyfile_path: str):
encryption_params = EncryptionParams.random()
pathlib.Path(keyfile_path).parent.mkdir(parents=True, exist_ok=True)
with open(keyfile_path, "wb") as f:
f.write(encryption_params.key)
@pytest.mark.skipif(not is_curl_installed(), reason="cURL is not installed")
def test_deserialized_encrypted_vllm_model_has_same_outputs(
model_ref, vllm_runner, tmp_path, model_path
):
args = EngineArgs(model=model_ref)
with vllm_runner(model_ref) as vllm_model:
key_path = tmp_path / model_ref / "model.key"
write_keyfile(key_path)
outputs = vllm_model.generate(prompts, sampling_params)
config_for_serializing = TensorizerConfig(
tensorizer_uri=str(model_path), encryption_keyfile=str(key_path)
)
tensorize_vllm_model(args, config_for_serializing)
config_for_deserializing = TensorizerConfig(
tensorizer_uri=str(model_path), encryption_keyfile=str(key_path)
)
with vllm_runner(
model_ref,
load_format="tensorizer",
model_loader_extra_config=config_for_deserializing,
) as loaded_vllm_model: # noqa: E501
deserialized_outputs = loaded_vllm_model.generate(prompts, sampling_params)
# noqa: E501
assert outputs == deserialized_outputs
def test_deserialized_hf_model_has_same_outputs(
hf_runner, vllm_runner, tmp_path, model_ref, model_path
):
with hf_runner(model_ref) as hf_model:
max_tokens = 50
outputs = hf_model.generate_greedy(prompts, max_tokens=max_tokens)
with open_stream(model_path, "wb+") as stream:
serializer = TensorSerializer(stream)
serializer.write_module(hf_model.model)
with vllm_runner(
model_ref,
load_format="tensorizer",
model_loader_extra_config=TensorizerConfig(
tensorizer_uri=str(model_path),
num_readers=1,
),
) as loaded_hf_model:
deserialized_outputs = loaded_hf_model.generate_greedy(
prompts, max_tokens=max_tokens
)
assert outputs == deserialized_outputs
def test_load_without_tensorizer_load_format(vllm_runner, capfd, model_ref):
model = None
try:
model = vllm_runner(
model_ref, model_loader_extra_config=TensorizerConfig(tensorizer_uri="test")
)
pytest.fail("Expected RuntimeError for extra config keys")
except RuntimeError:
out, err = capfd.readouterr()
combined_output = out + err
assert (
"ValueError: Unexpected extra config keys for load format auto"
) in combined_output
finally:
del model
gc.collect()
torch.accelerator.empty_cache()
def test_raise_value_error_on_invalid_load_format(vllm_runner, capfd, model_ref):
model = None
try:
model = vllm_runner(
model_ref,
load_format="safetensors",
model_loader_extra_config=TensorizerConfig(tensorizer_uri="test"),
)
pytest.fail("Expected RuntimeError for extra config keys")
except RuntimeError:
out, err = capfd.readouterr()
combined_output = out + err
assert (
"ValueError: Unexpected extra config keys for load format safetensors"
) in combined_output
finally:
del model
gc.collect()
torch.accelerator.empty_cache()
@pytest.mark.skipif(torch.accelerator.device_count() < 2, reason="Requires 2 GPUs")
def test_tensorizer_with_tp_path_without_template(vllm_runner, capfd):
try:
model_ref = "EleutherAI/pythia-1.4b"
tensorized_path = f"s3://tensorized/{model_ref}/fp16/model.tensors"
vllm_runner(
model_ref,
load_format="tensorizer",
model_loader_extra_config=TensorizerConfig(
tensorizer_uri=tensorized_path,
num_readers=1,
s3_endpoint="object.ord1.coreweave.com",
),
tensor_parallel_size=2,
disable_custom_all_reduce=True,
)
except RuntimeError:
out, err = capfd.readouterr()
combined_output = out + err
assert (
"ValueError: For a sharded model, tensorizer_uri "
"should include a string format template like '%04d' "
"to be formatted with the rank "
"of the shard"
) in combined_output
@pytest.mark.skipif(torch.accelerator.device_count() < 2, reason="Requires 2 GPUs")
def test_deserialized_encrypted_vllm_model_with_tp_has_same_outputs(
vllm_runner, tmp_path
):
model_ref = "EleutherAI/pythia-1.4b"
# record outputs from un-sharded un-tensorized model
with vllm_runner(
model_ref,
disable_custom_all_reduce=True,
enforce_eager=True,
) as base_model:
outputs = base_model.generate(prompts, sampling_params)
# load model with two shards and serialize with encryption
model_path = str(tmp_path / model_ref / "model-%02d.tensors")
key_path = tmp_path / (model_ref + ".key")
tensorizer_config = TensorizerConfig(
tensorizer_uri=model_path,
encryption_keyfile=str(key_path),
)
tensorize_vllm_model(
engine_args=EngineArgs(
model=model_ref,
tensor_parallel_size=2,
disable_custom_all_reduce=True,
enforce_eager=True,
),
tensorizer_config=tensorizer_config,
)
assert os.path.isfile(model_path % 0), "Serialization subprocess failed"
assert os.path.isfile(model_path % 1), "Serialization subprocess failed"
with vllm_runner(
model_ref,
tensor_parallel_size=2,
load_format="tensorizer",
disable_custom_all_reduce=True,
enforce_eager=True,
model_loader_extra_config=tensorizer_config,
) as loaded_vllm_model:
deserialized_outputs = loaded_vllm_model.generate(prompts, sampling_params)
assert outputs == deserialized_outputs
@pytest.mark.flaky(reruns=3)
def test_vllm_tensorized_model_has_same_outputs(
model_ref, vllm_runner, tmp_path, model_path
):
gc.collect()
torch.accelerator.empty_cache()
config = TensorizerConfig(tensorizer_uri=str(model_path))
args = EngineArgs(model=model_ref)
with vllm_runner(model_ref) as vllm_model:
outputs = vllm_model.generate(prompts, sampling_params)
tensorize_vllm_model(args, config)
assert is_vllm_tensorized(config)
with vllm_runner(
model_ref, load_format="tensorizer", model_loader_extra_config=config
) as loaded_vllm_model:
deserialized_outputs = loaded_vllm_model.generate(prompts, sampling_params)
# noqa: E501
assert outputs == deserialized_outputs
def test_load_with_just_model_tensors(just_serialize_model_tensors, model_ref):
# For backwards compatibility, ensure Tensorizer can be still be loaded
# for inference by passing the model reference name, not a local/S3 dir,
# and the location of the model tensors
model_dir = just_serialize_model_tensors
extra_config = {"tensorizer_uri": f"{model_dir}/model.tensors"}
## Start OpenAI API server
args = [
"--load-format",
"tensorizer",
"--model-loader-extra-config",
json.dumps(extra_config),
]
with RemoteOpenAIServer(model_ref, args):
# This test only concerns itself with being able to load the model
# and successfully initialize the server
pass
def test_assert_serialization_kwargs_passed_to_tensor_serializer(tmp_path, vllm_runner):
serialization_params = {
"limit_cpu_concurrency": 2,
}
model_ref = "facebook/opt-125m"
model_path = tmp_path / (model_ref + ".tensors")
config = TensorizerConfig(
tensorizer_uri=str(model_path), serialization_kwargs=serialization_params
)
def serialization_test(self, *args, **kwargs):
# This is performed in the ephemeral worker process, so monkey-patching
# will actually work, and cleanup is guaranteed so don't
# need to reset things
original_dict = serialization_params
to_compare = {}
original = tensorizer.serialization.TensorSerializer.__init__
def tensorizer_serializer_wrapper(self, *args, **kwargs):
nonlocal to_compare
to_compare = kwargs.copy()
return original(self, *args, **kwargs)
tensorizer.serialization.TensorSerializer.__init__ = (
tensorizer_serializer_wrapper
)
tensorizer_config = TensorizerConfig(**kwargs["tensorizer_config"])
self.save_tensorized_model(
tensorizer_config=tensorizer_config,
)
return to_compare | original_dict == to_compare
kwargs = {"tensorizer_config": config.to_serializable()}
with vllm_runner(model_ref) as runner:
assert assert_from_collective_rpc(runner.get_llm(), serialization_test, kwargs)
@pytest.mark.parametrize(
(
"serialization_error",
"renderer_shutdown_error",
"engine_shutdown_error",
"expected_error",
),
[
(None, None, None, None),
(RuntimeError("serialization failed"), None, None, "serialization failed"),
(
RuntimeError("serialization failed"),
ValueError("renderer shutdown failed"),
ValueError("engine shutdown failed"),
"serialization failed",
),
(
None,
ValueError("renderer shutdown failed"),
None,
"renderer shutdown failed",
),
(None, None, ValueError("engine shutdown failed"), "engine shutdown failed"),
],
)
def test_tensorize_vllm_model_shuts_down_engine(
monkeypatch,
serialization_error,
renderer_shutdown_error,
engine_shutdown_error,
expected_error,
):
from vllm.v1.engine.llm_engine import LLMEngine
engine_args = Mock()
config = Mock(encryption_keyfile=None)
engine = Mock()
engine.collective_rpc.side_effect = serialization_error
engine.renderer.shutdown.side_effect = renderer_shutdown_error
engine.engine_core.shutdown.side_effect = engine_shutdown_error
monkeypatch.setattr(LLMEngine, "from_vllm_config", Mock(return_value=engine))
monkeypatch.setenv("VLLM_WORKER_SHUTDOWN_TIMEOUT_SECONDS", "7")
if expected_error is None:
tensorize_vllm_model(engine_args, config)
else:
with pytest.raises(Exception, match=expected_error) as exc_info:
tensorize_vllm_model(engine_args, config)
notes = getattr(exc_info.value, "__notes__", ())
if serialization_error is not None and renderer_shutdown_error is not None:
assert any("renderer shutdown failed" in note for note in notes)
if serialization_error is not None and engine_shutdown_error is not None:
assert any("engine shutdown failed" in note for note in notes)
engine.engine_core.shutdown.assert_called_once_with(timeout=17.0)
engine.renderer.shutdown.assert_called_once_with()
def test_assert_deserialization_kwargs_passed_to_tensor_deserializer(tmp_path, capfd):
deserialization_kwargs = {
"num_readers": "bar", # illegal value
}
serialization_params = {
"limit_cpu_concurrency": 2,
}
model_ref = "facebook/opt-125m"
model_path = tmp_path / (model_ref + ".tensors")
config = TensorizerConfig(
tensorizer_uri=str(model_path), serialization_kwargs=serialization_params
)
args = EngineArgs(model=model_ref)
tensorize_vllm_model(args, config)
loader_tc = TensorizerConfig(
tensorizer_uri=str(model_path),
deserialization_kwargs=deserialization_kwargs,
)
engine_args = EngineArgs(
model="facebook/opt-125m",
load_format="tensorizer",
model_loader_extra_config=loader_tc.to_serializable(),
)
vllm_config = engine_args.create_engine_config()
executor = DummyExecutor(vllm_config)
assert_specific_tensorizer_error_is_raised(
executor,
tensorizer.serialization.TensorDeserializer,
"__init__",
TypeError,
)
def test_assert_stream_kwargs_passed_to_tensor_deserializer(tmp_path, capfd):
deserialization_kwargs = {
"num_readers": 1,
}
serialization_params = {
"limit_cpu_concurrency": 2,
}
model_ref = "facebook/opt-125m"
model_path = tmp_path / (model_ref + ".tensors")
config = TensorizerConfig(
tensorizer_uri=str(model_path), serialization_kwargs=serialization_params
)
args = EngineArgs(model=model_ref)
tensorize_vllm_model(args, config)
stream_kwargs = {"mode": "foo"}
loader_tc = TensorizerConfig(
tensorizer_uri=str(model_path),
deserialization_kwargs=deserialization_kwargs,
stream_kwargs=stream_kwargs,
)
engine_args = EngineArgs(
model="facebook/opt-125m",
load_format="tensorizer",
model_loader_extra_config=loader_tc.to_serializable(),
)
vllm_config = engine_args.create_engine_config()
executor = DummyExecutor(vllm_config)
assert_specific_tensorizer_error_is_raised(
executor,
vllm.model_executor.model_loader.tensorizer,
"open_stream",
ValueError,
)
@pytest.mark.asyncio
async def test_serialize_and_serve_entrypoints(tmp_path):
model_ref = "facebook/opt-125m"
suffix = "test"
try:
result = subprocess.run(
[
sys.executable,
f"{VLLM_PATH}/examples/features/tensorize_vllm_model.py",
"--model",
model_ref,
"serialize",
"--serialized-directory",
str(tmp_path),
"--suffix",
suffix,
"--serialization-kwargs",
'{"limit_cpu_concurrency": 4}',
],
check=True,
capture_output=True,
text=True,
)
except subprocess.CalledProcessError as e:
print("Tensorizing failed.")
print("STDOUT:\n", e.stdout)
print("STDERR:\n", e.stderr)
raise
assert "Successfully serialized" in result.stdout
# Next, try to serve with vllm serve
model_uri = tmp_path / "vllm" / model_ref / suffix / "model.tensors"
model_loader_extra_config = {
"tensorizer_uri": str(model_uri),
"stream_kwargs": {
"force_http": False,
},
"deserialization_kwargs": {
"verify_hash": True,
"num_readers": 8,
},
}
cmd = [
"-m",
"vllm.entrypoints.cli.main",
"serve",
"--host",
"localhost",
"--load-format",
"tensorizer",
model_ref,
"--model-loader-extra-config",
json.dumps(model_loader_extra_config, indent=2),
]
proc = await asyncio.create_subprocess_exec(
sys.executable,
*cmd,
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.STDOUT,
)
assert proc.stdout is not None
fut = proc.stdout.readuntil(b"Application startup complete.")
try:
await asyncio.wait_for(fut, 180)
except TimeoutError:
pytest.fail("Server did not start successfully")
finally:
proc.terminate()
await proc.communicate()
@pytest.mark.parametrize("illegal_value", BLACKLISTED_TENSORIZER_ARGS)
def test_blacklisted_parameter_for_loading(tmp_path, vllm_runner, capfd, illegal_value):
serialization_params = {
"limit_cpu_concurrency": 2,
}
model_ref = "facebook/opt-125m"
model_path = tmp_path / (model_ref + ".tensors")
config = TensorizerConfig(
tensorizer_uri=str(model_path), serialization_kwargs=serialization_params
)
args = EngineArgs(model=model_ref)
tensorize_vllm_model(args, config)
loader_tc = {"tensorizer_uri": str(model_path), illegal_value: "foo"}
try:
vllm_runner(
model_ref,
load_format="tensorizer",
model_loader_extra_config=loader_tc,
)
except RuntimeError:
out, err = capfd.readouterr()
combined_output = out + err
assert (
f"ValueError: {illegal_value} is not an allowed Tensorizer argument."
) in combined_output
@@ -1,603 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import fnmatch
import multiprocessing as mp
import os
import shutil
from tempfile import TemporaryDirectory
from types import SimpleNamespace
import pytest
import torch
from vllm import LLM, SamplingParams
from vllm.model_executor.model_loader import ShardedStateLoader
from vllm.transformers_utils.repo_utils import hf_api
prompts = [
"Hello, my name is",
"The president of the United States is",
"The capital of France is",
"The future of AI is",
]
# Create a sampling params object.
sampling_params = SamplingParams(
temperature=0,
max_tokens=256,
ignore_eos=True,
)
def test_filter_subtensors():
state_dict = {
"a": torch.empty(2),
"b": torch.empty((2, 4)),
"c": torch.empty((2, 4, 8)),
}
state_dict.update(
{
"x": state_dict["b"],
"y": state_dict["c"][1, 2, :],
"z": state_dict["c"][1, :, 4],
}
)
filtered_state_dict = ShardedStateLoader._filter_subtensors(state_dict)
assert tuple(filtered_state_dict.keys()) == ("a", "b", "c")
for key, tensor in filtered_state_dict.items():
# NOTE: don't use `equal` here, as the tensor might contain NaNs
assert tensor is state_dict[key]
def _mock_pp_group(
monkeypatch: pytest.MonkeyPatch,
world_size: int,
*,
pp_rank: int = 0,
tp_rank: int = 0,
tp_world_size: int = 1,
) -> None:
"""Stub the parallel-state accessors used by the sharded state loader."""
monkeypatch.setattr(
"vllm.distributed.get_pp_group",
lambda: SimpleNamespace(world_size=world_size, rank_in_group=pp_rank),
)
monkeypatch.setattr(
"vllm.distributed.get_tp_group",
lambda: SimpleNamespace(world_size=tp_world_size, rank_in_group=tp_rank),
)
monkeypatch.setattr(
"vllm.distributed.get_tensor_model_parallel_rank",
lambda: tp_rank,
)
monkeypatch.setattr(
"vllm.distributed.get_pipeline_model_parallel_rank",
lambda: pp_rank,
)
@pytest.mark.parametrize(
"world_size,expected",
[
(1, ShardedStateLoader.TP_ONLY_PATTERN),
(2, ShardedStateLoader.PP_AND_TP_PATTERN),
],
)
def test_get_default_pattern(monkeypatch, world_size, expected):
"""The default pattern is chosen from the pipeline parallel group size."""
_mock_pp_group(monkeypatch, world_size)
assert ShardedStateLoader._get_default_pattern() == expected
def test_render_default_pattern_default_tp(monkeypatch):
"""Without a custom pattern and PP=1, legacy TP-only names are rendered."""
_mock_pp_group(monkeypatch, 1)
assert (
ShardedStateLoader._render_default_pattern(part=0)
== "model-rank-0-part-0.safetensors"
)
def test_render_default_pattern_default_pp(monkeypatch):
"""Without a custom pattern and PP>1, pp/tp-aware names are rendered."""
_mock_pp_group(monkeypatch, 2)
assert (
ShardedStateLoader._render_default_pattern(part=0)
== "model-pp-0-tp-0-part-0.safetensors"
)
def test_render_default_pattern_custom_tp_only(monkeypatch):
"""A custom TP-only pattern is honored when pipeline parallelism is off."""
_mock_pp_group(monkeypatch, 1)
assert (
ShardedStateLoader._render_default_pattern(
ShardedStateLoader.TP_ONLY_PATTERN, part=0
)
== "model-rank-0-part-0.safetensors"
)
def test_render_default_pattern_tp_only_with_pp_fails(monkeypatch):
"""TP-only filenames would collide across pipeline stages; fail fast."""
_mock_pp_group(monkeypatch, 2)
with pytest.raises(ValueError, match="does not embed the pipeline parallel"):
ShardedStateLoader._render_default_pattern(
ShardedStateLoader.TP_ONLY_PATTERN, part=0
)
def test_render_default_pattern_pp_pattern_without_pp_fails(monkeypatch):
"""A pp/tp-aware pattern cannot be rendered when pipeline parallelism is off."""
_mock_pp_group(monkeypatch, 1)
with pytest.raises(ValueError, match="requires pipeline parallelism"):
ShardedStateLoader._render_default_pattern(
ShardedStateLoader.PP_AND_TP_PATTERN, part=0
)
def test_render_default_pattern_custom_pp(monkeypatch):
"""A custom pp/tp-aware pattern is honored when pipeline parallelism is on."""
_mock_pp_group(monkeypatch, 2)
assert (
ShardedStateLoader._render_default_pattern(
ShardedStateLoader.PP_AND_TP_PATTERN, part=0
)
== "model-pp-0-tp-0-part-0.safetensors"
)
@pytest.mark.parametrize(
"pattern,match",
[
# tp field present but pp field missing: pipeline stages with the
# same tp rank would write the same file.
(
"model-tp-{tp_rank}-part-{part}.safetensors",
"does not embed the pipeline parallel",
),
# pp field present but tp field missing: TP workers would overwrite
# each other's shards.
(
"model-pp-{pp_rank}-part-{part}.safetensors",
"does not embed the tensor parallel",
),
],
)
def test_render_default_pattern_incomplete_custom_patterns_fail(
monkeypatch, pattern, match
):
"""A custom pattern must distinguish every active rank (pp and tp)."""
_mock_pp_group(monkeypatch, 2, pp_rank=1, tp_rank=0, tp_world_size=2)
with pytest.raises(ValueError, match=match):
ShardedStateLoader._render_default_pattern(pattern, part=0)
def test_render_default_pattern_no_rank_field_with_tp_fails(monkeypatch):
"""Without any rank field, TP workers would collide even when PP is off."""
_mock_pp_group(monkeypatch, 1, tp_world_size=2)
with pytest.raises(ValueError, match="does not embed the tensor parallel"):
ShardedStateLoader._render_default_pattern(
"model-part-{part}.safetensors", part=0
)
def test_render_default_pattern_custom_pp_tp1_allows_pp_only(monkeypatch):
"""With TP=1 a pp-only custom pattern is safe and must be accepted."""
_mock_pp_group(monkeypatch, 2, pp_rank=1, tp_world_size=1)
assert (
ShardedStateLoader._render_default_pattern(
"model-pp-{pp_rank}-part-{part}.safetensors", part=0
)
== "model-pp-1-part-0.safetensors"
)
@pytest.mark.parametrize(
"pattern,match",
[
# {{pp_rank}} renders as literal text: both stages would write the
# same file, so it must not count as a pp field.
(
"model-{{pp_rank}}-tp-{tp_rank}-part-{part}.safetensors",
"does not embed the pipeline parallel",
),
# Same for an escaped tp field: TP workers would overwrite each
# other's shards.
(
"model-pp-{pp_rank}-tp-{{tp_rank}}-part-{part}.safetensors",
"does not embed the tensor parallel",
),
],
)
def test_render_default_pattern_escaped_field_not_rank_field(
monkeypatch, pattern, match
):
"""Escaped braces are literal text and cannot pose as rank fields."""
_mock_pp_group(monkeypatch, 2, pp_rank=1, tp_rank=0, tp_world_size=2)
with pytest.raises(ValueError, match=match):
ShardedStateLoader._render_default_pattern(pattern, part=0)
@pytest.mark.parametrize(
"pattern,world_size,tp_world_size,pp_rank,tp_rank,expected",
[
# Mixed tp alias works when pipeline parallelism is on.
(
"model-pp-{pp_rank}-rank-{rank}-part-{part}.safetensors",
2,
2,
1,
0,
"model-pp-1-rank-0-part-0.safetensors",
),
# tp_rank alias works for TP-only checkpoints too.
(
"model-tp-{tp_rank}-part-{part}.safetensors",
1,
2,
0,
3,
"model-tp-3-part-0.safetensors",
),
],
)
def test_render_default_pattern_tp_aliases_accepted(
monkeypatch, pattern, world_size, tp_world_size, pp_rank, tp_rank, expected
):
"""Both tp aliases are supplied at render time, so either works."""
_mock_pp_group(
monkeypatch,
world_size,
pp_rank=pp_rank,
tp_rank=tp_rank,
tp_world_size=tp_world_size,
)
assert ShardedStateLoader._render_default_pattern(pattern, part=0) == expected
def test_render_default_pattern_unsupported_field_fails(monkeypatch):
"""Unknown placeholders are rejected instead of KeyError-ing later."""
_mock_pp_group(monkeypatch, 2, pp_rank=1, tp_world_size=2)
with pytest.raises(ValueError, match="unsupported placeholder"):
ShardedStateLoader._render_default_pattern(
"model-{pp_rank}-{tp_rank}-{epoch}-part-{part}.safetensors", part=0
)
@pytest.mark.parametrize(
"pp_rank,tp_rank,expected",
[
(0, 0, "model-pp-0-tp-0-part-0.safetensors"),
(1, 2, "model-pp-1-tp-2-part-0.safetensors"),
(3, 5, "model-pp-3-tp-5-part-0.safetensors"),
],
)
def test_render_default_pattern_non_zero_ranks(monkeypatch, pp_rank, tp_rank, expected):
"""The rendered filename must embed the actual pp/tp ranks, not zeros."""
_mock_pp_group(monkeypatch, 2, pp_rank=pp_rank, tp_rank=tp_rank)
assert (
ShardedStateLoader._render_default_pattern(
ShardedStateLoader.PP_AND_TP_PATTERN, part=0
)
== expected
)
def test_render_default_pattern_part_wildcard(monkeypatch):
"""The glob wildcard '*' passes through as the part token for file lookup."""
_mock_pp_group(monkeypatch, 1, tp_rank=4)
assert (
ShardedStateLoader._render_default_pattern(part="*")
== "model-rank-4-part-*.safetensors"
)
def test_save_model_pp_aware_filenames(monkeypatch, tmp_path):
"""save_model writes pp/tp-aware filenames for pipeline-parallel models."""
_mock_pp_group(monkeypatch, 2, pp_rank=1, tp_rank=2)
saved: list[str] = []
def _capture(tensors, path, **kwargs):
saved.append(os.path.basename(path))
monkeypatch.setattr("safetensors.torch.save_file", _capture)
model = SimpleNamespace(
state_dict=lambda: {"w": torch.zeros(4, dtype=torch.float32)}
)
ShardedStateLoader.save_model(model, str(tmp_path))
assert saved == ["model-pp-1-tp-2-part-0.safetensors"]
def test_save_model_part_split(monkeypatch, tmp_path):
"""save_model rolls over to a new part when max_size is exceeded."""
_mock_pp_group(monkeypatch, 1, tp_rank=0)
saved: list[str] = []
def _capture(tensors, path, **kwargs):
saved.append(os.path.basename(path))
monkeypatch.setattr("safetensors.torch.save_file", _capture)
# Two float32[4] tensors (16 B each); max_size=20 fits one but not two.
model = SimpleNamespace(
state_dict=lambda: {
"a": torch.zeros(4, dtype=torch.float32),
"b": torch.zeros(4, dtype=torch.float32),
}
)
ShardedStateLoader.save_model(model, str(tmp_path), max_size=20)
assert saved == [
"model-rank-0-part-0.safetensors",
"model-rank-0-part-1.safetensors",
]
def test_save_model_rejects_bad_pattern_before_overwrite(monkeypatch, tmp_path):
"""An unsafe pattern is rejected before any shard file is written."""
_mock_pp_group(monkeypatch, 2, pp_rank=1, tp_rank=0, tp_world_size=2)
# With the escaped pp field this pattern renders both stages to the
# same literal name; the validation must fire before any write.
colliding = tmp_path / "model-{pp_rank}-tp-0-part-0.safetensors"
colliding.write_bytes(b"existing shard")
before = sorted(p.name for p in tmp_path.iterdir())
model = SimpleNamespace(
state_dict=lambda: {"w": torch.zeros(4, dtype=torch.float32)}
)
with pytest.raises(ValueError, match="does not embed the pipeline parallel"):
ShardedStateLoader.save_model(
model,
str(tmp_path),
pattern="model-{{pp_rank}}-tp-{tp_rank}-part-{part}.safetensors",
)
# The pre-existing shard is untouched and nothing new was written.
assert colliding.read_bytes() == b"existing shard"
assert sorted(p.name for p in tmp_path.iterdir()) == before
@pytest.fixture(scope="module")
def llama_3p2_1b_files():
input_dir = hf_api().snapshot_download(
"meta-llama/Llama-3.2-1B-Instruct", ignore_patterns=["*.bin*", "original/*"]
)
yield input_dir
def _run_writer(input_dir, output_dir, weights_patterns, **kwargs):
llm_sharded_writer = LLM(model=input_dir, **kwargs)
# Dump worker states to output directory
llm_sharded_writer.llm_engine.engine_core.save_sharded_state(path=output_dir)
# Copy metadata files to output directory
for file in os.listdir(input_dir):
if os.path.isdir(os.path.join(input_dir, file)):
shutil.copytree(
os.path.join(input_dir, file), os.path.join(output_dir, file)
)
elif not any(fnmatch.fnmatch(file, ext) for ext in weights_patterns):
shutil.copy(os.path.join(input_dir, file), output_dir)
def _run_generate(input_dir, queue: mp.Queue, **kwargs):
llm = LLM(model=input_dir, **kwargs)
gen = llm.generate(prompts, sampling_params)
queue.put([g.outputs[0].__dict__ for g in gen])
queue.close()
queue.join_thread()
@pytest.mark.parametrize("enable_lora", [False, True])
@pytest.mark.parametrize("tp_size", [1, 2])
def test_sharded_state_loader(
enable_lora, tp_size, num_gpus_available, llama_3p2_1b_files
):
if num_gpus_available < tp_size:
pytest.skip(f"Not enough GPUs for tensor parallelism {tp_size}")
weights_patterns = ("*.safetensors",)
gpu_memory_utilization = 0.8
input_dir = llama_3p2_1b_files
ctx = mp.get_context("spawn")
# Keep batching deterministic: this test compares exact greedy outputs
# across separate engine processes, whose schedulers may otherwise form
# different batches depending on prompt-processing timing.
platform_args = {"max_num_seqs": 1}
# Run in separate processes for memory & CUDA isolation
with TemporaryDirectory() as output_dir:
p = ctx.Process(
target=_run_writer,
args=(input_dir, output_dir, weights_patterns),
kwargs=dict(
tensor_parallel_size=tp_size,
gpu_memory_utilization=gpu_memory_utilization,
enforce_eager=True,
**platform_args,
),
)
p.start()
p.join()
queue = ctx.Queue()
p = ctx.Process(
target=_run_generate,
args=(input_dir, queue),
kwargs=dict(
enable_lora=enable_lora,
gpu_memory_utilization=gpu_memory_utilization,
tensor_parallel_size=tp_size,
**platform_args,
),
)
p.start()
# Call queue.get() before p.join() to prevent deadlock:
# If p.join() is called before queue.get() and the queue is full,
# the child process may block while writing to the queue and never
# terminate, causing the parent to wait indefinitely on p.join().
# See: https://github.com/vllm-project/vllm/pull/22371#discussion_r2257773814
out_before = queue.get()
p.join()
queue.close()
queue.join_thread()
queue = ctx.Queue()
p = ctx.Process(
target=_run_generate,
args=(output_dir, queue),
kwargs=dict(
enable_lora=enable_lora,
gpu_memory_utilization=gpu_memory_utilization,
tensor_parallel_size=tp_size,
load_format="sharded_state",
**platform_args,
),
)
p.start()
# Call queue.get() before p.join() to prevent deadlock:
# If p.join() is called before queue.get() and the queue is full,
# the child process may block while writing to the queue and never
# terminate, causing the parent to wait indefinitely on p.join().
# See: https://github.com/vllm-project/vllm/pull/22371#discussion_r2257773814
out_after = queue.get()
p.join()
queue.close()
queue.join_thread()
assert out_before == out_after
def _run_sharded_state_round_trip(
input_dir: str,
*,
tp_size: int,
pp_size: int,
num_gpus_available: int,
) -> None:
"""Save a TP/PP sharded checkpoint, reload it, and compare outputs.
Args:
input_dir: directory containing the original (un-sharded) model.
tp_size: tensor parallel size used for both save and load.
pp_size: pipeline parallel size used for both save and load.
num_gpus_available: number of GPUs on the current machine.
"""
if num_gpus_available < tp_size * pp_size:
pytest.skip(f"Not enough GPUs for TP={tp_size}, PP={pp_size}")
weights_patterns = ("*.safetensors",)
gpu_memory_utilization = 0.8
ctx = mp.get_context("spawn")
# Keep batching deterministic: this test compares exact greedy outputs
# across separate engine processes, whose schedulers may otherwise form
# different batches depending on prompt-processing timing.
platform_args = {"max_num_seqs": 1}
parallel_args = dict(
tensor_parallel_size=tp_size,
pipeline_parallel_size=pp_size,
)
with TemporaryDirectory() as output_dir:
p = ctx.Process(
target=_run_writer,
args=(input_dir, output_dir, weights_patterns),
kwargs=dict(
**parallel_args,
gpu_memory_utilization=gpu_memory_utilization,
enforce_eager=True,
**platform_args,
),
)
p.start()
p.join()
assert p.exitcode == 0, f"writer process failed with exit code {p.exitcode}"
if pp_size > 1:
# PP checkpoints must use the pp/tp-aware naming scheme so that
# different pipeline stages write distinct files.
sharded_files = [
f for f in os.listdir(output_dir) if f.endswith(".safetensors")
]
assert sharded_files, "no sharded checkpoint files were written"
assert any(
fnmatch.fnmatch(f, "model-pp-*-tp-*-part-*.safetensors")
for f in sharded_files
), f"expected PP-aware filenames, got {sharded_files}"
queue = ctx.Queue()
p = ctx.Process(
target=_run_generate,
args=(input_dir, queue),
kwargs=dict(
enable_lora=False,
gpu_memory_utilization=gpu_memory_utilization,
**parallel_args,
**platform_args,
),
)
p.start()
# Call queue.get() before p.join() to prevent deadlock:
# If p.join() is called before queue.get() and the queue is full,
# the child process may block while writing to the queue and never
# terminate, causing the parent to wait indefinitely on p.join().
# See: https://github.com/vllm-project/vllm/pull/22371#discussion_r2257773814
out_before = queue.get()
p.join()
assert p.exitcode == 0, f"generate process failed with exit code {p.exitcode}"
queue.close()
queue.join_thread()
queue = ctx.Queue()
p = ctx.Process(
target=_run_generate,
args=(output_dir, queue),
kwargs=dict(
enable_lora=False,
gpu_memory_utilization=gpu_memory_utilization,
load_format="sharded_state",
**parallel_args,
**platform_args,
),
)
p.start()
out_after = queue.get()
p.join()
assert p.exitcode == 0, f"generate process failed with exit code {p.exitcode}"
queue.close()
queue.join_thread()
assert out_before == out_after
def test_sharded_state_loader_pp(num_gpus_available, llama_3p2_1b_files):
"""Round-trip sharded state save/load with pipeline parallelism (PP=2)."""
_run_sharded_state_round_trip(
llama_3p2_1b_files,
tp_size=1,
pp_size=2,
num_gpus_available=num_gpus_available,
)
def test_sharded_state_loader_pp_tp(num_gpus_available, llama_3p2_1b_files):
"""Round-trip sharded state save/load with pipeline + tensor parallelism
(PP=2, TP=2)."""
_run_sharded_state_round_trip(
llama_3p2_1b_files,
tp_size=2,
pp_size=2,
num_gpus_available=num_gpus_available,
)
@@ -1,486 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""End-to-end tests for the IPC weight cache loader.
A cold start (no weight cache daemon, so the loader falls back to disk) and
warm restarts (weights mapped from the daemon via CUDA IPC) must both serve
identical outputs.
"""
import shutil
import subprocess
import sys
import tempfile
import threading
import time
import urllib.error
import urllib.request
from dataclasses import dataclass, field
from typing import Any
import pytest
import regex as re
from vllm import SamplingParams
from vllm.assets.image import ImageAsset
from vllm.platforms import current_platform
from vllm.utils.network_utils import get_open_port
class WeightCacheDaemon:
"""Context manager running the real weight cache daemon as a subprocess."""
def __init__(
self,
model: str,
tp_size: int,
extra_args: list[str] | None = None,
):
# Short base path: Unix socket paths are limited to ~107 characters.
self.socket_dir = tempfile.mkdtemp(prefix="vllm_ipc_")
self.health_port = get_open_port()
self._cmd = [
sys.executable,
"-m",
"vllm.entrypoints.cli.main",
"preload",
"--model",
model,
"--tensor-parallel-size",
str(tp_size),
"--weight-cache-socket-dir",
self.socket_dir,
"--enforce-eager",
"--weight-cache-health-host",
"127.0.0.1",
"--weight-cache-health-port",
str(self.health_port),
*(extra_args or []),
]
self._proc: subprocess.Popen | None = None
def __enter__(self) -> "WeightCacheDaemon":
self._proc = subprocess.Popen(
self._cmd, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, text=True
)
threading.Thread(target=self._drain_stderr, daemon=True).start()
return self
def __exit__(self, *exc_info) -> None:
self._stop()
def _drain_stderr(self) -> None:
assert self._proc is not None and self._proc.stderr is not None
self._proc.stderr.read()
def check_health(self) -> None:
url = f"http://127.0.0.1:{self.health_port}/health"
try:
with urllib.request.urlopen(url, timeout=5) as response:
assert response.status == 200
except urllib.error.HTTPError as error:
raise AssertionError(
f"Weight cache daemon is not ready: HTTP {error.code}"
) from error
def wait_until_ready(self, timeout_s: float = 900) -> None:
"""Poll until READY (which covers the autotune tuners) or the preload
process dies."""
deadline = time.monotonic() + timeout_s
while True:
try:
self.check_health()
return
except (AssertionError, urllib.error.URLError):
if time.monotonic() > deadline:
raise
if self._proc is not None and self._proc.poll() is not None:
raise AssertionError("vllm preload exited before ready") from None
time.sleep(2)
def _stop(self) -> None:
assert self._proc is not None
self._proc.terminate()
try:
self._proc.wait(timeout=60)
except subprocess.TimeoutExpired:
self._proc.kill()
self._proc.wait()
shutil.rmtree(self.socket_dir, ignore_errors=True)
@dataclass
class ModelCase:
model: str
prompts: list[str]
images: list | None = None
llm_kwargs: dict[str, Any] = field(default_factory=dict)
daemon_args: list[str] = field(default_factory=list)
# Additionally assert the daemons tune in place before serving and write
# the on-disk FlashInfer autotune cache, and the restart engine reuses it
# (needs SM90+/FlashInfer).
check_flashinfer_cache: bool = False
def generate(
vllm_runner,
case: ModelCase,
socket_dir: str | None,
fallback: bool,
):
extra_config = (
{} if socket_dir is None else {"socket_dir": socket_dir, "fallback": fallback}
)
with vllm_runner(
case.model,
load_format="auto" if socket_dir is None else "ipc_cache",
model_loader_extra_config=extra_config,
# Greedy outputs are compared across engine restarts; cap the batch
# size so scheduling cannot change the results.
max_num_seqs=1,
**case.llm_kwargs,
) as llm:
sampling_params = SamplingParams(temperature=0, max_tokens=16, ignore_eos=True)
return llm.generate(case.prompts, sampling_params, images=case.images)
# Both hybrid-recurrent models need chunked prefill for their mamba-style
# cache mode.
QWEN_CASE = ModelCase(
model="Qwen/Qwen3.5-0.8B",
prompts=[
"Hello, my name is",
"The capital of France is",
],
llm_kwargs=dict(
gpu_memory_utilization=0.3,
enforce_eager=True,
enable_chunked_prefill=True,
),
)
# Kimi K3 adds the mxfp4-pack MoE path (post-load kernel setup runs in
# pre-processed mode) and a vision tower, exercised with an image request.
K3_CASE = ModelCase(
model="tiny-random/kimi-k3",
prompts=[
"The capital of France is",
"def quicksort(arr):",
"<|kimi_image_placeholder|>What is shown in the image?",
],
images=[None, None, ImageAsset("stop_sign").pil_image],
llm_kwargs=dict(
gpu_memory_utilization=0.3,
enforce_eager=True,
trust_remote_code=True,
enable_chunked_prefill=True,
# The image placeholder expands to >1k tokens for the stop_sign asset.
max_model_len=4096,
),
daemon_args=["--trust-remote-code"],
# Kimi K3 has tunable FlashInfer ops (a dense model like Qwen2.5 tunes
# nothing and upstream never writes the cache file for it).
check_flashinfer_cache=True,
)
# Qwen3.5-0.8B ships one MTP layer in the target checkpoint, so method="mtp"
# loads the draft from the same model. The daemon must cache it in its draft
# group for the warm runs (fallback=False) to succeed.
QWEN_MTP_CASE = ModelCase(
model="Qwen/Qwen3.5-0.8B",
prompts=[
"Hello, my name is",
"The capital of France is",
],
llm_kwargs=dict(
gpu_memory_utilization=0.3,
enforce_eager=True,
enable_chunked_prefill=True,
speculative_config={"method": "mtp", "num_speculative_tokens": 1},
),
daemon_args=[
"--speculative-config",
'{"method": "mtp", "num_speculative_tokens": 1}',
],
)
@pytest.mark.parametrize(
"case",
[QWEN_CASE, K3_CASE, QWEN_MTP_CASE],
ids=["qwen3.5", "kimi-k3", "qwen3.5-mtp"],
)
def test_ipc_cache_cold_start_and_warm_restart(
vllm_runner, case: ModelCase, tmp_path, monkeypatch, capfd
):
"""Cold start falls back to disk; warm restarts load weights via CUDA IPC.
All runs must produce outputs identical to a default-loader baseline. The
warm runs disable the disk fallback, so they only pass if the weights
really came from the daemon — for the MTP case, both the target's and the
draft's daemon groups.
Cases with ``check_flashinfer_cache`` additionally cover in-daemon
autotune (default-on with `--enable-flashinfer-autotune`): each daemon
tunes before binding its socket and must write the on-disk FlashInfer
autotune cache, and the restart engine must load tactics from it instead
of re-profiling. The daemon's cache file is keyed with the
OPENAI_API_SERVER batch defaults while these engines run under LLM_CLASS,
so they never hit the daemon's file by design; reuse is proven against
the file the earlier in-process engines wrote.
"""
if not current_platform.is_cuda_alike():
pytest.skip("Weight cache IPC sharing requires CUDA or ROCm")
if case is K3_CASE and not current_platform.is_device_capability_family(100):
pytest.skip("Kimi K3 IPC weight cache requires an SM100 MXFP4 backend")
if case.check_flashinfer_cache:
from vllm.utils.flashinfer import has_flashinfer
if not (has_flashinfer() and current_platform.has_device_capability(90)):
pytest.skip("FlashInfer autotune requires FlashInfer and SM90+")
# Isolate the on-disk autotune cache. The engines are in-process
# (envs read lazily); the daemon subprocesses spawn later and
# inherit it.
monkeypatch.setenv("VLLM_FLASHINFER_AUTOTUNE_CACHE_DIR", str(tmp_path))
# Baseline: plain disk loading with the default loader.
baseline_outputs = generate(
vllm_runner,
case,
None,
fallback=True,
)
assert all(text for _, texts in baseline_outputs for text in texts)
# Cold start: no daemon is serving, so the loader falls back to disk.
with tempfile.TemporaryDirectory(prefix="vllm_ipc_empty_") as empty_socket_dir:
cold_outputs = generate(vllm_runner, case, empty_socket_dir, fallback=True)
if case.check_flashinfer_cache:
# The baseline engine already wrote its own (LLM_CLASS-keyed) table.
engine_cache_files = set(tmp_path.rglob("autotune_configs*.json"))
with WeightCacheDaemon(
case.model,
tp_size=1,
extra_args=case.daemon_args,
) as d:
warm_outputs = generate(vllm_runner, case, d.socket_dir, fallback=False)
d.wait_until_ready()
if case.check_flashinfer_cache:
# Each daemon tunes before its ready report; exactly one new file
# (their OPENAI_API_SERVER-keyed table) must appear.
tuner_files = set(tmp_path.rglob("autotune_configs*.json"))
tuner_files -= engine_cache_files
assert len(tuner_files) == 1, (
f"expected the tuner's tuned table, got {tuner_files}"
)
# Warm restart: a second engine lifetime against the same daemon.
if case.check_flashinfer_cache:
capfd.readouterr() # drain: the assertion reads the increment
restart_outputs = generate(vllm_runner, case, d.socket_dir, fallback=False)
if case.check_flashinfer_cache:
# The restart's EngineCore subprocess inherits pytest's fds; the
# flashinfer autotuner logs "[Autotuner]: Loaded N configs from
# <path>" (INFO, not once-deduplicated) on a disk cache hit.
out = capfd.readouterr()
m = re.search(
r"\[Autotuner\]: Loaded (\d+) configs from", out.out + out.err
)
assert m and int(m.group(1)) > 0, (
"restart engine did not load the on-disk autotune cache"
)
assert cold_outputs == baseline_outputs
assert warm_outputs == baseline_outputs
assert restart_outputs == baseline_outputs
def _parallel(**kw):
from types import SimpleNamespace
base = dict(
tensor_parallel_size=1,
pipeline_parallel_size=1,
data_parallel_size=1,
data_parallel_size_local=1,
data_parallel_rank=0,
nnodes=1,
node_rank=0,
)
base.update(kw)
return SimpleNamespace(**base)
def test_daemon_places_tp_and_dp_ranks_on_local_gpus():
"""Each node serves a contiguous global-rank block (DP-major), so without
DP local GPU i is TP rank r*local+i, and single-node DP launchers offset
by --data-parallel-start-rank."""
from vllm.model_executor.model_loader.weight_cache.daemon import plan_local_ranks
tp_second_node = _parallel(tensor_parallel_size=8, nnodes=2, node_rank=1)
assert plan_local_ranks(tp_second_node) == [(i, 0, 0, 4 + i) for i in range(4)]
dp_third_node = _parallel(
data_parallel_size=16, data_parallel_size_local=4, data_parallel_rank=8
)
assert plan_local_ranks(dp_third_node) == [(i, 8 + i, 0, 0) for i in range(4)]
dp_tp = _parallel(
tensor_parallel_size=2,
data_parallel_size=4,
data_parallel_size_local=2,
data_parallel_rank=2,
)
assert plan_local_ranks(dp_tp) == [
(0, 2, 0, 0),
(1, 2, 0, 1),
(2, 3, 0, 0),
(3, 3, 0, 1),
]
# DP + nnodes: node-local TP replicas
dp_tp_node0 = _parallel(tensor_parallel_size=8, data_parallel_size=2, nnodes=2)
assert plan_local_ranks(dp_tp_node0) == [(i, 0, 0, i) for i in range(8)]
dp_tp_node1 = _parallel(
tensor_parallel_size=8, data_parallel_size=2, nnodes=2, node_rank=1
)
assert plan_local_ranks(dp_tp_node1) == [(i, 1, 0, i) for i in range(8)]
# DP + nnodes: TP group spans nodes (TP8 x DP2 on 4 nodes)
tp_span_node1 = _parallel(
tensor_parallel_size=8, data_parallel_size=2, nnodes=4, node_rank=1
)
assert plan_local_ranks(tp_span_node1) == [(i, 0, 0, 4 + i) for i in range(4)]
tp_span_node2 = _parallel(
tensor_parallel_size=8, data_parallel_size=2, nnodes=4, node_rank=2
)
assert plan_local_ranks(tp_span_node2) == [(i, 1, 0, i) for i in range(4)]
# DP + nnodes: several DP replicas per node (TP4 x DP4 on 2 nodes)
dp_multi_node0 = _parallel(tensor_parallel_size=4, data_parallel_size=4, nnodes=2)
assert plan_local_ranks(dp_multi_node0) == [(i, i // 4, 0, i % 4) for i in range(8)]
# PP adds one daemon placement per stage while preserving TP rank order.
pp_tp = _parallel(tensor_parallel_size=2, pipeline_parallel_size=2)
assert plan_local_ranks(pp_tp) == [
(0, 0, 0, 0),
(1, 0, 0, 1),
(2, 0, 1, 0),
(3, 0, 1, 1),
]
def test_daemon_rejects_unmappable_parallelism():
from vllm.model_executor.model_loader.weight_cache.daemon import (
_reject_unsupported_parallelism,
)
_reject_unsupported_parallelism(
_parallel(
data_parallel_size=16, data_parallel_size_local=4, data_parallel_rank=12
)
)
# DP combines with --nnodes when the world size divides evenly.
_reject_unsupported_parallelism(
_parallel(tensor_parallel_size=4, data_parallel_size=4, nnodes=2)
)
_reject_unsupported_parallelism(_parallel(pipeline_parallel_size=2))
with pytest.raises(ValueError, match="evenly divide"):
_reject_unsupported_parallelism(_parallel(tensor_parallel_size=3, nnodes=2))
with pytest.raises(ValueError, match="evenly divide"):
_reject_unsupported_parallelism(
_parallel(tensor_parallel_size=2, data_parallel_size=3, nnodes=4)
)
with pytest.raises(ValueError, match="exceeds"):
_reject_unsupported_parallelism(
_parallel(
data_parallel_size=16, data_parallel_size_local=4, data_parallel_rank=13
)
)
def test_weight_cache_key_distinguishes_dp_ranks():
from dataclasses import replace
from vllm.model_executor.model_loader.weight_cache.protocol import WeightCacheKey
key = WeightCacheKey(
checkpoint="ckpt",
model_arch="Arch",
tp_size=1,
tp_rank=0,
dtype="bf16",
quantization=None,
quant_config_hash="h",
revision=None,
vllm_version="v",
dp_size=16,
dp_rank=3,
pp_size=2,
pp_rank=1,
)
assert key.mismatched_fields(replace(key, dp_rank=4)) == ["dp_rank"]
assert key.mismatched_fields(replace(key, dp_size=8, dp_rank=3)) == ["dp_size"]
assert key.mismatched_fields(replace(key, pp_rank=0)) == ["pp_rank"]
@pytest.mark.parametrize("hf_quant_config", [None, {"quant_algo": "NVFP4"}])
def test_weight_cache_key_distinguishes_nvfp4_activation_override(
tmp_path, hf_quant_config
):
"""Cache keys isolate activation modes and reject ambiguous legacy hashes."""
import json
from dataclasses import replace
from types import SimpleNamespace
import torch
from vllm.config.quantization import QuantizationConfigArgs
from vllm.model_executor.model_loader.weight_cache.protocol import WeightCacheKey
from vllm.utils.hashing import safe_hash
model_config = SimpleNamespace(
model=str(tmp_path),
dtype=torch.bfloat16,
quantization="modelopt_fp4",
quantization_config=None,
revision=None,
hf_config=SimpleNamespace(
architectures=["NemotronHForCausalLM"],
quantization_config=hf_quant_config,
),
)
static_key = WeightCacheKey.from_model_config(model_config, tp_size=1, tp_rank=0)
model_config.quantization_config = QuantizationConfigArgs(
moe={"activation": "nvfp4_per_token"}
)
per_token_key = WeightCacheKey.from_model_config(model_config, tp_size=1, tp_rank=0)
assert static_key.mismatched_fields(per_token_key) == ["quant_config_hash"]
legacy_hash = (
""
if hf_quant_config is None
else safe_hash(
json.dumps(hf_quant_config, sort_keys=True).encode(),
usedforsecurity=False,
).hexdigest()
)
assert static_key.mismatched_fields(
replace(static_key, quant_config_hash=legacy_hash)
) == ["quant_config_hash"]
def test_ipc_loader_copy_mode_reports_no_external_weight_memory():
"""Copy mode clones the weights into the engine, so nothing is external
(vllm_config is never touched)."""
from vllm.config import LoadConfig
from vllm.model_executor.model_loader.weight_cache.ipc_loader import (
IpcModelLoader,
)
copy_mode = LoadConfig(
load_format="ipc_cache", model_loader_extra_config={"mode": "copy"}
)
loader = IpcModelLoader(copy_mode)
assert loader.get_external_weight_memory(None) == 0
@@ -1,93 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from types import SimpleNamespace
import pytest
import torch
import torch.nn as nn
from torch.testing._internal.two_tensor import TwoTensor
from vllm.config import ParallelConfig
from vllm.model_executor import parameter
from vllm.model_executor.model_loader.weight_checksum import (
compute_tensor_digests,
zero_weights,
)
from vllm.v1.worker import gpu_worker
from vllm.v1.worker.gpu_worker import Worker
pytestmark = pytest.mark.cpu_test
class _Model(nn.Module):
def __init__(self):
super().__init__()
self.linear = nn.Linear(2, 2)
nn.init.ones_(self.linear.weight)
nn.init.ones_(self.linear.bias)
# A strided view, which cannot be reinterpreted as bytes in place.
self.strided = nn.Parameter(torch.arange(1.0, 7.0)[::2])
self.scalar = nn.Parameter(torch.tensor(2.0))
# Loading never restores buffers, so reset must leave them alone.
self.register_buffer("k_scale", torch.tensor(2.0))
def test_digests_cover_every_parameter():
assert set(compute_tensor_digests(_Model())) == {
"linear.weight",
"linear.bias",
"strided",
"scalar",
}
def test_zero_weights_changes_every_parameter_and_no_buffer():
model = _Model()
before = compute_tensor_digests(model)
zero_weights(model)
after = compute_tensor_digests(model)
assert all(before[name] != after[name] for name in before)
assert model.k_scale.item() == 2.0
def test_shared_weight_partitions_are_hashed_and_reset(monkeypatch):
monkeypatch.setattr(parameter, "get_tensor_model_parallel_rank", lambda: 0)
monkeypatch.setattr(parameter, "get_tensor_model_parallel_world_size", lambda: 1)
model = nn.Module()
model.transform = parameter.SharedWeightParameter(weight_loader=None)
model.transform.add_partition(0, object(), 2, 2)
model.transform.partitions[0].data.fill_(3.0)
before = compute_tensor_digests(model)
zero_weights(model)
assert set(before) == {"transform.0"}
assert compute_tensor_digests(model) != before
def test_tensor_subclass_inner_tensors_are_hashed_and_reset():
model = nn.Module()
model.weight = nn.Parameter(
TwoTensor(torch.ones(2, 2), torch.full((2, 2), 2.0)), requires_grad=False
)
before = compute_tensor_digests(model)
zero_weights(model)
assert set(before) == {"weight.a", "weight.b"}
assert all(
before[name] != digest for name, digest in compute_tensor_digests(model).items()
)
def test_dense_dp_replicas_get_distinct_key_prefixes(monkeypatch):
for name in ("get_tp_group", "get_pp_group", "get_pcp_group"):
monkeypatch.setattr(gpu_worker, name, lambda: SimpleNamespace(rank_in_group=0))
prefixes = []
for dp_rank in (0, 1):
worker = object.__new__(Worker)
worker.parallel_config = ParallelConfig(
data_parallel_size=2, data_parallel_rank=dp_rank
)
# Dense DP resets data_parallel_rank to 0 in each engine.
worker.parallel_config.reconfigure_for_independent_dp_rank()
prefixes.append(worker._weight_checksum_prefix())
assert prefixes == ["dp0:pp0:pcp0:tp0:", "dp1:pp0:pcp0:tp0:"]