[Mini-vLLM] Cut dead subsystems: models, entrypoints, backends, utils, tests
- Drop dead model modules, keeping files live imports require - Remove anthropic, cohere, codegen entrypoints and batch STT modes - Delete unused attention backends, MoE experts, and NPU/XPU util leaves - Strip dead third_party trees and ROCm leaves with no CUDA path - Remove mooncake EC connectors: dependency missing since branch base - Re-audit verified by full import sweep and Qwen3.5 smoke test Co-authored-by: Hinata <hinata@idihore.id> Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
-28
@@ -1,28 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm import SamplingParams
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
test_model = "openai-community/gpt2"
|
||||
|
||||
prompts = [
|
||||
"Hello, my name is",
|
||||
"The president of the United States is",
|
||||
"The capital of France is",
|
||||
"The future of AI is",
|
||||
]
|
||||
# Create a sampling params object.
|
||||
sampling_params = SamplingParams(temperature=0.8, top_p=0.95, seed=0)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda_alike(),
|
||||
reason="fastsafetensors requires NVIDIA/AMD GPUs",
|
||||
)
|
||||
def test_model_loader_download_files(vllm_runner):
|
||||
with vllm_runner(test_model, load_format="fastsafetensors") as llm:
|
||||
deserialized_outputs = llm.generate(prompts, sampling_params)
|
||||
assert deserialized_outputs
|
||||
@@ -1,49 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import glob
|
||||
import tempfile
|
||||
|
||||
import huggingface_hub.constants
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.model_loader.weight_utils import (
|
||||
download_weights_from_hf,
|
||||
fastsafetensors_weights_iterator,
|
||||
safetensors_weights_iterator,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda_alike(),
|
||||
reason="fastsafetensors requires NVIDIA/AMD GPUs",
|
||||
)
|
||||
@pytest.mark.parametrize("queue_size", [0, 1])
|
||||
def test_fastsafetensors_model_loader(monkeypatch, queue_size):
|
||||
monkeypatch.setenv("VLLM_FASTSAFETENSORS_QUEUE_SIZE", str(queue_size))
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
huggingface_hub.constants.HF_HUB_OFFLINE = False
|
||||
download_weights_from_hf(
|
||||
"openai-community/gpt2", allow_patterns=["*.safetensors"], cache_dir=tmpdir
|
||||
)
|
||||
safetensors = glob.glob(f"{tmpdir}/**/*.safetensors", recursive=True)
|
||||
assert len(safetensors) > 0
|
||||
|
||||
fastsafetensors_tensors = {}
|
||||
hf_safetensors_tensors = {}
|
||||
|
||||
for name, tensor in fastsafetensors_weights_iterator(safetensors, True):
|
||||
fastsafetensors_tensors[name] = tensor
|
||||
|
||||
for name, tensor in safetensors_weights_iterator(safetensors, True):
|
||||
hf_safetensors_tensors[name] = tensor
|
||||
|
||||
assert len(fastsafetensors_tensors) == len(hf_safetensors_tensors)
|
||||
|
||||
for name, fastsafetensors_tensor in fastsafetensors_tensors.items():
|
||||
fastsafetensors_tensor = fastsafetensors_tensor.to("cpu")
|
||||
assert fastsafetensors_tensor.dtype == hf_safetensors_tensors[name].dtype
|
||||
assert fastsafetensors_tensor.shape == hf_safetensors_tensors[name].shape
|
||||
assert torch.all(fastsafetensors_tensor.eq(hf_safetensors_tensors[name]))
|
||||
@@ -1,28 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm import SamplingParams
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
test_model = "openai-community/gpt2"
|
||||
|
||||
prompts = [
|
||||
"Hello, my name is",
|
||||
"The president of the United States is",
|
||||
"The capital of France is",
|
||||
"The future of AI is",
|
||||
]
|
||||
# Create a sampling params object.
|
||||
sampling_params = SamplingParams(temperature=0.8, top_p=0.95, seed=0)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda(),
|
||||
reason="InstantTensor requires NVIDIA GPUs",
|
||||
)
|
||||
def test_model_loader_download_files(vllm_runner):
|
||||
with vllm_runner(test_model, load_format="instanttensor") as llm:
|
||||
deserialized_outputs = llm.generate(prompts, sampling_params)
|
||||
assert deserialized_outputs
|
||||
@@ -1,104 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import glob
|
||||
import sys
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from safetensors.torch import save_file
|
||||
|
||||
from vllm.model_executor.model_loader.weight_utils import (
|
||||
download_weights_from_hf,
|
||||
instanttensor_weights_iterator,
|
||||
safetensors_weights_iterator,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("weight_name", "local_ids", "collective"),
|
||||
[
|
||||
("mlp.experts.gate_proj.weight", {0}, True),
|
||||
("mlp.experts.0.gate_proj.weight", {0}, False),
|
||||
("mlp.experts.0.gate_proj.weight", set(), False),
|
||||
],
|
||||
)
|
||||
def test_instanttensor_ep_collective_loading(
|
||||
monkeypatch, tmp_path, weight_name, local_ids, collective
|
||||
):
|
||||
"""Keep fused checkpoints collective and expert decisions rank-independent."""
|
||||
from vllm.model_executor.model_loader import weight_utils
|
||||
|
||||
path = tmp_path / "model.safetensors"
|
||||
save_file({weight_name: torch.ones(1)}, str(path))
|
||||
native_open = MagicMock()
|
||||
reader = native_open.return_value.__enter__.return_value
|
||||
reader.total_tensor_size = 0
|
||||
reader.tensors.return_value = ()
|
||||
monkeypatch.setitem(
|
||||
sys.modules, "instanttensor", SimpleNamespace(safe_open=native_open)
|
||||
)
|
||||
group = SimpleNamespace(world_size=2, device_group=object())
|
||||
monkeypatch.setattr(weight_utils, "get_world_group", lambda: group)
|
||||
monkeypatch.setattr(current_platform, "is_cuda", lambda: True)
|
||||
monkeypatch.setattr(current_platform, "current_device", lambda: 0)
|
||||
list(instanttensor_weights_iterator([str(path)], False, local_expert_ids=local_ids))
|
||||
kwargs = native_open.call_args.kwargs
|
||||
assert kwargs["process_group"] is (group.device_group if collective else None)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda(),
|
||||
reason="InstantTensor requires NVIDIA GPUs",
|
||||
)
|
||||
def test_instanttensor_ep_filter_cuda(tmp_path):
|
||||
weights = {
|
||||
"mlp.experts.0.gate_proj.weight": torch.ones(2),
|
||||
"mlp.experts.1.gate_proj.weight": torch.ones(2),
|
||||
"mlp.experts.1.gate_proj.weight_scale": torch.ones(1),
|
||||
"model.layers[0].attn.weight": torch.ones(2),
|
||||
}
|
||||
path = tmp_path / "model.safetensors"
|
||||
save_file(weights, str(path))
|
||||
loaded = dict(
|
||||
instanttensor_weights_iterator([str(path)], False, local_expert_ids={0})
|
||||
)
|
||||
del weights["mlp.experts.1.gate_proj.weight"]
|
||||
assert loaded.keys() == weights.keys()
|
||||
for name, tensor in loaded.items():
|
||||
torch.testing.assert_close(tensor.cpu(), weights[name], rtol=0, atol=0)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda(),
|
||||
reason="InstantTensor requires NVIDIA GPUs",
|
||||
)
|
||||
def test_instanttensor_model_loader():
|
||||
model_dir = download_weights_from_hf(
|
||||
"openai-community/gpt2", cache_dir=None, allow_patterns=["*.safetensors"]
|
||||
)
|
||||
safetensors = glob.glob(f"{model_dir}/*.safetensors")
|
||||
assert len(safetensors) > 0
|
||||
|
||||
instanttensor_tensors = {}
|
||||
hf_safetensors_tensors = {}
|
||||
|
||||
for name, tensor in instanttensor_weights_iterator(safetensors, True):
|
||||
instanttensor_tensors[name] = tensor.to("cpu")
|
||||
|
||||
for name, tensor in safetensors_weights_iterator(safetensors, True):
|
||||
hf_safetensors_tensors[name] = tensor
|
||||
|
||||
assert len(instanttensor_tensors) == len(hf_safetensors_tensors)
|
||||
|
||||
for name, instanttensor_tensor in instanttensor_tensors.items():
|
||||
assert instanttensor_tensor.dtype == hf_safetensors_tensors[name].dtype
|
||||
assert instanttensor_tensor.shape == hf_safetensors_tensors[name].shape
|
||||
assert torch.all(instanttensor_tensor.eq(hf_safetensors_tensors[name]))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
test_instanttensor_model_loader()
|
||||
-721
@@ -1,721 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import json
|
||||
import os
|
||||
import types
|
||||
from datetime import timedelta
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from safetensors.torch import load_file, save_file
|
||||
|
||||
from tests.utils import create_new_process_for_each_test
|
||||
from vllm import SamplingParams
|
||||
from vllm.config.load import LoadConfig
|
||||
from vllm.model_executor.model_loader import get_model_loader
|
||||
from vllm.model_executor.model_loader import runai_streamer_loader as rsl
|
||||
from vllm.utils.mem_constants import GiB_bytes
|
||||
|
||||
load_format = "runai_streamer"
|
||||
test_model = "openai-community/gpt2"
|
||||
# TODO(amacaskill): Replace with a GKE owned GCS bucket.
|
||||
test_gcs_model = "gs://vertex-model-garden-public-us/codegemma/codegemma-2b/"
|
||||
|
||||
prompts = [
|
||||
"Hello, my name is",
|
||||
"The president of the United States is",
|
||||
"The capital of France is",
|
||||
"The future of AI is",
|
||||
]
|
||||
# Create a sampling params object.
|
||||
sampling_params = SamplingParams(temperature=0.8, top_p=0.95, seed=0)
|
||||
|
||||
|
||||
def get_runai_model_loader():
|
||||
load_config = LoadConfig(load_format=load_format)
|
||||
return get_model_loader(load_config)
|
||||
|
||||
|
||||
def test_get_model_loader_with_runai_flag():
|
||||
model_loader = get_runai_model_loader()
|
||||
assert model_loader.__class__.__name__ == "RunaiModelStreamerLoader"
|
||||
|
||||
|
||||
def test_checkpoint_selection_hook_is_optional_for_model_detection():
|
||||
from vllm.model_executor.models.interfaces import SupportsMultiModal
|
||||
from vllm.model_executor.models.interfaces_base import (
|
||||
VllmModel,
|
||||
is_text_generation_model,
|
||||
)
|
||||
|
||||
class ModelWithoutHook(torch.nn.Module):
|
||||
def __init__(self, vllm_config, prefix=""):
|
||||
super().__init__()
|
||||
|
||||
def embed_input_ids(self, input_ids):
|
||||
return input_ids
|
||||
|
||||
def forward(self, input_ids, positions):
|
||||
return input_ids
|
||||
|
||||
def compute_logits(self, hidden_states):
|
||||
return hidden_states
|
||||
|
||||
assert is_text_generation_model(ModelWithoutHook)
|
||||
assert is_text_generation_model(ModelWithoutHook(None))
|
||||
|
||||
class ModelWithDefaultRule(ModelWithoutHook, VllmModel):
|
||||
pass
|
||||
|
||||
assert not ModelWithDefaultRule(None).is_unused_checkpoint_weight("any.weight")
|
||||
|
||||
# Structural models do not inherit the protocol's optional default hook.
|
||||
for names in ([], ["language_model"]):
|
||||
model = torch.nn.Module()
|
||||
model.language_model = ModelWithoutHook(None)
|
||||
model._language_model_names = names
|
||||
assert SupportsMultiModal.get_language_model(model) is model.language_model
|
||||
|
||||
|
||||
def test_runai_model_loader_download_files(vllm_runner):
|
||||
with vllm_runner(test_model, load_format=load_format) as llm:
|
||||
deserialized_outputs = llm.generate(prompts, sampling_params)
|
||||
assert deserialized_outputs
|
||||
|
||||
|
||||
# In-process MTP compilation must not retain GPU resources in the pytest worker.
|
||||
@create_new_process_for_each_test("spawn")
|
||||
def test_runai_mtp_real_checkpoint(vllm_runner, tmp_path, monkeypatch):
|
||||
"""Skip main-only shards without changing a real MTP model's generation."""
|
||||
from vllm.model_executor.models.qwen3_5_mtp import Qwen3_5MTP
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.transformers_utils.repo_utils import hf_api
|
||||
from vllm.v1.metrics.reader import Counter
|
||||
|
||||
if not current_platform.is_cuda_alike():
|
||||
pytest.skip("MTP serving requires CUDA or ROCm")
|
||||
|
||||
checkpoint = tmp_path / "checkpoint"
|
||||
hf_api().snapshot_download(
|
||||
repo_id="Qwen/Qwen3.5-0.8B-Base", local_dir=str(checkpoint)
|
||||
)
|
||||
original_files = list(checkpoint.glob("*.safetensors"))
|
||||
tensors = {}
|
||||
for path in original_files:
|
||||
tensors.update(load_file(path))
|
||||
# Reshard real, unmodified weights so selection cannot pass vacuously on
|
||||
# the published single mixed file. This predicate needs no model state.
|
||||
selector = Qwen3_5MTP.__new__(Qwen3_5MTP)
|
||||
main: dict[str, torch.Tensor] = {}
|
||||
draft: dict[str, torch.Tensor] = {}
|
||||
for name, tensor in tensors.items():
|
||||
(main if selector.is_unused_checkpoint_weight(name) else draft)[name] = tensor
|
||||
assert main and draft
|
||||
files = [checkpoint / "main.safetensors", checkpoint / "draft.safetensors"]
|
||||
for path, contents in zip(files, (main, draft)):
|
||||
save_file(contents, path, metadata={"format": "pt"})
|
||||
for path in original_files:
|
||||
path.unlink()
|
||||
(checkpoint / "model.safetensors.index.json").write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"weight_map": {
|
||||
name: path.name
|
||||
for path, data in zip(files, (main, draft))
|
||||
for name in data
|
||||
}
|
||||
}
|
||||
)
|
||||
)
|
||||
del tensors, main, draft
|
||||
|
||||
monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0")
|
||||
opened = []
|
||||
real_stream = rsl.runai_safetensors_weights_iterator
|
||||
|
||||
def stream(paths, *args):
|
||||
opened.append(list(paths))
|
||||
yield from real_stream(paths, *args)
|
||||
|
||||
monkeypatch.setattr(rsl, "runai_safetensors_weights_iterator", stream)
|
||||
results = []
|
||||
for load_format in ("auto", "runai_streamer"):
|
||||
with vllm_runner(
|
||||
str(checkpoint),
|
||||
load_format=load_format,
|
||||
block_size=None,
|
||||
enable_chunked_prefill=True,
|
||||
disable_log_stats=False,
|
||||
limit_mm_per_prompt={"image": 0, "video": 0},
|
||||
speculative_config={"method": "mtp", "num_speculative_tokens": 1},
|
||||
) as runner:
|
||||
outputs = runner.generate_greedy(prompts, max_tokens=32)
|
||||
results.append(outputs)
|
||||
accepted = next(
|
||||
m.value
|
||||
for m in runner.llm.get_metrics()
|
||||
if isinstance(m, Counter)
|
||||
and m.name == "vllm:spec_decode_num_accepted_tokens"
|
||||
)
|
||||
assert accepted > 0
|
||||
assert len(opened) == 2
|
||||
assert set(opened[0]) == {str(path) for path in files}
|
||||
assert opened[1] == [str(files[1])]
|
||||
matches = sum(a == b for a, b in zip(*results))
|
||||
assert matches >= 3, "RunAI MTP generation diverged from the Default loader"
|
||||
|
||||
|
||||
@pytest.mark.skip(
|
||||
reason="Temporarily disabled due to GCS access issues. "
|
||||
"TODO: Re-enable this test once the underlying issue is resolved."
|
||||
)
|
||||
def test_runai_model_loader_download_files_gcs(
|
||||
vllm_runner, monkeypatch: pytest.MonkeyPatch
|
||||
):
|
||||
monkeypatch.setenv("GOOGLE_CLOUD_PROJECT", "fake-project")
|
||||
monkeypatch.setenv("RUNAI_STREAMER_GCS_USE_ANONYMOUS_CREDENTIALS", "true")
|
||||
monkeypatch.setenv(
|
||||
"CLOUD_STORAGE_EMULATOR_ENDPOINT", "https://storage.googleapis.com"
|
||||
)
|
||||
with vllm_runner(test_gcs_model, load_format=load_format) as llm:
|
||||
deserialized_outputs = llm.generate(prompts, sampling_params)
|
||||
assert deserialized_outputs
|
||||
|
||||
|
||||
def test_runai_passes_revision_by_name():
|
||||
# revision must reach download_safetensors_index_file_from_hf as the
|
||||
# ``revision`` keyword, not the positional ``subfolder`` slot.
|
||||
fake_self = types.SimpleNamespace(
|
||||
load_config=types.SimpleNamespace(download_dir="/cache", ignore_patterns=[])
|
||||
)
|
||||
with (
|
||||
patch.object(rsl, "is_runai_obj_uri", return_value=False),
|
||||
patch.object(rsl, "download_weights_from_hf", return_value="/folder"),
|
||||
patch.object(
|
||||
rsl, "list_safetensors", return_value=["/folder/model.safetensors"]
|
||||
),
|
||||
patch.object(rsl, "download_safetensors_index_file_from_hf") as mock_idx,
|
||||
):
|
||||
rsl.RunaiModelStreamerLoader._prepare_weights(fake_self, "org/model", "myrev")
|
||||
|
||||
mock_idx.assert_called_once()
|
||||
assert mock_idx.call_args.kwargs.get("revision") == "myrev"
|
||||
assert "myrev" not in mock_idx.call_args.args
|
||||
|
||||
|
||||
def _runai_loader(extra):
|
||||
return rsl.RunaiModelStreamerLoader(
|
||||
LoadConfig(load_format="runai_streamer", model_loader_extra_config=extra)
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"extra, match",
|
||||
[
|
||||
({"typo_key": 1}, "Unexpected extra config"),
|
||||
({"distributed": "yes"}, "distributed must be a bool"),
|
||||
({"concurrency": "16"}, "concurrency must be a positive integer"),
|
||||
({"concurrency": -1}, "concurrency must be a positive integer"),
|
||||
],
|
||||
)
|
||||
def test_runai_rejects_invalid_extra_config(extra, match):
|
||||
# The loader used to silently drop unknown keys / wrong types / negatives.
|
||||
with pytest.raises(ValueError, match=match):
|
||||
_runai_loader(extra)
|
||||
|
||||
|
||||
def test_runai_accepts_valid_extra_config():
|
||||
with patch.dict(os.environ, {}, clear=False):
|
||||
os.environ.pop("RUNAI_STREAMER_CONCURRENCY", None)
|
||||
os.environ.pop("RUNAI_STREAMER_MEMORY_LIMIT", None)
|
||||
loader = _runai_loader(
|
||||
{"distributed": True, "concurrency": 16, "memory_limit": 1024}
|
||||
)
|
||||
assert loader._is_distributed is True
|
||||
assert os.environ["RUNAI_STREAMER_CONCURRENCY"] == "16"
|
||||
assert os.environ["RUNAI_STREAMER_MEMORY_LIMIT"] == "1024"
|
||||
|
||||
|
||||
def test_runai_invalid_extra_config_leaves_environ_untouched():
|
||||
# A later invalid key must not leave an earlier valid key applied to
|
||||
# os.environ (all values are validated before any global mutation).
|
||||
with patch.dict(os.environ, {}, clear=False):
|
||||
os.environ.pop("RUNAI_STREAMER_CONCURRENCY", None)
|
||||
with pytest.raises(ValueError, match="memory_limit must be an integer >= -1"):
|
||||
_runai_loader({"concurrency": 16, "memory_limit": -5})
|
||||
assert "RUNAI_STREAMER_CONCURRENCY" not in os.environ
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model_config,expected_source",
|
||||
[
|
||||
# Object storage: model_weights holds the URI, model the pulled config dir.
|
||||
(
|
||||
types.SimpleNamespace(
|
||||
model="/tmp/pulled-config-files",
|
||||
model_weights="s3://bucket/weights",
|
||||
revision="myrev",
|
||||
),
|
||||
"s3://bucket/weights",
|
||||
),
|
||||
# HF repo or local path: model_weights is empty, model is the source.
|
||||
(
|
||||
types.SimpleNamespace(
|
||||
model="org/model", model_weights="", revision="myrev"
|
||||
),
|
||||
"org/model",
|
||||
),
|
||||
],
|
||||
ids=["object_storage", "hf_repo"],
|
||||
)
|
||||
def test_runai_get_all_weights_resolves_source(model_config, expected_source):
|
||||
fake_self = types.SimpleNamespace(
|
||||
_get_weights_iterator=lambda path, revision, is_unused_weight=None: iter(
|
||||
[(f"{path}@{revision}", None)]
|
||||
)
|
||||
)
|
||||
|
||||
weights = list(
|
||||
rsl.RunaiModelStreamerLoader.get_all_weights(
|
||||
fake_self, model_config, model=None
|
||||
)
|
||||
)
|
||||
|
||||
assert weights == [(f"{expected_source}@myrev", None)]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("selection", ["draft", "unadapted", "reject_all"])
|
||||
def test_runai_selects_shards_before_streaming(tmp_path, monkeypatch, selection):
|
||||
"""Unused shards must not reach RunAI; mixed shards retain their tensors."""
|
||||
contents = [
|
||||
{"main.weight": torch.tensor([1.0])},
|
||||
{"main.bias": torch.tensor([2.0]), "draft.weight": torch.tensor([3.0])},
|
||||
{"draft.bias": torch.tensor([4.0])},
|
||||
]
|
||||
files = []
|
||||
for i, tensors in enumerate(contents):
|
||||
path = str(tmp_path / f"part-{i}.safetensors")
|
||||
save_file(tensors, path)
|
||||
files.append(path)
|
||||
monkeypatch.setattr(rsl, "list_safetensors", lambda path: files)
|
||||
|
||||
opened = []
|
||||
real_stream = rsl.runai_safetensors_weights_iterator
|
||||
|
||||
def stream(paths, *args):
|
||||
opened.extend(paths)
|
||||
yield from real_stream(paths, *args)
|
||||
|
||||
monkeypatch.setattr(rsl, "runai_safetensors_weights_iterator", stream)
|
||||
model = torch.nn.Module()
|
||||
if selection == "draft":
|
||||
model.is_unused_checkpoint_weight = lambda name: not name.startswith("draft.")
|
||||
elif selection == "reject_all":
|
||||
model.is_unused_checkpoint_weight = lambda name: True
|
||||
config = types.SimpleNamespace(model=str(tmp_path), model_weights="", revision=None)
|
||||
loaded = dict(_runai_loader({}).get_all_weights(config, model))
|
||||
|
||||
expected = files[1:] if selection == "draft" else files
|
||||
assert opened == expected
|
||||
assert set(loaded) == {
|
||||
name
|
||||
for tensors in (contents[1:] if selection == "draft" else contents)
|
||||
for name in tensors
|
||||
}
|
||||
for name, tensor in {**contents[1], **contents[2]}.items():
|
||||
torch.testing.assert_close(loaded[name], tensor)
|
||||
|
||||
|
||||
def _check_distributed_selection(rank, rendezvous, scenario):
|
||||
torch.distributed.init_process_group(
|
||||
"gloo",
|
||||
init_method=rendezvous,
|
||||
rank=rank,
|
||||
world_size=2,
|
||||
timeout=timedelta(seconds=30),
|
||||
)
|
||||
root = "s3://bucket/weights"
|
||||
if scenario == "different_roots":
|
||||
root += str(rank)
|
||||
files = [root + "/main.safetensors", root + "/draft.safetensors"]
|
||||
|
||||
def pull_files(source, destination, **kwargs):
|
||||
if scenario == "metadata_failure" and rank == 1:
|
||||
raise OSError("Index temporarily unavailable")
|
||||
(Path(destination) / "model.safetensors.index.json").write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"weight_map": {
|
||||
"main.weight": "main.safetensors",
|
||||
"draft.weight": "draft.safetensors",
|
||||
}
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
try:
|
||||
with pytest.MonkeyPatch.context() as monkeypatch:
|
||||
monkeypatch.setattr(rsl, "list_safetensors", lambda path: files)
|
||||
monkeypatch.setattr(rsl, "runai_pull_files", pull_files)
|
||||
monkeypatch.setattr(
|
||||
rsl,
|
||||
"_get_remote_checkpoint_keys",
|
||||
lambda paths: [
|
||||
["main.weight"]
|
||||
if path.endswith("main.safetensors")
|
||||
else ["draft.weight"]
|
||||
for path in paths
|
||||
],
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
rsl,
|
||||
"get_world_group",
|
||||
lambda: types.SimpleNamespace(
|
||||
world_size=2, cpu_group=torch.distributed.group.WORLD
|
||||
),
|
||||
raising=False,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
rsl,
|
||||
"get_pp_group",
|
||||
lambda: types.SimpleNamespace(
|
||||
world_size=2 if scenario == "pipeline_draft" else 1
|
||||
),
|
||||
raising=False,
|
||||
)
|
||||
if scenario == "pipeline_draft" and rank == 0:
|
||||
# Earlier PP stages never invoke the draft loader.
|
||||
torch.distributed.barrier()
|
||||
return
|
||||
# Exercise real selection and Gloo; observe the distributed stream input.
|
||||
monkeypatch.setattr(
|
||||
rsl,
|
||||
"runai_safetensors_weights_iterator",
|
||||
lambda paths, *args: iter((path, None) for path in paths),
|
||||
)
|
||||
model = torch.nn.Module()
|
||||
if not (scenario == "missing_hook" and rank == 1):
|
||||
needed = (
|
||||
"main." if scenario == "different_roles" and rank == 1 else "draft."
|
||||
)
|
||||
model.is_unused_checkpoint_weight = lambda name: (
|
||||
not name.startswith(needed)
|
||||
)
|
||||
config = types.SimpleNamespace(model=root, model_weights="", revision=None)
|
||||
selected = [
|
||||
path
|
||||
for path, _ in _runai_loader({"distributed": True}).get_all_weights(
|
||||
config, model
|
||||
)
|
||||
]
|
||||
expected = files[1:] if scenario == "healthy" else files
|
||||
assert selected == expected
|
||||
if scenario == "pipeline_draft":
|
||||
torch.distributed.barrier()
|
||||
finally:
|
||||
torch.distributed.destroy_process_group()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"scenario",
|
||||
[
|
||||
"healthy",
|
||||
"metadata_failure",
|
||||
"different_roles",
|
||||
"missing_hook",
|
||||
"different_roots",
|
||||
"pipeline_draft",
|
||||
],
|
||||
)
|
||||
def test_distributed_runai_preserves_every_ranks_required_shards(tmp_path, scenario):
|
||||
"""All ranks must stream the same union even if one cannot filter safely."""
|
||||
torch.multiprocessing.spawn(
|
||||
_check_distributed_selection,
|
||||
args=((tmp_path / "rendezvous").as_uri(), scenario),
|
||||
nprocs=2,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("scheme", ["s3", "gs", "az"])
|
||||
@pytest.mark.parametrize(
|
||||
"index_kind",
|
||||
[
|
||||
"valid",
|
||||
"missing",
|
||||
"malformed",
|
||||
"unavailable",
|
||||
"unadapted",
|
||||
"missing_shard",
|
||||
"stale",
|
||||
"header_unavailable",
|
||||
],
|
||||
)
|
||||
def test_runai_remote_selection_uses_actual_weight_source(
|
||||
tmp_path, monkeypatch, scheme, index_kind
|
||||
):
|
||||
root = f"{scheme}://bucket/weights"
|
||||
files = [
|
||||
root + "/main/part.safetensors",
|
||||
root + "/draft/part.safetensors",
|
||||
root + "/mixed.safetensors",
|
||||
root + "/extra.safetensors",
|
||||
]
|
||||
monkeypatch.setattr(rsl, "list_safetensors", lambda path: files)
|
||||
pulled = []
|
||||
|
||||
def pull_files(source, destination, allow_pattern, ignore_pattern=None):
|
||||
pulled.append((source.rstrip("/"), allow_pattern))
|
||||
if index_kind == "unavailable":
|
||||
# Storage SDKs do not necessarily derive errors from OSError.
|
||||
raise Exception("Object metadata unavailable")
|
||||
index = Path(destination) / "model.safetensors.index.json"
|
||||
if index_kind in ("valid", "missing_shard", "stale", "header_unavailable"):
|
||||
index.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"weight_map": {
|
||||
"main.weight": "main/part.safetensors",
|
||||
"draft.weight": (
|
||||
"absent.safetensors"
|
||||
if index_kind == "missing_shard"
|
||||
else "draft/part.safetensors"
|
||||
),
|
||||
"main.bias": "mixed.safetensors",
|
||||
"draft.bias": "mixed.safetensors",
|
||||
}
|
||||
}
|
||||
)
|
||||
)
|
||||
elif index_kind == "malformed":
|
||||
index.write_text("not JSON")
|
||||
|
||||
monkeypatch.setattr(rsl, "runai_pull_files", pull_files, raising=False)
|
||||
headers_checked = []
|
||||
|
||||
def checkpoint_keys(paths):
|
||||
headers_checked.extend(paths)
|
||||
if index_kind == "header_unavailable":
|
||||
raise OSError("Header temporarily unavailable")
|
||||
return [
|
||||
["main.weight", "draft.unindexed_weight"]
|
||||
if index_kind == "stale"
|
||||
else ["main.weight"]
|
||||
for _ in paths
|
||||
]
|
||||
|
||||
monkeypatch.setattr(
|
||||
rsl, "_get_remote_checkpoint_keys", checkpoint_keys, raising=False
|
||||
)
|
||||
opened = []
|
||||
|
||||
def stream(paths, *args):
|
||||
opened.extend(paths)
|
||||
return iter(())
|
||||
|
||||
monkeypatch.setattr(rsl, "runai_safetensors_weights_iterator", stream)
|
||||
model = torch.nn.Module()
|
||||
if index_kind != "unadapted":
|
||||
model.is_unused_checkpoint_weight = lambda name: not name.startswith("draft.")
|
||||
# The config directory is not the checkpoint: using its index could drop
|
||||
# tensors from an independent model_weights source.
|
||||
(tmp_path / "model.safetensors.index.json").write_text("not the weights index")
|
||||
config = types.SimpleNamespace(
|
||||
model=str(tmp_path), model_weights=root, revision=None
|
||||
)
|
||||
list(_runai_loader({}).get_all_weights(config, model))
|
||||
|
||||
assert opened == (files[1:] if index_kind == "valid" else files)
|
||||
assert headers_checked == (
|
||||
files[:1] if index_kind in ("valid", "stale", "header_unavailable") else []
|
||||
)
|
||||
assert pulled == (
|
||||
[]
|
||||
if index_kind == "unadapted"
|
||||
else [
|
||||
(
|
||||
root,
|
||||
[
|
||||
"weights/model.safetensors.index.json",
|
||||
"model.safetensors.index.json",
|
||||
],
|
||||
)
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"weight_map",
|
||||
[None, {}, [], {"draft.weight": None}, {"draft.weight": "../part.safetensors"}],
|
||||
)
|
||||
def test_runai_invalid_weight_map_keeps_original_files(monkeypatch, weight_map):
|
||||
files = ["s3://bucket/model/part.safetensors"]
|
||||
monkeypatch.setattr(rsl, "list_safetensors", lambda path: files)
|
||||
|
||||
def pull_files(source, directory, **kwargs):
|
||||
(Path(directory) / "model.safetensors.index.json").write_text(
|
||||
json.dumps({"weight_map": weight_map})
|
||||
)
|
||||
|
||||
monkeypatch.setattr(rsl, "runai_pull_files", pull_files)
|
||||
monkeypatch.setattr(
|
||||
rsl, "runai_safetensors_weights_iterator", lambda paths, *args: iter(paths)
|
||||
)
|
||||
model = torch.nn.Module()
|
||||
model.is_unused_checkpoint_weight = lambda name: True
|
||||
config = types.SimpleNamespace(
|
||||
model="s3://bucket/model", model_weights="", revision=None
|
||||
)
|
||||
assert list(_runai_loader({}).get_all_weights(config, model)) == files
|
||||
|
||||
|
||||
def test_runai_model_rule_error_is_not_hidden_by_metadata_fallback(monkeypatch):
|
||||
files = ["s3://bucket/model/part.safetensors"]
|
||||
monkeypatch.setattr(rsl, "list_safetensors", lambda path: files)
|
||||
|
||||
def pull_files(source, directory, **kwargs):
|
||||
(Path(directory) / "model.safetensors.index.json").write_text(
|
||||
json.dumps({"weight_map": {"draft.weight": "part.safetensors"}})
|
||||
)
|
||||
|
||||
monkeypatch.setattr(rsl, "runai_pull_files", pull_files)
|
||||
model = torch.nn.Module()
|
||||
|
||||
def invalid_rule(name):
|
||||
raise ValueError("Invalid model rule")
|
||||
|
||||
model.is_unused_checkpoint_weight = invalid_rule
|
||||
config = types.SimpleNamespace(
|
||||
model="s3://bucket/model", model_weights="", revision=None
|
||||
)
|
||||
with pytest.raises(ValueError, match="Invalid model rule"):
|
||||
list(_runai_loader({}).get_all_weights(config, model))
|
||||
|
||||
|
||||
def test_runai_header_reader_uses_actual_tensor_names(tmp_path):
|
||||
"""The RunAI header reader sees names even when an index omits them."""
|
||||
path = tmp_path / "mixed.safetensors"
|
||||
save_file(
|
||||
{
|
||||
"main.weight": torch.tensor([1.0]),
|
||||
"draft.unindexed_weight": torch.tensor([2.0]),
|
||||
},
|
||||
path,
|
||||
metadata={"format": "pt"},
|
||||
)
|
||||
(names,) = rsl._get_remote_checkpoint_keys([str(path)])
|
||||
assert set(names) == {"main.weight", "draft.unindexed_weight"}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("stale", [False, True])
|
||||
def test_runai_stale_index_keeps_actual_draft_weights(tmp_path, monkeypatch, stale):
|
||||
"""Verify excluded headers before dropping a shard with real draft weights."""
|
||||
root = "s3://bucket/checkpoint"
|
||||
main_path = tmp_path / "main.safetensors"
|
||||
draft_path = tmp_path / "draft.safetensors"
|
||||
main_tensors = {"main.weight": torch.tensor([1.0])}
|
||||
if stale:
|
||||
main_tensors["draft.unindexed_weight"] = torch.tensor([2.0])
|
||||
save_file(main_tensors, main_path)
|
||||
save_file({"draft.weight": torch.tensor([3.0])}, draft_path)
|
||||
paths = {
|
||||
root + "/main.safetensors": str(main_path),
|
||||
root + "/draft.safetensors": str(draft_path),
|
||||
}
|
||||
monkeypatch.setattr(rsl, "list_safetensors", lambda path: list(paths))
|
||||
|
||||
def pull_files(source, directory, **kwargs):
|
||||
(Path(directory) / "model.safetensors.index.json").write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"weight_map": {
|
||||
"main.weight": "main.safetensors",
|
||||
"draft.weight": "draft.safetensors",
|
||||
}
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
monkeypatch.setattr(rsl, "runai_pull_files", pull_files)
|
||||
read_headers = rsl._get_remote_checkpoint_keys
|
||||
monkeypatch.setattr(
|
||||
rsl,
|
||||
"_get_remote_checkpoint_keys",
|
||||
lambda files: read_headers([paths[path] for path in files]),
|
||||
)
|
||||
stream = rsl.runai_safetensors_weights_iterator
|
||||
|
||||
def stream_local(files, *args):
|
||||
for name, tensor in stream([paths[path] for path in files], *args):
|
||||
yield name, tensor.clone()
|
||||
|
||||
monkeypatch.setattr(rsl, "runai_safetensors_weights_iterator", stream_local)
|
||||
model = torch.nn.Module()
|
||||
model.is_unused_checkpoint_weight = lambda name: not name.startswith("draft.")
|
||||
config = types.SimpleNamespace(model=root, model_weights="", revision=None)
|
||||
loaded = dict(_runai_loader({}).get_all_weights(config, model))
|
||||
assert set(loaded) == (
|
||||
{"main.weight", "draft.unindexed_weight", "draft.weight"}
|
||||
if stale
|
||||
else {"draft.weight"}
|
||||
)
|
||||
torch.testing.assert_close(loaded["draft.weight"], torch.tensor([3.0]))
|
||||
if stale:
|
||||
torch.testing.assert_close(
|
||||
loaded["draft.unindexed_weight"], torch.tensor([2.0])
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"base_model,mul_model,add_model",
|
||||
[
|
||||
(
|
||||
"Qwen/Qwen3-0.6B",
|
||||
"inference-optimization/Qwen3-0.6B-debug-multiply",
|
||||
"inference-optimization/Qwen3-0.6B-debug-add",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_runai_deep_sleep_reload_weights(base_model, mul_model, add_model, vllm_runner):
|
||||
free, total = torch.accelerator.get_memory_info()
|
||||
used_bytes_baseline = total - free
|
||||
|
||||
def sleep_and_reload(llm, path):
|
||||
# Level 2 discards the parameter memory, so after wake_up the weights
|
||||
# can only come from the reload: a tensor the streamer fails to
|
||||
# deliver changes the output instead of keeping a stale valid value.
|
||||
llm.get_llm().sleep(level=2)
|
||||
free, total = torch.accelerator.get_memory_info()
|
||||
assert total - free - used_bytes_baseline < 3 * GiB_bytes
|
||||
llm.get_llm().wake_up(tags=["weights"])
|
||||
llm.collective_rpc("reload_weights", kwargs={"weights_path": path})
|
||||
free, total = torch.accelerator.get_memory_info()
|
||||
assert total - free - used_bytes_baseline < 4 * GiB_bytes
|
||||
llm.get_llm().wake_up(tags=["kv_cache"])
|
||||
|
||||
with vllm_runner(
|
||||
model_name=base_model,
|
||||
load_format=load_format,
|
||||
enable_sleep_mode=True,
|
||||
enable_prefix_caching=False,
|
||||
max_model_len=16,
|
||||
max_num_seqs=1,
|
||||
) as llm:
|
||||
base_output = llm.generate_greedy(["3 4 ="], max_tokens=4)
|
||||
|
||||
sleep_and_reload(llm, mul_model)
|
||||
mul_perp = llm.generate_prompt_perplexity(["3 4 = 12"], mask=["3 4 ="])[0]
|
||||
add_perp = llm.generate_prompt_perplexity(["3 4 = 7"], mask=["3 4 ="])[0]
|
||||
assert mul_perp < add_perp
|
||||
|
||||
sleep_and_reload(llm, add_model)
|
||||
mul_perp = llm.generate_prompt_perplexity(["3 4 = 12"], mask=["3 4 ="])[0]
|
||||
add_perp = llm.generate_prompt_perplexity(["3 4 = 7"], mask=["3 4 ="])[0]
|
||||
assert add_perp < mul_perp
|
||||
|
||||
# Round trip to the original weights over discarded memory must
|
||||
# reproduce the pre-sleep output.
|
||||
sleep_and_reload(llm, base_model)
|
||||
assert llm.generate_greedy(["3 4 ="], max_tokens=4) == base_output
|
||||
@@ -1,617 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import asyncio
|
||||
import gc
|
||||
import json
|
||||
import os
|
||||
import pathlib
|
||||
import subprocess
|
||||
import sys
|
||||
from typing import Any
|
||||
from unittest.mock import Mock
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
import vllm.model_executor.model_loader.tensorizer
|
||||
from tests.utils import VLLM_PATH, RemoteOpenAIServer
|
||||
from vllm import SamplingParams
|
||||
from vllm.engine.arg_utils import EngineArgs
|
||||
from vllm.model_executor.model_loader.tensorizer import (
|
||||
TensorizerConfig,
|
||||
TensorSerializer,
|
||||
is_vllm_tensorized,
|
||||
open_stream,
|
||||
tensorize_vllm_model,
|
||||
)
|
||||
from vllm.model_executor.model_loader.tensorizer_loader import (
|
||||
BLACKLISTED_TENSORIZER_ARGS,
|
||||
)
|
||||
from vllm.utils.import_utils import PlaceholderModule
|
||||
|
||||
from .conftest import DummyExecutor, assert_from_collective_rpc
|
||||
|
||||
try:
|
||||
import tensorizer
|
||||
from tensorizer import EncryptionParams
|
||||
except ImportError:
|
||||
tensorizer = PlaceholderModule("tensorizer")
|
||||
EncryptionParams = tensorizer.placeholder_attr("EncryptionParams")
|
||||
|
||||
|
||||
class TensorizerCaughtError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
EXAMPLES_PATH = VLLM_PATH / "examples"
|
||||
|
||||
pytest_plugins = ("pytest_asyncio",)
|
||||
|
||||
prompts = [
|
||||
"Hello, my name is",
|
||||
"The president of the United States is",
|
||||
"The capital of France is",
|
||||
"The future of AI is",
|
||||
]
|
||||
# Create a sampling params object.
|
||||
sampling_params = SamplingParams(temperature=0.8, top_p=0.95, seed=0)
|
||||
|
||||
|
||||
def patch_init_and_catch_error(self, obj, method_name, expected_error: type[Exception]):
|
||||
original = getattr(obj, method_name, None)
|
||||
if original is None:
|
||||
raise ValueError("Method '{}' not found.".format(method_name))
|
||||
|
||||
def wrapper(*args, **kwargs):
|
||||
try:
|
||||
return original(*args, **kwargs)
|
||||
except expected_error as err:
|
||||
raise TensorizerCaughtError from err
|
||||
|
||||
setattr(obj, method_name, wrapper)
|
||||
|
||||
self.load_model()
|
||||
|
||||
|
||||
def assert_specific_tensorizer_error_is_raised(
|
||||
executor,
|
||||
obj: Any,
|
||||
method_name: str,
|
||||
expected_error: type[Exception],
|
||||
):
|
||||
with pytest.raises(TensorizerCaughtError):
|
||||
executor.collective_rpc(
|
||||
patch_init_and_catch_error,
|
||||
args=(
|
||||
obj,
|
||||
method_name,
|
||||
expected_error,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def is_curl_installed():
|
||||
try:
|
||||
subprocess.check_call(["curl", "--version"])
|
||||
return True
|
||||
except (subprocess.CalledProcessError, FileNotFoundError):
|
||||
return False
|
||||
|
||||
|
||||
def write_keyfile(keyfile_path: str):
|
||||
encryption_params = EncryptionParams.random()
|
||||
pathlib.Path(keyfile_path).parent.mkdir(parents=True, exist_ok=True)
|
||||
with open(keyfile_path, "wb") as f:
|
||||
f.write(encryption_params.key)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not is_curl_installed(), reason="cURL is not installed")
|
||||
def test_deserialized_encrypted_vllm_model_has_same_outputs(
|
||||
model_ref, vllm_runner, tmp_path, model_path
|
||||
):
|
||||
args = EngineArgs(model=model_ref)
|
||||
with vllm_runner(model_ref) as vllm_model:
|
||||
key_path = tmp_path / model_ref / "model.key"
|
||||
write_keyfile(key_path)
|
||||
|
||||
outputs = vllm_model.generate(prompts, sampling_params)
|
||||
|
||||
config_for_serializing = TensorizerConfig(
|
||||
tensorizer_uri=str(model_path), encryption_keyfile=str(key_path)
|
||||
)
|
||||
|
||||
tensorize_vllm_model(args, config_for_serializing)
|
||||
|
||||
config_for_deserializing = TensorizerConfig(
|
||||
tensorizer_uri=str(model_path), encryption_keyfile=str(key_path)
|
||||
)
|
||||
|
||||
with vllm_runner(
|
||||
model_ref,
|
||||
load_format="tensorizer",
|
||||
model_loader_extra_config=config_for_deserializing,
|
||||
) as loaded_vllm_model: # noqa: E501
|
||||
deserialized_outputs = loaded_vllm_model.generate(prompts, sampling_params)
|
||||
# noqa: E501
|
||||
|
||||
assert outputs == deserialized_outputs
|
||||
|
||||
|
||||
def test_deserialized_hf_model_has_same_outputs(
|
||||
hf_runner, vllm_runner, tmp_path, model_ref, model_path
|
||||
):
|
||||
with hf_runner(model_ref) as hf_model:
|
||||
max_tokens = 50
|
||||
outputs = hf_model.generate_greedy(prompts, max_tokens=max_tokens)
|
||||
with open_stream(model_path, "wb+") as stream:
|
||||
serializer = TensorSerializer(stream)
|
||||
serializer.write_module(hf_model.model)
|
||||
|
||||
with vllm_runner(
|
||||
model_ref,
|
||||
load_format="tensorizer",
|
||||
model_loader_extra_config=TensorizerConfig(
|
||||
tensorizer_uri=str(model_path),
|
||||
num_readers=1,
|
||||
),
|
||||
) as loaded_hf_model:
|
||||
deserialized_outputs = loaded_hf_model.generate_greedy(
|
||||
prompts, max_tokens=max_tokens
|
||||
)
|
||||
|
||||
assert outputs == deserialized_outputs
|
||||
|
||||
|
||||
def test_load_without_tensorizer_load_format(vllm_runner, capfd, model_ref):
|
||||
model = None
|
||||
try:
|
||||
model = vllm_runner(
|
||||
model_ref, model_loader_extra_config=TensorizerConfig(tensorizer_uri="test")
|
||||
)
|
||||
pytest.fail("Expected RuntimeError for extra config keys")
|
||||
except RuntimeError:
|
||||
out, err = capfd.readouterr()
|
||||
combined_output = out + err
|
||||
assert (
|
||||
"ValueError: Unexpected extra config keys for load format auto"
|
||||
) in combined_output
|
||||
finally:
|
||||
del model
|
||||
gc.collect()
|
||||
torch.accelerator.empty_cache()
|
||||
|
||||
|
||||
def test_raise_value_error_on_invalid_load_format(vllm_runner, capfd, model_ref):
|
||||
model = None
|
||||
try:
|
||||
model = vllm_runner(
|
||||
model_ref,
|
||||
load_format="safetensors",
|
||||
model_loader_extra_config=TensorizerConfig(tensorizer_uri="test"),
|
||||
)
|
||||
pytest.fail("Expected RuntimeError for extra config keys")
|
||||
except RuntimeError:
|
||||
out, err = capfd.readouterr()
|
||||
|
||||
combined_output = out + err
|
||||
assert (
|
||||
"ValueError: Unexpected extra config keys for load format safetensors"
|
||||
) in combined_output
|
||||
finally:
|
||||
del model
|
||||
gc.collect()
|
||||
torch.accelerator.empty_cache()
|
||||
|
||||
|
||||
@pytest.mark.skipif(torch.accelerator.device_count() < 2, reason="Requires 2 GPUs")
|
||||
def test_tensorizer_with_tp_path_without_template(vllm_runner, capfd):
|
||||
try:
|
||||
model_ref = "EleutherAI/pythia-1.4b"
|
||||
tensorized_path = f"s3://tensorized/{model_ref}/fp16/model.tensors"
|
||||
|
||||
vllm_runner(
|
||||
model_ref,
|
||||
load_format="tensorizer",
|
||||
model_loader_extra_config=TensorizerConfig(
|
||||
tensorizer_uri=tensorized_path,
|
||||
num_readers=1,
|
||||
s3_endpoint="object.ord1.coreweave.com",
|
||||
),
|
||||
tensor_parallel_size=2,
|
||||
disable_custom_all_reduce=True,
|
||||
)
|
||||
except RuntimeError:
|
||||
out, err = capfd.readouterr()
|
||||
combined_output = out + err
|
||||
assert (
|
||||
"ValueError: For a sharded model, tensorizer_uri "
|
||||
"should include a string format template like '%04d' "
|
||||
"to be formatted with the rank "
|
||||
"of the shard"
|
||||
) in combined_output
|
||||
|
||||
|
||||
@pytest.mark.skipif(torch.accelerator.device_count() < 2, reason="Requires 2 GPUs")
|
||||
def test_deserialized_encrypted_vllm_model_with_tp_has_same_outputs(
|
||||
vllm_runner, tmp_path
|
||||
):
|
||||
model_ref = "EleutherAI/pythia-1.4b"
|
||||
# record outputs from un-sharded un-tensorized model
|
||||
with vllm_runner(
|
||||
model_ref,
|
||||
disable_custom_all_reduce=True,
|
||||
enforce_eager=True,
|
||||
) as base_model:
|
||||
outputs = base_model.generate(prompts, sampling_params)
|
||||
|
||||
# load model with two shards and serialize with encryption
|
||||
model_path = str(tmp_path / model_ref / "model-%02d.tensors")
|
||||
key_path = tmp_path / (model_ref + ".key")
|
||||
|
||||
tensorizer_config = TensorizerConfig(
|
||||
tensorizer_uri=model_path,
|
||||
encryption_keyfile=str(key_path),
|
||||
)
|
||||
|
||||
tensorize_vllm_model(
|
||||
engine_args=EngineArgs(
|
||||
model=model_ref,
|
||||
tensor_parallel_size=2,
|
||||
disable_custom_all_reduce=True,
|
||||
enforce_eager=True,
|
||||
),
|
||||
tensorizer_config=tensorizer_config,
|
||||
)
|
||||
assert os.path.isfile(model_path % 0), "Serialization subprocess failed"
|
||||
assert os.path.isfile(model_path % 1), "Serialization subprocess failed"
|
||||
|
||||
with vllm_runner(
|
||||
model_ref,
|
||||
tensor_parallel_size=2,
|
||||
load_format="tensorizer",
|
||||
disable_custom_all_reduce=True,
|
||||
enforce_eager=True,
|
||||
model_loader_extra_config=tensorizer_config,
|
||||
) as loaded_vllm_model:
|
||||
deserialized_outputs = loaded_vllm_model.generate(prompts, sampling_params)
|
||||
|
||||
assert outputs == deserialized_outputs
|
||||
|
||||
|
||||
@pytest.mark.flaky(reruns=3)
|
||||
def test_vllm_tensorized_model_has_same_outputs(
|
||||
model_ref, vllm_runner, tmp_path, model_path
|
||||
):
|
||||
gc.collect()
|
||||
torch.accelerator.empty_cache()
|
||||
config = TensorizerConfig(tensorizer_uri=str(model_path))
|
||||
args = EngineArgs(model=model_ref)
|
||||
|
||||
with vllm_runner(model_ref) as vllm_model:
|
||||
outputs = vllm_model.generate(prompts, sampling_params)
|
||||
|
||||
tensorize_vllm_model(args, config)
|
||||
assert is_vllm_tensorized(config)
|
||||
|
||||
with vllm_runner(
|
||||
model_ref, load_format="tensorizer", model_loader_extra_config=config
|
||||
) as loaded_vllm_model:
|
||||
deserialized_outputs = loaded_vllm_model.generate(prompts, sampling_params)
|
||||
# noqa: E501
|
||||
|
||||
assert outputs == deserialized_outputs
|
||||
|
||||
|
||||
def test_load_with_just_model_tensors(just_serialize_model_tensors, model_ref):
|
||||
# For backwards compatibility, ensure Tensorizer can be still be loaded
|
||||
# for inference by passing the model reference name, not a local/S3 dir,
|
||||
# and the location of the model tensors
|
||||
|
||||
model_dir = just_serialize_model_tensors
|
||||
|
||||
extra_config = {"tensorizer_uri": f"{model_dir}/model.tensors"}
|
||||
|
||||
## Start OpenAI API server
|
||||
args = [
|
||||
"--load-format",
|
||||
"tensorizer",
|
||||
"--model-loader-extra-config",
|
||||
json.dumps(extra_config),
|
||||
]
|
||||
|
||||
with RemoteOpenAIServer(model_ref, args):
|
||||
# This test only concerns itself with being able to load the model
|
||||
# and successfully initialize the server
|
||||
pass
|
||||
|
||||
|
||||
def test_assert_serialization_kwargs_passed_to_tensor_serializer(tmp_path, vllm_runner):
|
||||
serialization_params = {
|
||||
"limit_cpu_concurrency": 2,
|
||||
}
|
||||
model_ref = "facebook/opt-125m"
|
||||
model_path = tmp_path / (model_ref + ".tensors")
|
||||
config = TensorizerConfig(
|
||||
tensorizer_uri=str(model_path), serialization_kwargs=serialization_params
|
||||
)
|
||||
|
||||
def serialization_test(self, *args, **kwargs):
|
||||
# This is performed in the ephemeral worker process, so monkey-patching
|
||||
# will actually work, and cleanup is guaranteed so don't
|
||||
# need to reset things
|
||||
|
||||
original_dict = serialization_params
|
||||
to_compare = {}
|
||||
|
||||
original = tensorizer.serialization.TensorSerializer.__init__
|
||||
|
||||
def tensorizer_serializer_wrapper(self, *args, **kwargs):
|
||||
nonlocal to_compare
|
||||
to_compare = kwargs.copy()
|
||||
return original(self, *args, **kwargs)
|
||||
|
||||
tensorizer.serialization.TensorSerializer.__init__ = (
|
||||
tensorizer_serializer_wrapper
|
||||
)
|
||||
|
||||
tensorizer_config = TensorizerConfig(**kwargs["tensorizer_config"])
|
||||
self.save_tensorized_model(
|
||||
tensorizer_config=tensorizer_config,
|
||||
)
|
||||
return to_compare | original_dict == to_compare
|
||||
|
||||
kwargs = {"tensorizer_config": config.to_serializable()}
|
||||
|
||||
with vllm_runner(model_ref) as runner:
|
||||
assert assert_from_collective_rpc(runner.get_llm(), serialization_test, kwargs)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
(
|
||||
"serialization_error",
|
||||
"renderer_shutdown_error",
|
||||
"engine_shutdown_error",
|
||||
"expected_error",
|
||||
),
|
||||
[
|
||||
(None, None, None, None),
|
||||
(RuntimeError("serialization failed"), None, None, "serialization failed"),
|
||||
(
|
||||
RuntimeError("serialization failed"),
|
||||
ValueError("renderer shutdown failed"),
|
||||
ValueError("engine shutdown failed"),
|
||||
"serialization failed",
|
||||
),
|
||||
(
|
||||
None,
|
||||
ValueError("renderer shutdown failed"),
|
||||
None,
|
||||
"renderer shutdown failed",
|
||||
),
|
||||
(None, None, ValueError("engine shutdown failed"), "engine shutdown failed"),
|
||||
],
|
||||
)
|
||||
def test_tensorize_vllm_model_shuts_down_engine(
|
||||
monkeypatch,
|
||||
serialization_error,
|
||||
renderer_shutdown_error,
|
||||
engine_shutdown_error,
|
||||
expected_error,
|
||||
):
|
||||
from vllm.v1.engine.llm_engine import LLMEngine
|
||||
|
||||
engine_args = Mock()
|
||||
config = Mock(encryption_keyfile=None)
|
||||
engine = Mock()
|
||||
engine.collective_rpc.side_effect = serialization_error
|
||||
engine.renderer.shutdown.side_effect = renderer_shutdown_error
|
||||
engine.engine_core.shutdown.side_effect = engine_shutdown_error
|
||||
monkeypatch.setattr(LLMEngine, "from_vllm_config", Mock(return_value=engine))
|
||||
monkeypatch.setenv("VLLM_WORKER_SHUTDOWN_TIMEOUT_SECONDS", "7")
|
||||
|
||||
if expected_error is None:
|
||||
tensorize_vllm_model(engine_args, config)
|
||||
else:
|
||||
with pytest.raises(Exception, match=expected_error) as exc_info:
|
||||
tensorize_vllm_model(engine_args, config)
|
||||
notes = getattr(exc_info.value, "__notes__", ())
|
||||
if serialization_error is not None and renderer_shutdown_error is not None:
|
||||
assert any("renderer shutdown failed" in note for note in notes)
|
||||
if serialization_error is not None and engine_shutdown_error is not None:
|
||||
assert any("engine shutdown failed" in note for note in notes)
|
||||
|
||||
engine.engine_core.shutdown.assert_called_once_with(timeout=17.0)
|
||||
engine.renderer.shutdown.assert_called_once_with()
|
||||
|
||||
|
||||
def test_assert_deserialization_kwargs_passed_to_tensor_deserializer(tmp_path, capfd):
|
||||
deserialization_kwargs = {
|
||||
"num_readers": "bar", # illegal value
|
||||
}
|
||||
|
||||
serialization_params = {
|
||||
"limit_cpu_concurrency": 2,
|
||||
}
|
||||
|
||||
model_ref = "facebook/opt-125m"
|
||||
model_path = tmp_path / (model_ref + ".tensors")
|
||||
config = TensorizerConfig(
|
||||
tensorizer_uri=str(model_path), serialization_kwargs=serialization_params
|
||||
)
|
||||
|
||||
args = EngineArgs(model=model_ref)
|
||||
tensorize_vllm_model(args, config)
|
||||
|
||||
loader_tc = TensorizerConfig(
|
||||
tensorizer_uri=str(model_path),
|
||||
deserialization_kwargs=deserialization_kwargs,
|
||||
)
|
||||
|
||||
engine_args = EngineArgs(
|
||||
model="facebook/opt-125m",
|
||||
load_format="tensorizer",
|
||||
model_loader_extra_config=loader_tc.to_serializable(),
|
||||
)
|
||||
|
||||
vllm_config = engine_args.create_engine_config()
|
||||
executor = DummyExecutor(vllm_config)
|
||||
|
||||
assert_specific_tensorizer_error_is_raised(
|
||||
executor,
|
||||
tensorizer.serialization.TensorDeserializer,
|
||||
"__init__",
|
||||
TypeError,
|
||||
)
|
||||
|
||||
|
||||
def test_assert_stream_kwargs_passed_to_tensor_deserializer(tmp_path, capfd):
|
||||
deserialization_kwargs = {
|
||||
"num_readers": 1,
|
||||
}
|
||||
|
||||
serialization_params = {
|
||||
"limit_cpu_concurrency": 2,
|
||||
}
|
||||
|
||||
model_ref = "facebook/opt-125m"
|
||||
model_path = tmp_path / (model_ref + ".tensors")
|
||||
config = TensorizerConfig(
|
||||
tensorizer_uri=str(model_path), serialization_kwargs=serialization_params
|
||||
)
|
||||
|
||||
args = EngineArgs(model=model_ref)
|
||||
tensorize_vllm_model(args, config)
|
||||
|
||||
stream_kwargs = {"mode": "foo"}
|
||||
|
||||
loader_tc = TensorizerConfig(
|
||||
tensorizer_uri=str(model_path),
|
||||
deserialization_kwargs=deserialization_kwargs,
|
||||
stream_kwargs=stream_kwargs,
|
||||
)
|
||||
|
||||
engine_args = EngineArgs(
|
||||
model="facebook/opt-125m",
|
||||
load_format="tensorizer",
|
||||
model_loader_extra_config=loader_tc.to_serializable(),
|
||||
)
|
||||
|
||||
vllm_config = engine_args.create_engine_config()
|
||||
executor = DummyExecutor(vllm_config)
|
||||
|
||||
assert_specific_tensorizer_error_is_raised(
|
||||
executor,
|
||||
vllm.model_executor.model_loader.tensorizer,
|
||||
"open_stream",
|
||||
ValueError,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_serialize_and_serve_entrypoints(tmp_path):
|
||||
model_ref = "facebook/opt-125m"
|
||||
|
||||
suffix = "test"
|
||||
try:
|
||||
result = subprocess.run(
|
||||
[
|
||||
sys.executable,
|
||||
f"{VLLM_PATH}/examples/features/tensorize_vllm_model.py",
|
||||
"--model",
|
||||
model_ref,
|
||||
"serialize",
|
||||
"--serialized-directory",
|
||||
str(tmp_path),
|
||||
"--suffix",
|
||||
suffix,
|
||||
"--serialization-kwargs",
|
||||
'{"limit_cpu_concurrency": 4}',
|
||||
],
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
except subprocess.CalledProcessError as e:
|
||||
print("Tensorizing failed.")
|
||||
print("STDOUT:\n", e.stdout)
|
||||
print("STDERR:\n", e.stderr)
|
||||
raise
|
||||
|
||||
assert "Successfully serialized" in result.stdout
|
||||
|
||||
# Next, try to serve with vllm serve
|
||||
model_uri = tmp_path / "vllm" / model_ref / suffix / "model.tensors"
|
||||
|
||||
model_loader_extra_config = {
|
||||
"tensorizer_uri": str(model_uri),
|
||||
"stream_kwargs": {
|
||||
"force_http": False,
|
||||
},
|
||||
"deserialization_kwargs": {
|
||||
"verify_hash": True,
|
||||
"num_readers": 8,
|
||||
},
|
||||
}
|
||||
|
||||
cmd = [
|
||||
"-m",
|
||||
"vllm.entrypoints.cli.main",
|
||||
"serve",
|
||||
"--host",
|
||||
"localhost",
|
||||
"--load-format",
|
||||
"tensorizer",
|
||||
model_ref,
|
||||
"--model-loader-extra-config",
|
||||
json.dumps(model_loader_extra_config, indent=2),
|
||||
]
|
||||
|
||||
proc = await asyncio.create_subprocess_exec(
|
||||
sys.executable,
|
||||
*cmd,
|
||||
stdout=asyncio.subprocess.PIPE,
|
||||
stderr=asyncio.subprocess.STDOUT,
|
||||
)
|
||||
|
||||
assert proc.stdout is not None
|
||||
fut = proc.stdout.readuntil(b"Application startup complete.")
|
||||
|
||||
try:
|
||||
await asyncio.wait_for(fut, 180)
|
||||
except TimeoutError:
|
||||
pytest.fail("Server did not start successfully")
|
||||
finally:
|
||||
proc.terminate()
|
||||
await proc.communicate()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("illegal_value", BLACKLISTED_TENSORIZER_ARGS)
|
||||
def test_blacklisted_parameter_for_loading(tmp_path, vllm_runner, capfd, illegal_value):
|
||||
serialization_params = {
|
||||
"limit_cpu_concurrency": 2,
|
||||
}
|
||||
|
||||
model_ref = "facebook/opt-125m"
|
||||
model_path = tmp_path / (model_ref + ".tensors")
|
||||
config = TensorizerConfig(
|
||||
tensorizer_uri=str(model_path), serialization_kwargs=serialization_params
|
||||
)
|
||||
|
||||
args = EngineArgs(model=model_ref)
|
||||
tensorize_vllm_model(args, config)
|
||||
|
||||
loader_tc = {"tensorizer_uri": str(model_path), illegal_value: "foo"}
|
||||
|
||||
try:
|
||||
vllm_runner(
|
||||
model_ref,
|
||||
load_format="tensorizer",
|
||||
model_loader_extra_config=loader_tc,
|
||||
)
|
||||
except RuntimeError:
|
||||
out, err = capfd.readouterr()
|
||||
combined_output = out + err
|
||||
assert (
|
||||
f"ValueError: {illegal_value} is not an allowed Tensorizer argument."
|
||||
) in combined_output
|
||||
@@ -1,603 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import fnmatch
|
||||
import multiprocessing as mp
|
||||
import os
|
||||
import shutil
|
||||
from tempfile import TemporaryDirectory
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm import LLM, SamplingParams
|
||||
from vllm.model_executor.model_loader import ShardedStateLoader
|
||||
from vllm.transformers_utils.repo_utils import hf_api
|
||||
|
||||
prompts = [
|
||||
"Hello, my name is",
|
||||
"The president of the United States is",
|
||||
"The capital of France is",
|
||||
"The future of AI is",
|
||||
]
|
||||
|
||||
# Create a sampling params object.
|
||||
sampling_params = SamplingParams(
|
||||
temperature=0,
|
||||
max_tokens=256,
|
||||
ignore_eos=True,
|
||||
)
|
||||
|
||||
|
||||
def test_filter_subtensors():
|
||||
state_dict = {
|
||||
"a": torch.empty(2),
|
||||
"b": torch.empty((2, 4)),
|
||||
"c": torch.empty((2, 4, 8)),
|
||||
}
|
||||
state_dict.update(
|
||||
{
|
||||
"x": state_dict["b"],
|
||||
"y": state_dict["c"][1, 2, :],
|
||||
"z": state_dict["c"][1, :, 4],
|
||||
}
|
||||
)
|
||||
filtered_state_dict = ShardedStateLoader._filter_subtensors(state_dict)
|
||||
assert tuple(filtered_state_dict.keys()) == ("a", "b", "c")
|
||||
for key, tensor in filtered_state_dict.items():
|
||||
# NOTE: don't use `equal` here, as the tensor might contain NaNs
|
||||
assert tensor is state_dict[key]
|
||||
|
||||
|
||||
def _mock_pp_group(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
world_size: int,
|
||||
*,
|
||||
pp_rank: int = 0,
|
||||
tp_rank: int = 0,
|
||||
tp_world_size: int = 1,
|
||||
) -> None:
|
||||
"""Stub the parallel-state accessors used by the sharded state loader."""
|
||||
monkeypatch.setattr(
|
||||
"vllm.distributed.get_pp_group",
|
||||
lambda: SimpleNamespace(world_size=world_size, rank_in_group=pp_rank),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"vllm.distributed.get_tp_group",
|
||||
lambda: SimpleNamespace(world_size=tp_world_size, rank_in_group=tp_rank),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"vllm.distributed.get_tensor_model_parallel_rank",
|
||||
lambda: tp_rank,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"vllm.distributed.get_pipeline_model_parallel_rank",
|
||||
lambda: pp_rank,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"world_size,expected",
|
||||
[
|
||||
(1, ShardedStateLoader.TP_ONLY_PATTERN),
|
||||
(2, ShardedStateLoader.PP_AND_TP_PATTERN),
|
||||
],
|
||||
)
|
||||
def test_get_default_pattern(monkeypatch, world_size, expected):
|
||||
"""The default pattern is chosen from the pipeline parallel group size."""
|
||||
_mock_pp_group(monkeypatch, world_size)
|
||||
assert ShardedStateLoader._get_default_pattern() == expected
|
||||
|
||||
|
||||
def test_render_default_pattern_default_tp(monkeypatch):
|
||||
"""Without a custom pattern and PP=1, legacy TP-only names are rendered."""
|
||||
_mock_pp_group(monkeypatch, 1)
|
||||
assert (
|
||||
ShardedStateLoader._render_default_pattern(part=0)
|
||||
== "model-rank-0-part-0.safetensors"
|
||||
)
|
||||
|
||||
|
||||
def test_render_default_pattern_default_pp(monkeypatch):
|
||||
"""Without a custom pattern and PP>1, pp/tp-aware names are rendered."""
|
||||
_mock_pp_group(monkeypatch, 2)
|
||||
assert (
|
||||
ShardedStateLoader._render_default_pattern(part=0)
|
||||
== "model-pp-0-tp-0-part-0.safetensors"
|
||||
)
|
||||
|
||||
|
||||
def test_render_default_pattern_custom_tp_only(monkeypatch):
|
||||
"""A custom TP-only pattern is honored when pipeline parallelism is off."""
|
||||
_mock_pp_group(monkeypatch, 1)
|
||||
assert (
|
||||
ShardedStateLoader._render_default_pattern(
|
||||
ShardedStateLoader.TP_ONLY_PATTERN, part=0
|
||||
)
|
||||
== "model-rank-0-part-0.safetensors"
|
||||
)
|
||||
|
||||
|
||||
def test_render_default_pattern_tp_only_with_pp_fails(monkeypatch):
|
||||
"""TP-only filenames would collide across pipeline stages; fail fast."""
|
||||
_mock_pp_group(monkeypatch, 2)
|
||||
with pytest.raises(ValueError, match="does not embed the pipeline parallel"):
|
||||
ShardedStateLoader._render_default_pattern(
|
||||
ShardedStateLoader.TP_ONLY_PATTERN, part=0
|
||||
)
|
||||
|
||||
|
||||
def test_render_default_pattern_pp_pattern_without_pp_fails(monkeypatch):
|
||||
"""A pp/tp-aware pattern cannot be rendered when pipeline parallelism is off."""
|
||||
_mock_pp_group(monkeypatch, 1)
|
||||
with pytest.raises(ValueError, match="requires pipeline parallelism"):
|
||||
ShardedStateLoader._render_default_pattern(
|
||||
ShardedStateLoader.PP_AND_TP_PATTERN, part=0
|
||||
)
|
||||
|
||||
|
||||
def test_render_default_pattern_custom_pp(monkeypatch):
|
||||
"""A custom pp/tp-aware pattern is honored when pipeline parallelism is on."""
|
||||
_mock_pp_group(monkeypatch, 2)
|
||||
assert (
|
||||
ShardedStateLoader._render_default_pattern(
|
||||
ShardedStateLoader.PP_AND_TP_PATTERN, part=0
|
||||
)
|
||||
== "model-pp-0-tp-0-part-0.safetensors"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"pattern,match",
|
||||
[
|
||||
# tp field present but pp field missing: pipeline stages with the
|
||||
# same tp rank would write the same file.
|
||||
(
|
||||
"model-tp-{tp_rank}-part-{part}.safetensors",
|
||||
"does not embed the pipeline parallel",
|
||||
),
|
||||
# pp field present but tp field missing: TP workers would overwrite
|
||||
# each other's shards.
|
||||
(
|
||||
"model-pp-{pp_rank}-part-{part}.safetensors",
|
||||
"does not embed the tensor parallel",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_render_default_pattern_incomplete_custom_patterns_fail(
|
||||
monkeypatch, pattern, match
|
||||
):
|
||||
"""A custom pattern must distinguish every active rank (pp and tp)."""
|
||||
_mock_pp_group(monkeypatch, 2, pp_rank=1, tp_rank=0, tp_world_size=2)
|
||||
with pytest.raises(ValueError, match=match):
|
||||
ShardedStateLoader._render_default_pattern(pattern, part=0)
|
||||
|
||||
|
||||
def test_render_default_pattern_no_rank_field_with_tp_fails(monkeypatch):
|
||||
"""Without any rank field, TP workers would collide even when PP is off."""
|
||||
_mock_pp_group(monkeypatch, 1, tp_world_size=2)
|
||||
with pytest.raises(ValueError, match="does not embed the tensor parallel"):
|
||||
ShardedStateLoader._render_default_pattern(
|
||||
"model-part-{part}.safetensors", part=0
|
||||
)
|
||||
|
||||
|
||||
def test_render_default_pattern_custom_pp_tp1_allows_pp_only(monkeypatch):
|
||||
"""With TP=1 a pp-only custom pattern is safe and must be accepted."""
|
||||
_mock_pp_group(monkeypatch, 2, pp_rank=1, tp_world_size=1)
|
||||
assert (
|
||||
ShardedStateLoader._render_default_pattern(
|
||||
"model-pp-{pp_rank}-part-{part}.safetensors", part=0
|
||||
)
|
||||
== "model-pp-1-part-0.safetensors"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"pattern,match",
|
||||
[
|
||||
# {{pp_rank}} renders as literal text: both stages would write the
|
||||
# same file, so it must not count as a pp field.
|
||||
(
|
||||
"model-{{pp_rank}}-tp-{tp_rank}-part-{part}.safetensors",
|
||||
"does not embed the pipeline parallel",
|
||||
),
|
||||
# Same for an escaped tp field: TP workers would overwrite each
|
||||
# other's shards.
|
||||
(
|
||||
"model-pp-{pp_rank}-tp-{{tp_rank}}-part-{part}.safetensors",
|
||||
"does not embed the tensor parallel",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_render_default_pattern_escaped_field_not_rank_field(
|
||||
monkeypatch, pattern, match
|
||||
):
|
||||
"""Escaped braces are literal text and cannot pose as rank fields."""
|
||||
_mock_pp_group(monkeypatch, 2, pp_rank=1, tp_rank=0, tp_world_size=2)
|
||||
with pytest.raises(ValueError, match=match):
|
||||
ShardedStateLoader._render_default_pattern(pattern, part=0)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"pattern,world_size,tp_world_size,pp_rank,tp_rank,expected",
|
||||
[
|
||||
# Mixed tp alias works when pipeline parallelism is on.
|
||||
(
|
||||
"model-pp-{pp_rank}-rank-{rank}-part-{part}.safetensors",
|
||||
2,
|
||||
2,
|
||||
1,
|
||||
0,
|
||||
"model-pp-1-rank-0-part-0.safetensors",
|
||||
),
|
||||
# tp_rank alias works for TP-only checkpoints too.
|
||||
(
|
||||
"model-tp-{tp_rank}-part-{part}.safetensors",
|
||||
1,
|
||||
2,
|
||||
0,
|
||||
3,
|
||||
"model-tp-3-part-0.safetensors",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_render_default_pattern_tp_aliases_accepted(
|
||||
monkeypatch, pattern, world_size, tp_world_size, pp_rank, tp_rank, expected
|
||||
):
|
||||
"""Both tp aliases are supplied at render time, so either works."""
|
||||
_mock_pp_group(
|
||||
monkeypatch,
|
||||
world_size,
|
||||
pp_rank=pp_rank,
|
||||
tp_rank=tp_rank,
|
||||
tp_world_size=tp_world_size,
|
||||
)
|
||||
assert ShardedStateLoader._render_default_pattern(pattern, part=0) == expected
|
||||
|
||||
|
||||
def test_render_default_pattern_unsupported_field_fails(monkeypatch):
|
||||
"""Unknown placeholders are rejected instead of KeyError-ing later."""
|
||||
_mock_pp_group(monkeypatch, 2, pp_rank=1, tp_world_size=2)
|
||||
with pytest.raises(ValueError, match="unsupported placeholder"):
|
||||
ShardedStateLoader._render_default_pattern(
|
||||
"model-{pp_rank}-{tp_rank}-{epoch}-part-{part}.safetensors", part=0
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"pp_rank,tp_rank,expected",
|
||||
[
|
||||
(0, 0, "model-pp-0-tp-0-part-0.safetensors"),
|
||||
(1, 2, "model-pp-1-tp-2-part-0.safetensors"),
|
||||
(3, 5, "model-pp-3-tp-5-part-0.safetensors"),
|
||||
],
|
||||
)
|
||||
def test_render_default_pattern_non_zero_ranks(monkeypatch, pp_rank, tp_rank, expected):
|
||||
"""The rendered filename must embed the actual pp/tp ranks, not zeros."""
|
||||
_mock_pp_group(monkeypatch, 2, pp_rank=pp_rank, tp_rank=tp_rank)
|
||||
assert (
|
||||
ShardedStateLoader._render_default_pattern(
|
||||
ShardedStateLoader.PP_AND_TP_PATTERN, part=0
|
||||
)
|
||||
== expected
|
||||
)
|
||||
|
||||
|
||||
def test_render_default_pattern_part_wildcard(monkeypatch):
|
||||
"""The glob wildcard '*' passes through as the part token for file lookup."""
|
||||
_mock_pp_group(monkeypatch, 1, tp_rank=4)
|
||||
assert (
|
||||
ShardedStateLoader._render_default_pattern(part="*")
|
||||
== "model-rank-4-part-*.safetensors"
|
||||
)
|
||||
|
||||
|
||||
def test_save_model_pp_aware_filenames(monkeypatch, tmp_path):
|
||||
"""save_model writes pp/tp-aware filenames for pipeline-parallel models."""
|
||||
_mock_pp_group(monkeypatch, 2, pp_rank=1, tp_rank=2)
|
||||
|
||||
saved: list[str] = []
|
||||
|
||||
def _capture(tensors, path, **kwargs):
|
||||
saved.append(os.path.basename(path))
|
||||
|
||||
monkeypatch.setattr("safetensors.torch.save_file", _capture)
|
||||
|
||||
model = SimpleNamespace(
|
||||
state_dict=lambda: {"w": torch.zeros(4, dtype=torch.float32)}
|
||||
)
|
||||
ShardedStateLoader.save_model(model, str(tmp_path))
|
||||
assert saved == ["model-pp-1-tp-2-part-0.safetensors"]
|
||||
|
||||
|
||||
def test_save_model_part_split(monkeypatch, tmp_path):
|
||||
"""save_model rolls over to a new part when max_size is exceeded."""
|
||||
_mock_pp_group(monkeypatch, 1, tp_rank=0)
|
||||
|
||||
saved: list[str] = []
|
||||
|
||||
def _capture(tensors, path, **kwargs):
|
||||
saved.append(os.path.basename(path))
|
||||
|
||||
monkeypatch.setattr("safetensors.torch.save_file", _capture)
|
||||
|
||||
# Two float32[4] tensors (16 B each); max_size=20 fits one but not two.
|
||||
model = SimpleNamespace(
|
||||
state_dict=lambda: {
|
||||
"a": torch.zeros(4, dtype=torch.float32),
|
||||
"b": torch.zeros(4, dtype=torch.float32),
|
||||
}
|
||||
)
|
||||
ShardedStateLoader.save_model(model, str(tmp_path), max_size=20)
|
||||
assert saved == [
|
||||
"model-rank-0-part-0.safetensors",
|
||||
"model-rank-0-part-1.safetensors",
|
||||
]
|
||||
|
||||
|
||||
def test_save_model_rejects_bad_pattern_before_overwrite(monkeypatch, tmp_path):
|
||||
"""An unsafe pattern is rejected before any shard file is written."""
|
||||
_mock_pp_group(monkeypatch, 2, pp_rank=1, tp_rank=0, tp_world_size=2)
|
||||
|
||||
# With the escaped pp field this pattern renders both stages to the
|
||||
# same literal name; the validation must fire before any write.
|
||||
colliding = tmp_path / "model-{pp_rank}-tp-0-part-0.safetensors"
|
||||
colliding.write_bytes(b"existing shard")
|
||||
before = sorted(p.name for p in tmp_path.iterdir())
|
||||
|
||||
model = SimpleNamespace(
|
||||
state_dict=lambda: {"w": torch.zeros(4, dtype=torch.float32)}
|
||||
)
|
||||
with pytest.raises(ValueError, match="does not embed the pipeline parallel"):
|
||||
ShardedStateLoader.save_model(
|
||||
model,
|
||||
str(tmp_path),
|
||||
pattern="model-{{pp_rank}}-tp-{tp_rank}-part-{part}.safetensors",
|
||||
)
|
||||
|
||||
# The pre-existing shard is untouched and nothing new was written.
|
||||
assert colliding.read_bytes() == b"existing shard"
|
||||
assert sorted(p.name for p in tmp_path.iterdir()) == before
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def llama_3p2_1b_files():
|
||||
input_dir = hf_api().snapshot_download(
|
||||
"meta-llama/Llama-3.2-1B-Instruct", ignore_patterns=["*.bin*", "original/*"]
|
||||
)
|
||||
|
||||
yield input_dir
|
||||
|
||||
|
||||
def _run_writer(input_dir, output_dir, weights_patterns, **kwargs):
|
||||
llm_sharded_writer = LLM(model=input_dir, **kwargs)
|
||||
|
||||
# Dump worker states to output directory
|
||||
llm_sharded_writer.llm_engine.engine_core.save_sharded_state(path=output_dir)
|
||||
|
||||
# Copy metadata files to output directory
|
||||
for file in os.listdir(input_dir):
|
||||
if os.path.isdir(os.path.join(input_dir, file)):
|
||||
shutil.copytree(
|
||||
os.path.join(input_dir, file), os.path.join(output_dir, file)
|
||||
)
|
||||
elif not any(fnmatch.fnmatch(file, ext) for ext in weights_patterns):
|
||||
shutil.copy(os.path.join(input_dir, file), output_dir)
|
||||
|
||||
|
||||
def _run_generate(input_dir, queue: mp.Queue, **kwargs):
|
||||
llm = LLM(model=input_dir, **kwargs)
|
||||
gen = llm.generate(prompts, sampling_params)
|
||||
queue.put([g.outputs[0].__dict__ for g in gen])
|
||||
queue.close()
|
||||
queue.join_thread()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("enable_lora", [False, True])
|
||||
@pytest.mark.parametrize("tp_size", [1, 2])
|
||||
def test_sharded_state_loader(
|
||||
enable_lora, tp_size, num_gpus_available, llama_3p2_1b_files
|
||||
):
|
||||
if num_gpus_available < tp_size:
|
||||
pytest.skip(f"Not enough GPUs for tensor parallelism {tp_size}")
|
||||
|
||||
weights_patterns = ("*.safetensors",)
|
||||
gpu_memory_utilization = 0.8
|
||||
input_dir = llama_3p2_1b_files
|
||||
ctx = mp.get_context("spawn")
|
||||
|
||||
# Keep batching deterministic: this test compares exact greedy outputs
|
||||
# across separate engine processes, whose schedulers may otherwise form
|
||||
# different batches depending on prompt-processing timing.
|
||||
platform_args = {"max_num_seqs": 1}
|
||||
|
||||
# Run in separate processes for memory & CUDA isolation
|
||||
with TemporaryDirectory() as output_dir:
|
||||
p = ctx.Process(
|
||||
target=_run_writer,
|
||||
args=(input_dir, output_dir, weights_patterns),
|
||||
kwargs=dict(
|
||||
tensor_parallel_size=tp_size,
|
||||
gpu_memory_utilization=gpu_memory_utilization,
|
||||
enforce_eager=True,
|
||||
**platform_args,
|
||||
),
|
||||
)
|
||||
p.start()
|
||||
p.join()
|
||||
|
||||
queue = ctx.Queue()
|
||||
|
||||
p = ctx.Process(
|
||||
target=_run_generate,
|
||||
args=(input_dir, queue),
|
||||
kwargs=dict(
|
||||
enable_lora=enable_lora,
|
||||
gpu_memory_utilization=gpu_memory_utilization,
|
||||
tensor_parallel_size=tp_size,
|
||||
**platform_args,
|
||||
),
|
||||
)
|
||||
p.start()
|
||||
# Call queue.get() before p.join() to prevent deadlock:
|
||||
# If p.join() is called before queue.get() and the queue is full,
|
||||
# the child process may block while writing to the queue and never
|
||||
# terminate, causing the parent to wait indefinitely on p.join().
|
||||
# See: https://github.com/vllm-project/vllm/pull/22371#discussion_r2257773814
|
||||
out_before = queue.get()
|
||||
p.join()
|
||||
queue.close()
|
||||
queue.join_thread()
|
||||
|
||||
queue = ctx.Queue()
|
||||
|
||||
p = ctx.Process(
|
||||
target=_run_generate,
|
||||
args=(output_dir, queue),
|
||||
kwargs=dict(
|
||||
enable_lora=enable_lora,
|
||||
gpu_memory_utilization=gpu_memory_utilization,
|
||||
tensor_parallel_size=tp_size,
|
||||
load_format="sharded_state",
|
||||
**platform_args,
|
||||
),
|
||||
)
|
||||
p.start()
|
||||
# Call queue.get() before p.join() to prevent deadlock:
|
||||
# If p.join() is called before queue.get() and the queue is full,
|
||||
# the child process may block while writing to the queue and never
|
||||
# terminate, causing the parent to wait indefinitely on p.join().
|
||||
# See: https://github.com/vllm-project/vllm/pull/22371#discussion_r2257773814
|
||||
out_after = queue.get()
|
||||
p.join()
|
||||
queue.close()
|
||||
queue.join_thread()
|
||||
|
||||
assert out_before == out_after
|
||||
|
||||
|
||||
def _run_sharded_state_round_trip(
|
||||
input_dir: str,
|
||||
*,
|
||||
tp_size: int,
|
||||
pp_size: int,
|
||||
num_gpus_available: int,
|
||||
) -> None:
|
||||
"""Save a TP/PP sharded checkpoint, reload it, and compare outputs.
|
||||
|
||||
Args:
|
||||
input_dir: directory containing the original (un-sharded) model.
|
||||
tp_size: tensor parallel size used for both save and load.
|
||||
pp_size: pipeline parallel size used for both save and load.
|
||||
num_gpus_available: number of GPUs on the current machine.
|
||||
|
||||
"""
|
||||
if num_gpus_available < tp_size * pp_size:
|
||||
pytest.skip(f"Not enough GPUs for TP={tp_size}, PP={pp_size}")
|
||||
|
||||
weights_patterns = ("*.safetensors",)
|
||||
gpu_memory_utilization = 0.8
|
||||
ctx = mp.get_context("spawn")
|
||||
|
||||
# Keep batching deterministic: this test compares exact greedy outputs
|
||||
# across separate engine processes, whose schedulers may otherwise form
|
||||
# different batches depending on prompt-processing timing.
|
||||
platform_args = {"max_num_seqs": 1}
|
||||
parallel_args = dict(
|
||||
tensor_parallel_size=tp_size,
|
||||
pipeline_parallel_size=pp_size,
|
||||
)
|
||||
|
||||
with TemporaryDirectory() as output_dir:
|
||||
p = ctx.Process(
|
||||
target=_run_writer,
|
||||
args=(input_dir, output_dir, weights_patterns),
|
||||
kwargs=dict(
|
||||
**parallel_args,
|
||||
gpu_memory_utilization=gpu_memory_utilization,
|
||||
enforce_eager=True,
|
||||
**platform_args,
|
||||
),
|
||||
)
|
||||
p.start()
|
||||
p.join()
|
||||
assert p.exitcode == 0, f"writer process failed with exit code {p.exitcode}"
|
||||
|
||||
if pp_size > 1:
|
||||
# PP checkpoints must use the pp/tp-aware naming scheme so that
|
||||
# different pipeline stages write distinct files.
|
||||
sharded_files = [
|
||||
f for f in os.listdir(output_dir) if f.endswith(".safetensors")
|
||||
]
|
||||
assert sharded_files, "no sharded checkpoint files were written"
|
||||
assert any(
|
||||
fnmatch.fnmatch(f, "model-pp-*-tp-*-part-*.safetensors")
|
||||
for f in sharded_files
|
||||
), f"expected PP-aware filenames, got {sharded_files}"
|
||||
|
||||
queue = ctx.Queue()
|
||||
p = ctx.Process(
|
||||
target=_run_generate,
|
||||
args=(input_dir, queue),
|
||||
kwargs=dict(
|
||||
enable_lora=False,
|
||||
gpu_memory_utilization=gpu_memory_utilization,
|
||||
**parallel_args,
|
||||
**platform_args,
|
||||
),
|
||||
)
|
||||
p.start()
|
||||
# Call queue.get() before p.join() to prevent deadlock:
|
||||
# If p.join() is called before queue.get() and the queue is full,
|
||||
# the child process may block while writing to the queue and never
|
||||
# terminate, causing the parent to wait indefinitely on p.join().
|
||||
# See: https://github.com/vllm-project/vllm/pull/22371#discussion_r2257773814
|
||||
out_before = queue.get()
|
||||
p.join()
|
||||
assert p.exitcode == 0, f"generate process failed with exit code {p.exitcode}"
|
||||
queue.close()
|
||||
queue.join_thread()
|
||||
|
||||
queue = ctx.Queue()
|
||||
p = ctx.Process(
|
||||
target=_run_generate,
|
||||
args=(output_dir, queue),
|
||||
kwargs=dict(
|
||||
enable_lora=False,
|
||||
gpu_memory_utilization=gpu_memory_utilization,
|
||||
load_format="sharded_state",
|
||||
**parallel_args,
|
||||
**platform_args,
|
||||
),
|
||||
)
|
||||
p.start()
|
||||
out_after = queue.get()
|
||||
p.join()
|
||||
assert p.exitcode == 0, f"generate process failed with exit code {p.exitcode}"
|
||||
queue.close()
|
||||
queue.join_thread()
|
||||
|
||||
assert out_before == out_after
|
||||
|
||||
|
||||
def test_sharded_state_loader_pp(num_gpus_available, llama_3p2_1b_files):
|
||||
"""Round-trip sharded state save/load with pipeline parallelism (PP=2)."""
|
||||
_run_sharded_state_round_trip(
|
||||
llama_3p2_1b_files,
|
||||
tp_size=1,
|
||||
pp_size=2,
|
||||
num_gpus_available=num_gpus_available,
|
||||
)
|
||||
|
||||
|
||||
def test_sharded_state_loader_pp_tp(num_gpus_available, llama_3p2_1b_files):
|
||||
"""Round-trip sharded state save/load with pipeline + tensor parallelism
|
||||
(PP=2, TP=2)."""
|
||||
_run_sharded_state_round_trip(
|
||||
llama_3p2_1b_files,
|
||||
tp_size=2,
|
||||
pp_size=2,
|
||||
num_gpus_available=num_gpus_available,
|
||||
)
|
||||
@@ -1,486 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""End-to-end tests for the IPC weight cache loader.
|
||||
|
||||
A cold start (no weight cache daemon, so the loader falls back to disk) and
|
||||
warm restarts (weights mapped from the daemon via CUDA IPC) must both serve
|
||||
identical outputs.
|
||||
"""
|
||||
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import threading
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
import regex as re
|
||||
|
||||
from vllm import SamplingParams
|
||||
from vllm.assets.image import ImageAsset
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.network_utils import get_open_port
|
||||
|
||||
|
||||
class WeightCacheDaemon:
|
||||
"""Context manager running the real weight cache daemon as a subprocess."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
model: str,
|
||||
tp_size: int,
|
||||
extra_args: list[str] | None = None,
|
||||
):
|
||||
# Short base path: Unix socket paths are limited to ~107 characters.
|
||||
self.socket_dir = tempfile.mkdtemp(prefix="vllm_ipc_")
|
||||
self.health_port = get_open_port()
|
||||
self._cmd = [
|
||||
sys.executable,
|
||||
"-m",
|
||||
"vllm.entrypoints.cli.main",
|
||||
"preload",
|
||||
"--model",
|
||||
model,
|
||||
"--tensor-parallel-size",
|
||||
str(tp_size),
|
||||
"--weight-cache-socket-dir",
|
||||
self.socket_dir,
|
||||
"--enforce-eager",
|
||||
"--weight-cache-health-host",
|
||||
"127.0.0.1",
|
||||
"--weight-cache-health-port",
|
||||
str(self.health_port),
|
||||
*(extra_args or []),
|
||||
]
|
||||
self._proc: subprocess.Popen | None = None
|
||||
|
||||
def __enter__(self) -> "WeightCacheDaemon":
|
||||
self._proc = subprocess.Popen(
|
||||
self._cmd, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, text=True
|
||||
)
|
||||
threading.Thread(target=self._drain_stderr, daemon=True).start()
|
||||
return self
|
||||
|
||||
def __exit__(self, *exc_info) -> None:
|
||||
self._stop()
|
||||
|
||||
def _drain_stderr(self) -> None:
|
||||
assert self._proc is not None and self._proc.stderr is not None
|
||||
self._proc.stderr.read()
|
||||
|
||||
def check_health(self) -> None:
|
||||
url = f"http://127.0.0.1:{self.health_port}/health"
|
||||
try:
|
||||
with urllib.request.urlopen(url, timeout=5) as response:
|
||||
assert response.status == 200
|
||||
except urllib.error.HTTPError as error:
|
||||
raise AssertionError(
|
||||
f"Weight cache daemon is not ready: HTTP {error.code}"
|
||||
) from error
|
||||
|
||||
def wait_until_ready(self, timeout_s: float = 900) -> None:
|
||||
"""Poll until READY (which covers the autotune tuners) or the preload
|
||||
process dies."""
|
||||
deadline = time.monotonic() + timeout_s
|
||||
while True:
|
||||
try:
|
||||
self.check_health()
|
||||
return
|
||||
except (AssertionError, urllib.error.URLError):
|
||||
if time.monotonic() > deadline:
|
||||
raise
|
||||
if self._proc is not None and self._proc.poll() is not None:
|
||||
raise AssertionError("vllm preload exited before ready") from None
|
||||
time.sleep(2)
|
||||
|
||||
def _stop(self) -> None:
|
||||
assert self._proc is not None
|
||||
self._proc.terminate()
|
||||
try:
|
||||
self._proc.wait(timeout=60)
|
||||
except subprocess.TimeoutExpired:
|
||||
self._proc.kill()
|
||||
self._proc.wait()
|
||||
shutil.rmtree(self.socket_dir, ignore_errors=True)
|
||||
|
||||
|
||||
@dataclass
|
||||
class ModelCase:
|
||||
model: str
|
||||
prompts: list[str]
|
||||
images: list | None = None
|
||||
llm_kwargs: dict[str, Any] = field(default_factory=dict)
|
||||
daemon_args: list[str] = field(default_factory=list)
|
||||
# Additionally assert the daemons tune in place before serving and write
|
||||
# the on-disk FlashInfer autotune cache, and the restart engine reuses it
|
||||
# (needs SM90+/FlashInfer).
|
||||
check_flashinfer_cache: bool = False
|
||||
|
||||
|
||||
def generate(
|
||||
vllm_runner,
|
||||
case: ModelCase,
|
||||
socket_dir: str | None,
|
||||
fallback: bool,
|
||||
):
|
||||
extra_config = (
|
||||
{} if socket_dir is None else {"socket_dir": socket_dir, "fallback": fallback}
|
||||
)
|
||||
with vllm_runner(
|
||||
case.model,
|
||||
load_format="auto" if socket_dir is None else "ipc_cache",
|
||||
model_loader_extra_config=extra_config,
|
||||
# Greedy outputs are compared across engine restarts; cap the batch
|
||||
# size so scheduling cannot change the results.
|
||||
max_num_seqs=1,
|
||||
**case.llm_kwargs,
|
||||
) as llm:
|
||||
sampling_params = SamplingParams(temperature=0, max_tokens=16, ignore_eos=True)
|
||||
return llm.generate(case.prompts, sampling_params, images=case.images)
|
||||
|
||||
|
||||
# Both hybrid-recurrent models need chunked prefill for their mamba-style
|
||||
# cache mode.
|
||||
QWEN_CASE = ModelCase(
|
||||
model="Qwen/Qwen3.5-0.8B",
|
||||
prompts=[
|
||||
"Hello, my name is",
|
||||
"The capital of France is",
|
||||
],
|
||||
llm_kwargs=dict(
|
||||
gpu_memory_utilization=0.3,
|
||||
enforce_eager=True,
|
||||
enable_chunked_prefill=True,
|
||||
),
|
||||
)
|
||||
|
||||
# Kimi K3 adds the mxfp4-pack MoE path (post-load kernel setup runs in
|
||||
# pre-processed mode) and a vision tower, exercised with an image request.
|
||||
K3_CASE = ModelCase(
|
||||
model="tiny-random/kimi-k3",
|
||||
prompts=[
|
||||
"The capital of France is",
|
||||
"def quicksort(arr):",
|
||||
"<|kimi_image_placeholder|>What is shown in the image?",
|
||||
],
|
||||
images=[None, None, ImageAsset("stop_sign").pil_image],
|
||||
llm_kwargs=dict(
|
||||
gpu_memory_utilization=0.3,
|
||||
enforce_eager=True,
|
||||
trust_remote_code=True,
|
||||
enable_chunked_prefill=True,
|
||||
# The image placeholder expands to >1k tokens for the stop_sign asset.
|
||||
max_model_len=4096,
|
||||
),
|
||||
daemon_args=["--trust-remote-code"],
|
||||
# Kimi K3 has tunable FlashInfer ops (a dense model like Qwen2.5 tunes
|
||||
# nothing and upstream never writes the cache file for it).
|
||||
check_flashinfer_cache=True,
|
||||
)
|
||||
|
||||
# Qwen3.5-0.8B ships one MTP layer in the target checkpoint, so method="mtp"
|
||||
# loads the draft from the same model. The daemon must cache it in its draft
|
||||
# group for the warm runs (fallback=False) to succeed.
|
||||
QWEN_MTP_CASE = ModelCase(
|
||||
model="Qwen/Qwen3.5-0.8B",
|
||||
prompts=[
|
||||
"Hello, my name is",
|
||||
"The capital of France is",
|
||||
],
|
||||
llm_kwargs=dict(
|
||||
gpu_memory_utilization=0.3,
|
||||
enforce_eager=True,
|
||||
enable_chunked_prefill=True,
|
||||
speculative_config={"method": "mtp", "num_speculative_tokens": 1},
|
||||
),
|
||||
daemon_args=[
|
||||
"--speculative-config",
|
||||
'{"method": "mtp", "num_speculative_tokens": 1}',
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"case",
|
||||
[QWEN_CASE, K3_CASE, QWEN_MTP_CASE],
|
||||
ids=["qwen3.5", "kimi-k3", "qwen3.5-mtp"],
|
||||
)
|
||||
def test_ipc_cache_cold_start_and_warm_restart(
|
||||
vllm_runner, case: ModelCase, tmp_path, monkeypatch, capfd
|
||||
):
|
||||
"""Cold start falls back to disk; warm restarts load weights via CUDA IPC.
|
||||
|
||||
All runs must produce outputs identical to a default-loader baseline. The
|
||||
warm runs disable the disk fallback, so they only pass if the weights
|
||||
really came from the daemon — for the MTP case, both the target's and the
|
||||
draft's daemon groups.
|
||||
|
||||
Cases with ``check_flashinfer_cache`` additionally cover in-daemon
|
||||
autotune (default-on with `--enable-flashinfer-autotune`): each daemon
|
||||
tunes before binding its socket and must write the on-disk FlashInfer
|
||||
autotune cache, and the restart engine must load tactics from it instead
|
||||
of re-profiling. The daemon's cache file is keyed with the
|
||||
OPENAI_API_SERVER batch defaults while these engines run under LLM_CLASS,
|
||||
so they never hit the daemon's file by design; reuse is proven against
|
||||
the file the earlier in-process engines wrote.
|
||||
"""
|
||||
if not current_platform.is_cuda_alike():
|
||||
pytest.skip("Weight cache IPC sharing requires CUDA or ROCm")
|
||||
if case is K3_CASE and not current_platform.is_device_capability_family(100):
|
||||
pytest.skip("Kimi K3 IPC weight cache requires an SM100 MXFP4 backend")
|
||||
if case.check_flashinfer_cache:
|
||||
from vllm.utils.flashinfer import has_flashinfer
|
||||
|
||||
if not (has_flashinfer() and current_platform.has_device_capability(90)):
|
||||
pytest.skip("FlashInfer autotune requires FlashInfer and SM90+")
|
||||
# Isolate the on-disk autotune cache. The engines are in-process
|
||||
# (envs read lazily); the daemon subprocesses spawn later and
|
||||
# inherit it.
|
||||
monkeypatch.setenv("VLLM_FLASHINFER_AUTOTUNE_CACHE_DIR", str(tmp_path))
|
||||
|
||||
# Baseline: plain disk loading with the default loader.
|
||||
baseline_outputs = generate(
|
||||
vllm_runner,
|
||||
case,
|
||||
None,
|
||||
fallback=True,
|
||||
)
|
||||
assert all(text for _, texts in baseline_outputs for text in texts)
|
||||
|
||||
# Cold start: no daemon is serving, so the loader falls back to disk.
|
||||
with tempfile.TemporaryDirectory(prefix="vllm_ipc_empty_") as empty_socket_dir:
|
||||
cold_outputs = generate(vllm_runner, case, empty_socket_dir, fallback=True)
|
||||
|
||||
if case.check_flashinfer_cache:
|
||||
# The baseline engine already wrote its own (LLM_CLASS-keyed) table.
|
||||
engine_cache_files = set(tmp_path.rglob("autotune_configs*.json"))
|
||||
|
||||
with WeightCacheDaemon(
|
||||
case.model,
|
||||
tp_size=1,
|
||||
extra_args=case.daemon_args,
|
||||
) as d:
|
||||
warm_outputs = generate(vllm_runner, case, d.socket_dir, fallback=False)
|
||||
d.wait_until_ready()
|
||||
if case.check_flashinfer_cache:
|
||||
# Each daemon tunes before its ready report; exactly one new file
|
||||
# (their OPENAI_API_SERVER-keyed table) must appear.
|
||||
tuner_files = set(tmp_path.rglob("autotune_configs*.json"))
|
||||
tuner_files -= engine_cache_files
|
||||
assert len(tuner_files) == 1, (
|
||||
f"expected the tuner's tuned table, got {tuner_files}"
|
||||
)
|
||||
# Warm restart: a second engine lifetime against the same daemon.
|
||||
if case.check_flashinfer_cache:
|
||||
capfd.readouterr() # drain: the assertion reads the increment
|
||||
restart_outputs = generate(vllm_runner, case, d.socket_dir, fallback=False)
|
||||
if case.check_flashinfer_cache:
|
||||
# The restart's EngineCore subprocess inherits pytest's fds; the
|
||||
# flashinfer autotuner logs "[Autotuner]: Loaded N configs from
|
||||
# <path>" (INFO, not once-deduplicated) on a disk cache hit.
|
||||
out = capfd.readouterr()
|
||||
m = re.search(
|
||||
r"\[Autotuner\]: Loaded (\d+) configs from", out.out + out.err
|
||||
)
|
||||
assert m and int(m.group(1)) > 0, (
|
||||
"restart engine did not load the on-disk autotune cache"
|
||||
)
|
||||
|
||||
assert cold_outputs == baseline_outputs
|
||||
assert warm_outputs == baseline_outputs
|
||||
assert restart_outputs == baseline_outputs
|
||||
|
||||
|
||||
def _parallel(**kw):
|
||||
from types import SimpleNamespace
|
||||
|
||||
base = dict(
|
||||
tensor_parallel_size=1,
|
||||
pipeline_parallel_size=1,
|
||||
data_parallel_size=1,
|
||||
data_parallel_size_local=1,
|
||||
data_parallel_rank=0,
|
||||
nnodes=1,
|
||||
node_rank=0,
|
||||
)
|
||||
base.update(kw)
|
||||
return SimpleNamespace(**base)
|
||||
|
||||
|
||||
def test_daemon_places_tp_and_dp_ranks_on_local_gpus():
|
||||
"""Each node serves a contiguous global-rank block (DP-major), so without
|
||||
DP local GPU i is TP rank r*local+i, and single-node DP launchers offset
|
||||
by --data-parallel-start-rank."""
|
||||
from vllm.model_executor.model_loader.weight_cache.daemon import plan_local_ranks
|
||||
|
||||
tp_second_node = _parallel(tensor_parallel_size=8, nnodes=2, node_rank=1)
|
||||
assert plan_local_ranks(tp_second_node) == [(i, 0, 0, 4 + i) for i in range(4)]
|
||||
|
||||
dp_third_node = _parallel(
|
||||
data_parallel_size=16, data_parallel_size_local=4, data_parallel_rank=8
|
||||
)
|
||||
assert plan_local_ranks(dp_third_node) == [(i, 8 + i, 0, 0) for i in range(4)]
|
||||
|
||||
dp_tp = _parallel(
|
||||
tensor_parallel_size=2,
|
||||
data_parallel_size=4,
|
||||
data_parallel_size_local=2,
|
||||
data_parallel_rank=2,
|
||||
)
|
||||
assert plan_local_ranks(dp_tp) == [
|
||||
(0, 2, 0, 0),
|
||||
(1, 2, 0, 1),
|
||||
(2, 3, 0, 0),
|
||||
(3, 3, 0, 1),
|
||||
]
|
||||
|
||||
# DP + nnodes: node-local TP replicas
|
||||
dp_tp_node0 = _parallel(tensor_parallel_size=8, data_parallel_size=2, nnodes=2)
|
||||
assert plan_local_ranks(dp_tp_node0) == [(i, 0, 0, i) for i in range(8)]
|
||||
dp_tp_node1 = _parallel(
|
||||
tensor_parallel_size=8, data_parallel_size=2, nnodes=2, node_rank=1
|
||||
)
|
||||
assert plan_local_ranks(dp_tp_node1) == [(i, 1, 0, i) for i in range(8)]
|
||||
|
||||
# DP + nnodes: TP group spans nodes (TP8 x DP2 on 4 nodes)
|
||||
tp_span_node1 = _parallel(
|
||||
tensor_parallel_size=8, data_parallel_size=2, nnodes=4, node_rank=1
|
||||
)
|
||||
assert plan_local_ranks(tp_span_node1) == [(i, 0, 0, 4 + i) for i in range(4)]
|
||||
tp_span_node2 = _parallel(
|
||||
tensor_parallel_size=8, data_parallel_size=2, nnodes=4, node_rank=2
|
||||
)
|
||||
assert plan_local_ranks(tp_span_node2) == [(i, 1, 0, i) for i in range(4)]
|
||||
|
||||
# DP + nnodes: several DP replicas per node (TP4 x DP4 on 2 nodes)
|
||||
dp_multi_node0 = _parallel(tensor_parallel_size=4, data_parallel_size=4, nnodes=2)
|
||||
assert plan_local_ranks(dp_multi_node0) == [(i, i // 4, 0, i % 4) for i in range(8)]
|
||||
|
||||
# PP adds one daemon placement per stage while preserving TP rank order.
|
||||
pp_tp = _parallel(tensor_parallel_size=2, pipeline_parallel_size=2)
|
||||
assert plan_local_ranks(pp_tp) == [
|
||||
(0, 0, 0, 0),
|
||||
(1, 0, 0, 1),
|
||||
(2, 0, 1, 0),
|
||||
(3, 0, 1, 1),
|
||||
]
|
||||
|
||||
|
||||
def test_daemon_rejects_unmappable_parallelism():
|
||||
from vllm.model_executor.model_loader.weight_cache.daemon import (
|
||||
_reject_unsupported_parallelism,
|
||||
)
|
||||
|
||||
_reject_unsupported_parallelism(
|
||||
_parallel(
|
||||
data_parallel_size=16, data_parallel_size_local=4, data_parallel_rank=12
|
||||
)
|
||||
)
|
||||
# DP combines with --nnodes when the world size divides evenly.
|
||||
_reject_unsupported_parallelism(
|
||||
_parallel(tensor_parallel_size=4, data_parallel_size=4, nnodes=2)
|
||||
)
|
||||
_reject_unsupported_parallelism(_parallel(pipeline_parallel_size=2))
|
||||
with pytest.raises(ValueError, match="evenly divide"):
|
||||
_reject_unsupported_parallelism(_parallel(tensor_parallel_size=3, nnodes=2))
|
||||
with pytest.raises(ValueError, match="evenly divide"):
|
||||
_reject_unsupported_parallelism(
|
||||
_parallel(tensor_parallel_size=2, data_parallel_size=3, nnodes=4)
|
||||
)
|
||||
with pytest.raises(ValueError, match="exceeds"):
|
||||
_reject_unsupported_parallelism(
|
||||
_parallel(
|
||||
data_parallel_size=16, data_parallel_size_local=4, data_parallel_rank=13
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def test_weight_cache_key_distinguishes_dp_ranks():
|
||||
from dataclasses import replace
|
||||
|
||||
from vllm.model_executor.model_loader.weight_cache.protocol import WeightCacheKey
|
||||
|
||||
key = WeightCacheKey(
|
||||
checkpoint="ckpt",
|
||||
model_arch="Arch",
|
||||
tp_size=1,
|
||||
tp_rank=0,
|
||||
dtype="bf16",
|
||||
quantization=None,
|
||||
quant_config_hash="h",
|
||||
revision=None,
|
||||
vllm_version="v",
|
||||
dp_size=16,
|
||||
dp_rank=3,
|
||||
pp_size=2,
|
||||
pp_rank=1,
|
||||
)
|
||||
assert key.mismatched_fields(replace(key, dp_rank=4)) == ["dp_rank"]
|
||||
assert key.mismatched_fields(replace(key, dp_size=8, dp_rank=3)) == ["dp_size"]
|
||||
assert key.mismatched_fields(replace(key, pp_rank=0)) == ["pp_rank"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("hf_quant_config", [None, {"quant_algo": "NVFP4"}])
|
||||
def test_weight_cache_key_distinguishes_nvfp4_activation_override(
|
||||
tmp_path, hf_quant_config
|
||||
):
|
||||
"""Cache keys isolate activation modes and reject ambiguous legacy hashes."""
|
||||
import json
|
||||
from dataclasses import replace
|
||||
from types import SimpleNamespace
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.config.quantization import QuantizationConfigArgs
|
||||
from vllm.model_executor.model_loader.weight_cache.protocol import WeightCacheKey
|
||||
from vllm.utils.hashing import safe_hash
|
||||
|
||||
model_config = SimpleNamespace(
|
||||
model=str(tmp_path),
|
||||
dtype=torch.bfloat16,
|
||||
quantization="modelopt_fp4",
|
||||
quantization_config=None,
|
||||
revision=None,
|
||||
hf_config=SimpleNamespace(
|
||||
architectures=["NemotronHForCausalLM"],
|
||||
quantization_config=hf_quant_config,
|
||||
),
|
||||
)
|
||||
static_key = WeightCacheKey.from_model_config(model_config, tp_size=1, tp_rank=0)
|
||||
model_config.quantization_config = QuantizationConfigArgs(
|
||||
moe={"activation": "nvfp4_per_token"}
|
||||
)
|
||||
per_token_key = WeightCacheKey.from_model_config(model_config, tp_size=1, tp_rank=0)
|
||||
assert static_key.mismatched_fields(per_token_key) == ["quant_config_hash"]
|
||||
|
||||
legacy_hash = (
|
||||
""
|
||||
if hf_quant_config is None
|
||||
else safe_hash(
|
||||
json.dumps(hf_quant_config, sort_keys=True).encode(),
|
||||
usedforsecurity=False,
|
||||
).hexdigest()
|
||||
)
|
||||
assert static_key.mismatched_fields(
|
||||
replace(static_key, quant_config_hash=legacy_hash)
|
||||
) == ["quant_config_hash"]
|
||||
|
||||
|
||||
def test_ipc_loader_copy_mode_reports_no_external_weight_memory():
|
||||
"""Copy mode clones the weights into the engine, so nothing is external
|
||||
(vllm_config is never touched)."""
|
||||
from vllm.config import LoadConfig
|
||||
from vllm.model_executor.model_loader.weight_cache.ipc_loader import (
|
||||
IpcModelLoader,
|
||||
)
|
||||
|
||||
copy_mode = LoadConfig(
|
||||
load_format="ipc_cache", model_loader_extra_config={"mode": "copy"}
|
||||
)
|
||||
loader = IpcModelLoader(copy_mode)
|
||||
assert loader.get_external_weight_memory(None) == 0
|
||||
@@ -1,93 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
from torch.testing._internal.two_tensor import TwoTensor
|
||||
|
||||
from vllm.config import ParallelConfig
|
||||
from vllm.model_executor import parameter
|
||||
from vllm.model_executor.model_loader.weight_checksum import (
|
||||
compute_tensor_digests,
|
||||
zero_weights,
|
||||
)
|
||||
from vllm.v1.worker import gpu_worker
|
||||
from vllm.v1.worker.gpu_worker import Worker
|
||||
|
||||
pytestmark = pytest.mark.cpu_test
|
||||
|
||||
|
||||
class _Model(nn.Module):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.linear = nn.Linear(2, 2)
|
||||
nn.init.ones_(self.linear.weight)
|
||||
nn.init.ones_(self.linear.bias)
|
||||
# A strided view, which cannot be reinterpreted as bytes in place.
|
||||
self.strided = nn.Parameter(torch.arange(1.0, 7.0)[::2])
|
||||
self.scalar = nn.Parameter(torch.tensor(2.0))
|
||||
# Loading never restores buffers, so reset must leave them alone.
|
||||
self.register_buffer("k_scale", torch.tensor(2.0))
|
||||
|
||||
|
||||
def test_digests_cover_every_parameter():
|
||||
assert set(compute_tensor_digests(_Model())) == {
|
||||
"linear.weight",
|
||||
"linear.bias",
|
||||
"strided",
|
||||
"scalar",
|
||||
}
|
||||
|
||||
|
||||
def test_zero_weights_changes_every_parameter_and_no_buffer():
|
||||
model = _Model()
|
||||
before = compute_tensor_digests(model)
|
||||
zero_weights(model)
|
||||
after = compute_tensor_digests(model)
|
||||
assert all(before[name] != after[name] for name in before)
|
||||
assert model.k_scale.item() == 2.0
|
||||
|
||||
|
||||
def test_shared_weight_partitions_are_hashed_and_reset(monkeypatch):
|
||||
monkeypatch.setattr(parameter, "get_tensor_model_parallel_rank", lambda: 0)
|
||||
monkeypatch.setattr(parameter, "get_tensor_model_parallel_world_size", lambda: 1)
|
||||
model = nn.Module()
|
||||
model.transform = parameter.SharedWeightParameter(weight_loader=None)
|
||||
model.transform.add_partition(0, object(), 2, 2)
|
||||
model.transform.partitions[0].data.fill_(3.0)
|
||||
|
||||
before = compute_tensor_digests(model)
|
||||
zero_weights(model)
|
||||
assert set(before) == {"transform.0"}
|
||||
assert compute_tensor_digests(model) != before
|
||||
|
||||
|
||||
def test_tensor_subclass_inner_tensors_are_hashed_and_reset():
|
||||
model = nn.Module()
|
||||
model.weight = nn.Parameter(
|
||||
TwoTensor(torch.ones(2, 2), torch.full((2, 2), 2.0)), requires_grad=False
|
||||
)
|
||||
|
||||
before = compute_tensor_digests(model)
|
||||
zero_weights(model)
|
||||
assert set(before) == {"weight.a", "weight.b"}
|
||||
assert all(
|
||||
before[name] != digest for name, digest in compute_tensor_digests(model).items()
|
||||
)
|
||||
|
||||
|
||||
def test_dense_dp_replicas_get_distinct_key_prefixes(monkeypatch):
|
||||
for name in ("get_tp_group", "get_pp_group", "get_pcp_group"):
|
||||
monkeypatch.setattr(gpu_worker, name, lambda: SimpleNamespace(rank_in_group=0))
|
||||
prefixes = []
|
||||
for dp_rank in (0, 1):
|
||||
worker = object.__new__(Worker)
|
||||
worker.parallel_config = ParallelConfig(
|
||||
data_parallel_size=2, data_parallel_rank=dp_rank
|
||||
)
|
||||
# Dense DP resets data_parallel_rank to 0 in each engine.
|
||||
worker.parallel_config.reconfigure_for_independent_dp_rank()
|
||||
prefixes.append(worker._weight_checksum_prefix())
|
||||
assert prefixes == ["dp0:pp0:pcp0:tp0:", "dp1:pp0:pcp0:tp0:"]
|
||||
Reference in New Issue
Block a user