[Mini-vLLM] Cut dead subsystems: models, entrypoints, backends, utils, tests

- Drop dead model modules, keeping files live imports require
- Remove anthropic, cohere, codegen entrypoints and batch STT modes
- Delete unused attention backends, MoE experts, and NPU/XPU util leaves
- Strip dead third_party trees and ROCm leaves with no CUDA path
- Remove mooncake EC connectors: dependency missing since branch base
- Re-audit verified by full import sweep and Qwen3.5 smoke test

Co-authored-by: Hinata <hinata@idihore.id>
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-10-19 00:20:48 +07:00
co-authored by Hinata
parent 6a95f62e2f
commit 81074e94a3
940 changed files with 11 additions and 380533 deletions
@@ -1,483 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import gc
import pytest
import torch
import vllm.device_allocator.cumem as cumem
from vllm import LLM, SamplingParams
from vllm.device_allocator import get_mem_allocator_instance
from vllm.platforms import current_platform
from ....utils import create_new_process_for_each_test
DEVICE_TYPE = current_platform.device_type
def mapped_usage(allocator) -> int:
"""Bytes the allocator still has mapped on the device.
`torch.accelerator.get_memory_info()` is a device-wide `cudaMemGetInfo` /
`hipMemGetInfo` query, so its delta also carries whatever else on the GPU
allocates or frees while we measure. The allocator's own bookkeeping is
process-local and exact.
"""
return sum(
data.handle[1]
for data in allocator.pointer_to_data.values()
if not data.is_asleep
)
def _wake_up_with_poisoned_mappings(allocator, byte_value: int = 0xA5) -> None:
"""Wake discarded allocations with deterministic nonzero contents."""
original_create_and_map = cumem.create_and_map
def create_and_map_with_poison(handle) -> None:
original_create_and_map(handle)
_, size, ptr, _ = handle
cumem.libcudart.cudaMemset(ptr, byte_value, size)
cumem.create_and_map = create_and_map_with_poison
try:
allocator.wake_up()
finally:
cumem.create_and_map = original_create_and_map
@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn")
def test_python_error():
"""Test if Python error occurs when there's low-level
error happening from the C++ side.
"""
allocator = get_mem_allocator_instance()
total_bytes = torch.accelerator.get_memory_info()[1]
alloc_bytes = int(total_bytes * 0.7)
tensors = []
with allocator.use_memory_pool():
# allocate 70% of the total memory
x = torch.empty(alloc_bytes, dtype=torch.uint8, device=DEVICE_TYPE)
tensors.append(x)
# release the memory
allocator.sleep(offload_tags=())
# allocate more memory than the total memory
y = torch.empty(alloc_bytes, dtype=torch.uint8, device=DEVICE_TYPE)
tensors.append(y)
with pytest.raises(RuntimeError):
# when the allocator is woken up, it should raise an error
# because we don't have enough memory
allocator.wake_up()
@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn")
def test_basic_cumem():
# some tensors from default memory pool
shape = (1024, 1024)
x = torch.empty(shape, device=DEVICE_TYPE)
x.zero_()
# some tensors from custom memory pool
allocator = get_mem_allocator_instance()
with allocator.use_memory_pool():
# custom memory pool
y = torch.empty(shape, device=DEVICE_TYPE)
y.zero_()
y += 1
z = torch.empty(shape, device=DEVICE_TYPE)
z.zero_()
z += 2
# they can be used together
output = x + y + z
assert torch.allclose(output, torch.ones_like(output) * 3)
# Track the allocator's own mapped bytes, not device-wide free memory.
mapped_bytes = mapped_usage(allocator)
assert mapped_bytes >= y.nbytes + z.nbytes
allocator.sleep()
assert mapped_usage(allocator) == 0
allocator.wake_up()
assert mapped_usage(allocator) == mapped_bytes
# they can be used together
output = x + y + z
assert torch.allclose(output, torch.ones_like(output) * 3)
@pytest.mark.parametrize("full_sleep", [0, 1, 2], ids=["kv-only", "sleep-1", "sleep-2"])
@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn")
def test_release_kv_cache_memory_preserves_generation(full_sleep, monkeypatch):
"""Rejected release and KV-only/full sleep cycles preserve greedy tokens."""
monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
kv_cache_memory_bytes = 256 * 1024 * 1024
llm = LLM(
"Qwen/Qwen3-0.6B",
enable_sleep_mode=True,
enforce_eager=True,
kv_cache_memory_bytes=kv_cache_memory_bytes,
max_model_len=1024,
max_num_seqs=4,
gpu_memory_utilization=0.05,
)
prompt = "How are you?"
sampling_params = SamplingParams(temperature=0, max_tokens=10)
expected = llm.generate(prompt, sampling_params)[0].outputs[0].token_ids
def get_mapped_bytes_by_tag(worker):
allocator = get_mem_allocator_instance()
mapped_bytes = mapped_usage(allocator)
kv_cache_bytes = sum(
data.handle[1]
for data in allocator.pointer_to_data.values()
if data.tag == "kv_cache" and not data.is_asleep
)
return mapped_bytes, kv_cache_bytes
mapped_before, kv_cache_before = llm.collective_rpc(get_mapped_bytes_by_tag)[0]
# Utility RPC transports the engine error as a plain Exception.
with pytest.raises(Exception, match="requires a completed pause"):
llm.release_kv_cache_memory()
assert llm.collective_rpc(get_mapped_bytes_by_tag)[0][0] == mapped_before
assert not llm.llm_engine.is_sleeping()
assert llm.generate(prompt, sampling_params)[0].outputs[0].token_ids == expected
llm.sleep(level=0)
llm.release_kv_cache_memory()
mapped_after, kv_cache_after = llm.collective_rpc(get_mapped_bytes_by_tag)[0]
assert kv_cache_before > 0
assert kv_cache_after == 0
assert mapped_before - mapped_after >= kv_cache_before
assert mapped_after > 0
assert llm.llm_engine.is_sleeping()
if full_sleep:
llm.sleep(level=full_sleep)
assert llm.collective_rpc(get_mapped_bytes_by_tag)[0][0] == 0
llm.wake_up(tags=None if full_sleep else ["kv_cache"])
if full_sleep == 2:
llm.collective_rpc("reload_weights")
assert llm.collective_rpc(get_mapped_bytes_by_tag)[0][0] == mapped_before
assert not llm.llm_engine.is_sleeping()
actual = llm.generate(prompt, sampling_params)[0].outputs[0].token_ids
assert actual == expected
@pytest.mark.parametrize("first_level", [1, 2])
@pytest.mark.parametrize("second_level", [1, 2])
@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn")
def test_sleep_with_only_weights_asleep(first_level, second_level, monkeypatch):
"""Repeated sleep after KV-only wake preserves mappings and recoverability."""
monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
llm = LLM(
"Qwen/Qwen3-0.6B",
enable_sleep_mode=True,
enforce_eager=True,
kv_cache_memory_bytes=256 * 1024 * 1024,
max_model_len=1024,
max_num_seqs=4,
gpu_memory_utilization=0.05,
)
prompt = "How are you?"
sampling_params = SamplingParams(temperature=0, max_tokens=10)
expected = llm.generate(prompt, sampling_params)[0].outputs[0].token_ids
def get_mapped_bytes(worker):
return mapped_usage(get_mem_allocator_instance())
llm.sleep(level=first_level)
llm.wake_up(tags=["kv_cache"])
mapped_before = llm.collective_rpc(get_mapped_bytes)[0]
assert mapped_before > 0
llm.sleep(level=second_level)
assert llm.collective_rpc(get_mapped_bytes)[0] == mapped_before
llm.wake_up(tags=["weights"])
if first_level == 2:
llm.collective_rpc("reload_weights")
assert not llm.llm_engine.is_sleeping()
actual = llm.generate(prompt, sampling_params)[0].outputs[0].token_ids
assert actual == expected
@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn")
def test_discard_tags():
"""Test that discard(tags) selectively frees GPU memory for specific
tags while keeping other tags mapped and usable."""
allocator = get_mem_allocator_instance()
with allocator.use_memory_pool("weights"):
weights = torch.ones(1024, 1024, device=DEVICE_TYPE)
with allocator.use_memory_pool("kv_cache"):
kv = torch.ones(512, 512, device=DEVICE_TYPE)
mapped_bytes = mapped_usage(allocator)
# Discard kv_cache only — weights should remain valid
allocator.discard("kv_cache")
mapped_bytes_after_discard = mapped_usage(allocator)
assert mapped_bytes - mapped_bytes_after_discard >= kv.nbytes
assert mapped_bytes_after_discard >= weights.nbytes
# Weights are still usable
assert torch.allclose(weights, torch.ones_like(weights))
# Wake up and verify kv_cache is remapped; discarded contents are undefined.
allocator.wake_up()
assert kv.shape == (512, 512)
# Full sleep/wake cycle still works after discard
allocator.sleep(offload_tags="weights")
allocator.wake_up()
assert torch.allclose(weights, torch.ones_like(weights))
@pytest.mark.parametrize("tag", ["workspace", None], ids=["workspace", "default"])
@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn")
def test_selective_wake_restores_internal_tags(tag):
"""A selective wake defers only weights/kv_cache; other tags always wake."""
allocator = get_mem_allocator_instance()
with allocator.use_memory_pool(tag):
internal = torch.empty(16 << 20, dtype=torch.uint8, device=DEVICE_TYPE)
with allocator.use_memory_pool("kv_cache"):
kv = torch.empty(32 << 20, dtype=torch.uint8, device=DEVICE_TYPE)
allocator.sleep(offload_tags=())
assert mapped_usage(allocator) == 0
allocator.wake_up(tags=["weights"])
assert mapped_usage(allocator) == internal.nbytes
allocator.wake_up(tags=["kv_cache"])
assert mapped_usage(allocator) == internal.nbytes + kv.nbytes
@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn")
@pytest.mark.skipif(current_platform.is_xpu(), reason="Uses the CuMem allocator")
def test_workspace_scratch_discarded_on_sleep():
"""Workspace scratch lives in the sleep-mode pool: sleep discards it, any
(selective) wake remaps it at the same address so captured graphs replay,
and a buffer outgrown by a resize stays usable until it is released."""
from vllm.device_allocator.sleep_mode_backend import CuMemBackend
from vllm.v1.worker.workspace import WorkspaceManager
allocator = get_mem_allocator_instance()
manager = WorkspaceManager(
torch.device(DEVICE_TYPE),
alloc_context=lambda: allocator.use_memory_pool("workspace"),
)
(small,) = manager.get_simultaneous(((32 << 20,), torch.uint8))
# Resize while a view of the old buffer is alive: the tag is re-entered.
(scratch,) = manager.get_simultaneous(((64 << 20,), torch.uint8))
assert len(allocator.allocator_and_pools["workspace"]) == 2
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
scratch.add_(1)
backend = CuMemBackend()
backend.suspend(level=1)
assert mapped_usage(allocator) == 0
# Discarded, not copied.
assert all(
d.tag == "workspace" and d.cpu_backup_tensor is None
for d in allocator.pointer_to_data.values()
)
backend.resume(tags=["kv_cache"])
assert mapped_usage(allocator) == small.nbytes + scratch.nbytes
scratch.zero_()
graph.replay()
assert int(scratch.sum()) == scratch.numel()
small.fill_(1)
# The next resize releases the outgrown buffer; wake does not remap it.
del small
(big,) = manager.get_simultaneous(((128 << 20,), torch.uint8))
assert mapped_usage(allocator) == scratch.nbytes + big.nbytes
backend.suspend(level=1)
backend.resume(tags=["weights"])
assert mapped_usage(allocator) == scratch.nbytes + big.nbytes
@pytest.mark.parametrize("free_x_early", [True, False], ids=["x-freed", "x-alive"])
@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn")
@pytest.mark.skipif(current_platform.is_xpu(), reason="Uses the CuMem allocator")
def test_reentered_tag(free_x_early):
"""Enter a tag twice (two workspace ubatches): freeing X returns its memory
by the next entry, without touching Y or breaking sleep/wake."""
allocator = get_mem_allocator_instance()
nbytes = 64 << 20
with allocator.use_memory_pool("t"):
xs = [torch.empty(nbytes, dtype=torch.uint8, device=DEVICE_TYPE)]
if free_x_early:
xs.clear()
with allocator.use_memory_pool("t"):
y = torch.empty(nbytes, dtype=torch.uint8, device=DEVICE_TYPE)
xs.clear()
gc.collect()
with allocator.use_memory_pool("t"):
pass
assert mapped_usage(allocator) == nbytes
y.fill_(1)
allocator.sleep(offload_tags="t")
allocator.wake_up()
assert int(y.sum()) == y.numel()
allocator.release_pools()
@pytest.mark.parametrize("level", [1, 2], ids=["sleep-1", "sleep-2"])
@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn")
@pytest.mark.skipif(current_platform.is_xpu(), reason="Uses the CuMem allocator")
def test_runtime_state_survives_sleep(level):
"""Runtime state is offloaded at both levels and restored by the first
selective wake; a nested kv_cache pool keeps its own tag."""
from vllm.device_allocator.sleep_mode_backend import CuMemBackend
allocator = get_mem_allocator_instance()
with allocator.use_memory_pool("runtime"):
state = torch.arange(1 << 20, device=DEVICE_TYPE)
with allocator.use_memory_pool("kv_cache"):
kv = torch.zeros(1 << 20, device=DEVICE_TYPE)
assert {d.tag for d in allocator.pointer_to_data.values()} == {
"runtime",
"kv_cache",
}
backend = CuMemBackend()
backend.suspend(level=level)
assert mapped_usage(allocator) == 0
backend.resume(tags=["weights"])
assert torch.equal(state, torch.arange(1 << 20, device=DEVICE_TYPE))
backend.resume(tags=["kv_cache"])
kv.fill_(1)
assert int(kv.sum()) == kv.numel()
@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn")
@pytest.mark.skipif(current_platform.is_xpu(), reason="Uses the CuMem allocator")
def test_level2_discards_ordinary_tensor_with_weights_tag():
"""Discarded weights-tag memory is remapped; ROCm zeroes it over stale pages."""
allocator = get_mem_allocator_instance()
with allocator.use_memory_pool("weights"):
fake_weight = torch.full((4096,), 0x44, dtype=torch.uint8, device=DEVICE_TYPE)
ordinary_tensor = torch.full(
(4096,), 0x55, dtype=torch.uint8, device=DEVICE_TYPE
)
pointers = (fake_weight.data_ptr(), ordinary_tensor.data_ptr())
allocator.sleep(offload_tags=())
_wake_up_with_poisoned_mappings(allocator)
torch.accelerator.synchronize()
assert (fake_weight.data_ptr(), ordinary_tensor.data_ptr()) == pointers
expected = 0 if current_platform.is_rocm() else 0xA5
assert torch.all(fake_weight == expected)
assert torch.all(ordinary_tensor == expected)
@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn")
@pytest.mark.skipif(current_platform.is_xpu(), reason="CUDA graph not supported on XPU")
def test_cumem_with_cudagraph():
allocator = get_mem_allocator_instance()
with allocator.use_memory_pool():
weight = torch.eye(1024, device=DEVICE_TYPE)
with allocator.use_memory_pool(tag="discard"):
cache = torch.empty(1024, 1024, device=DEVICE_TYPE)
def model(x):
out = x @ weight
cache[: out.size(0)].copy_(out)
return out + 1
x = torch.empty(128, 1024, device=DEVICE_TYPE)
# warmup
model(x)
# capture cudagraph
model_graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(model_graph):
y = model(x)
mapped_bytes = mapped_usage(allocator)
assert mapped_bytes >= weight.nbytes + cache.nbytes
allocator.sleep()
assert mapped_usage(allocator) == 0
allocator.wake_up()
assert mapped_usage(allocator) == mapped_bytes
# after waking up, the content in the weight tensor
# should be restored, but the content in the cache tensor
# should be discarded
# this operation is also compatible with cudagraph
x.random_()
model_graph.replay()
# cache content is as expected
assert torch.allclose(x, cache[: x.size(0)])
# output content is as expected
assert torch.allclose(y, x + 1)
@pytest.mark.parametrize("level", [1, 2], ids=["sleep-1", "sleep-2"])
@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn")
@pytest.mark.skipif(
not current_platform.is_cuda_alike(), reason="cuMem CUDA graph pool"
)
def test_cudagraph_pool_sleep(level):
"""Routing, and the graph pool backed up at both sleep levels and restored
in place by any wake."""
from contextlib import nullcontext
from types import SimpleNamespace
import vllm.distributed.device_communicators.pynccl_allocator as nccl_alloc
from vllm.compilation.cudagraph_pool import (
capture_outside_cumem_pool,
capture_pool,
)
from vllm.device_allocator.sleep_mode_backend import CuMemBackend
allocator = get_mem_allocator_instance()
on, off = (SimpleNamespace(use_cumem_cudagraph_pool=v) for v in (True, False))
handle = current_platform.graph_pool_handle()
for cfg, ctx in ((off, nullcontext()), (on, capture_outside_cumem_pool())):
with ctx, capture_pool(handle, cfg) as used:
assert used == handle and allocator.current_tag != "cudagraph"
with allocator.use_memory_pool("weights"):
weight = torch.full((1 << 20,), 2.0, device=DEVICE_TYPE)
x = torch.ones_like(weight)
def capture() -> list:
graph = torch.cuda.CUDAGraph()
stream = torch.cuda.Stream()
with (
capture_pool(handle, on) as pool,
torch.cuda.graph(graph, pool=pool, stream=stream),
):
assert nccl_alloc._graph_pool_id == pool != handle
const = torch.empty_like(x) # Never written by replay: needs a backup.
y = x * weight + const
return [graph, y, const.fill_(3.0)]
def graph_pool() -> dict[int, int]:
data = allocator.pointer_to_data.items()
return {
p: d.handle[1] for p, d in data if d.tag == "cudagraph" and not d.is_asleep
}
held, backend = capture(), CuMemBackend()
mapped = graph_pool()
backend.suspend(level=level)
backend.resume(tags=["kv_cache"])
assert graph_pool() == mapped
backend.resume(tags=["weights"])
weight.fill_(2.0) # Level 2 discards weights; emulate the reload.
held[0].replay()
assert torch.equal(held[1], torch.full_like(x, 5.0))
@@ -1,483 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import asyncio
import math
import os
import pytest
import torch
import vllm.device_allocator.cumem as cumem
import vllm.envs as envs
from vllm import LLM, AsyncEngineArgs, AsyncLLMEngine, SamplingParams
from vllm.model_executor.model_loader import get_model_loader
from vllm.platforms import current_platform
from vllm.utils.mem_constants import GiB_bytes
from ....utils import (
create_new_process_for_each_test,
multi_gpu_marks,
multi_gpu_test,
requires_fp8,
)
@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn")
@pytest.mark.parametrize(
"model",
[
# sleep mode with safetensors
"hmellor/tiny-random-LlamaForCausalLM",
# sleep mode with pytorch checkpoint
"facebook/opt-125m",
],
)
def test_end_to_end(model: str):
free, total = torch.accelerator.get_memory_info()
used_bytes_baseline = total - free # in case other process is running
llm = LLM(model, enable_sleep_mode=True)
prompt = "How are you?"
sampling_params = SamplingParams(temperature=0, max_tokens=10)
output = llm.generate(prompt, sampling_params)
# the benefit of `llm.sleep(level=2)` is mainly CPU memory usage,
# which is difficult to measure in the test. therefore, we only
# test sleep level 1 here.
llm.sleep(level=1)
free_gpu_bytes_after_sleep, total = torch.accelerator.get_memory_info()
used_bytes = total - free_gpu_bytes_after_sleep - used_bytes_baseline
# now the memory usage is mostly cudagraph memory pool,
# and it should be less than the model weights (1B model, 2GiB weights)
# NOTE: In V1, the memory buffer for logits (max_num_reqs x vocab_size)
# is captured but cannot be releasesd from PyTorch due to a known bug,
# therefore high memory usage after `llm.sleep` is called is expected.
# FIXME(youkaichao & ywang96): Fix memory buffer issue with sleep mode
# in V1.
assert used_bytes < 7 * GiB_bytes
llm.wake_up()
output2 = llm.generate(prompt, sampling_params)
# cmp output
assert output[0].outputs[0].text == output2[0].outputs[0].text
llm.sleep(level=1)
llm.wake_up(tags=["weights"])
free_gpu_bytes_wake_up_w, total = torch.accelerator.get_memory_info()
used_bytes = total - free_gpu_bytes_wake_up_w - used_bytes_baseline
# should just reallocate memory for weights (1B model, ~2GiB weights)
assert used_bytes < 10 * GiB_bytes
# now allocate kv cache memory
llm.wake_up(tags=["kv_cache"])
output3 = llm.generate(prompt, sampling_params)
# cmp output
assert output[0].outputs[0].text == output3[0].outputs[0].text
@create_new_process_for_each_test()
def test_deep_sleep(gpu_memory_cleared):
model = "hmellor/tiny-random-LlamaForCausalLM"
free, total = torch.accelerator.get_memory_info()
used_bytes_baseline = total - free # in case other process is running
llm = LLM(model, enable_sleep_mode=True)
prompt = "How are you?"
sampling_params = SamplingParams(temperature=0, max_tokens=10)
output = llm.generate(prompt, sampling_params)
# Put the engine to deep sleep
llm.sleep(level=2)
free_gpu_bytes_after_sleep, total = torch.accelerator.get_memory_info()
used_bytes = total - free_gpu_bytes_after_sleep - used_bytes_baseline
assert used_bytes < 3 * GiB_bytes
llm.wake_up(tags=["weights"])
llm.collective_rpc("reload_weights")
free_gpu_bytes_wake_up_w, total = torch.accelerator.get_memory_info()
used_bytes = total - free_gpu_bytes_wake_up_w - used_bytes_baseline
assert used_bytes < 4 * GiB_bytes
# now allocate kv cache and cuda graph memory
llm.wake_up(tags=["kv_cache"])
output2 = llm.generate(prompt, sampling_params)
# cmp output
assert output[0].outputs[0].text == output2[0].outputs[0].text
@create_new_process_for_each_test()
def test_deep_sleep_lora(gpu_memory_cleared):
"""Level-2 sleep/wake/reload with enable_lora=True.
LoRA wrapping moves parameters under base_layer and adds LoRA
stacked tensors that are plain attributes, not restored by the
reload machinery — reload must forward checkpoint weights through
the wrappers and reset the LoRA state afterwards.
"""
model = "hmellor/tiny-random-LlamaForCausalLM"
llm = LLM(
model,
enable_sleep_mode=True,
enable_lora=True,
max_lora_rank=8,
enforce_eager=True,
)
prompt = "How are you?"
sampling_params = SamplingParams(temperature=0, max_tokens=10)
output = llm.generate(prompt, sampling_params)
# Level-2 sleep discards all GPU memory
llm.sleep(level=2)
# Reload weights from checkpoint
llm.wake_up(tags=["weights"])
llm.collective_rpc("reload_weights")
llm.wake_up(tags=["kv_cache"])
output2 = llm.generate(prompt, sampling_params)
assert output[0].outputs[0].text == output2[0].outputs[0].text
# Multiple cycles should not accumulate corruption
for _ in range(3):
llm.sleep(level=2)
llm.wake_up(tags=["weights"])
llm.collective_rpc("reload_weights")
llm.wake_up(tags=["kv_cache"])
output3 = llm.generate(prompt, sampling_params)
assert output[0].outputs[0].text == output3[0].outputs[0].text
def _lora_logits_mapping_present(model) -> bool:
from vllm.lora.layers.logits_processor import LogitsProcessorWithLoRA
return any(
isinstance(m, LogitsProcessorWithLoRA)
and m.sharded_to_full_mapping_gpu is not None
for m in model.modules()
)
@multi_gpu_test(num_gpus=2)
def test_deep_sleep_lora_tp2(monkeypatch):
"""Level-2 sleep/wake/reload with enable_lora=True and TP=2.
With TP > 1 the LoRA logits processor carries
``sharded_to_full_mapping_gpu``, a permanent index mapping used to
reorder gathered logits. Like the LoRA stacked tensors it is a plain
attribute allocated in the sleep-mode pool, so level-2 sleep destroys
its contents — it must be restored after reload.
"""
# Needed for apply_model to reach the multiproc TP workers below.
monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
model = "hmellor/tiny-random-LlamaForCausalLM"
llm = LLM(
model,
enable_sleep_mode=True,
enable_lora=True,
max_lora_rank=8,
tensor_parallel_size=2,
enforce_eager=True,
)
# Guard against this test silently not exercising the TP>1 reindex
# path (e.g. if lm_head wrapping conditions change).
assert all(llm.apply_model(_lora_logits_mapping_present))
prompt = "How are you?"
sampling_params = SamplingParams(temperature=0, max_tokens=10)
output = llm.generate(prompt, sampling_params)
llm.sleep(level=2)
llm.wake_up(tags=["weights"])
llm.collective_rpc("reload_weights")
llm.wake_up(tags=["kv_cache"])
output2 = llm.generate(prompt, sampling_params)
assert output[0].outputs[0].text == output2[0].outputs[0].text
def _cudagraph_bytes(worker) -> int:
data = cumem.CuMemAllocator.get_instance().pointer_to_data.values()
return sum(d.handle[1] for d in data if d.tag == "cudagraph")
def _custom_ar_active(worker) -> bool:
from vllm.distributed.parallel_state import get_tp_group
comm = get_tp_group().device_communicator
return any(
c is not None and not c.disabled for c in (comm.ca_comm, comm.aiter_ar_comm)
)
def _capture_registers_buffers(worker) -> bool:
from vllm.distributed.parallel_state import get_tp_group
comm = get_tp_group().device_communicator
ca, aiter = comm.ca_comm, comm.aiter_ar_comm
return (ca is not None and ca._capture_registered) or (
aiter is not None and aiter.aiter_ca.enable_register_for_capturing
)
@pytest.mark.parametrize(
("mode", "breakable", "tp", "offload"),
[
("FULL", False, 1, True),
("PIECEWISE", False, 1, True),
("PIECEWISE", True, 1, True),
pytest.param(
"FULL_AND_PIECEWISE", False, 2, True, marks=multi_gpu_marks(num_gpus=2)
),
("FULL_AND_PIECEWISE", False, 1, False),
],
ids=["full", "piecewise", "breakable", "custom-ar-tp2", "off-by-default"],
)
@create_new_process_for_each_test()
@pytest.mark.skipif(
not current_platform.is_cuda_alike(), reason="cuMem CUDA graph pool"
)
def test_sleep_cudagraph_pool(monkeypatch, mode, breakable, tp, offload):
"""Each capture site uses the pool, exact across sleeps; TP=2 needs
unregistered custom AR; off changes nothing."""
for name, value in [
("VLLM_USE_V2_MODEL_RUNNER", "1"),
("VLLM_USE_BREAKABLE_CUDAGRAPH", "1" if breakable else "0"),
("VLLM_ALLOW_INSECURE_SERIALIZATION", "1"),
("VLLM_ALLREDUCE_USE_FLASHINFER", "0"),
("VLLM_ALLREDUCE_USE_SYMM_MEM", "0"),
]:
monkeypatch.setenv(name, value)
monkeypatch.delenv("NCCL_GRAPH_REGISTER", raising=False)
llm = LLM(
"hmellor/tiny-random-LlamaForCausalLM",
enable_sleep_mode=True,
sleep_mode_offload_cudagraph=offload,
tensor_parallel_size=tp,
compilation_config={
"cudagraph_mode": mode,
"pass_config": {"fuse_allreduce_rms": False},
},
)
graph_bytes = llm.collective_rpc(_cudagraph_bytes)
assert os.environ.get("NCCL_GRAPH_REGISTER") == ("0" if offload else None)
assert all(graph_bytes) if offload else not any(graph_bytes)
if not offload:
return
if tp > 1:
if not all(llm.collective_rpc(_custom_ar_active)):
pytest.skip("Custom allreduce is unavailable on these GPUs")
assert not any(llm.collective_rpc(_capture_registers_buffers))
prompt, params = "How are you?", SamplingParams(temperature=0, max_tokens=10)
expected = llm.generate(prompt, params)[0].outputs[0].text
for level in (1, 2):
llm.sleep(level=level)
llm.wake_up(tags=["weights"])
if level == 2:
llm.collective_rpc("reload_weights")
llm.wake_up(tags=["kv_cache"])
assert llm.generate(prompt, params)[0].outputs[0].text == expected
@multi_gpu_test(num_gpus=2)
@create_new_process_for_each_test()
@pytest.mark.skipif(not current_platform.is_rocm(), reason="AITER fused allreduce")
def test_sleep_cudagraph_pool_aiter_fused_ar_group_quant(monkeypatch):
"""The fused AITER AR + RMSNorm + per-group FP8 quant captures into the
cuMem pool without IPC-registering its input, across sleeps."""
for name, value in [
("VLLM_USE_V2_MODEL_RUNNER", "1"),
("VLLM_ROCM_USE_AITER", "1"),
("VLLM_ALLOW_INSECURE_SERIALIZATION", "1"),
]:
monkeypatch.setenv(name, value)
monkeypatch.delenv("NCCL_GRAPH_REGISTER", raising=False)
llm = LLM(
"Qwen/Qwen3-0.6B-FP8",
enable_sleep_mode=True,
sleep_mode_offload_cudagraph=True,
tensor_parallel_size=2,
max_model_len=1024,
compilation_config={"pass_config": {"fuse_allreduce_rms": True}},
)
if not all(llm.collective_rpc(_custom_ar_active)):
pytest.skip("AITER custom allreduce is unavailable on these GPUs")
assert not any(llm.collective_rpc(_capture_registers_buffers))
assert all(llm.collective_rpc(_cudagraph_bytes))
prompt, params = "How are you?", SamplingParams(temperature=0, max_tokens=10)
assert llm.generate(prompt, params)[0].outputs[0].text
for level in (1, 2):
llm.sleep(level=level)
llm.wake_up(tags=["weights"])
if level == 2:
llm.collective_rpc("reload_weights")
llm.wake_up(tags=["kv_cache"])
assert llm.generate(prompt, params)[0].outputs[0].text
@create_new_process_for_each_test()
def test_deep_sleep_async(gpu_memory_cleared):
async def test():
model = "hmellor/tiny-random-LlamaForCausalLM"
free, total = torch.accelerator.get_memory_info()
used_bytes_baseline = total - free # in case other process is running
engine_args = AsyncEngineArgs(
model=model,
enable_sleep_mode=True,
)
llm = AsyncLLMEngine.from_engine_args(engine_args)
prompt = "How are you?"
sampling_params = SamplingParams(temperature=0, max_tokens=10)
outputs = llm.generate(prompt, sampling_params, request_id="test_request_id1")
async for output in outputs:
pass
# Put the engine to deep sleep
await llm.sleep(level=2)
await llm.wake_up(tags=["weights"])
await llm.collective_rpc("reload_weights")
free_gpu_bytes_wake_up_w, total = torch.accelerator.get_memory_info()
used_bytes = total - free_gpu_bytes_wake_up_w - used_bytes_baseline
assert used_bytes < 4 * GiB_bytes
# now allocate kv cache and cuda graph memory
await llm.wake_up(tags=["kv_cache"])
outputs2 = llm.generate(prompt, sampling_params, request_id="test_request_id2")
async for output2 in outputs2:
pass
# cmp output
assert output.outputs[0].text == output2.outputs[0].text
asyncio.run(test())
def _load_weights(worker) -> None:
# Not reload_weights(): its layerwise finalize re-derives the KV scales.
model = worker.model_runner.get_model()
loader = get_model_loader(worker.load_config)
model.load_weights(loader.get_all_weights(worker.model_config, model))
def _attn_kv_scales(model) -> dict[str, list[float]]:
return {
f"{name}.{attr}": getattr(module, attr).flatten().tolist()
for name, module in model.named_modules()
for attr in ("_q_scale", "_k_scale", "_v_scale")
if isinstance(getattr(module, attr, None), torch.Tensor)
}
@requires_fp8
@create_new_process_for_each_test()
def test_deep_sleep_compressed_tensors_kv_scales(
monkeypatch: pytest.MonkeyPatch, gpu_memory_cleared
):
"""KV scales must survive level-2 sleep plus a weights reload (else NaN)."""
monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
llm = LLM(
"nm-testing/TinyLlama-1.1B-Chat-v1.0-kvcache-fp8-tensor",
enable_sleep_mode=True,
kv_cache_dtype="fp8",
enforce_eager=True,
gpu_memory_utilization=0.6,
)
sampling_params = SamplingParams(temperature=0, max_tokens=10, logprobs=0)
scales = llm.apply_model(_attn_kv_scales)[0]
assert any(v != 1.0 for values in scales.values() for v in values)
expected = llm.generate("How are you?", sampling_params)[0].outputs[0]
llm.sleep(level=2)
llm.wake_up(tags=["weights"])
llm.collective_rpc(_load_weights)
llm.wake_up(tags=["kv_cache"])
assert llm.apply_model(_attn_kv_scales)[0] == scales
actual = llm.generate("How are you?", sampling_params)[0].outputs[0]
assert actual.token_ids == expected.token_ids
logprobs = [
[lp[t].logprob for lp, t in zip(o.logprobs, o.token_ids)]
for o in (expected, actual)
]
assert all(math.isfinite(x) for x in logprobs[1])
assert logprobs[0] == logprobs[1]
@requires_fp8
def test_deep_sleep_fp8_kvcache(gpu_memory_cleared):
# Regression test for https://github.com/vllm-project/vllm/pull/28783:
# fp8 KV cache output must be restored correctly after level-2 sleep.
model = "Qwen/Qwen2-0.5B"
used_bytes_baseline = current_platform.get_current_memory_usage()
llm = LLM(model, enable_sleep_mode=True, kv_cache_dtype="fp8")
prompt = "How are you?"
sampling_params = SamplingParams(temperature=0, max_tokens=10)
output = llm.generate(prompt, sampling_params)
# Put the engine to deep sleep
llm.sleep(level=2)
used_bytes = current_platform.get_current_memory_usage() - used_bytes_baseline
# Rocm uses more memory for CudaGraphs, so we add 2 GiB more for the threshold
rocm_extra_mem_bytes = 2 * GiB_bytes if current_platform.is_rocm() else 0
mem_threshold_after_sleep = 3 * GiB_bytes + rocm_extra_mem_bytes
assert used_bytes < mem_threshold_after_sleep
llm.wake_up(tags=["weights"])
llm.collective_rpc("reload_weights")
used_bytes = current_platform.get_current_memory_usage() - used_bytes_baseline
mem_threshold_after_wake_up = 4 * GiB_bytes + rocm_extra_mem_bytes
assert used_bytes < mem_threshold_after_wake_up
# now allocate kv cache and cuda graph memory
llm.wake_up(tags=["kv_cache"])
output2 = llm.generate(prompt, sampling_params)
# cmp output
assert output[0].outputs[0].text == output2[0].outputs[0].text
@requires_fp8
def test_deep_sleep_fp8_kvcache_with_undefined_remap(
monkeypatch: pytest.MonkeyPatch,
):
monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0")
envs.disable_envs_cache()
llm = LLM(
"Qwen/Qwen2-0.5B",
enable_sleep_mode=True,
kv_cache_dtype="fp8",
)
prompt = "How are you?"
sampling_params = SamplingParams(temperature=0, max_tokens=10)
expected = llm.generate(prompt, sampling_params)
llm.sleep(level=2)
llm.wake_up(tags=["weights"])
llm.collective_rpc("reload_weights")
original_create_and_map = cumem.create_and_map
def create_and_map_with_poison(handle) -> None:
original_create_and_map(handle)
_, size, ptr, _ = handle
cumem.libcudart.cudaMemset(ptr, 0xA5, size)
monkeypatch.setattr(cumem, "create_and_map", create_and_map_with_poison)
# New requests must overwrite undefined remapped KV bytes before reading them.
llm.wake_up(tags=["kv_cache"])
actual = llm.generate(prompt, sampling_params)
assert expected[0].outputs[0].text == actual[0].outputs[0].text
@@ -1,316 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Compare the short outputs of HF and vLLM when using greedy sampling.
Run `pytest tests/basic_correctness/models/test_basic_correctness.py`.
"""
import os
import weakref
from unittest.mock import Mock
import pytest
import torch
import vllm.envs as envs
from vllm import LLM
from vllm.platforms import current_platform
from vllm.v1.engine.llm_engine import LLMEngine
from ...conftest import HfRunner, VllmRunner
from ...models.utils import check_outputs_equal
from ...utils import multi_gpu_test
ATTN_BACKEND = ["ROCM_ATTN"] if current_platform.is_rocm() else ["FLASH_ATTN"]
MODELS = [
"hmellor/tiny-random-Gemma2ForCausalLM",
"meta-llama/Llama-3.2-1B-Instruct",
]
TARGET_TEST_SUITE_ENV = "VLLM_TARGET_TEST_SUITE"
LEGACY_TARGET_TEST_SUITE_ENV = "TARGET_TEST_SUITE"
GENERIC_DISTRIBUTED_TEST_SUITES = ("L4", "MI250", "MI300", "MI325", "MI355")
ALL_DISTRIBUTED_TEST_SUITES = (*GENERIC_DISTRIBUTED_TEST_SUITES, "A100")
def _default_target_test_suite() -> str:
if not current_platform.is_rocm():
return "L4"
try:
device_name = current_platform.get_device_name().upper()
except Exception:
device_name = ""
if "MI355" in device_name:
return "MI355"
if "MI300" in device_name:
return "MI300"
if "MI325" in device_name:
return "MI325"
if "MI250" in device_name:
return "MI250"
try:
from vllm.platforms import rocm as rocm_platform
if rocm_platform.on_gfx950():
return "MI355"
if rocm_platform.on_gfx942():
return "MI300"
except Exception:
pass
return "MI250"
def _resolve_target_test_suite() -> str:
for env_name in (TARGET_TEST_SUITE_ENV, LEGACY_TARGET_TEST_SUITE_ENV):
value = os.environ.get(env_name, "").strip().upper()
if value:
return value
return _default_target_test_suite()
TARGET_TEST_SUITE = _resolve_target_test_suite()
# ROCm can occasionally retain the object until fixture teardown. Retry only
# that assertion after cleanup; collecting cyclic garbage here would mask the
# reference cycles this test is intended to catch.
@pytest.mark.flaky(
reruns=2,
reruns_delay=5,
only_rerun="AssertionError",
condition=current_platform.is_rocm(),
)
def test_vllm_gc_ed():
"""Verify vllm instance is GC'ed when it is deleted."""
llm = LLM("hmellor/tiny-random-LlamaForCausalLM")
weak_llm = weakref.ref(llm)
del llm
# If there's any circular reference to vllm, this fails
# because llm instance is not GC'ed.
assert weak_llm() is None
def _fix_prompt_embed_outputs(
vllm_outputs: list[tuple[list[int], str]],
hf_model: HfRunner,
example_prompts: list[str],
) -> list[tuple[list[int], str]]:
fixed_vllm_outputs = []
for vllm_output, hf_input, prompt in zip(
vllm_outputs, hf_model.get_inputs(example_prompts), example_prompts
):
hf_input_ids = hf_input["input_ids"].tolist()[0]
fixed_vllm_outputs.append(
(
hf_input_ids + vllm_output[0][len(hf_input_ids) :],
prompt + vllm_output[1],
)
)
return fixed_vllm_outputs
@pytest.mark.parametrize("model", MODELS)
@pytest.mark.parametrize("backend", ATTN_BACKEND)
@pytest.mark.parametrize("max_tokens", [5])
@pytest.mark.parametrize("enforce_eager", [False])
@pytest.mark.parametrize("async_scheduling", [True, False])
@pytest.mark.parametrize("model_executor", ["uni", "mp"])
@pytest.mark.parametrize("enable_prompt_embeds", [True, False])
def test_models(
hf_runner,
model: str,
backend: str,
max_tokens: int,
enforce_eager: bool,
async_scheduling: bool,
model_executor: str,
enable_prompt_embeds: bool,
) -> None:
# 5042 tokens for gemma2
# gemma2 has alternating sliding window size of 4096
# we need a prompt with more than 4096 tokens to test the sliding window
prompt = (
"The following numbers of the sequence "
+ ", ".join(str(i) for i in range(1024))
+ " are:"
)
example_prompts = [prompt]
with hf_runner(model) as hf_model:
hf_outputs = hf_model.generate_greedy(example_prompts, max_tokens)
if enable_prompt_embeds:
with torch.no_grad():
prompt_embeds = hf_model.get_prompt_embeddings(example_prompts)
with VllmRunner(
model,
max_model_len=8192,
enforce_eager=enforce_eager,
enable_prompt_embeds=enable_prompt_embeds,
gpu_memory_utilization=0.7,
async_scheduling=async_scheduling,
distributed_executor_backend=model_executor,
attention_config={"backend": backend},
) as vllm_model:
if enable_prompt_embeds:
vllm_outputs = vllm_model.generate_greedy(prompt_embeds, max_tokens)
vllm_outputs = _fix_prompt_embed_outputs(
vllm_outputs, hf_model, example_prompts
)
else:
vllm_outputs = vllm_model.generate_greedy(example_prompts, max_tokens)
check_outputs_equal(
outputs_0_lst=hf_outputs,
outputs_1_lst=vllm_outputs,
name_0="hf",
name_1="vllm",
)
@multi_gpu_test(num_gpus=2)
@pytest.mark.parametrize(
(
"model, distributed_executor_backend, attention_backend, "
"target_test_suites, extra_env"
),
[
("facebook/opt-125m", "ray", "", ALL_DISTRIBUTED_TEST_SUITES, {}),
("facebook/opt-125m", "mp", "", ALL_DISTRIBUTED_TEST_SUITES, {}),
(
"meta-llama/Llama-3.2-1B-Instruct",
"ray",
"",
GENERIC_DISTRIBUTED_TEST_SUITES,
{},
),
(
"meta-llama/Llama-3.2-1B-Instruct",
"mp",
"",
GENERIC_DISTRIBUTED_TEST_SUITES,
{},
),
],
)
@pytest.mark.parametrize("enable_prompt_embeds", [True, False])
def test_models_distributed(
monkeypatch: pytest.MonkeyPatch,
hf_runner,
vllm_runner,
example_prompts,
model: str,
distributed_executor_backend: str,
attention_backend: str,
target_test_suites: tuple[str, ...],
extra_env: dict[str, str],
enable_prompt_embeds: bool,
) -> None:
if TARGET_TEST_SUITE and TARGET_TEST_SUITE not in target_test_suites:
pytest.skip(f"Skip test for {TARGET_TEST_SUITE}")
with monkeypatch.context() as monkeypatch_context:
for k, v in extra_env.items():
monkeypatch_context.setenv(k, v)
dtype = "half"
max_tokens = 5
# NOTE: take care of the order. run vLLM first, and then run HF.
# vLLM needs a fresh new process without cuda initialization.
# if we run HF first, the cuda initialization will be done and it
# will hurt multiprocessing backend with fork method
# (the default method).
attention_config = {"backend": attention_backend} if attention_backend else None
with vllm_runner(
model,
dtype=dtype,
tensor_parallel_size=2,
distributed_executor_backend=distributed_executor_backend,
enable_prompt_embeds=enable_prompt_embeds,
gpu_memory_utilization=0.7,
attention_config=attention_config,
) as vllm_model:
if enable_prompt_embeds:
with hf_runner(model, dtype=dtype) as hf_model:
with torch.no_grad():
prompt_embeds = hf_model.get_prompt_embeddings(example_prompts)
vllm_outputs = vllm_model.generate_greedy(prompt_embeds, max_tokens)
vllm_outputs = _fix_prompt_embed_outputs(
vllm_outputs, hf_model, example_prompts
)
hf_outputs = hf_model.generate_greedy(example_prompts, max_tokens)
else:
vllm_outputs = vllm_model.generate_greedy(example_prompts, max_tokens)
with hf_runner(model, dtype=dtype) as hf_model:
hf_outputs = hf_model.generate_greedy(example_prompts, max_tokens)
check_outputs_equal(
outputs_0_lst=hf_outputs,
outputs_1_lst=vllm_outputs,
name_0="hf",
name_1="vllm",
)
def test_failed_model_execution(vllm_runner, monkeypatch) -> None:
# Needed to mock an error in the same process
monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0")
with vllm_runner("facebook/opt-125m", enforce_eager=True) as vllm_model:
if isinstance(vllm_model.llm.llm_engine, LLMEngine):
v1_test_failed_model_execution(vllm_model)
@pytest.mark.parametrize("use_v2_model_runner", [False, True], ids=["v1", "v2"])
def test_raise_on_logit_nans(
vllm_runner, monkeypatch, use_v2_model_runner: bool
) -> None:
monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0")
monkeypatch.setenv("VLLM_RAISE_ON_LOGIT_NANS", "1")
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", str(int(use_v2_model_runner)))
envs.disable_envs_cache()
try:
with vllm_runner("facebook/opt-125m", enforce_eager=True) as vllm_model:
engine_core = vllm_model.llm.llm_engine.engine_core.engine_core
model_runner = engine_core.model_executor.driver_worker.worker.model_runner
assert model_runner.vllm_config.use_v2_model_runner is use_v2_model_runner
model_cls = type(model_runner.model)
original_compute_logits = model_cls.compute_logits
def compute_nan_logits(self, *args, **kwargs):
logits = original_compute_logits(self, *args, **kwargs)
# `logits[0, 0] = ...` copies the scalar H2D and blocks.
logits[0, 0].fill_(float("nan"))
return logits
monkeypatch.setattr(model_cls, "compute_logits", compute_nan_logits)
with pytest.raises(RuntimeError, match="NaNs detected in logits"):
vllm_model.generate_greedy(["Hello, my name is"], 1, use_tqdm=False)
finally:
envs.disable_envs_cache()
def v1_test_failed_model_execution(vllm_model):
engine = vllm_model.llm.llm_engine
mocked_execute_model = Mock(side_effect=RuntimeError("Mocked Critical Error"))
engine.engine_core.engine_core.model_executor.execute_model = mocked_execute_model
with pytest.raises(RuntimeError) as exc_info:
prompts = [
"Hello, my name is",
"The president of the United States is",
"The capital of France is",
"The future of AI is",
]
vllm_model.generate_greedy(prompts, 200, use_tqdm=False)
assert isinstance(exc_info.value, RuntimeError)
assert "Mocked Critical Error" in str(exc_info.value)