[Mini-vLLM] Cut dead subsystems: models, entrypoints, backends, utils, tests
- Drop dead model modules, keeping files live imports require - Remove anthropic, cohere, codegen entrypoints and batch STT modes - Delete unused attention backends, MoE experts, and NPU/XPU util leaves - Strip dead third_party trees and ROCm leaves with no CUDA path - Remove mooncake EC connectors: dependency missing since branch base - Re-audit verified by full import sweep and Qwen3.5 smoke test Co-authored-by: Hinata <hinata@idihore.id> Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
@@ -1,483 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import gc
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
import vllm.device_allocator.cumem as cumem
|
||||
from vllm import LLM, SamplingParams
|
||||
from vllm.device_allocator import get_mem_allocator_instance
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
from ....utils import create_new_process_for_each_test
|
||||
|
||||
DEVICE_TYPE = current_platform.device_type
|
||||
|
||||
|
||||
def mapped_usage(allocator) -> int:
|
||||
"""Bytes the allocator still has mapped on the device.
|
||||
|
||||
`torch.accelerator.get_memory_info()` is a device-wide `cudaMemGetInfo` /
|
||||
`hipMemGetInfo` query, so its delta also carries whatever else on the GPU
|
||||
allocates or frees while we measure. The allocator's own bookkeeping is
|
||||
process-local and exact.
|
||||
"""
|
||||
return sum(
|
||||
data.handle[1]
|
||||
for data in allocator.pointer_to_data.values()
|
||||
if not data.is_asleep
|
||||
)
|
||||
|
||||
|
||||
def _wake_up_with_poisoned_mappings(allocator, byte_value: int = 0xA5) -> None:
|
||||
"""Wake discarded allocations with deterministic nonzero contents."""
|
||||
original_create_and_map = cumem.create_and_map
|
||||
|
||||
def create_and_map_with_poison(handle) -> None:
|
||||
original_create_and_map(handle)
|
||||
_, size, ptr, _ = handle
|
||||
cumem.libcudart.cudaMemset(ptr, byte_value, size)
|
||||
|
||||
cumem.create_and_map = create_and_map_with_poison
|
||||
try:
|
||||
allocator.wake_up()
|
||||
finally:
|
||||
cumem.create_and_map = original_create_and_map
|
||||
|
||||
|
||||
@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn")
|
||||
def test_python_error():
|
||||
"""Test if Python error occurs when there's low-level
|
||||
error happening from the C++ side.
|
||||
"""
|
||||
allocator = get_mem_allocator_instance()
|
||||
total_bytes = torch.accelerator.get_memory_info()[1]
|
||||
alloc_bytes = int(total_bytes * 0.7)
|
||||
tensors = []
|
||||
with allocator.use_memory_pool():
|
||||
# allocate 70% of the total memory
|
||||
x = torch.empty(alloc_bytes, dtype=torch.uint8, device=DEVICE_TYPE)
|
||||
tensors.append(x)
|
||||
# release the memory
|
||||
allocator.sleep(offload_tags=())
|
||||
|
||||
# allocate more memory than the total memory
|
||||
y = torch.empty(alloc_bytes, dtype=torch.uint8, device=DEVICE_TYPE)
|
||||
tensors.append(y)
|
||||
with pytest.raises(RuntimeError):
|
||||
# when the allocator is woken up, it should raise an error
|
||||
# because we don't have enough memory
|
||||
allocator.wake_up()
|
||||
|
||||
|
||||
@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn")
|
||||
def test_basic_cumem():
|
||||
# some tensors from default memory pool
|
||||
shape = (1024, 1024)
|
||||
x = torch.empty(shape, device=DEVICE_TYPE)
|
||||
x.zero_()
|
||||
|
||||
# some tensors from custom memory pool
|
||||
allocator = get_mem_allocator_instance()
|
||||
with allocator.use_memory_pool():
|
||||
# custom memory pool
|
||||
y = torch.empty(shape, device=DEVICE_TYPE)
|
||||
y.zero_()
|
||||
y += 1
|
||||
z = torch.empty(shape, device=DEVICE_TYPE)
|
||||
z.zero_()
|
||||
z += 2
|
||||
|
||||
# they can be used together
|
||||
output = x + y + z
|
||||
assert torch.allclose(output, torch.ones_like(output) * 3)
|
||||
|
||||
# Track the allocator's own mapped bytes, not device-wide free memory.
|
||||
mapped_bytes = mapped_usage(allocator)
|
||||
assert mapped_bytes >= y.nbytes + z.nbytes
|
||||
allocator.sleep()
|
||||
assert mapped_usage(allocator) == 0
|
||||
allocator.wake_up()
|
||||
assert mapped_usage(allocator) == mapped_bytes
|
||||
|
||||
# they can be used together
|
||||
output = x + y + z
|
||||
assert torch.allclose(output, torch.ones_like(output) * 3)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("full_sleep", [0, 1, 2], ids=["kv-only", "sleep-1", "sleep-2"])
|
||||
@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn")
|
||||
def test_release_kv_cache_memory_preserves_generation(full_sleep, monkeypatch):
|
||||
"""Rejected release and KV-only/full sleep cycles preserve greedy tokens."""
|
||||
monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
|
||||
kv_cache_memory_bytes = 256 * 1024 * 1024
|
||||
llm = LLM(
|
||||
"Qwen/Qwen3-0.6B",
|
||||
enable_sleep_mode=True,
|
||||
enforce_eager=True,
|
||||
kv_cache_memory_bytes=kv_cache_memory_bytes,
|
||||
max_model_len=1024,
|
||||
max_num_seqs=4,
|
||||
gpu_memory_utilization=0.05,
|
||||
)
|
||||
prompt = "How are you?"
|
||||
sampling_params = SamplingParams(temperature=0, max_tokens=10)
|
||||
expected = llm.generate(prompt, sampling_params)[0].outputs[0].token_ids
|
||||
|
||||
def get_mapped_bytes_by_tag(worker):
|
||||
allocator = get_mem_allocator_instance()
|
||||
mapped_bytes = mapped_usage(allocator)
|
||||
kv_cache_bytes = sum(
|
||||
data.handle[1]
|
||||
for data in allocator.pointer_to_data.values()
|
||||
if data.tag == "kv_cache" and not data.is_asleep
|
||||
)
|
||||
return mapped_bytes, kv_cache_bytes
|
||||
|
||||
mapped_before, kv_cache_before = llm.collective_rpc(get_mapped_bytes_by_tag)[0]
|
||||
# Utility RPC transports the engine error as a plain Exception.
|
||||
with pytest.raises(Exception, match="requires a completed pause"):
|
||||
llm.release_kv_cache_memory()
|
||||
assert llm.collective_rpc(get_mapped_bytes_by_tag)[0][0] == mapped_before
|
||||
assert not llm.llm_engine.is_sleeping()
|
||||
assert llm.generate(prompt, sampling_params)[0].outputs[0].token_ids == expected
|
||||
|
||||
llm.sleep(level=0)
|
||||
llm.release_kv_cache_memory()
|
||||
mapped_after, kv_cache_after = llm.collective_rpc(get_mapped_bytes_by_tag)[0]
|
||||
assert kv_cache_before > 0
|
||||
assert kv_cache_after == 0
|
||||
assert mapped_before - mapped_after >= kv_cache_before
|
||||
assert mapped_after > 0
|
||||
assert llm.llm_engine.is_sleeping()
|
||||
|
||||
if full_sleep:
|
||||
llm.sleep(level=full_sleep)
|
||||
assert llm.collective_rpc(get_mapped_bytes_by_tag)[0][0] == 0
|
||||
llm.wake_up(tags=None if full_sleep else ["kv_cache"])
|
||||
if full_sleep == 2:
|
||||
llm.collective_rpc("reload_weights")
|
||||
assert llm.collective_rpc(get_mapped_bytes_by_tag)[0][0] == mapped_before
|
||||
assert not llm.llm_engine.is_sleeping()
|
||||
actual = llm.generate(prompt, sampling_params)[0].outputs[0].token_ids
|
||||
assert actual == expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize("first_level", [1, 2])
|
||||
@pytest.mark.parametrize("second_level", [1, 2])
|
||||
@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn")
|
||||
def test_sleep_with_only_weights_asleep(first_level, second_level, monkeypatch):
|
||||
"""Repeated sleep after KV-only wake preserves mappings and recoverability."""
|
||||
monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
|
||||
llm = LLM(
|
||||
"Qwen/Qwen3-0.6B",
|
||||
enable_sleep_mode=True,
|
||||
enforce_eager=True,
|
||||
kv_cache_memory_bytes=256 * 1024 * 1024,
|
||||
max_model_len=1024,
|
||||
max_num_seqs=4,
|
||||
gpu_memory_utilization=0.05,
|
||||
)
|
||||
prompt = "How are you?"
|
||||
sampling_params = SamplingParams(temperature=0, max_tokens=10)
|
||||
expected = llm.generate(prompt, sampling_params)[0].outputs[0].token_ids
|
||||
|
||||
def get_mapped_bytes(worker):
|
||||
return mapped_usage(get_mem_allocator_instance())
|
||||
|
||||
llm.sleep(level=first_level)
|
||||
llm.wake_up(tags=["kv_cache"])
|
||||
mapped_before = llm.collective_rpc(get_mapped_bytes)[0]
|
||||
assert mapped_before > 0
|
||||
llm.sleep(level=second_level)
|
||||
assert llm.collective_rpc(get_mapped_bytes)[0] == mapped_before
|
||||
llm.wake_up(tags=["weights"])
|
||||
if first_level == 2:
|
||||
llm.collective_rpc("reload_weights")
|
||||
assert not llm.llm_engine.is_sleeping()
|
||||
actual = llm.generate(prompt, sampling_params)[0].outputs[0].token_ids
|
||||
assert actual == expected
|
||||
|
||||
|
||||
@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn")
|
||||
def test_discard_tags():
|
||||
"""Test that discard(tags) selectively frees GPU memory for specific
|
||||
tags while keeping other tags mapped and usable."""
|
||||
allocator = get_mem_allocator_instance()
|
||||
|
||||
with allocator.use_memory_pool("weights"):
|
||||
weights = torch.ones(1024, 1024, device=DEVICE_TYPE)
|
||||
|
||||
with allocator.use_memory_pool("kv_cache"):
|
||||
kv = torch.ones(512, 512, device=DEVICE_TYPE)
|
||||
|
||||
mapped_bytes = mapped_usage(allocator)
|
||||
|
||||
# Discard kv_cache only — weights should remain valid
|
||||
allocator.discard("kv_cache")
|
||||
|
||||
mapped_bytes_after_discard = mapped_usage(allocator)
|
||||
assert mapped_bytes - mapped_bytes_after_discard >= kv.nbytes
|
||||
assert mapped_bytes_after_discard >= weights.nbytes
|
||||
|
||||
# Weights are still usable
|
||||
assert torch.allclose(weights, torch.ones_like(weights))
|
||||
|
||||
# Wake up and verify kv_cache is remapped; discarded contents are undefined.
|
||||
allocator.wake_up()
|
||||
assert kv.shape == (512, 512)
|
||||
|
||||
# Full sleep/wake cycle still works after discard
|
||||
allocator.sleep(offload_tags="weights")
|
||||
allocator.wake_up()
|
||||
assert torch.allclose(weights, torch.ones_like(weights))
|
||||
|
||||
|
||||
@pytest.mark.parametrize("tag", ["workspace", None], ids=["workspace", "default"])
|
||||
@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn")
|
||||
def test_selective_wake_restores_internal_tags(tag):
|
||||
"""A selective wake defers only weights/kv_cache; other tags always wake."""
|
||||
allocator = get_mem_allocator_instance()
|
||||
with allocator.use_memory_pool(tag):
|
||||
internal = torch.empty(16 << 20, dtype=torch.uint8, device=DEVICE_TYPE)
|
||||
with allocator.use_memory_pool("kv_cache"):
|
||||
kv = torch.empty(32 << 20, dtype=torch.uint8, device=DEVICE_TYPE)
|
||||
|
||||
allocator.sleep(offload_tags=())
|
||||
assert mapped_usage(allocator) == 0
|
||||
|
||||
allocator.wake_up(tags=["weights"])
|
||||
assert mapped_usage(allocator) == internal.nbytes
|
||||
allocator.wake_up(tags=["kv_cache"])
|
||||
assert mapped_usage(allocator) == internal.nbytes + kv.nbytes
|
||||
|
||||
|
||||
@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn")
|
||||
@pytest.mark.skipif(current_platform.is_xpu(), reason="Uses the CuMem allocator")
|
||||
def test_workspace_scratch_discarded_on_sleep():
|
||||
"""Workspace scratch lives in the sleep-mode pool: sleep discards it, any
|
||||
(selective) wake remaps it at the same address so captured graphs replay,
|
||||
and a buffer outgrown by a resize stays usable until it is released."""
|
||||
from vllm.device_allocator.sleep_mode_backend import CuMemBackend
|
||||
from vllm.v1.worker.workspace import WorkspaceManager
|
||||
|
||||
allocator = get_mem_allocator_instance()
|
||||
manager = WorkspaceManager(
|
||||
torch.device(DEVICE_TYPE),
|
||||
alloc_context=lambda: allocator.use_memory_pool("workspace"),
|
||||
)
|
||||
(small,) = manager.get_simultaneous(((32 << 20,), torch.uint8))
|
||||
# Resize while a view of the old buffer is alive: the tag is re-entered.
|
||||
(scratch,) = manager.get_simultaneous(((64 << 20,), torch.uint8))
|
||||
assert len(allocator.allocator_and_pools["workspace"]) == 2
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph):
|
||||
scratch.add_(1)
|
||||
|
||||
backend = CuMemBackend()
|
||||
backend.suspend(level=1)
|
||||
assert mapped_usage(allocator) == 0
|
||||
# Discarded, not copied.
|
||||
assert all(
|
||||
d.tag == "workspace" and d.cpu_backup_tensor is None
|
||||
for d in allocator.pointer_to_data.values()
|
||||
)
|
||||
|
||||
backend.resume(tags=["kv_cache"])
|
||||
assert mapped_usage(allocator) == small.nbytes + scratch.nbytes
|
||||
scratch.zero_()
|
||||
graph.replay()
|
||||
assert int(scratch.sum()) == scratch.numel()
|
||||
small.fill_(1)
|
||||
|
||||
# The next resize releases the outgrown buffer; wake does not remap it.
|
||||
del small
|
||||
(big,) = manager.get_simultaneous(((128 << 20,), torch.uint8))
|
||||
assert mapped_usage(allocator) == scratch.nbytes + big.nbytes
|
||||
backend.suspend(level=1)
|
||||
backend.resume(tags=["weights"])
|
||||
assert mapped_usage(allocator) == scratch.nbytes + big.nbytes
|
||||
|
||||
|
||||
@pytest.mark.parametrize("free_x_early", [True, False], ids=["x-freed", "x-alive"])
|
||||
@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn")
|
||||
@pytest.mark.skipif(current_platform.is_xpu(), reason="Uses the CuMem allocator")
|
||||
def test_reentered_tag(free_x_early):
|
||||
"""Enter a tag twice (two workspace ubatches): freeing X returns its memory
|
||||
by the next entry, without touching Y or breaking sleep/wake."""
|
||||
allocator = get_mem_allocator_instance()
|
||||
nbytes = 64 << 20
|
||||
with allocator.use_memory_pool("t"):
|
||||
xs = [torch.empty(nbytes, dtype=torch.uint8, device=DEVICE_TYPE)]
|
||||
if free_x_early:
|
||||
xs.clear()
|
||||
with allocator.use_memory_pool("t"):
|
||||
y = torch.empty(nbytes, dtype=torch.uint8, device=DEVICE_TYPE)
|
||||
xs.clear()
|
||||
gc.collect()
|
||||
with allocator.use_memory_pool("t"):
|
||||
pass
|
||||
assert mapped_usage(allocator) == nbytes
|
||||
y.fill_(1)
|
||||
allocator.sleep(offload_tags="t")
|
||||
allocator.wake_up()
|
||||
assert int(y.sum()) == y.numel()
|
||||
allocator.release_pools()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("level", [1, 2], ids=["sleep-1", "sleep-2"])
|
||||
@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn")
|
||||
@pytest.mark.skipif(current_platform.is_xpu(), reason="Uses the CuMem allocator")
|
||||
def test_runtime_state_survives_sleep(level):
|
||||
"""Runtime state is offloaded at both levels and restored by the first
|
||||
selective wake; a nested kv_cache pool keeps its own tag."""
|
||||
from vllm.device_allocator.sleep_mode_backend import CuMemBackend
|
||||
|
||||
allocator = get_mem_allocator_instance()
|
||||
with allocator.use_memory_pool("runtime"):
|
||||
state = torch.arange(1 << 20, device=DEVICE_TYPE)
|
||||
with allocator.use_memory_pool("kv_cache"):
|
||||
kv = torch.zeros(1 << 20, device=DEVICE_TYPE)
|
||||
assert {d.tag for d in allocator.pointer_to_data.values()} == {
|
||||
"runtime",
|
||||
"kv_cache",
|
||||
}
|
||||
|
||||
backend = CuMemBackend()
|
||||
backend.suspend(level=level)
|
||||
assert mapped_usage(allocator) == 0
|
||||
|
||||
backend.resume(tags=["weights"])
|
||||
assert torch.equal(state, torch.arange(1 << 20, device=DEVICE_TYPE))
|
||||
backend.resume(tags=["kv_cache"])
|
||||
kv.fill_(1)
|
||||
assert int(kv.sum()) == kv.numel()
|
||||
|
||||
|
||||
@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn")
|
||||
@pytest.mark.skipif(current_platform.is_xpu(), reason="Uses the CuMem allocator")
|
||||
def test_level2_discards_ordinary_tensor_with_weights_tag():
|
||||
"""Discarded weights-tag memory is remapped; ROCm zeroes it over stale pages."""
|
||||
allocator = get_mem_allocator_instance()
|
||||
|
||||
with allocator.use_memory_pool("weights"):
|
||||
fake_weight = torch.full((4096,), 0x44, dtype=torch.uint8, device=DEVICE_TYPE)
|
||||
ordinary_tensor = torch.full(
|
||||
(4096,), 0x55, dtype=torch.uint8, device=DEVICE_TYPE
|
||||
)
|
||||
|
||||
pointers = (fake_weight.data_ptr(), ordinary_tensor.data_ptr())
|
||||
allocator.sleep(offload_tags=())
|
||||
_wake_up_with_poisoned_mappings(allocator)
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
assert (fake_weight.data_ptr(), ordinary_tensor.data_ptr()) == pointers
|
||||
expected = 0 if current_platform.is_rocm() else 0xA5
|
||||
assert torch.all(fake_weight == expected)
|
||||
assert torch.all(ordinary_tensor == expected)
|
||||
|
||||
|
||||
@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn")
|
||||
@pytest.mark.skipif(current_platform.is_xpu(), reason="CUDA graph not supported on XPU")
|
||||
def test_cumem_with_cudagraph():
|
||||
allocator = get_mem_allocator_instance()
|
||||
with allocator.use_memory_pool():
|
||||
weight = torch.eye(1024, device=DEVICE_TYPE)
|
||||
with allocator.use_memory_pool(tag="discard"):
|
||||
cache = torch.empty(1024, 1024, device=DEVICE_TYPE)
|
||||
|
||||
def model(x):
|
||||
out = x @ weight
|
||||
cache[: out.size(0)].copy_(out)
|
||||
return out + 1
|
||||
|
||||
x = torch.empty(128, 1024, device=DEVICE_TYPE)
|
||||
|
||||
# warmup
|
||||
model(x)
|
||||
|
||||
# capture cudagraph
|
||||
model_graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(model_graph):
|
||||
y = model(x)
|
||||
|
||||
mapped_bytes = mapped_usage(allocator)
|
||||
assert mapped_bytes >= weight.nbytes + cache.nbytes
|
||||
allocator.sleep()
|
||||
assert mapped_usage(allocator) == 0
|
||||
allocator.wake_up()
|
||||
assert mapped_usage(allocator) == mapped_bytes
|
||||
|
||||
# after waking up, the content in the weight tensor
|
||||
# should be restored, but the content in the cache tensor
|
||||
# should be discarded
|
||||
|
||||
# this operation is also compatible with cudagraph
|
||||
|
||||
x.random_()
|
||||
model_graph.replay()
|
||||
|
||||
# cache content is as expected
|
||||
assert torch.allclose(x, cache[: x.size(0)])
|
||||
|
||||
# output content is as expected
|
||||
assert torch.allclose(y, x + 1)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("level", [1, 2], ids=["sleep-1", "sleep-2"])
|
||||
@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn")
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda_alike(), reason="cuMem CUDA graph pool"
|
||||
)
|
||||
def test_cudagraph_pool_sleep(level):
|
||||
"""Routing, and the graph pool backed up at both sleep levels and restored
|
||||
in place by any wake."""
|
||||
from contextlib import nullcontext
|
||||
from types import SimpleNamespace
|
||||
|
||||
import vllm.distributed.device_communicators.pynccl_allocator as nccl_alloc
|
||||
from vllm.compilation.cudagraph_pool import (
|
||||
capture_outside_cumem_pool,
|
||||
capture_pool,
|
||||
)
|
||||
from vllm.device_allocator.sleep_mode_backend import CuMemBackend
|
||||
|
||||
allocator = get_mem_allocator_instance()
|
||||
on, off = (SimpleNamespace(use_cumem_cudagraph_pool=v) for v in (True, False))
|
||||
handle = current_platform.graph_pool_handle()
|
||||
for cfg, ctx in ((off, nullcontext()), (on, capture_outside_cumem_pool())):
|
||||
with ctx, capture_pool(handle, cfg) as used:
|
||||
assert used == handle and allocator.current_tag != "cudagraph"
|
||||
with allocator.use_memory_pool("weights"):
|
||||
weight = torch.full((1 << 20,), 2.0, device=DEVICE_TYPE)
|
||||
x = torch.ones_like(weight)
|
||||
|
||||
def capture() -> list:
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
stream = torch.cuda.Stream()
|
||||
with (
|
||||
capture_pool(handle, on) as pool,
|
||||
torch.cuda.graph(graph, pool=pool, stream=stream),
|
||||
):
|
||||
assert nccl_alloc._graph_pool_id == pool != handle
|
||||
const = torch.empty_like(x) # Never written by replay: needs a backup.
|
||||
y = x * weight + const
|
||||
return [graph, y, const.fill_(3.0)]
|
||||
|
||||
def graph_pool() -> dict[int, int]:
|
||||
data = allocator.pointer_to_data.items()
|
||||
return {
|
||||
p: d.handle[1] for p, d in data if d.tag == "cudagraph" and not d.is_asleep
|
||||
}
|
||||
|
||||
held, backend = capture(), CuMemBackend()
|
||||
mapped = graph_pool()
|
||||
backend.suspend(level=level)
|
||||
backend.resume(tags=["kv_cache"])
|
||||
assert graph_pool() == mapped
|
||||
backend.resume(tags=["weights"])
|
||||
weight.fill_(2.0) # Level 2 discards weights; emulate the reload.
|
||||
held[0].replay()
|
||||
assert torch.equal(held[1], torch.full_like(x, 5.0))
|
||||
@@ -1,483 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import asyncio
|
||||
import math
|
||||
import os
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
import vllm.device_allocator.cumem as cumem
|
||||
import vllm.envs as envs
|
||||
from vllm import LLM, AsyncEngineArgs, AsyncLLMEngine, SamplingParams
|
||||
from vllm.model_executor.model_loader import get_model_loader
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.mem_constants import GiB_bytes
|
||||
|
||||
from ....utils import (
|
||||
create_new_process_for_each_test,
|
||||
multi_gpu_marks,
|
||||
multi_gpu_test,
|
||||
requires_fp8,
|
||||
)
|
||||
|
||||
|
||||
@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn")
|
||||
@pytest.mark.parametrize(
|
||||
"model",
|
||||
[
|
||||
# sleep mode with safetensors
|
||||
"hmellor/tiny-random-LlamaForCausalLM",
|
||||
# sleep mode with pytorch checkpoint
|
||||
"facebook/opt-125m",
|
||||
],
|
||||
)
|
||||
def test_end_to_end(model: str):
|
||||
free, total = torch.accelerator.get_memory_info()
|
||||
used_bytes_baseline = total - free # in case other process is running
|
||||
llm = LLM(model, enable_sleep_mode=True)
|
||||
prompt = "How are you?"
|
||||
sampling_params = SamplingParams(temperature=0, max_tokens=10)
|
||||
output = llm.generate(prompt, sampling_params)
|
||||
|
||||
# the benefit of `llm.sleep(level=2)` is mainly CPU memory usage,
|
||||
# which is difficult to measure in the test. therefore, we only
|
||||
# test sleep level 1 here.
|
||||
llm.sleep(level=1)
|
||||
|
||||
free_gpu_bytes_after_sleep, total = torch.accelerator.get_memory_info()
|
||||
used_bytes = total - free_gpu_bytes_after_sleep - used_bytes_baseline
|
||||
# now the memory usage is mostly cudagraph memory pool,
|
||||
# and it should be less than the model weights (1B model, 2GiB weights)
|
||||
|
||||
# NOTE: In V1, the memory buffer for logits (max_num_reqs x vocab_size)
|
||||
# is captured but cannot be releasesd from PyTorch due to a known bug,
|
||||
# therefore high memory usage after `llm.sleep` is called is expected.
|
||||
# FIXME(youkaichao & ywang96): Fix memory buffer issue with sleep mode
|
||||
# in V1.
|
||||
assert used_bytes < 7 * GiB_bytes
|
||||
|
||||
llm.wake_up()
|
||||
output2 = llm.generate(prompt, sampling_params)
|
||||
# cmp output
|
||||
assert output[0].outputs[0].text == output2[0].outputs[0].text
|
||||
|
||||
llm.sleep(level=1)
|
||||
llm.wake_up(tags=["weights"])
|
||||
|
||||
free_gpu_bytes_wake_up_w, total = torch.accelerator.get_memory_info()
|
||||
used_bytes = total - free_gpu_bytes_wake_up_w - used_bytes_baseline
|
||||
|
||||
# should just reallocate memory for weights (1B model, ~2GiB weights)
|
||||
assert used_bytes < 10 * GiB_bytes
|
||||
|
||||
# now allocate kv cache memory
|
||||
llm.wake_up(tags=["kv_cache"])
|
||||
output3 = llm.generate(prompt, sampling_params)
|
||||
|
||||
# cmp output
|
||||
assert output[0].outputs[0].text == output3[0].outputs[0].text
|
||||
|
||||
|
||||
@create_new_process_for_each_test()
|
||||
def test_deep_sleep(gpu_memory_cleared):
|
||||
model = "hmellor/tiny-random-LlamaForCausalLM"
|
||||
free, total = torch.accelerator.get_memory_info()
|
||||
used_bytes_baseline = total - free # in case other process is running
|
||||
llm = LLM(model, enable_sleep_mode=True)
|
||||
prompt = "How are you?"
|
||||
sampling_params = SamplingParams(temperature=0, max_tokens=10)
|
||||
output = llm.generate(prompt, sampling_params)
|
||||
|
||||
# Put the engine to deep sleep
|
||||
llm.sleep(level=2)
|
||||
|
||||
free_gpu_bytes_after_sleep, total = torch.accelerator.get_memory_info()
|
||||
used_bytes = total - free_gpu_bytes_after_sleep - used_bytes_baseline
|
||||
assert used_bytes < 3 * GiB_bytes
|
||||
|
||||
llm.wake_up(tags=["weights"])
|
||||
llm.collective_rpc("reload_weights")
|
||||
free_gpu_bytes_wake_up_w, total = torch.accelerator.get_memory_info()
|
||||
used_bytes = total - free_gpu_bytes_wake_up_w - used_bytes_baseline
|
||||
assert used_bytes < 4 * GiB_bytes
|
||||
|
||||
# now allocate kv cache and cuda graph memory
|
||||
llm.wake_up(tags=["kv_cache"])
|
||||
output2 = llm.generate(prompt, sampling_params)
|
||||
|
||||
# cmp output
|
||||
assert output[0].outputs[0].text == output2[0].outputs[0].text
|
||||
|
||||
|
||||
@create_new_process_for_each_test()
|
||||
def test_deep_sleep_lora(gpu_memory_cleared):
|
||||
"""Level-2 sleep/wake/reload with enable_lora=True.
|
||||
|
||||
LoRA wrapping moves parameters under base_layer and adds LoRA
|
||||
stacked tensors that are plain attributes, not restored by the
|
||||
reload machinery — reload must forward checkpoint weights through
|
||||
the wrappers and reset the LoRA state afterwards.
|
||||
"""
|
||||
model = "hmellor/tiny-random-LlamaForCausalLM"
|
||||
llm = LLM(
|
||||
model,
|
||||
enable_sleep_mode=True,
|
||||
enable_lora=True,
|
||||
max_lora_rank=8,
|
||||
enforce_eager=True,
|
||||
)
|
||||
prompt = "How are you?"
|
||||
sampling_params = SamplingParams(temperature=0, max_tokens=10)
|
||||
output = llm.generate(prompt, sampling_params)
|
||||
|
||||
# Level-2 sleep discards all GPU memory
|
||||
llm.sleep(level=2)
|
||||
|
||||
# Reload weights from checkpoint
|
||||
llm.wake_up(tags=["weights"])
|
||||
llm.collective_rpc("reload_weights")
|
||||
llm.wake_up(tags=["kv_cache"])
|
||||
output2 = llm.generate(prompt, sampling_params)
|
||||
assert output[0].outputs[0].text == output2[0].outputs[0].text
|
||||
|
||||
# Multiple cycles should not accumulate corruption
|
||||
for _ in range(3):
|
||||
llm.sleep(level=2)
|
||||
llm.wake_up(tags=["weights"])
|
||||
llm.collective_rpc("reload_weights")
|
||||
llm.wake_up(tags=["kv_cache"])
|
||||
output3 = llm.generate(prompt, sampling_params)
|
||||
assert output[0].outputs[0].text == output3[0].outputs[0].text
|
||||
|
||||
|
||||
def _lora_logits_mapping_present(model) -> bool:
|
||||
from vllm.lora.layers.logits_processor import LogitsProcessorWithLoRA
|
||||
|
||||
return any(
|
||||
isinstance(m, LogitsProcessorWithLoRA)
|
||||
and m.sharded_to_full_mapping_gpu is not None
|
||||
for m in model.modules()
|
||||
)
|
||||
|
||||
|
||||
@multi_gpu_test(num_gpus=2)
|
||||
def test_deep_sleep_lora_tp2(monkeypatch):
|
||||
"""Level-2 sleep/wake/reload with enable_lora=True and TP=2.
|
||||
|
||||
With TP > 1 the LoRA logits processor carries
|
||||
``sharded_to_full_mapping_gpu``, a permanent index mapping used to
|
||||
reorder gathered logits. Like the LoRA stacked tensors it is a plain
|
||||
attribute allocated in the sleep-mode pool, so level-2 sleep destroys
|
||||
its contents — it must be restored after reload.
|
||||
"""
|
||||
# Needed for apply_model to reach the multiproc TP workers below.
|
||||
monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
|
||||
|
||||
model = "hmellor/tiny-random-LlamaForCausalLM"
|
||||
llm = LLM(
|
||||
model,
|
||||
enable_sleep_mode=True,
|
||||
enable_lora=True,
|
||||
max_lora_rank=8,
|
||||
tensor_parallel_size=2,
|
||||
enforce_eager=True,
|
||||
)
|
||||
|
||||
# Guard against this test silently not exercising the TP>1 reindex
|
||||
# path (e.g. if lm_head wrapping conditions change).
|
||||
assert all(llm.apply_model(_lora_logits_mapping_present))
|
||||
|
||||
prompt = "How are you?"
|
||||
sampling_params = SamplingParams(temperature=0, max_tokens=10)
|
||||
output = llm.generate(prompt, sampling_params)
|
||||
|
||||
llm.sleep(level=2)
|
||||
llm.wake_up(tags=["weights"])
|
||||
llm.collective_rpc("reload_weights")
|
||||
llm.wake_up(tags=["kv_cache"])
|
||||
output2 = llm.generate(prompt, sampling_params)
|
||||
assert output[0].outputs[0].text == output2[0].outputs[0].text
|
||||
|
||||
|
||||
def _cudagraph_bytes(worker) -> int:
|
||||
data = cumem.CuMemAllocator.get_instance().pointer_to_data.values()
|
||||
return sum(d.handle[1] for d in data if d.tag == "cudagraph")
|
||||
|
||||
|
||||
def _custom_ar_active(worker) -> bool:
|
||||
from vllm.distributed.parallel_state import get_tp_group
|
||||
|
||||
comm = get_tp_group().device_communicator
|
||||
return any(
|
||||
c is not None and not c.disabled for c in (comm.ca_comm, comm.aiter_ar_comm)
|
||||
)
|
||||
|
||||
|
||||
def _capture_registers_buffers(worker) -> bool:
|
||||
from vllm.distributed.parallel_state import get_tp_group
|
||||
|
||||
comm = get_tp_group().device_communicator
|
||||
ca, aiter = comm.ca_comm, comm.aiter_ar_comm
|
||||
return (ca is not None and ca._capture_registered) or (
|
||||
aiter is not None and aiter.aiter_ca.enable_register_for_capturing
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("mode", "breakable", "tp", "offload"),
|
||||
[
|
||||
("FULL", False, 1, True),
|
||||
("PIECEWISE", False, 1, True),
|
||||
("PIECEWISE", True, 1, True),
|
||||
pytest.param(
|
||||
"FULL_AND_PIECEWISE", False, 2, True, marks=multi_gpu_marks(num_gpus=2)
|
||||
),
|
||||
("FULL_AND_PIECEWISE", False, 1, False),
|
||||
],
|
||||
ids=["full", "piecewise", "breakable", "custom-ar-tp2", "off-by-default"],
|
||||
)
|
||||
@create_new_process_for_each_test()
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda_alike(), reason="cuMem CUDA graph pool"
|
||||
)
|
||||
def test_sleep_cudagraph_pool(monkeypatch, mode, breakable, tp, offload):
|
||||
"""Each capture site uses the pool, exact across sleeps; TP=2 needs
|
||||
unregistered custom AR; off changes nothing."""
|
||||
for name, value in [
|
||||
("VLLM_USE_V2_MODEL_RUNNER", "1"),
|
||||
("VLLM_USE_BREAKABLE_CUDAGRAPH", "1" if breakable else "0"),
|
||||
("VLLM_ALLOW_INSECURE_SERIALIZATION", "1"),
|
||||
("VLLM_ALLREDUCE_USE_FLASHINFER", "0"),
|
||||
("VLLM_ALLREDUCE_USE_SYMM_MEM", "0"),
|
||||
]:
|
||||
monkeypatch.setenv(name, value)
|
||||
monkeypatch.delenv("NCCL_GRAPH_REGISTER", raising=False)
|
||||
llm = LLM(
|
||||
"hmellor/tiny-random-LlamaForCausalLM",
|
||||
enable_sleep_mode=True,
|
||||
sleep_mode_offload_cudagraph=offload,
|
||||
tensor_parallel_size=tp,
|
||||
compilation_config={
|
||||
"cudagraph_mode": mode,
|
||||
"pass_config": {"fuse_allreduce_rms": False},
|
||||
},
|
||||
)
|
||||
graph_bytes = llm.collective_rpc(_cudagraph_bytes)
|
||||
assert os.environ.get("NCCL_GRAPH_REGISTER") == ("0" if offload else None)
|
||||
assert all(graph_bytes) if offload else not any(graph_bytes)
|
||||
if not offload:
|
||||
return
|
||||
if tp > 1:
|
||||
if not all(llm.collective_rpc(_custom_ar_active)):
|
||||
pytest.skip("Custom allreduce is unavailable on these GPUs")
|
||||
assert not any(llm.collective_rpc(_capture_registers_buffers))
|
||||
prompt, params = "How are you?", SamplingParams(temperature=0, max_tokens=10)
|
||||
expected = llm.generate(prompt, params)[0].outputs[0].text
|
||||
for level in (1, 2):
|
||||
llm.sleep(level=level)
|
||||
llm.wake_up(tags=["weights"])
|
||||
if level == 2:
|
||||
llm.collective_rpc("reload_weights")
|
||||
llm.wake_up(tags=["kv_cache"])
|
||||
assert llm.generate(prompt, params)[0].outputs[0].text == expected
|
||||
|
||||
|
||||
@multi_gpu_test(num_gpus=2)
|
||||
@create_new_process_for_each_test()
|
||||
@pytest.mark.skipif(not current_platform.is_rocm(), reason="AITER fused allreduce")
|
||||
def test_sleep_cudagraph_pool_aiter_fused_ar_group_quant(monkeypatch):
|
||||
"""The fused AITER AR + RMSNorm + per-group FP8 quant captures into the
|
||||
cuMem pool without IPC-registering its input, across sleeps."""
|
||||
for name, value in [
|
||||
("VLLM_USE_V2_MODEL_RUNNER", "1"),
|
||||
("VLLM_ROCM_USE_AITER", "1"),
|
||||
("VLLM_ALLOW_INSECURE_SERIALIZATION", "1"),
|
||||
]:
|
||||
monkeypatch.setenv(name, value)
|
||||
monkeypatch.delenv("NCCL_GRAPH_REGISTER", raising=False)
|
||||
llm = LLM(
|
||||
"Qwen/Qwen3-0.6B-FP8",
|
||||
enable_sleep_mode=True,
|
||||
sleep_mode_offload_cudagraph=True,
|
||||
tensor_parallel_size=2,
|
||||
max_model_len=1024,
|
||||
compilation_config={"pass_config": {"fuse_allreduce_rms": True}},
|
||||
)
|
||||
if not all(llm.collective_rpc(_custom_ar_active)):
|
||||
pytest.skip("AITER custom allreduce is unavailable on these GPUs")
|
||||
assert not any(llm.collective_rpc(_capture_registers_buffers))
|
||||
assert all(llm.collective_rpc(_cudagraph_bytes))
|
||||
prompt, params = "How are you?", SamplingParams(temperature=0, max_tokens=10)
|
||||
assert llm.generate(prompt, params)[0].outputs[0].text
|
||||
for level in (1, 2):
|
||||
llm.sleep(level=level)
|
||||
llm.wake_up(tags=["weights"])
|
||||
if level == 2:
|
||||
llm.collective_rpc("reload_weights")
|
||||
llm.wake_up(tags=["kv_cache"])
|
||||
assert llm.generate(prompt, params)[0].outputs[0].text
|
||||
|
||||
|
||||
@create_new_process_for_each_test()
|
||||
def test_deep_sleep_async(gpu_memory_cleared):
|
||||
async def test():
|
||||
model = "hmellor/tiny-random-LlamaForCausalLM"
|
||||
free, total = torch.accelerator.get_memory_info()
|
||||
used_bytes_baseline = total - free # in case other process is running
|
||||
engine_args = AsyncEngineArgs(
|
||||
model=model,
|
||||
enable_sleep_mode=True,
|
||||
)
|
||||
|
||||
llm = AsyncLLMEngine.from_engine_args(engine_args)
|
||||
prompt = "How are you?"
|
||||
sampling_params = SamplingParams(temperature=0, max_tokens=10)
|
||||
outputs = llm.generate(prompt, sampling_params, request_id="test_request_id1")
|
||||
async for output in outputs:
|
||||
pass
|
||||
|
||||
# Put the engine to deep sleep
|
||||
await llm.sleep(level=2)
|
||||
|
||||
await llm.wake_up(tags=["weights"])
|
||||
await llm.collective_rpc("reload_weights")
|
||||
free_gpu_bytes_wake_up_w, total = torch.accelerator.get_memory_info()
|
||||
used_bytes = total - free_gpu_bytes_wake_up_w - used_bytes_baseline
|
||||
assert used_bytes < 4 * GiB_bytes
|
||||
|
||||
# now allocate kv cache and cuda graph memory
|
||||
await llm.wake_up(tags=["kv_cache"])
|
||||
outputs2 = llm.generate(prompt, sampling_params, request_id="test_request_id2")
|
||||
async for output2 in outputs2:
|
||||
pass
|
||||
|
||||
# cmp output
|
||||
assert output.outputs[0].text == output2.outputs[0].text
|
||||
|
||||
asyncio.run(test())
|
||||
|
||||
|
||||
def _load_weights(worker) -> None:
|
||||
# Not reload_weights(): its layerwise finalize re-derives the KV scales.
|
||||
model = worker.model_runner.get_model()
|
||||
loader = get_model_loader(worker.load_config)
|
||||
model.load_weights(loader.get_all_weights(worker.model_config, model))
|
||||
|
||||
|
||||
def _attn_kv_scales(model) -> dict[str, list[float]]:
|
||||
return {
|
||||
f"{name}.{attr}": getattr(module, attr).flatten().tolist()
|
||||
for name, module in model.named_modules()
|
||||
for attr in ("_q_scale", "_k_scale", "_v_scale")
|
||||
if isinstance(getattr(module, attr, None), torch.Tensor)
|
||||
}
|
||||
|
||||
|
||||
@requires_fp8
|
||||
@create_new_process_for_each_test()
|
||||
def test_deep_sleep_compressed_tensors_kv_scales(
|
||||
monkeypatch: pytest.MonkeyPatch, gpu_memory_cleared
|
||||
):
|
||||
"""KV scales must survive level-2 sleep plus a weights reload (else NaN)."""
|
||||
monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
|
||||
llm = LLM(
|
||||
"nm-testing/TinyLlama-1.1B-Chat-v1.0-kvcache-fp8-tensor",
|
||||
enable_sleep_mode=True,
|
||||
kv_cache_dtype="fp8",
|
||||
enforce_eager=True,
|
||||
gpu_memory_utilization=0.6,
|
||||
)
|
||||
sampling_params = SamplingParams(temperature=0, max_tokens=10, logprobs=0)
|
||||
scales = llm.apply_model(_attn_kv_scales)[0]
|
||||
assert any(v != 1.0 for values in scales.values() for v in values)
|
||||
expected = llm.generate("How are you?", sampling_params)[0].outputs[0]
|
||||
|
||||
llm.sleep(level=2)
|
||||
llm.wake_up(tags=["weights"])
|
||||
llm.collective_rpc(_load_weights)
|
||||
llm.wake_up(tags=["kv_cache"])
|
||||
|
||||
assert llm.apply_model(_attn_kv_scales)[0] == scales
|
||||
actual = llm.generate("How are you?", sampling_params)[0].outputs[0]
|
||||
assert actual.token_ids == expected.token_ids
|
||||
logprobs = [
|
||||
[lp[t].logprob for lp, t in zip(o.logprobs, o.token_ids)]
|
||||
for o in (expected, actual)
|
||||
]
|
||||
assert all(math.isfinite(x) for x in logprobs[1])
|
||||
assert logprobs[0] == logprobs[1]
|
||||
|
||||
|
||||
@requires_fp8
|
||||
def test_deep_sleep_fp8_kvcache(gpu_memory_cleared):
|
||||
# Regression test for https://github.com/vllm-project/vllm/pull/28783:
|
||||
# fp8 KV cache output must be restored correctly after level-2 sleep.
|
||||
model = "Qwen/Qwen2-0.5B"
|
||||
used_bytes_baseline = current_platform.get_current_memory_usage()
|
||||
|
||||
llm = LLM(model, enable_sleep_mode=True, kv_cache_dtype="fp8")
|
||||
prompt = "How are you?"
|
||||
sampling_params = SamplingParams(temperature=0, max_tokens=10)
|
||||
output = llm.generate(prompt, sampling_params)
|
||||
|
||||
# Put the engine to deep sleep
|
||||
llm.sleep(level=2)
|
||||
|
||||
used_bytes = current_platform.get_current_memory_usage() - used_bytes_baseline
|
||||
|
||||
# Rocm uses more memory for CudaGraphs, so we add 2 GiB more for the threshold
|
||||
rocm_extra_mem_bytes = 2 * GiB_bytes if current_platform.is_rocm() else 0
|
||||
mem_threshold_after_sleep = 3 * GiB_bytes + rocm_extra_mem_bytes
|
||||
assert used_bytes < mem_threshold_after_sleep
|
||||
|
||||
llm.wake_up(tags=["weights"])
|
||||
llm.collective_rpc("reload_weights")
|
||||
|
||||
used_bytes = current_platform.get_current_memory_usage() - used_bytes_baseline
|
||||
mem_threshold_after_wake_up = 4 * GiB_bytes + rocm_extra_mem_bytes
|
||||
assert used_bytes < mem_threshold_after_wake_up
|
||||
|
||||
# now allocate kv cache and cuda graph memory
|
||||
llm.wake_up(tags=["kv_cache"])
|
||||
output2 = llm.generate(prompt, sampling_params)
|
||||
|
||||
# cmp output
|
||||
assert output[0].outputs[0].text == output2[0].outputs[0].text
|
||||
|
||||
|
||||
@requires_fp8
|
||||
def test_deep_sleep_fp8_kvcache_with_undefined_remap(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0")
|
||||
envs.disable_envs_cache()
|
||||
|
||||
llm = LLM(
|
||||
"Qwen/Qwen2-0.5B",
|
||||
enable_sleep_mode=True,
|
||||
kv_cache_dtype="fp8",
|
||||
)
|
||||
prompt = "How are you?"
|
||||
sampling_params = SamplingParams(temperature=0, max_tokens=10)
|
||||
expected = llm.generate(prompt, sampling_params)
|
||||
|
||||
llm.sleep(level=2)
|
||||
llm.wake_up(tags=["weights"])
|
||||
llm.collective_rpc("reload_weights")
|
||||
|
||||
original_create_and_map = cumem.create_and_map
|
||||
|
||||
def create_and_map_with_poison(handle) -> None:
|
||||
original_create_and_map(handle)
|
||||
_, size, ptr, _ = handle
|
||||
cumem.libcudart.cudaMemset(ptr, 0xA5, size)
|
||||
|
||||
monkeypatch.setattr(cumem, "create_and_map", create_and_map_with_poison)
|
||||
|
||||
# New requests must overwrite undefined remapped KV bytes before reading them.
|
||||
llm.wake_up(tags=["kv_cache"])
|
||||
actual = llm.generate(prompt, sampling_params)
|
||||
|
||||
assert expected[0].outputs[0].text == actual[0].outputs[0].text
|
||||
Reference in New Issue
Block a user