[CI] Split (H200 MIG/MI300) Basic Correctness into named jobs (#57054)

Signed-off-by: Thang Nguyen <thangnguyenvn647@gmail.com>
Signed-off-by: Andreas Karatzas <Andreas.Karatzas@amd.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: Andreas Karatzas <Andreas.Karatzas@amd.com>
Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
Thang Nguyen
2026-09-25 17:05:33 -07:00
committed by GitHub
co-authored by Claude Opus 5.5 Andreas Karatzas Codex
parent 9bf44c4013
commit da1aaec31f
18 changed files with 461 additions and 330 deletions
@@ -18,11 +18,12 @@ steps:
VLLM_TEST_DEVICE: "xpu" VLLM_TEST_DEVICE: "xpu"
source_file_dependencies: source_file_dependencies:
- vllm/ - vllm/
- tests/basic_correctness/test_cumem.py - tests/basic_correctness/memory/
- tests/basic_correctness/cpu_offload/
commands: commands:
- >- - >-
bash .buildkite/scripts/hardware_ci/run-intel-test.sh bash .buildkite/scripts/hardware_ci/run-intel-test.sh
'cd tests && 'cd tests &&
export VLLM_WORKER_MULTIPROC_METHOD=spawn && export VLLM_WORKER_MULTIPROC_METHOD=spawn &&
pytest -v -s basic_correctness/test_cpu_offload.py && pytest -v -s basic_correctness/cpu_offload &&
pytest -v -s basic_correctness/test_mem.py' pytest -v -s basic_correctness/memory'
+66 -9
View File
@@ -1030,14 +1030,14 @@ steps:
- tests/distributed/test_ray_v2_executor.py - tests/distributed/test_ray_v2_executor.py
- tests/distributed/test_ray_v2_executor_e2e.py - tests/distributed/test_ray_v2_executor_e2e.py
- tests/distributed/test_pipeline_parallel.py - tests/distributed/test_pipeline_parallel.py
- tests/basic_correctness/test_basic_correctness.py - tests/basic_correctness/models/test_basic_correctness.py
- vllm/platforms/rocm.py - vllm/platforms/rocm.py
commands: commands:
- export VLLM_USE_RAY_V2_EXECUTOR_BACKEND=1 - export VLLM_USE_RAY_V2_EXECUTOR_BACKEND=1
- pytest -v -s distributed/test_ray_v2_executor.py - pytest -v -s distributed/test_ray_v2_executor.py
- pytest -v -s distributed/test_ray_v2_executor_e2e.py - pytest -v -s distributed/test_ray_v2_executor_e2e.py
- pytest -v -s distributed/test_pipeline_parallel.py -k "ray" - pytest -v -s distributed/test_pipeline_parallel.py -k "ray"
- TARGET_TEST_SUITE=L4 pytest -v -s basic_correctness/test_basic_correctness.py -k "ray" - TARGET_TEST_SUITE=L4 pytest -v -s basic_correctness/models/test_basic_correctness.py -k "ray"
- label: ":amd: (MI300) Distributed DP + EP" # TBD - label: ":amd: (MI300) Distributed DP + EP" # TBD
timeout_in_minutes: 50 timeout_in_minutes: 50
@@ -2762,7 +2762,23 @@ steps:
#----------------------------------------------------- mi355 · basic_correctness -----------------------------------------------------# #----------------------------------------------------- mi355 · basic_correctness -----------------------------------------------------#
- label: ":amd: (MI355 DPX) Basic Correctness" # TBD - label: ":amd: (MI355 DPX) Basic Correctness Models %N" # TBD
timeout_in_minutes: 60
parallelism: 2
mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
dind: false
agent_pool: mi355_dpx
fast_check: true
working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- vllm/
- tests/basic_correctness/
optional: true
commands:
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
- VLLM_TARGET_TEST_SUITE=MI355 pytest -v -s basic_correctness/models --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB
- label: ":amd: (MI355 DPX) Basic Correctness CuMem" # TBD
timeout_in_minutes: 60 timeout_in_minutes: 60
mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
dind: false dind: false
@@ -2771,15 +2787,56 @@ steps:
working_dir: "/vllm-workspace/tests" working_dir: "/vllm-workspace/tests"
source_file_dependencies: source_file_dependencies:
- vllm/ - vllm/
- tests/basic_correctness/test_basic_correctness - tests/basic_correctness/
- tests/basic_correctness/test_cpu_offload
- tests/basic_correctness/test_mem.py
optional: true optional: true
commands: commands:
- export VLLM_WORKER_MULTIPROC_METHOD=spawn - export VLLM_WORKER_MULTIPROC_METHOD=spawn
- pytest -v -s basic_correctness/test_mem.py - pytest -v -s basic_correctness/memory/cumem
- VLLM_TARGET_TEST_SUITE=MI355 pytest -v -s basic_correctness/test_basic_correctness.py
- pytest -v -s basic_correctness/test_cpu_offload.py - label: ":amd: (MI355 DPX) Basic Correctness Sleep Mode" # TBD
timeout_in_minutes: 60
mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
dind: false
agent_pool: mi355_dpx
fast_check: true
working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- vllm/
- tests/basic_correctness/
optional: true
commands:
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
- pytest -v -s basic_correctness/memory/sleep_mode
- label: ":amd: (MI355 DPX) Basic Correctness CPU Offload" # TBD
timeout_in_minutes: 60
mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
dind: false
agent_pool: mi355_dpx
fast_check: true
working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- vllm/
- tests/basic_correctness/
optional: true
commands:
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
- pytest -v -s basic_correctness/cpu_offload
- label: ":amd: (MI355 DPX) Basic Correctness Prefetch Offload" # TBD
timeout_in_minutes: 60
mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
dind: false
agent_pool: mi355_dpx
fast_check: true
working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- vllm/
- tests/basic_correctness/
optional: true
commands:
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
- pytest -v -s basic_correctness/prefetch_offload
#-------------------------------------------------------- mi355 · benchmarks ---------------------------------------------------------# #-------------------------------------------------------- mi355 · benchmarks ---------------------------------------------------------#
+74 -9
View File
@@ -2,24 +2,89 @@ group: Basic Correctness
depends_on: depends_on:
- image-build - image-build
steps: steps:
- label: ":nvidia: (H200 MIG 18GB) Basic Correctness" - label: ":nvidia: (H200 MIG 18GB) Basic Correctness Models %N"
key: basic-correctness key: basic-correctness
timeout_in_minutes: 68 timeout_in_minutes: 68
device: h200_18gb device: h200_18gb
source_file_dependencies: parallelism: 2
source_file_dependencies: &basic-correctness-deps
- vllm/ - vllm/
- "!vllm/distributed/kv_transfer/" - "!vllm/distributed/kv_transfer/"
- tests/basic_correctness/test_basic_correctness - tests/basic_correctness/
- tests/basic_correctness/test_cpu_offload
- tests/basic_correctness/test_mem.py
commands: commands:
- export VLLM_WORKER_MULTIPROC_METHOD=spawn - export VLLM_WORKER_MULTIPROC_METHOD=spawn
- pytest -v -s basic_correctness/test_mem.py - pytest -v -s basic_correctness/models --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB
- pytest -v -s basic_correctness/test_basic_correctness.py
- pytest -v -s basic_correctness/test_cpu_offload.py
mirror: mirror:
amd: amd:
label: ":amd: (MI355 DPX) Basic Correctness" label: ":amd: (MI355 DPX) Basic Correctness Models %N"
dind: false
device: mi355_dpx
timeout_in_minutes: 60
depends_on:
- image-build-amd
- label: ":nvidia: (H200 MIG 18GB) Basic Correctness CuMem"
key: basic-correctness-cumem
timeout_in_minutes: 68
device: h200_18gb
source_file_dependencies: *basic-correctness-deps
commands:
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
- pytest -v -s basic_correctness/memory/cumem
mirror:
amd:
label: ":amd: (MI355 DPX) Basic Correctness CuMem"
dind: false
device: mi355_dpx
timeout_in_minutes: 60
depends_on:
- image-build-amd
- label: ":nvidia: (H200 MIG 18GB) Basic Correctness Sleep Mode"
key: basic-correctness-sleep-mode
timeout_in_minutes: 68
device: h200_18gb
source_file_dependencies: *basic-correctness-deps
commands:
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
- pytest -v -s basic_correctness/memory/sleep_mode
mirror:
amd:
label: ":amd: (MI355 DPX) Basic Correctness Sleep Mode"
dind: false
device: mi355_dpx
timeout_in_minutes: 60
depends_on:
- image-build-amd
- label: ":nvidia: (H200 MIG 18GB) Basic Correctness CPU Offload"
key: basic-correctness-cpu-offload
timeout_in_minutes: 68
device: h200_18gb
source_file_dependencies: *basic-correctness-deps
commands:
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
- pytest -v -s basic_correctness/cpu_offload
mirror:
amd:
label: ":amd: (MI355 DPX) Basic Correctness CPU Offload"
dind: false
device: mi355_dpx
timeout_in_minutes: 60
depends_on:
- image-build-amd
- label: ":nvidia: (H200 MIG 18GB) Basic Correctness Prefetch Offload"
key: basic-correctness-prefetch-offload
timeout_in_minutes: 68
device: h200_18gb
source_file_dependencies: *basic-correctness-deps
commands:
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
- pytest -v -s basic_correctness/prefetch_offload
mirror:
amd:
label: ":amd: (MI355 DPX) Basic Correctness Prefetch Offload"
dind: false dind: false
device: mi355_dpx device: mi355_dpx
timeout_in_minutes: 60 timeout_in_minutes: 60
+2 -2
View File
@@ -525,11 +525,11 @@ steps:
- tests/distributed/test_ray_v2_executor.py - tests/distributed/test_ray_v2_executor.py
- tests/distributed/test_ray_v2_executor_e2e.py - tests/distributed/test_ray_v2_executor_e2e.py
- tests/distributed/test_pipeline_parallel.py - tests/distributed/test_pipeline_parallel.py
- tests/basic_correctness/test_basic_correctness.py - tests/basic_correctness/models/test_basic_correctness.py
commands: commands:
- export VLLM_USE_RAY_V2_EXECUTOR_BACKEND=1 - export VLLM_USE_RAY_V2_EXECUTOR_BACKEND=1
- export NCCL_CUMEM_HOST_ENABLE=0 - export NCCL_CUMEM_HOST_ENABLE=0
- pytest -v -s distributed/test_ray_v2_executor.py - pytest -v -s distributed/test_ray_v2_executor.py
- pytest -v -s distributed/test_ray_v2_executor_e2e.py - pytest -v -s distributed/test_ray_v2_executor_e2e.py
- pytest -v -s distributed/test_pipeline_parallel.py -k "ray" - pytest -v -s distributed/test_pipeline_parallel.py -k "ray"
- TARGET_TEST_SUITE=L4 pytest -v -s basic_correctness/test_basic_correctness.py -k "ray" - TARGET_TEST_SUITE=L4 pytest -v -s basic_correctness/models/test_basic_correctness.py -k "ray"
@@ -15,7 +15,7 @@ from vllm.model_executor.offloader import (
) )
from vllm.v1.worker.gpu.model_runner import GPUModelRunner from vllm.v1.worker.gpu.model_runner import GPUModelRunner
from ..utils import compare_two_settings from ...utils import compare_two_settings
@pytest.mark.parametrize("disable_pin_memory", [False, True]) @pytest.mark.parametrize("disable_pin_memory", [False, True])
@@ -0,0 +1,302 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import torch
import vllm.device_allocator.cumem as cumem
from vllm import LLM, SamplingParams
from vllm.device_allocator import get_mem_allocator_instance
from vllm.platforms import current_platform
from ....utils import create_new_process_for_each_test
DEVICE_TYPE = current_platform.device_type
def mapped_usage(allocator) -> int:
"""Bytes the allocator still has mapped on the device.
`torch.accelerator.get_memory_info()` is a device-wide `cudaMemGetInfo` /
`hipMemGetInfo` query, so its delta also carries whatever else on the GPU
allocates or frees while we measure. The allocator's own bookkeeping is
process-local and exact.
"""
return sum(
data.handle[1]
for data in allocator.pointer_to_data.values()
if not data.is_asleep
)
def _wake_up_with_poisoned_mappings(allocator, byte_value: int = 0xA5) -> None:
"""Wake discarded allocations with deterministic nonzero contents."""
original_create_and_map = cumem.create_and_map
def create_and_map_with_poison(handle) -> None:
original_create_and_map(handle)
_, size, ptr, _ = handle
cumem.libcudart.cudaMemset(ptr, byte_value, size)
cumem.create_and_map = create_and_map_with_poison
try:
allocator.wake_up()
finally:
cumem.create_and_map = original_create_and_map
@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn")
def test_python_error():
"""Test if Python error occurs when there's low-level
error happening from the C++ side.
"""
allocator = get_mem_allocator_instance()
total_bytes = torch.accelerator.get_memory_info()[1]
alloc_bytes = int(total_bytes * 0.7)
tensors = []
with allocator.use_memory_pool():
# allocate 70% of the total memory
x = torch.empty(alloc_bytes, dtype=torch.uint8, device=DEVICE_TYPE)
tensors.append(x)
# release the memory
allocator.sleep(offload_tags=())
# allocate more memory than the total memory
y = torch.empty(alloc_bytes, dtype=torch.uint8, device=DEVICE_TYPE)
tensors.append(y)
with pytest.raises(RuntimeError):
# when the allocator is woken up, it should raise an error
# because we don't have enough memory
allocator.wake_up()
@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn")
def test_basic_cumem():
# some tensors from default memory pool
shape = (1024, 1024)
x = torch.empty(shape, device=DEVICE_TYPE)
x.zero_()
# some tensors from custom memory pool
allocator = get_mem_allocator_instance()
with allocator.use_memory_pool():
# custom memory pool
y = torch.empty(shape, device=DEVICE_TYPE)
y.zero_()
y += 1
z = torch.empty(shape, device=DEVICE_TYPE)
z.zero_()
z += 2
# they can be used together
output = x + y + z
assert torch.allclose(output, torch.ones_like(output) * 3)
# Track the allocator's own mapped bytes, not device-wide free memory.
mapped_bytes = mapped_usage(allocator)
assert mapped_bytes >= y.nbytes + z.nbytes
allocator.sleep()
assert mapped_usage(allocator) == 0
allocator.wake_up()
assert mapped_usage(allocator) == mapped_bytes
# they can be used together
output = x + y + z
assert torch.allclose(output, torch.ones_like(output) * 3)
@pytest.mark.parametrize("full_sleep", [0, 1, 2], ids=["kv-only", "sleep-1", "sleep-2"])
@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn")
def test_release_kv_cache_memory_preserves_generation(full_sleep, monkeypatch):
"""Rejected release and KV-only/full sleep cycles preserve greedy tokens."""
monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
kv_cache_memory_bytes = 256 * 1024 * 1024
llm = LLM(
"Qwen/Qwen3-0.6B",
enable_sleep_mode=True,
enforce_eager=True,
kv_cache_memory_bytes=kv_cache_memory_bytes,
max_model_len=1024,
max_num_seqs=4,
gpu_memory_utilization=0.05,
)
prompt = "How are you?"
sampling_params = SamplingParams(temperature=0, max_tokens=10)
expected = llm.generate(prompt, sampling_params)[0].outputs[0].token_ids
def get_mapped_bytes_by_tag(worker):
allocator = get_mem_allocator_instance()
mapped_bytes = mapped_usage(allocator)
kv_cache_bytes = sum(
data.handle[1]
for data in allocator.pointer_to_data.values()
if data.tag == "kv_cache" and not data.is_asleep
)
return mapped_bytes, kv_cache_bytes
mapped_before, kv_cache_before = llm.collective_rpc(get_mapped_bytes_by_tag)[0]
# Utility RPC transports the engine error as a plain Exception.
with pytest.raises(Exception, match="requires a completed pause"):
llm.release_kv_cache_memory()
assert llm.collective_rpc(get_mapped_bytes_by_tag)[0][0] == mapped_before
assert not llm.llm_engine.is_sleeping()
assert llm.generate(prompt, sampling_params)[0].outputs[0].token_ids == expected
llm.sleep(level=0)
llm.release_kv_cache_memory()
mapped_after, kv_cache_after = llm.collective_rpc(get_mapped_bytes_by_tag)[0]
assert kv_cache_before > 0
assert kv_cache_after == 0
assert mapped_before - mapped_after >= kv_cache_before
assert mapped_after > 0
assert llm.llm_engine.is_sleeping()
if full_sleep:
llm.sleep(level=full_sleep)
assert llm.collective_rpc(get_mapped_bytes_by_tag)[0][0] == 0
llm.wake_up(tags=None if full_sleep else ["kv_cache"])
if full_sleep == 2:
llm.collective_rpc("reload_weights")
assert llm.collective_rpc(get_mapped_bytes_by_tag)[0][0] == mapped_before
assert not llm.llm_engine.is_sleeping()
actual = llm.generate(prompt, sampling_params)[0].outputs[0].token_ids
assert actual == expected
@pytest.mark.parametrize("first_level", [1, 2])
@pytest.mark.parametrize("second_level", [1, 2])
@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn")
def test_sleep_with_only_weights_asleep(first_level, second_level, monkeypatch):
"""Repeated sleep after KV-only wake preserves mappings and recoverability."""
monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
llm = LLM(
"Qwen/Qwen3-0.6B",
enable_sleep_mode=True,
enforce_eager=True,
kv_cache_memory_bytes=256 * 1024 * 1024,
max_model_len=1024,
max_num_seqs=4,
gpu_memory_utilization=0.05,
)
prompt = "How are you?"
sampling_params = SamplingParams(temperature=0, max_tokens=10)
expected = llm.generate(prompt, sampling_params)[0].outputs[0].token_ids
def get_mapped_bytes(worker):
return mapped_usage(get_mem_allocator_instance())
llm.sleep(level=first_level)
llm.wake_up(tags=["kv_cache"])
mapped_before = llm.collective_rpc(get_mapped_bytes)[0]
assert mapped_before > 0
llm.sleep(level=second_level)
assert llm.collective_rpc(get_mapped_bytes)[0] == mapped_before
llm.wake_up(tags=["weights"])
if first_level == 2:
llm.collective_rpc("reload_weights")
assert not llm.llm_engine.is_sleeping()
actual = llm.generate(prompt, sampling_params)[0].outputs[0].token_ids
assert actual == expected
@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn")
def test_discard_tags():
"""Test that discard(tags) selectively frees GPU memory for specific
tags while keeping other tags mapped and usable."""
allocator = get_mem_allocator_instance()
with allocator.use_memory_pool("weights"):
weights = torch.ones(1024, 1024, device=DEVICE_TYPE)
with allocator.use_memory_pool("kv_cache"):
kv = torch.ones(512, 512, device=DEVICE_TYPE)
mapped_bytes = mapped_usage(allocator)
# Discard kv_cache only — weights should remain valid
allocator.discard("kv_cache")
mapped_bytes_after_discard = mapped_usage(allocator)
assert mapped_bytes - mapped_bytes_after_discard >= kv.nbytes
assert mapped_bytes_after_discard >= weights.nbytes
# Weights are still usable
assert torch.allclose(weights, torch.ones_like(weights))
# Wake up and verify kv_cache is remapped; discarded contents are undefined.
allocator.wake_up()
assert kv.shape == (512, 512)
# Full sleep/wake cycle still works after discard
allocator.sleep(offload_tags="weights")
allocator.wake_up()
assert torch.allclose(weights, torch.ones_like(weights))
@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn")
@pytest.mark.skipif(current_platform.is_xpu(), reason="Uses the CuMem allocator")
def test_level2_discards_ordinary_tensor_with_weights_tag():
"""Reproduce the level-2 variant for an ordinary tensor in weights."""
allocator = get_mem_allocator_instance()
with allocator.use_memory_pool("weights"):
fake_weight = torch.full((4096,), 0x44, dtype=torch.uint8, device=DEVICE_TYPE)
ordinary_tensor = torch.full(
(4096,), 0x55, dtype=torch.uint8, device=DEVICE_TYPE
)
pointers = (fake_weight.data_ptr(), ordinary_tensor.data_ptr())
allocator.sleep(offload_tags=())
_wake_up_with_poisoned_mappings(allocator)
torch.accelerator.synchronize()
assert (fake_weight.data_ptr(), ordinary_tensor.data_ptr()) == pointers
assert torch.all(fake_weight == 0xA5)
assert torch.all(ordinary_tensor == 0xA5)
@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn")
@pytest.mark.skipif(current_platform.is_xpu(), reason="CUDA graph not supported on XPU")
def test_cumem_with_cudagraph():
allocator = get_mem_allocator_instance()
with allocator.use_memory_pool():
weight = torch.eye(1024, device=DEVICE_TYPE)
with allocator.use_memory_pool(tag="discard"):
cache = torch.empty(1024, 1024, device=DEVICE_TYPE)
def model(x):
out = x @ weight
cache[: out.size(0)].copy_(out)
return out + 1
x = torch.empty(128, 1024, device=DEVICE_TYPE)
# warmup
model(x)
# capture cudagraph
model_graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(model_graph):
y = model(x)
mapped_bytes = mapped_usage(allocator)
assert mapped_bytes >= weight.nbytes + cache.nbytes
allocator.sleep()
assert mapped_usage(allocator) == 0
allocator.wake_up()
assert mapped_usage(allocator) == mapped_bytes
# after waking up, the content in the weight tensor
# should be restored, but the content in the cache tensor
# should be discarded
# this operation is also compatible with cudagraph
x.random_()
model_graph.replay()
# cache content is as expected
assert torch.allclose(x, cache[: x.size(0)])
# output content is as expected
assert torch.allclose(y, x + 1)
@@ -9,301 +9,10 @@ import torch
import vllm.device_allocator.cumem as cumem import vllm.device_allocator.cumem as cumem
import vllm.envs as envs import vllm.envs as envs
from vllm import LLM, AsyncEngineArgs, AsyncLLMEngine, SamplingParams from vllm import LLM, AsyncEngineArgs, AsyncLLMEngine, SamplingParams
from vllm.device_allocator import get_mem_allocator_instance
from vllm.platforms import current_platform from vllm.platforms import current_platform
from vllm.utils.mem_constants import GiB_bytes from vllm.utils.mem_constants import GiB_bytes
from ..utils import create_new_process_for_each_test, requires_fp8 from ....utils import create_new_process_for_each_test, multi_gpu_test, requires_fp8
DEVICE_TYPE = current_platform.device_type
def mapped_usage(allocator) -> int:
"""Bytes the allocator still has mapped on the device.
`torch.accelerator.get_memory_info()` is a device-wide `cudaMemGetInfo` /
`hipMemGetInfo` query, so its delta also carries whatever else on the GPU
allocates or frees while we measure. The allocator's own bookkeeping is
process-local and exact.
"""
return sum(
data.handle[1]
for data in allocator.pointer_to_data.values()
if not data.is_asleep
)
def _wake_up_with_poisoned_mappings(allocator, byte_value: int = 0xA5) -> None:
"""Wake discarded allocations with deterministic nonzero contents."""
original_create_and_map = cumem.create_and_map
def create_and_map_with_poison(handle) -> None:
original_create_and_map(handle)
_, size, ptr, _ = handle
cumem.libcudart.cudaMemset(ptr, byte_value, size)
cumem.create_and_map = create_and_map_with_poison
try:
allocator.wake_up()
finally:
cumem.create_and_map = original_create_and_map
@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn")
def test_python_error():
"""Test if Python error occurs when there's low-level
error happening from the C++ side.
"""
allocator = get_mem_allocator_instance()
total_bytes = torch.accelerator.get_memory_info()[1]
alloc_bytes = int(total_bytes * 0.7)
tensors = []
with allocator.use_memory_pool():
# allocate 70% of the total memory
x = torch.empty(alloc_bytes, dtype=torch.uint8, device=DEVICE_TYPE)
tensors.append(x)
# release the memory
allocator.sleep(offload_tags=())
# allocate more memory than the total memory
y = torch.empty(alloc_bytes, dtype=torch.uint8, device=DEVICE_TYPE)
tensors.append(y)
with pytest.raises(RuntimeError):
# when the allocator is woken up, it should raise an error
# because we don't have enough memory
allocator.wake_up()
@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn")
def test_basic_cumem():
# some tensors from default memory pool
shape = (1024, 1024)
x = torch.empty(shape, device=DEVICE_TYPE)
x.zero_()
# some tensors from custom memory pool
allocator = get_mem_allocator_instance()
with allocator.use_memory_pool():
# custom memory pool
y = torch.empty(shape, device=DEVICE_TYPE)
y.zero_()
y += 1
z = torch.empty(shape, device=DEVICE_TYPE)
z.zero_()
z += 2
# they can be used together
output = x + y + z
assert torch.allclose(output, torch.ones_like(output) * 3)
# Track the allocator's own mapped bytes, not device-wide free memory.
mapped_bytes = mapped_usage(allocator)
assert mapped_bytes >= y.nbytes + z.nbytes
allocator.sleep()
assert mapped_usage(allocator) == 0
allocator.wake_up()
assert mapped_usage(allocator) == mapped_bytes
# they can be used together
output = x + y + z
assert torch.allclose(output, torch.ones_like(output) * 3)
@pytest.mark.parametrize("full_sleep", [0, 1, 2], ids=["kv-only", "sleep-1", "sleep-2"])
@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn")
def test_release_kv_cache_memory_preserves_generation(full_sleep, monkeypatch):
"""Rejected release and KV-only/full sleep cycles preserve greedy tokens."""
monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
kv_cache_memory_bytes = 256 * 1024 * 1024
llm = LLM(
"Qwen/Qwen3-0.6B",
enable_sleep_mode=True,
enforce_eager=True,
kv_cache_memory_bytes=kv_cache_memory_bytes,
max_model_len=1024,
max_num_seqs=4,
gpu_memory_utilization=0.05,
)
prompt = "How are you?"
sampling_params = SamplingParams(temperature=0, max_tokens=10)
expected = llm.generate(prompt, sampling_params)[0].outputs[0].token_ids
def get_mapped_bytes_by_tag(worker):
allocator = get_mem_allocator_instance()
mapped_bytes = mapped_usage(allocator)
kv_cache_bytes = sum(
data.handle[1]
for data in allocator.pointer_to_data.values()
if data.tag == "kv_cache" and not data.is_asleep
)
return mapped_bytes, kv_cache_bytes
mapped_before, kv_cache_before = llm.collective_rpc(get_mapped_bytes_by_tag)[0]
# Utility RPC transports the engine error as a plain Exception.
with pytest.raises(Exception, match="requires a completed pause"):
llm.release_kv_cache_memory()
assert llm.collective_rpc(get_mapped_bytes_by_tag)[0][0] == mapped_before
assert not llm.llm_engine.is_sleeping()
assert llm.generate(prompt, sampling_params)[0].outputs[0].token_ids == expected
llm.sleep(level=0)
llm.release_kv_cache_memory()
mapped_after, kv_cache_after = llm.collective_rpc(get_mapped_bytes_by_tag)[0]
assert kv_cache_before > 0
assert kv_cache_after == 0
assert mapped_before - mapped_after >= kv_cache_before
assert mapped_after > 0
assert llm.llm_engine.is_sleeping()
if full_sleep:
llm.sleep(level=full_sleep)
assert llm.collective_rpc(get_mapped_bytes_by_tag)[0][0] == 0
llm.wake_up(tags=None if full_sleep else ["kv_cache"])
if full_sleep == 2:
llm.collective_rpc("reload_weights")
assert llm.collective_rpc(get_mapped_bytes_by_tag)[0][0] == mapped_before
assert not llm.llm_engine.is_sleeping()
actual = llm.generate(prompt, sampling_params)[0].outputs[0].token_ids
assert actual == expected
@pytest.mark.parametrize("first_level", [1, 2])
@pytest.mark.parametrize("second_level", [1, 2])
@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn")
def test_sleep_with_only_weights_asleep(first_level, second_level, monkeypatch):
"""Repeated sleep after KV-only wake preserves mappings and recoverability."""
monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
llm = LLM(
"Qwen/Qwen3-0.6B",
enable_sleep_mode=True,
enforce_eager=True,
kv_cache_memory_bytes=256 * 1024 * 1024,
max_model_len=1024,
max_num_seqs=4,
gpu_memory_utilization=0.05,
)
prompt = "How are you?"
sampling_params = SamplingParams(temperature=0, max_tokens=10)
expected = llm.generate(prompt, sampling_params)[0].outputs[0].token_ids
def get_mapped_bytes(worker):
return mapped_usage(get_mem_allocator_instance())
llm.sleep(level=first_level)
llm.wake_up(tags=["kv_cache"])
mapped_before = llm.collective_rpc(get_mapped_bytes)[0]
assert mapped_before > 0
llm.sleep(level=second_level)
assert llm.collective_rpc(get_mapped_bytes)[0] == mapped_before
llm.wake_up(tags=["weights"])
if first_level == 2:
llm.collective_rpc("reload_weights")
assert not llm.llm_engine.is_sleeping()
actual = llm.generate(prompt, sampling_params)[0].outputs[0].token_ids
assert actual == expected
@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn")
def test_discard_tags():
"""Test that discard(tags) selectively frees GPU memory for specific
tags while keeping other tags mapped and usable."""
allocator = get_mem_allocator_instance()
with allocator.use_memory_pool("weights"):
weights = torch.ones(1024, 1024, device=DEVICE_TYPE)
with allocator.use_memory_pool("kv_cache"):
kv = torch.ones(512, 512, device=DEVICE_TYPE)
mapped_bytes = mapped_usage(allocator)
# Discard kv_cache only — weights should remain valid
allocator.discard("kv_cache")
mapped_bytes_after_discard = mapped_usage(allocator)
assert mapped_bytes - mapped_bytes_after_discard >= kv.nbytes
assert mapped_bytes_after_discard >= weights.nbytes
# Weights are still usable
assert torch.allclose(weights, torch.ones_like(weights))
# Wake up and verify kv_cache is remapped; discarded contents are undefined.
allocator.wake_up()
assert kv.shape == (512, 512)
# Full sleep/wake cycle still works after discard
allocator.sleep(offload_tags="weights")
allocator.wake_up()
assert torch.allclose(weights, torch.ones_like(weights))
@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn")
@pytest.mark.skipif(current_platform.is_xpu(), reason="Uses the CuMem allocator")
def test_level2_discards_ordinary_tensor_with_weights_tag():
"""Reproduce the level-2 variant for an ordinary tensor in weights."""
allocator = get_mem_allocator_instance()
with allocator.use_memory_pool("weights"):
fake_weight = torch.full((4096,), 0x44, dtype=torch.uint8, device=DEVICE_TYPE)
ordinary_tensor = torch.full(
(4096,), 0x55, dtype=torch.uint8, device=DEVICE_TYPE
)
pointers = (fake_weight.data_ptr(), ordinary_tensor.data_ptr())
allocator.sleep(offload_tags=())
_wake_up_with_poisoned_mappings(allocator)
torch.accelerator.synchronize()
assert (fake_weight.data_ptr(), ordinary_tensor.data_ptr()) == pointers
assert torch.all(fake_weight == 0xA5)
assert torch.all(ordinary_tensor == 0xA5)
@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn")
@pytest.mark.skipif(current_platform.is_xpu(), reason="CUDA graph not supported on XPU")
def test_cumem_with_cudagraph():
allocator = get_mem_allocator_instance()
with allocator.use_memory_pool():
weight = torch.eye(1024, device=DEVICE_TYPE)
with allocator.use_memory_pool(tag="discard"):
cache = torch.empty(1024, 1024, device=DEVICE_TYPE)
def model(x):
out = x @ weight
cache[: out.size(0)].copy_(out)
return out + 1
x = torch.empty(128, 1024, device=DEVICE_TYPE)
# warmup
model(x)
# capture cudagraph
model_graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(model_graph):
y = model(x)
mapped_bytes = mapped_usage(allocator)
assert mapped_bytes >= weight.nbytes + cache.nbytes
allocator.sleep()
assert mapped_usage(allocator) == 0
allocator.wake_up()
assert mapped_usage(allocator) == mapped_bytes
# after waking up, the content in the weight tensor
# should be restored, but the content in the cache tensor
# should be discarded
# this operation is also compatible with cudagraph
x.random_()
model_graph.replay()
# cache content is as expected
assert torch.allclose(x, cache[: x.size(0)])
# output content is as expected
assert torch.allclose(y, x + 1)
@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn") @create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn")
@@ -445,8 +154,8 @@ def _lora_logits_mapping_present(model) -> bool:
) )
@create_new_process_for_each_test() @multi_gpu_test(num_gpus=2)
def test_deep_sleep_lora_tp2(num_gpus_available, monkeypatch): def test_deep_sleep_lora_tp2(monkeypatch):
"""Level-2 sleep/wake/reload with enable_lora=True and TP=2. """Level-2 sleep/wake/reload with enable_lora=True and TP=2.
With TP > 1 the LoRA logits processor carries With TP > 1 the LoRA logits processor carries
@@ -455,9 +164,6 @@ def test_deep_sleep_lora_tp2(num_gpus_available, monkeypatch):
attribute allocated in the sleep-mode pool, so level-2 sleep destroys attribute allocated in the sleep-mode pool, so level-2 sleep destroys
its contents — it must be restored after reload. its contents — it must be restored after reload.
""" """
if num_gpus_available < 2:
pytest.skip("Requires at least 2 GPUs")
# Needed for apply_model to reach the multiproc TP workers below. # Needed for apply_model to reach the multiproc TP workers below.
monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1") monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
@@ -2,7 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Compare the short outputs of HF and vLLM when using greedy sampling. """Compare the short outputs of HF and vLLM when using greedy sampling.
Run `pytest tests/basic_correctness/test_basic_correctness.py`. Run `pytest tests/basic_correctness/models/test_basic_correctness.py`.
""" """
import os import os
@@ -17,9 +17,9 @@ from vllm import LLM
from vllm.platforms import current_platform from vllm.platforms import current_platform
from vllm.v1.engine.llm_engine import LLMEngine from vllm.v1.engine.llm_engine import LLMEngine
from ..conftest import HfRunner, VllmRunner from ...conftest import HfRunner, VllmRunner
from ..models.utils import check_outputs_equal from ...models.utils import check_outputs_equal
from ..utils import multi_gpu_test from ...utils import multi_gpu_test
ATTN_BACKEND = ["ROCM_ATTN"] if current_platform.is_rocm() else ["FLASH_ATTN"] ATTN_BACKEND = ["ROCM_ATTN"] if current_platform.is_rocm() else ["FLASH_ATTN"]
@@ -2,7 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Test prefetch offloading correctness with Llama model.""" """Test prefetch offloading correctness with Llama model."""
from ..utils import compare_two_settings from ...utils import compare_two_settings
def test_prefetch_offload_llama(): def test_prefetch_offload_llama():
+1 -1
View File
@@ -928,7 +928,7 @@ class HfRunner:
# shutdown. This is helpful in cases where the HfRunner is # shutdown. This is helpful in cases where the HfRunner is
# initialized after significant GPU memory is already occupied, # initialized after significant GPU memory is already occupied,
# e.g. in # e.g. in
# tests/basic_correctness/test_basic_correctness.py::test_models_distributed # tests/basic_correctness/models/test_basic_correctness.py::test_models_distributed
# where vllm worker processes are still alive and holding GPU # where vllm worker processes are still alive and holding GPU
# memory when hf_runner.__exit__ is called. # memory when hf_runner.__exit__ is called.
from tests.utils import record_gpu_memory_usage_stats from tests.utils import record_gpu_memory_usage_stats
+1 -1
View File
@@ -2,7 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# Expanded quantized model tests for CPU offloading # Expanded quantized model tests for CPU offloading
# Base tests: tests/basic_correctness/test_cpu_offload.py # Base tests: tests/basic_correctness/cpu_offload/test_cpu_offload.py
import pytest import pytest
+1 -1
View File
@@ -4,7 +4,7 @@
These cover the registry/factory contract and capability flags. They do not These cover the registry/factory contract and capability flags. They do not
touch CUDA - the ``cumem`` suspend/resume path is exercised end-to-end on GPU touch CUDA - the ``cumem`` suspend/resume path is exercised end-to-end on GPU
in ``tests/basic_correctness/test_cumem.py``. in ``tests/basic_correctness/memory/``.
""" """
import pytest import pytest