[CI] Split (H200 MIG/MI300) Basic Correctness into named jobs (#57054)
Signed-off-by: Thang Nguyen <thangnguyenvn647@gmail.com> Signed-off-by: Andreas Karatzas <Andreas.Karatzas@amd.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: Andreas Karatzas <Andreas.Karatzas@amd.com> Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
Thang Nguyen
co-authored by
Claude Opus 5.5
Andreas Karatzas
Codex
Claude Opus 5.5
Andreas Karatzas
Codex
parent
9bf44c4013
commit
da1aaec31f
@@ -18,11 +18,12 @@ steps:
|
||||
VLLM_TEST_DEVICE: "xpu"
|
||||
source_file_dependencies:
|
||||
- vllm/
|
||||
- tests/basic_correctness/test_cumem.py
|
||||
- tests/basic_correctness/memory/
|
||||
- tests/basic_correctness/cpu_offload/
|
||||
commands:
|
||||
- >-
|
||||
bash .buildkite/scripts/hardware_ci/run-intel-test.sh
|
||||
'cd tests &&
|
||||
export VLLM_WORKER_MULTIPROC_METHOD=spawn &&
|
||||
pytest -v -s basic_correctness/test_cpu_offload.py &&
|
||||
pytest -v -s basic_correctness/test_mem.py'
|
||||
pytest -v -s basic_correctness/cpu_offload &&
|
||||
pytest -v -s basic_correctness/memory'
|
||||
|
||||
@@ -1030,14 +1030,14 @@ steps:
|
||||
- tests/distributed/test_ray_v2_executor.py
|
||||
- tests/distributed/test_ray_v2_executor_e2e.py
|
||||
- tests/distributed/test_pipeline_parallel.py
|
||||
- tests/basic_correctness/test_basic_correctness.py
|
||||
- tests/basic_correctness/models/test_basic_correctness.py
|
||||
- vllm/platforms/rocm.py
|
||||
commands:
|
||||
- export VLLM_USE_RAY_V2_EXECUTOR_BACKEND=1
|
||||
- pytest -v -s distributed/test_ray_v2_executor.py
|
||||
- pytest -v -s distributed/test_ray_v2_executor_e2e.py
|
||||
- pytest -v -s distributed/test_pipeline_parallel.py -k "ray"
|
||||
- TARGET_TEST_SUITE=L4 pytest -v -s basic_correctness/test_basic_correctness.py -k "ray"
|
||||
- TARGET_TEST_SUITE=L4 pytest -v -s basic_correctness/models/test_basic_correctness.py -k "ray"
|
||||
|
||||
- label: ":amd: (MI300) Distributed DP + EP" # TBD
|
||||
timeout_in_minutes: 50
|
||||
@@ -2762,7 +2762,23 @@ steps:
|
||||
|
||||
#----------------------------------------------------- mi355 · basic_correctness -----------------------------------------------------#
|
||||
|
||||
- label: ":amd: (MI355 DPX) Basic Correctness" # TBD
|
||||
- label: ":amd: (MI355 DPX) Basic Correctness Models %N" # TBD
|
||||
timeout_in_minutes: 60
|
||||
parallelism: 2
|
||||
mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
|
||||
dind: false
|
||||
agent_pool: mi355_dpx
|
||||
fast_check: true
|
||||
working_dir: "/vllm-workspace/tests"
|
||||
source_file_dependencies:
|
||||
- vllm/
|
||||
- tests/basic_correctness/
|
||||
optional: true
|
||||
commands:
|
||||
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
- VLLM_TARGET_TEST_SUITE=MI355 pytest -v -s basic_correctness/models --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB
|
||||
|
||||
- label: ":amd: (MI355 DPX) Basic Correctness CuMem" # TBD
|
||||
timeout_in_minutes: 60
|
||||
mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
|
||||
dind: false
|
||||
@@ -2771,15 +2787,56 @@ steps:
|
||||
working_dir: "/vllm-workspace/tests"
|
||||
source_file_dependencies:
|
||||
- vllm/
|
||||
- tests/basic_correctness/test_basic_correctness
|
||||
- tests/basic_correctness/test_cpu_offload
|
||||
- tests/basic_correctness/test_mem.py
|
||||
- tests/basic_correctness/
|
||||
optional: true
|
||||
commands:
|
||||
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
- pytest -v -s basic_correctness/test_mem.py
|
||||
- VLLM_TARGET_TEST_SUITE=MI355 pytest -v -s basic_correctness/test_basic_correctness.py
|
||||
- pytest -v -s basic_correctness/test_cpu_offload.py
|
||||
- pytest -v -s basic_correctness/memory/cumem
|
||||
|
||||
- label: ":amd: (MI355 DPX) Basic Correctness Sleep Mode" # TBD
|
||||
timeout_in_minutes: 60
|
||||
mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
|
||||
dind: false
|
||||
agent_pool: mi355_dpx
|
||||
fast_check: true
|
||||
working_dir: "/vllm-workspace/tests"
|
||||
source_file_dependencies:
|
||||
- vllm/
|
||||
- tests/basic_correctness/
|
||||
optional: true
|
||||
commands:
|
||||
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
- pytest -v -s basic_correctness/memory/sleep_mode
|
||||
|
||||
- label: ":amd: (MI355 DPX) Basic Correctness CPU Offload" # TBD
|
||||
timeout_in_minutes: 60
|
||||
mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
|
||||
dind: false
|
||||
agent_pool: mi355_dpx
|
||||
fast_check: true
|
||||
working_dir: "/vllm-workspace/tests"
|
||||
source_file_dependencies:
|
||||
- vllm/
|
||||
- tests/basic_correctness/
|
||||
optional: true
|
||||
commands:
|
||||
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
- pytest -v -s basic_correctness/cpu_offload
|
||||
|
||||
- label: ":amd: (MI355 DPX) Basic Correctness Prefetch Offload" # TBD
|
||||
timeout_in_minutes: 60
|
||||
mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
|
||||
dind: false
|
||||
agent_pool: mi355_dpx
|
||||
fast_check: true
|
||||
working_dir: "/vllm-workspace/tests"
|
||||
source_file_dependencies:
|
||||
- vllm/
|
||||
- tests/basic_correctness/
|
||||
optional: true
|
||||
commands:
|
||||
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
- pytest -v -s basic_correctness/prefetch_offload
|
||||
|
||||
#-------------------------------------------------------- mi355 · benchmarks ---------------------------------------------------------#
|
||||
|
||||
|
||||
@@ -1,25 +1,90 @@
|
||||
group: Basic Correctness
|
||||
depends_on:
|
||||
depends_on:
|
||||
- image-build
|
||||
steps:
|
||||
- label: ":nvidia: (H200 MIG 18GB) Basic Correctness"
|
||||
- label: ":nvidia: (H200 MIG 18GB) Basic Correctness Models %N"
|
||||
key: basic-correctness
|
||||
timeout_in_minutes: 68
|
||||
device: h200_18gb
|
||||
source_file_dependencies:
|
||||
parallelism: 2
|
||||
source_file_dependencies: &basic-correctness-deps
|
||||
- vllm/
|
||||
- "!vllm/distributed/kv_transfer/"
|
||||
- tests/basic_correctness/test_basic_correctness
|
||||
- tests/basic_correctness/test_cpu_offload
|
||||
- tests/basic_correctness/test_mem.py
|
||||
- tests/basic_correctness/
|
||||
commands:
|
||||
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
- pytest -v -s basic_correctness/test_mem.py
|
||||
- pytest -v -s basic_correctness/test_basic_correctness.py
|
||||
- pytest -v -s basic_correctness/test_cpu_offload.py
|
||||
- pytest -v -s basic_correctness/models --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB
|
||||
mirror:
|
||||
amd:
|
||||
label: ":amd: (MI355 DPX) Basic Correctness"
|
||||
label: ":amd: (MI355 DPX) Basic Correctness Models %N"
|
||||
dind: false
|
||||
device: mi355_dpx
|
||||
timeout_in_minutes: 60
|
||||
depends_on:
|
||||
- image-build-amd
|
||||
|
||||
- label: ":nvidia: (H200 MIG 18GB) Basic Correctness CuMem"
|
||||
key: basic-correctness-cumem
|
||||
timeout_in_minutes: 68
|
||||
device: h200_18gb
|
||||
source_file_dependencies: *basic-correctness-deps
|
||||
commands:
|
||||
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
- pytest -v -s basic_correctness/memory/cumem
|
||||
mirror:
|
||||
amd:
|
||||
label: ":amd: (MI355 DPX) Basic Correctness CuMem"
|
||||
dind: false
|
||||
device: mi355_dpx
|
||||
timeout_in_minutes: 60
|
||||
depends_on:
|
||||
- image-build-amd
|
||||
|
||||
- label: ":nvidia: (H200 MIG 18GB) Basic Correctness Sleep Mode"
|
||||
key: basic-correctness-sleep-mode
|
||||
timeout_in_minutes: 68
|
||||
device: h200_18gb
|
||||
source_file_dependencies: *basic-correctness-deps
|
||||
commands:
|
||||
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
- pytest -v -s basic_correctness/memory/sleep_mode
|
||||
mirror:
|
||||
amd:
|
||||
label: ":amd: (MI355 DPX) Basic Correctness Sleep Mode"
|
||||
dind: false
|
||||
device: mi355_dpx
|
||||
timeout_in_minutes: 60
|
||||
depends_on:
|
||||
- image-build-amd
|
||||
|
||||
- label: ":nvidia: (H200 MIG 18GB) Basic Correctness CPU Offload"
|
||||
key: basic-correctness-cpu-offload
|
||||
timeout_in_minutes: 68
|
||||
device: h200_18gb
|
||||
source_file_dependencies: *basic-correctness-deps
|
||||
commands:
|
||||
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
- pytest -v -s basic_correctness/cpu_offload
|
||||
mirror:
|
||||
amd:
|
||||
label: ":amd: (MI355 DPX) Basic Correctness CPU Offload"
|
||||
dind: false
|
||||
device: mi355_dpx
|
||||
timeout_in_minutes: 60
|
||||
depends_on:
|
||||
- image-build-amd
|
||||
|
||||
- label: ":nvidia: (H200 MIG 18GB) Basic Correctness Prefetch Offload"
|
||||
key: basic-correctness-prefetch-offload
|
||||
timeout_in_minutes: 68
|
||||
device: h200_18gb
|
||||
source_file_dependencies: *basic-correctness-deps
|
||||
commands:
|
||||
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
- pytest -v -s basic_correctness/prefetch_offload
|
||||
mirror:
|
||||
amd:
|
||||
label: ":amd: (MI355 DPX) Basic Correctness Prefetch Offload"
|
||||
dind: false
|
||||
device: mi355_dpx
|
||||
timeout_in_minutes: 60
|
||||
|
||||
@@ -525,11 +525,11 @@ steps:
|
||||
- tests/distributed/test_ray_v2_executor.py
|
||||
- tests/distributed/test_ray_v2_executor_e2e.py
|
||||
- tests/distributed/test_pipeline_parallel.py
|
||||
- tests/basic_correctness/test_basic_correctness.py
|
||||
- tests/basic_correctness/models/test_basic_correctness.py
|
||||
commands:
|
||||
- export VLLM_USE_RAY_V2_EXECUTOR_BACKEND=1
|
||||
- export NCCL_CUMEM_HOST_ENABLE=0
|
||||
- pytest -v -s distributed/test_ray_v2_executor.py
|
||||
- pytest -v -s distributed/test_ray_v2_executor_e2e.py
|
||||
- pytest -v -s distributed/test_pipeline_parallel.py -k "ray"
|
||||
- TARGET_TEST_SUITE=L4 pytest -v -s basic_correctness/test_basic_correctness.py -k "ray"
|
||||
- TARGET_TEST_SUITE=L4 pytest -v -s basic_correctness/models/test_basic_correctness.py -k "ray"
|
||||
|
||||
+1
-1
@@ -15,7 +15,7 @@ from vllm.model_executor.offloader import (
|
||||
)
|
||||
from vllm.v1.worker.gpu.model_runner import GPUModelRunner
|
||||
|
||||
from ..utils import compare_two_settings
|
||||
from ...utils import compare_two_settings
|
||||
|
||||
|
||||
@pytest.mark.parametrize("disable_pin_memory", [False, True])
|
||||
@@ -0,0 +1,302 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
import vllm.device_allocator.cumem as cumem
|
||||
from vllm import LLM, SamplingParams
|
||||
from vllm.device_allocator import get_mem_allocator_instance
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
from ....utils import create_new_process_for_each_test
|
||||
|
||||
DEVICE_TYPE = current_platform.device_type
|
||||
|
||||
|
||||
def mapped_usage(allocator) -> int:
|
||||
"""Bytes the allocator still has mapped on the device.
|
||||
|
||||
`torch.accelerator.get_memory_info()` is a device-wide `cudaMemGetInfo` /
|
||||
`hipMemGetInfo` query, so its delta also carries whatever else on the GPU
|
||||
allocates or frees while we measure. The allocator's own bookkeeping is
|
||||
process-local and exact.
|
||||
"""
|
||||
return sum(
|
||||
data.handle[1]
|
||||
for data in allocator.pointer_to_data.values()
|
||||
if not data.is_asleep
|
||||
)
|
||||
|
||||
|
||||
def _wake_up_with_poisoned_mappings(allocator, byte_value: int = 0xA5) -> None:
|
||||
"""Wake discarded allocations with deterministic nonzero contents."""
|
||||
original_create_and_map = cumem.create_and_map
|
||||
|
||||
def create_and_map_with_poison(handle) -> None:
|
||||
original_create_and_map(handle)
|
||||
_, size, ptr, _ = handle
|
||||
cumem.libcudart.cudaMemset(ptr, byte_value, size)
|
||||
|
||||
cumem.create_and_map = create_and_map_with_poison
|
||||
try:
|
||||
allocator.wake_up()
|
||||
finally:
|
||||
cumem.create_and_map = original_create_and_map
|
||||
|
||||
|
||||
@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn")
|
||||
def test_python_error():
|
||||
"""Test if Python error occurs when there's low-level
|
||||
error happening from the C++ side.
|
||||
"""
|
||||
allocator = get_mem_allocator_instance()
|
||||
total_bytes = torch.accelerator.get_memory_info()[1]
|
||||
alloc_bytes = int(total_bytes * 0.7)
|
||||
tensors = []
|
||||
with allocator.use_memory_pool():
|
||||
# allocate 70% of the total memory
|
||||
x = torch.empty(alloc_bytes, dtype=torch.uint8, device=DEVICE_TYPE)
|
||||
tensors.append(x)
|
||||
# release the memory
|
||||
allocator.sleep(offload_tags=())
|
||||
|
||||
# allocate more memory than the total memory
|
||||
y = torch.empty(alloc_bytes, dtype=torch.uint8, device=DEVICE_TYPE)
|
||||
tensors.append(y)
|
||||
with pytest.raises(RuntimeError):
|
||||
# when the allocator is woken up, it should raise an error
|
||||
# because we don't have enough memory
|
||||
allocator.wake_up()
|
||||
|
||||
|
||||
@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn")
|
||||
def test_basic_cumem():
|
||||
# some tensors from default memory pool
|
||||
shape = (1024, 1024)
|
||||
x = torch.empty(shape, device=DEVICE_TYPE)
|
||||
x.zero_()
|
||||
|
||||
# some tensors from custom memory pool
|
||||
allocator = get_mem_allocator_instance()
|
||||
with allocator.use_memory_pool():
|
||||
# custom memory pool
|
||||
y = torch.empty(shape, device=DEVICE_TYPE)
|
||||
y.zero_()
|
||||
y += 1
|
||||
z = torch.empty(shape, device=DEVICE_TYPE)
|
||||
z.zero_()
|
||||
z += 2
|
||||
|
||||
# they can be used together
|
||||
output = x + y + z
|
||||
assert torch.allclose(output, torch.ones_like(output) * 3)
|
||||
|
||||
# Track the allocator's own mapped bytes, not device-wide free memory.
|
||||
mapped_bytes = mapped_usage(allocator)
|
||||
assert mapped_bytes >= y.nbytes + z.nbytes
|
||||
allocator.sleep()
|
||||
assert mapped_usage(allocator) == 0
|
||||
allocator.wake_up()
|
||||
assert mapped_usage(allocator) == mapped_bytes
|
||||
|
||||
# they can be used together
|
||||
output = x + y + z
|
||||
assert torch.allclose(output, torch.ones_like(output) * 3)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("full_sleep", [0, 1, 2], ids=["kv-only", "sleep-1", "sleep-2"])
|
||||
@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn")
|
||||
def test_release_kv_cache_memory_preserves_generation(full_sleep, monkeypatch):
|
||||
"""Rejected release and KV-only/full sleep cycles preserve greedy tokens."""
|
||||
monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
|
||||
kv_cache_memory_bytes = 256 * 1024 * 1024
|
||||
llm = LLM(
|
||||
"Qwen/Qwen3-0.6B",
|
||||
enable_sleep_mode=True,
|
||||
enforce_eager=True,
|
||||
kv_cache_memory_bytes=kv_cache_memory_bytes,
|
||||
max_model_len=1024,
|
||||
max_num_seqs=4,
|
||||
gpu_memory_utilization=0.05,
|
||||
)
|
||||
prompt = "How are you?"
|
||||
sampling_params = SamplingParams(temperature=0, max_tokens=10)
|
||||
expected = llm.generate(prompt, sampling_params)[0].outputs[0].token_ids
|
||||
|
||||
def get_mapped_bytes_by_tag(worker):
|
||||
allocator = get_mem_allocator_instance()
|
||||
mapped_bytes = mapped_usage(allocator)
|
||||
kv_cache_bytes = sum(
|
||||
data.handle[1]
|
||||
for data in allocator.pointer_to_data.values()
|
||||
if data.tag == "kv_cache" and not data.is_asleep
|
||||
)
|
||||
return mapped_bytes, kv_cache_bytes
|
||||
|
||||
mapped_before, kv_cache_before = llm.collective_rpc(get_mapped_bytes_by_tag)[0]
|
||||
# Utility RPC transports the engine error as a plain Exception.
|
||||
with pytest.raises(Exception, match="requires a completed pause"):
|
||||
llm.release_kv_cache_memory()
|
||||
assert llm.collective_rpc(get_mapped_bytes_by_tag)[0][0] == mapped_before
|
||||
assert not llm.llm_engine.is_sleeping()
|
||||
assert llm.generate(prompt, sampling_params)[0].outputs[0].token_ids == expected
|
||||
|
||||
llm.sleep(level=0)
|
||||
llm.release_kv_cache_memory()
|
||||
mapped_after, kv_cache_after = llm.collective_rpc(get_mapped_bytes_by_tag)[0]
|
||||
assert kv_cache_before > 0
|
||||
assert kv_cache_after == 0
|
||||
assert mapped_before - mapped_after >= kv_cache_before
|
||||
assert mapped_after > 0
|
||||
assert llm.llm_engine.is_sleeping()
|
||||
|
||||
if full_sleep:
|
||||
llm.sleep(level=full_sleep)
|
||||
assert llm.collective_rpc(get_mapped_bytes_by_tag)[0][0] == 0
|
||||
llm.wake_up(tags=None if full_sleep else ["kv_cache"])
|
||||
if full_sleep == 2:
|
||||
llm.collective_rpc("reload_weights")
|
||||
assert llm.collective_rpc(get_mapped_bytes_by_tag)[0][0] == mapped_before
|
||||
assert not llm.llm_engine.is_sleeping()
|
||||
actual = llm.generate(prompt, sampling_params)[0].outputs[0].token_ids
|
||||
assert actual == expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize("first_level", [1, 2])
|
||||
@pytest.mark.parametrize("second_level", [1, 2])
|
||||
@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn")
|
||||
def test_sleep_with_only_weights_asleep(first_level, second_level, monkeypatch):
|
||||
"""Repeated sleep after KV-only wake preserves mappings and recoverability."""
|
||||
monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
|
||||
llm = LLM(
|
||||
"Qwen/Qwen3-0.6B",
|
||||
enable_sleep_mode=True,
|
||||
enforce_eager=True,
|
||||
kv_cache_memory_bytes=256 * 1024 * 1024,
|
||||
max_model_len=1024,
|
||||
max_num_seqs=4,
|
||||
gpu_memory_utilization=0.05,
|
||||
)
|
||||
prompt = "How are you?"
|
||||
sampling_params = SamplingParams(temperature=0, max_tokens=10)
|
||||
expected = llm.generate(prompt, sampling_params)[0].outputs[0].token_ids
|
||||
|
||||
def get_mapped_bytes(worker):
|
||||
return mapped_usage(get_mem_allocator_instance())
|
||||
|
||||
llm.sleep(level=first_level)
|
||||
llm.wake_up(tags=["kv_cache"])
|
||||
mapped_before = llm.collective_rpc(get_mapped_bytes)[0]
|
||||
assert mapped_before > 0
|
||||
llm.sleep(level=second_level)
|
||||
assert llm.collective_rpc(get_mapped_bytes)[0] == mapped_before
|
||||
llm.wake_up(tags=["weights"])
|
||||
if first_level == 2:
|
||||
llm.collective_rpc("reload_weights")
|
||||
assert not llm.llm_engine.is_sleeping()
|
||||
actual = llm.generate(prompt, sampling_params)[0].outputs[0].token_ids
|
||||
assert actual == expected
|
||||
|
||||
|
||||
@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn")
|
||||
def test_discard_tags():
|
||||
"""Test that discard(tags) selectively frees GPU memory for specific
|
||||
tags while keeping other tags mapped and usable."""
|
||||
allocator = get_mem_allocator_instance()
|
||||
|
||||
with allocator.use_memory_pool("weights"):
|
||||
weights = torch.ones(1024, 1024, device=DEVICE_TYPE)
|
||||
|
||||
with allocator.use_memory_pool("kv_cache"):
|
||||
kv = torch.ones(512, 512, device=DEVICE_TYPE)
|
||||
|
||||
mapped_bytes = mapped_usage(allocator)
|
||||
|
||||
# Discard kv_cache only — weights should remain valid
|
||||
allocator.discard("kv_cache")
|
||||
|
||||
mapped_bytes_after_discard = mapped_usage(allocator)
|
||||
assert mapped_bytes - mapped_bytes_after_discard >= kv.nbytes
|
||||
assert mapped_bytes_after_discard >= weights.nbytes
|
||||
|
||||
# Weights are still usable
|
||||
assert torch.allclose(weights, torch.ones_like(weights))
|
||||
|
||||
# Wake up and verify kv_cache is remapped; discarded contents are undefined.
|
||||
allocator.wake_up()
|
||||
assert kv.shape == (512, 512)
|
||||
|
||||
# Full sleep/wake cycle still works after discard
|
||||
allocator.sleep(offload_tags="weights")
|
||||
allocator.wake_up()
|
||||
assert torch.allclose(weights, torch.ones_like(weights))
|
||||
|
||||
|
||||
@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn")
|
||||
@pytest.mark.skipif(current_platform.is_xpu(), reason="Uses the CuMem allocator")
|
||||
def test_level2_discards_ordinary_tensor_with_weights_tag():
|
||||
"""Reproduce the level-2 variant for an ordinary tensor in weights."""
|
||||
allocator = get_mem_allocator_instance()
|
||||
|
||||
with allocator.use_memory_pool("weights"):
|
||||
fake_weight = torch.full((4096,), 0x44, dtype=torch.uint8, device=DEVICE_TYPE)
|
||||
ordinary_tensor = torch.full(
|
||||
(4096,), 0x55, dtype=torch.uint8, device=DEVICE_TYPE
|
||||
)
|
||||
|
||||
pointers = (fake_weight.data_ptr(), ordinary_tensor.data_ptr())
|
||||
allocator.sleep(offload_tags=())
|
||||
_wake_up_with_poisoned_mappings(allocator)
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
assert (fake_weight.data_ptr(), ordinary_tensor.data_ptr()) == pointers
|
||||
assert torch.all(fake_weight == 0xA5)
|
||||
assert torch.all(ordinary_tensor == 0xA5)
|
||||
|
||||
|
||||
@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn")
|
||||
@pytest.mark.skipif(current_platform.is_xpu(), reason="CUDA graph not supported on XPU")
|
||||
def test_cumem_with_cudagraph():
|
||||
allocator = get_mem_allocator_instance()
|
||||
with allocator.use_memory_pool():
|
||||
weight = torch.eye(1024, device=DEVICE_TYPE)
|
||||
with allocator.use_memory_pool(tag="discard"):
|
||||
cache = torch.empty(1024, 1024, device=DEVICE_TYPE)
|
||||
|
||||
def model(x):
|
||||
out = x @ weight
|
||||
cache[: out.size(0)].copy_(out)
|
||||
return out + 1
|
||||
|
||||
x = torch.empty(128, 1024, device=DEVICE_TYPE)
|
||||
|
||||
# warmup
|
||||
model(x)
|
||||
|
||||
# capture cudagraph
|
||||
model_graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(model_graph):
|
||||
y = model(x)
|
||||
|
||||
mapped_bytes = mapped_usage(allocator)
|
||||
assert mapped_bytes >= weight.nbytes + cache.nbytes
|
||||
allocator.sleep()
|
||||
assert mapped_usage(allocator) == 0
|
||||
allocator.wake_up()
|
||||
assert mapped_usage(allocator) == mapped_bytes
|
||||
|
||||
# after waking up, the content in the weight tensor
|
||||
# should be restored, but the content in the cache tensor
|
||||
# should be discarded
|
||||
|
||||
# this operation is also compatible with cudagraph
|
||||
|
||||
x.random_()
|
||||
model_graph.replay()
|
||||
|
||||
# cache content is as expected
|
||||
assert torch.allclose(x, cache[: x.size(0)])
|
||||
|
||||
# output content is as expected
|
||||
assert torch.allclose(y, x + 1)
|
||||
+3
-297
@@ -9,301 +9,10 @@ import torch
|
||||
import vllm.device_allocator.cumem as cumem
|
||||
import vllm.envs as envs
|
||||
from vllm import LLM, AsyncEngineArgs, AsyncLLMEngine, SamplingParams
|
||||
from vllm.device_allocator import get_mem_allocator_instance
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.mem_constants import GiB_bytes
|
||||
|
||||
from ..utils import create_new_process_for_each_test, requires_fp8
|
||||
|
||||
DEVICE_TYPE = current_platform.device_type
|
||||
|
||||
|
||||
def mapped_usage(allocator) -> int:
|
||||
"""Bytes the allocator still has mapped on the device.
|
||||
|
||||
`torch.accelerator.get_memory_info()` is a device-wide `cudaMemGetInfo` /
|
||||
`hipMemGetInfo` query, so its delta also carries whatever else on the GPU
|
||||
allocates or frees while we measure. The allocator's own bookkeeping is
|
||||
process-local and exact.
|
||||
"""
|
||||
return sum(
|
||||
data.handle[1]
|
||||
for data in allocator.pointer_to_data.values()
|
||||
if not data.is_asleep
|
||||
)
|
||||
|
||||
|
||||
def _wake_up_with_poisoned_mappings(allocator, byte_value: int = 0xA5) -> None:
|
||||
"""Wake discarded allocations with deterministic nonzero contents."""
|
||||
original_create_and_map = cumem.create_and_map
|
||||
|
||||
def create_and_map_with_poison(handle) -> None:
|
||||
original_create_and_map(handle)
|
||||
_, size, ptr, _ = handle
|
||||
cumem.libcudart.cudaMemset(ptr, byte_value, size)
|
||||
|
||||
cumem.create_and_map = create_and_map_with_poison
|
||||
try:
|
||||
allocator.wake_up()
|
||||
finally:
|
||||
cumem.create_and_map = original_create_and_map
|
||||
|
||||
|
||||
@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn")
|
||||
def test_python_error():
|
||||
"""Test if Python error occurs when there's low-level
|
||||
error happening from the C++ side.
|
||||
"""
|
||||
allocator = get_mem_allocator_instance()
|
||||
total_bytes = torch.accelerator.get_memory_info()[1]
|
||||
alloc_bytes = int(total_bytes * 0.7)
|
||||
tensors = []
|
||||
with allocator.use_memory_pool():
|
||||
# allocate 70% of the total memory
|
||||
x = torch.empty(alloc_bytes, dtype=torch.uint8, device=DEVICE_TYPE)
|
||||
tensors.append(x)
|
||||
# release the memory
|
||||
allocator.sleep(offload_tags=())
|
||||
|
||||
# allocate more memory than the total memory
|
||||
y = torch.empty(alloc_bytes, dtype=torch.uint8, device=DEVICE_TYPE)
|
||||
tensors.append(y)
|
||||
with pytest.raises(RuntimeError):
|
||||
# when the allocator is woken up, it should raise an error
|
||||
# because we don't have enough memory
|
||||
allocator.wake_up()
|
||||
|
||||
|
||||
@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn")
|
||||
def test_basic_cumem():
|
||||
# some tensors from default memory pool
|
||||
shape = (1024, 1024)
|
||||
x = torch.empty(shape, device=DEVICE_TYPE)
|
||||
x.zero_()
|
||||
|
||||
# some tensors from custom memory pool
|
||||
allocator = get_mem_allocator_instance()
|
||||
with allocator.use_memory_pool():
|
||||
# custom memory pool
|
||||
y = torch.empty(shape, device=DEVICE_TYPE)
|
||||
y.zero_()
|
||||
y += 1
|
||||
z = torch.empty(shape, device=DEVICE_TYPE)
|
||||
z.zero_()
|
||||
z += 2
|
||||
|
||||
# they can be used together
|
||||
output = x + y + z
|
||||
assert torch.allclose(output, torch.ones_like(output) * 3)
|
||||
|
||||
# Track the allocator's own mapped bytes, not device-wide free memory.
|
||||
mapped_bytes = mapped_usage(allocator)
|
||||
assert mapped_bytes >= y.nbytes + z.nbytes
|
||||
allocator.sleep()
|
||||
assert mapped_usage(allocator) == 0
|
||||
allocator.wake_up()
|
||||
assert mapped_usage(allocator) == mapped_bytes
|
||||
|
||||
# they can be used together
|
||||
output = x + y + z
|
||||
assert torch.allclose(output, torch.ones_like(output) * 3)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("full_sleep", [0, 1, 2], ids=["kv-only", "sleep-1", "sleep-2"])
|
||||
@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn")
|
||||
def test_release_kv_cache_memory_preserves_generation(full_sleep, monkeypatch):
|
||||
"""Rejected release and KV-only/full sleep cycles preserve greedy tokens."""
|
||||
monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
|
||||
kv_cache_memory_bytes = 256 * 1024 * 1024
|
||||
llm = LLM(
|
||||
"Qwen/Qwen3-0.6B",
|
||||
enable_sleep_mode=True,
|
||||
enforce_eager=True,
|
||||
kv_cache_memory_bytes=kv_cache_memory_bytes,
|
||||
max_model_len=1024,
|
||||
max_num_seqs=4,
|
||||
gpu_memory_utilization=0.05,
|
||||
)
|
||||
prompt = "How are you?"
|
||||
sampling_params = SamplingParams(temperature=0, max_tokens=10)
|
||||
expected = llm.generate(prompt, sampling_params)[0].outputs[0].token_ids
|
||||
|
||||
def get_mapped_bytes_by_tag(worker):
|
||||
allocator = get_mem_allocator_instance()
|
||||
mapped_bytes = mapped_usage(allocator)
|
||||
kv_cache_bytes = sum(
|
||||
data.handle[1]
|
||||
for data in allocator.pointer_to_data.values()
|
||||
if data.tag == "kv_cache" and not data.is_asleep
|
||||
)
|
||||
return mapped_bytes, kv_cache_bytes
|
||||
|
||||
mapped_before, kv_cache_before = llm.collective_rpc(get_mapped_bytes_by_tag)[0]
|
||||
# Utility RPC transports the engine error as a plain Exception.
|
||||
with pytest.raises(Exception, match="requires a completed pause"):
|
||||
llm.release_kv_cache_memory()
|
||||
assert llm.collective_rpc(get_mapped_bytes_by_tag)[0][0] == mapped_before
|
||||
assert not llm.llm_engine.is_sleeping()
|
||||
assert llm.generate(prompt, sampling_params)[0].outputs[0].token_ids == expected
|
||||
|
||||
llm.sleep(level=0)
|
||||
llm.release_kv_cache_memory()
|
||||
mapped_after, kv_cache_after = llm.collective_rpc(get_mapped_bytes_by_tag)[0]
|
||||
assert kv_cache_before > 0
|
||||
assert kv_cache_after == 0
|
||||
assert mapped_before - mapped_after >= kv_cache_before
|
||||
assert mapped_after > 0
|
||||
assert llm.llm_engine.is_sleeping()
|
||||
|
||||
if full_sleep:
|
||||
llm.sleep(level=full_sleep)
|
||||
assert llm.collective_rpc(get_mapped_bytes_by_tag)[0][0] == 0
|
||||
llm.wake_up(tags=None if full_sleep else ["kv_cache"])
|
||||
if full_sleep == 2:
|
||||
llm.collective_rpc("reload_weights")
|
||||
assert llm.collective_rpc(get_mapped_bytes_by_tag)[0][0] == mapped_before
|
||||
assert not llm.llm_engine.is_sleeping()
|
||||
actual = llm.generate(prompt, sampling_params)[0].outputs[0].token_ids
|
||||
assert actual == expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize("first_level", [1, 2])
|
||||
@pytest.mark.parametrize("second_level", [1, 2])
|
||||
@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn")
|
||||
def test_sleep_with_only_weights_asleep(first_level, second_level, monkeypatch):
|
||||
"""Repeated sleep after KV-only wake preserves mappings and recoverability."""
|
||||
monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
|
||||
llm = LLM(
|
||||
"Qwen/Qwen3-0.6B",
|
||||
enable_sleep_mode=True,
|
||||
enforce_eager=True,
|
||||
kv_cache_memory_bytes=256 * 1024 * 1024,
|
||||
max_model_len=1024,
|
||||
max_num_seqs=4,
|
||||
gpu_memory_utilization=0.05,
|
||||
)
|
||||
prompt = "How are you?"
|
||||
sampling_params = SamplingParams(temperature=0, max_tokens=10)
|
||||
expected = llm.generate(prompt, sampling_params)[0].outputs[0].token_ids
|
||||
|
||||
def get_mapped_bytes(worker):
|
||||
return mapped_usage(get_mem_allocator_instance())
|
||||
|
||||
llm.sleep(level=first_level)
|
||||
llm.wake_up(tags=["kv_cache"])
|
||||
mapped_before = llm.collective_rpc(get_mapped_bytes)[0]
|
||||
assert mapped_before > 0
|
||||
llm.sleep(level=second_level)
|
||||
assert llm.collective_rpc(get_mapped_bytes)[0] == mapped_before
|
||||
llm.wake_up(tags=["weights"])
|
||||
if first_level == 2:
|
||||
llm.collective_rpc("reload_weights")
|
||||
assert not llm.llm_engine.is_sleeping()
|
||||
actual = llm.generate(prompt, sampling_params)[0].outputs[0].token_ids
|
||||
assert actual == expected
|
||||
|
||||
|
||||
@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn")
|
||||
def test_discard_tags():
|
||||
"""Test that discard(tags) selectively frees GPU memory for specific
|
||||
tags while keeping other tags mapped and usable."""
|
||||
allocator = get_mem_allocator_instance()
|
||||
|
||||
with allocator.use_memory_pool("weights"):
|
||||
weights = torch.ones(1024, 1024, device=DEVICE_TYPE)
|
||||
|
||||
with allocator.use_memory_pool("kv_cache"):
|
||||
kv = torch.ones(512, 512, device=DEVICE_TYPE)
|
||||
|
||||
mapped_bytes = mapped_usage(allocator)
|
||||
|
||||
# Discard kv_cache only — weights should remain valid
|
||||
allocator.discard("kv_cache")
|
||||
|
||||
mapped_bytes_after_discard = mapped_usage(allocator)
|
||||
assert mapped_bytes - mapped_bytes_after_discard >= kv.nbytes
|
||||
assert mapped_bytes_after_discard >= weights.nbytes
|
||||
|
||||
# Weights are still usable
|
||||
assert torch.allclose(weights, torch.ones_like(weights))
|
||||
|
||||
# Wake up and verify kv_cache is remapped; discarded contents are undefined.
|
||||
allocator.wake_up()
|
||||
assert kv.shape == (512, 512)
|
||||
|
||||
# Full sleep/wake cycle still works after discard
|
||||
allocator.sleep(offload_tags="weights")
|
||||
allocator.wake_up()
|
||||
assert torch.allclose(weights, torch.ones_like(weights))
|
||||
|
||||
|
||||
@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn")
|
||||
@pytest.mark.skipif(current_platform.is_xpu(), reason="Uses the CuMem allocator")
|
||||
def test_level2_discards_ordinary_tensor_with_weights_tag():
|
||||
"""Reproduce the level-2 variant for an ordinary tensor in weights."""
|
||||
allocator = get_mem_allocator_instance()
|
||||
|
||||
with allocator.use_memory_pool("weights"):
|
||||
fake_weight = torch.full((4096,), 0x44, dtype=torch.uint8, device=DEVICE_TYPE)
|
||||
ordinary_tensor = torch.full(
|
||||
(4096,), 0x55, dtype=torch.uint8, device=DEVICE_TYPE
|
||||
)
|
||||
|
||||
pointers = (fake_weight.data_ptr(), ordinary_tensor.data_ptr())
|
||||
allocator.sleep(offload_tags=())
|
||||
_wake_up_with_poisoned_mappings(allocator)
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
assert (fake_weight.data_ptr(), ordinary_tensor.data_ptr()) == pointers
|
||||
assert torch.all(fake_weight == 0xA5)
|
||||
assert torch.all(ordinary_tensor == 0xA5)
|
||||
|
||||
|
||||
@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn")
|
||||
@pytest.mark.skipif(current_platform.is_xpu(), reason="CUDA graph not supported on XPU")
|
||||
def test_cumem_with_cudagraph():
|
||||
allocator = get_mem_allocator_instance()
|
||||
with allocator.use_memory_pool():
|
||||
weight = torch.eye(1024, device=DEVICE_TYPE)
|
||||
with allocator.use_memory_pool(tag="discard"):
|
||||
cache = torch.empty(1024, 1024, device=DEVICE_TYPE)
|
||||
|
||||
def model(x):
|
||||
out = x @ weight
|
||||
cache[: out.size(0)].copy_(out)
|
||||
return out + 1
|
||||
|
||||
x = torch.empty(128, 1024, device=DEVICE_TYPE)
|
||||
|
||||
# warmup
|
||||
model(x)
|
||||
|
||||
# capture cudagraph
|
||||
model_graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(model_graph):
|
||||
y = model(x)
|
||||
|
||||
mapped_bytes = mapped_usage(allocator)
|
||||
assert mapped_bytes >= weight.nbytes + cache.nbytes
|
||||
allocator.sleep()
|
||||
assert mapped_usage(allocator) == 0
|
||||
allocator.wake_up()
|
||||
assert mapped_usage(allocator) == mapped_bytes
|
||||
|
||||
# after waking up, the content in the weight tensor
|
||||
# should be restored, but the content in the cache tensor
|
||||
# should be discarded
|
||||
|
||||
# this operation is also compatible with cudagraph
|
||||
|
||||
x.random_()
|
||||
model_graph.replay()
|
||||
|
||||
# cache content is as expected
|
||||
assert torch.allclose(x, cache[: x.size(0)])
|
||||
|
||||
# output content is as expected
|
||||
assert torch.allclose(y, x + 1)
|
||||
from ....utils import create_new_process_for_each_test, multi_gpu_test, requires_fp8
|
||||
|
||||
|
||||
@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn")
|
||||
@@ -445,8 +154,8 @@ def _lora_logits_mapping_present(model) -> bool:
|
||||
)
|
||||
|
||||
|
||||
@create_new_process_for_each_test()
|
||||
def test_deep_sleep_lora_tp2(num_gpus_available, monkeypatch):
|
||||
@multi_gpu_test(num_gpus=2)
|
||||
def test_deep_sleep_lora_tp2(monkeypatch):
|
||||
"""Level-2 sleep/wake/reload with enable_lora=True and TP=2.
|
||||
|
||||
With TP > 1 the LoRA logits processor carries
|
||||
@@ -455,9 +164,6 @@ def test_deep_sleep_lora_tp2(num_gpus_available, monkeypatch):
|
||||
attribute allocated in the sleep-mode pool, so level-2 sleep destroys
|
||||
its contents — it must be restored after reload.
|
||||
"""
|
||||
if num_gpus_available < 2:
|
||||
pytest.skip("Requires at least 2 GPUs")
|
||||
|
||||
# Needed for apply_model to reach the multiproc TP workers below.
|
||||
monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
|
||||
|
||||
+4
-4
@@ -2,7 +2,7 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Compare the short outputs of HF and vLLM when using greedy sampling.
|
||||
|
||||
Run `pytest tests/basic_correctness/test_basic_correctness.py`.
|
||||
Run `pytest tests/basic_correctness/models/test_basic_correctness.py`.
|
||||
"""
|
||||
|
||||
import os
|
||||
@@ -17,9 +17,9 @@ from vllm import LLM
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.v1.engine.llm_engine import LLMEngine
|
||||
|
||||
from ..conftest import HfRunner, VllmRunner
|
||||
from ..models.utils import check_outputs_equal
|
||||
from ..utils import multi_gpu_test
|
||||
from ...conftest import HfRunner, VllmRunner
|
||||
from ...models.utils import check_outputs_equal
|
||||
from ...utils import multi_gpu_test
|
||||
|
||||
ATTN_BACKEND = ["ROCM_ATTN"] if current_platform.is_rocm() else ["FLASH_ATTN"]
|
||||
|
||||
+1
-1
@@ -2,7 +2,7 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Test prefetch offloading correctness with Llama model."""
|
||||
|
||||
from ..utils import compare_two_settings
|
||||
from ...utils import compare_two_settings
|
||||
|
||||
|
||||
def test_prefetch_offload_llama():
|
||||
+1
-1
@@ -928,7 +928,7 @@ class HfRunner:
|
||||
# shutdown. This is helpful in cases where the HfRunner is
|
||||
# initialized after significant GPU memory is already occupied,
|
||||
# e.g. in
|
||||
# tests/basic_correctness/test_basic_correctness.py::test_models_distributed
|
||||
# tests/basic_correctness/models/test_basic_correctness.py::test_models_distributed
|
||||
# where vllm worker processes are still alive and holding GPU
|
||||
# memory when hf_runner.__exit__ is called.
|
||||
from tests.utils import record_gpu_memory_usage_stats
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
# Expanded quantized model tests for CPU offloading
|
||||
# Base tests: tests/basic_correctness/test_cpu_offload.py
|
||||
# Base tests: tests/basic_correctness/cpu_offload/test_cpu_offload.py
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
|
||||
These cover the registry/factory contract and capability flags. They do not
|
||||
touch CUDA - the ``cumem`` suspend/resume path is exercised end-to-end on GPU
|
||||
in ``tests/basic_correctness/test_cumem.py``.
|
||||
in ``tests/basic_correctness/memory/``.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
|
||||
Reference in New Issue
Block a user