[CI] Split (H200 MIG/MI300) Basic Correctness into named jobs (#57054)
Signed-off-by: Thang Nguyen <thangnguyenvn647@gmail.com> Signed-off-by: Andreas Karatzas <Andreas.Karatzas@amd.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: Andreas Karatzas <Andreas.Karatzas@amd.com> Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
Andreas Karatzas
Codex
parent
9bf44c4013
commit
da1aaec31f
@@ -18,11 +18,12 @@ steps:
|
|||||||
VLLM_TEST_DEVICE: "xpu"
|
VLLM_TEST_DEVICE: "xpu"
|
||||||
source_file_dependencies:
|
source_file_dependencies:
|
||||||
- vllm/
|
- vllm/
|
||||||
- tests/basic_correctness/test_cumem.py
|
- tests/basic_correctness/memory/
|
||||||
|
- tests/basic_correctness/cpu_offload/
|
||||||
commands:
|
commands:
|
||||||
- >-
|
- >-
|
||||||
bash .buildkite/scripts/hardware_ci/run-intel-test.sh
|
bash .buildkite/scripts/hardware_ci/run-intel-test.sh
|
||||||
'cd tests &&
|
'cd tests &&
|
||||||
export VLLM_WORKER_MULTIPROC_METHOD=spawn &&
|
export VLLM_WORKER_MULTIPROC_METHOD=spawn &&
|
||||||
pytest -v -s basic_correctness/test_cpu_offload.py &&
|
pytest -v -s basic_correctness/cpu_offload &&
|
||||||
pytest -v -s basic_correctness/test_mem.py'
|
pytest -v -s basic_correctness/memory'
|
||||||
|
|||||||
@@ -1030,14 +1030,14 @@ steps:
|
|||||||
- tests/distributed/test_ray_v2_executor.py
|
- tests/distributed/test_ray_v2_executor.py
|
||||||
- tests/distributed/test_ray_v2_executor_e2e.py
|
- tests/distributed/test_ray_v2_executor_e2e.py
|
||||||
- tests/distributed/test_pipeline_parallel.py
|
- tests/distributed/test_pipeline_parallel.py
|
||||||
- tests/basic_correctness/test_basic_correctness.py
|
- tests/basic_correctness/models/test_basic_correctness.py
|
||||||
- vllm/platforms/rocm.py
|
- vllm/platforms/rocm.py
|
||||||
commands:
|
commands:
|
||||||
- export VLLM_USE_RAY_V2_EXECUTOR_BACKEND=1
|
- export VLLM_USE_RAY_V2_EXECUTOR_BACKEND=1
|
||||||
- pytest -v -s distributed/test_ray_v2_executor.py
|
- pytest -v -s distributed/test_ray_v2_executor.py
|
||||||
- pytest -v -s distributed/test_ray_v2_executor_e2e.py
|
- pytest -v -s distributed/test_ray_v2_executor_e2e.py
|
||||||
- pytest -v -s distributed/test_pipeline_parallel.py -k "ray"
|
- pytest -v -s distributed/test_pipeline_parallel.py -k "ray"
|
||||||
- TARGET_TEST_SUITE=L4 pytest -v -s basic_correctness/test_basic_correctness.py -k "ray"
|
- TARGET_TEST_SUITE=L4 pytest -v -s basic_correctness/models/test_basic_correctness.py -k "ray"
|
||||||
|
|
||||||
- label: ":amd: (MI300) Distributed DP + EP" # TBD
|
- label: ":amd: (MI300) Distributed DP + EP" # TBD
|
||||||
timeout_in_minutes: 50
|
timeout_in_minutes: 50
|
||||||
@@ -2762,7 +2762,23 @@ steps:
|
|||||||
|
|
||||||
#----------------------------------------------------- mi355 · basic_correctness -----------------------------------------------------#
|
#----------------------------------------------------- mi355 · basic_correctness -----------------------------------------------------#
|
||||||
|
|
||||||
- label: ":amd: (MI355 DPX) Basic Correctness" # TBD
|
- label: ":amd: (MI355 DPX) Basic Correctness Models %N" # TBD
|
||||||
|
timeout_in_minutes: 60
|
||||||
|
parallelism: 2
|
||||||
|
mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
|
||||||
|
dind: false
|
||||||
|
agent_pool: mi355_dpx
|
||||||
|
fast_check: true
|
||||||
|
working_dir: "/vllm-workspace/tests"
|
||||||
|
source_file_dependencies:
|
||||||
|
- vllm/
|
||||||
|
- tests/basic_correctness/
|
||||||
|
optional: true
|
||||||
|
commands:
|
||||||
|
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||||
|
- VLLM_TARGET_TEST_SUITE=MI355 pytest -v -s basic_correctness/models --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB
|
||||||
|
|
||||||
|
- label: ":amd: (MI355 DPX) Basic Correctness CuMem" # TBD
|
||||||
timeout_in_minutes: 60
|
timeout_in_minutes: 60
|
||||||
mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
|
mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
|
||||||
dind: false
|
dind: false
|
||||||
@@ -2771,15 +2787,56 @@ steps:
|
|||||||
working_dir: "/vllm-workspace/tests"
|
working_dir: "/vllm-workspace/tests"
|
||||||
source_file_dependencies:
|
source_file_dependencies:
|
||||||
- vllm/
|
- vllm/
|
||||||
- tests/basic_correctness/test_basic_correctness
|
- tests/basic_correctness/
|
||||||
- tests/basic_correctness/test_cpu_offload
|
|
||||||
- tests/basic_correctness/test_mem.py
|
|
||||||
optional: true
|
optional: true
|
||||||
commands:
|
commands:
|
||||||
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||||
- pytest -v -s basic_correctness/test_mem.py
|
- pytest -v -s basic_correctness/memory/cumem
|
||||||
- VLLM_TARGET_TEST_SUITE=MI355 pytest -v -s basic_correctness/test_basic_correctness.py
|
|
||||||
- pytest -v -s basic_correctness/test_cpu_offload.py
|
- label: ":amd: (MI355 DPX) Basic Correctness Sleep Mode" # TBD
|
||||||
|
timeout_in_minutes: 60
|
||||||
|
mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
|
||||||
|
dind: false
|
||||||
|
agent_pool: mi355_dpx
|
||||||
|
fast_check: true
|
||||||
|
working_dir: "/vllm-workspace/tests"
|
||||||
|
source_file_dependencies:
|
||||||
|
- vllm/
|
||||||
|
- tests/basic_correctness/
|
||||||
|
optional: true
|
||||||
|
commands:
|
||||||
|
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||||
|
- pytest -v -s basic_correctness/memory/sleep_mode
|
||||||
|
|
||||||
|
- label: ":amd: (MI355 DPX) Basic Correctness CPU Offload" # TBD
|
||||||
|
timeout_in_minutes: 60
|
||||||
|
mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
|
||||||
|
dind: false
|
||||||
|
agent_pool: mi355_dpx
|
||||||
|
fast_check: true
|
||||||
|
working_dir: "/vllm-workspace/tests"
|
||||||
|
source_file_dependencies:
|
||||||
|
- vllm/
|
||||||
|
- tests/basic_correctness/
|
||||||
|
optional: true
|
||||||
|
commands:
|
||||||
|
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||||
|
- pytest -v -s basic_correctness/cpu_offload
|
||||||
|
|
||||||
|
- label: ":amd: (MI355 DPX) Basic Correctness Prefetch Offload" # TBD
|
||||||
|
timeout_in_minutes: 60
|
||||||
|
mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
|
||||||
|
dind: false
|
||||||
|
agent_pool: mi355_dpx
|
||||||
|
fast_check: true
|
||||||
|
working_dir: "/vllm-workspace/tests"
|
||||||
|
source_file_dependencies:
|
||||||
|
- vllm/
|
||||||
|
- tests/basic_correctness/
|
||||||
|
optional: true
|
||||||
|
commands:
|
||||||
|
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||||
|
- pytest -v -s basic_correctness/prefetch_offload
|
||||||
|
|
||||||
#-------------------------------------------------------- mi355 · benchmarks ---------------------------------------------------------#
|
#-------------------------------------------------------- mi355 · benchmarks ---------------------------------------------------------#
|
||||||
|
|
||||||
|
|||||||
@@ -1,25 +1,90 @@
|
|||||||
group: Basic Correctness
|
group: Basic Correctness
|
||||||
depends_on:
|
depends_on:
|
||||||
- image-build
|
- image-build
|
||||||
steps:
|
steps:
|
||||||
- label: ":nvidia: (H200 MIG 18GB) Basic Correctness"
|
- label: ":nvidia: (H200 MIG 18GB) Basic Correctness Models %N"
|
||||||
key: basic-correctness
|
key: basic-correctness
|
||||||
timeout_in_minutes: 68
|
timeout_in_minutes: 68
|
||||||
device: h200_18gb
|
device: h200_18gb
|
||||||
source_file_dependencies:
|
parallelism: 2
|
||||||
|
source_file_dependencies: &basic-correctness-deps
|
||||||
- vllm/
|
- vllm/
|
||||||
- "!vllm/distributed/kv_transfer/"
|
- "!vllm/distributed/kv_transfer/"
|
||||||
- tests/basic_correctness/test_basic_correctness
|
- tests/basic_correctness/
|
||||||
- tests/basic_correctness/test_cpu_offload
|
|
||||||
- tests/basic_correctness/test_mem.py
|
|
||||||
commands:
|
commands:
|
||||||
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||||
- pytest -v -s basic_correctness/test_mem.py
|
- pytest -v -s basic_correctness/models --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB
|
||||||
- pytest -v -s basic_correctness/test_basic_correctness.py
|
|
||||||
- pytest -v -s basic_correctness/test_cpu_offload.py
|
|
||||||
mirror:
|
mirror:
|
||||||
amd:
|
amd:
|
||||||
label: ":amd: (MI355 DPX) Basic Correctness"
|
label: ":amd: (MI355 DPX) Basic Correctness Models %N"
|
||||||
|
dind: false
|
||||||
|
device: mi355_dpx
|
||||||
|
timeout_in_minutes: 60
|
||||||
|
depends_on:
|
||||||
|
- image-build-amd
|
||||||
|
|
||||||
|
- label: ":nvidia: (H200 MIG 18GB) Basic Correctness CuMem"
|
||||||
|
key: basic-correctness-cumem
|
||||||
|
timeout_in_minutes: 68
|
||||||
|
device: h200_18gb
|
||||||
|
source_file_dependencies: *basic-correctness-deps
|
||||||
|
commands:
|
||||||
|
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||||
|
- pytest -v -s basic_correctness/memory/cumem
|
||||||
|
mirror:
|
||||||
|
amd:
|
||||||
|
label: ":amd: (MI355 DPX) Basic Correctness CuMem"
|
||||||
|
dind: false
|
||||||
|
device: mi355_dpx
|
||||||
|
timeout_in_minutes: 60
|
||||||
|
depends_on:
|
||||||
|
- image-build-amd
|
||||||
|
|
||||||
|
- label: ":nvidia: (H200 MIG 18GB) Basic Correctness Sleep Mode"
|
||||||
|
key: basic-correctness-sleep-mode
|
||||||
|
timeout_in_minutes: 68
|
||||||
|
device: h200_18gb
|
||||||
|
source_file_dependencies: *basic-correctness-deps
|
||||||
|
commands:
|
||||||
|
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||||
|
- pytest -v -s basic_correctness/memory/sleep_mode
|
||||||
|
mirror:
|
||||||
|
amd:
|
||||||
|
label: ":amd: (MI355 DPX) Basic Correctness Sleep Mode"
|
||||||
|
dind: false
|
||||||
|
device: mi355_dpx
|
||||||
|
timeout_in_minutes: 60
|
||||||
|
depends_on:
|
||||||
|
- image-build-amd
|
||||||
|
|
||||||
|
- label: ":nvidia: (H200 MIG 18GB) Basic Correctness CPU Offload"
|
||||||
|
key: basic-correctness-cpu-offload
|
||||||
|
timeout_in_minutes: 68
|
||||||
|
device: h200_18gb
|
||||||
|
source_file_dependencies: *basic-correctness-deps
|
||||||
|
commands:
|
||||||
|
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||||
|
- pytest -v -s basic_correctness/cpu_offload
|
||||||
|
mirror:
|
||||||
|
amd:
|
||||||
|
label: ":amd: (MI355 DPX) Basic Correctness CPU Offload"
|
||||||
|
dind: false
|
||||||
|
device: mi355_dpx
|
||||||
|
timeout_in_minutes: 60
|
||||||
|
depends_on:
|
||||||
|
- image-build-amd
|
||||||
|
|
||||||
|
- label: ":nvidia: (H200 MIG 18GB) Basic Correctness Prefetch Offload"
|
||||||
|
key: basic-correctness-prefetch-offload
|
||||||
|
timeout_in_minutes: 68
|
||||||
|
device: h200_18gb
|
||||||
|
source_file_dependencies: *basic-correctness-deps
|
||||||
|
commands:
|
||||||
|
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||||
|
- pytest -v -s basic_correctness/prefetch_offload
|
||||||
|
mirror:
|
||||||
|
amd:
|
||||||
|
label: ":amd: (MI355 DPX) Basic Correctness Prefetch Offload"
|
||||||
dind: false
|
dind: false
|
||||||
device: mi355_dpx
|
device: mi355_dpx
|
||||||
timeout_in_minutes: 60
|
timeout_in_minutes: 60
|
||||||
|
|||||||
@@ -525,11 +525,11 @@ steps:
|
|||||||
- tests/distributed/test_ray_v2_executor.py
|
- tests/distributed/test_ray_v2_executor.py
|
||||||
- tests/distributed/test_ray_v2_executor_e2e.py
|
- tests/distributed/test_ray_v2_executor_e2e.py
|
||||||
- tests/distributed/test_pipeline_parallel.py
|
- tests/distributed/test_pipeline_parallel.py
|
||||||
- tests/basic_correctness/test_basic_correctness.py
|
- tests/basic_correctness/models/test_basic_correctness.py
|
||||||
commands:
|
commands:
|
||||||
- export VLLM_USE_RAY_V2_EXECUTOR_BACKEND=1
|
- export VLLM_USE_RAY_V2_EXECUTOR_BACKEND=1
|
||||||
- export NCCL_CUMEM_HOST_ENABLE=0
|
- export NCCL_CUMEM_HOST_ENABLE=0
|
||||||
- pytest -v -s distributed/test_ray_v2_executor.py
|
- pytest -v -s distributed/test_ray_v2_executor.py
|
||||||
- pytest -v -s distributed/test_ray_v2_executor_e2e.py
|
- pytest -v -s distributed/test_ray_v2_executor_e2e.py
|
||||||
- pytest -v -s distributed/test_pipeline_parallel.py -k "ray"
|
- pytest -v -s distributed/test_pipeline_parallel.py -k "ray"
|
||||||
- TARGET_TEST_SUITE=L4 pytest -v -s basic_correctness/test_basic_correctness.py -k "ray"
|
- TARGET_TEST_SUITE=L4 pytest -v -s basic_correctness/models/test_basic_correctness.py -k "ray"
|
||||||
|
|||||||
+1
-1
@@ -15,7 +15,7 @@ from vllm.model_executor.offloader import (
|
|||||||
)
|
)
|
||||||
from vllm.v1.worker.gpu.model_runner import GPUModelRunner
|
from vllm.v1.worker.gpu.model_runner import GPUModelRunner
|
||||||
|
|
||||||
from ..utils import compare_two_settings
|
from ...utils import compare_two_settings
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize("disable_pin_memory", [False, True])
|
@pytest.mark.parametrize("disable_pin_memory", [False, True])
|
||||||
@@ -0,0 +1,302 @@
|
|||||||
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
import torch
|
||||||
|
|
||||||
|
import vllm.device_allocator.cumem as cumem
|
||||||
|
from vllm import LLM, SamplingParams
|
||||||
|
from vllm.device_allocator import get_mem_allocator_instance
|
||||||
|
from vllm.platforms import current_platform
|
||||||
|
|
||||||
|
from ....utils import create_new_process_for_each_test
|
||||||
|
|
||||||
|
DEVICE_TYPE = current_platform.device_type
|
||||||
|
|
||||||
|
|
||||||
|
def mapped_usage(allocator) -> int:
|
||||||
|
"""Bytes the allocator still has mapped on the device.
|
||||||
|
|
||||||
|
`torch.accelerator.get_memory_info()` is a device-wide `cudaMemGetInfo` /
|
||||||
|
`hipMemGetInfo` query, so its delta also carries whatever else on the GPU
|
||||||
|
allocates or frees while we measure. The allocator's own bookkeeping is
|
||||||
|
process-local and exact.
|
||||||
|
"""
|
||||||
|
return sum(
|
||||||
|
data.handle[1]
|
||||||
|
for data in allocator.pointer_to_data.values()
|
||||||
|
if not data.is_asleep
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _wake_up_with_poisoned_mappings(allocator, byte_value: int = 0xA5) -> None:
|
||||||
|
"""Wake discarded allocations with deterministic nonzero contents."""
|
||||||
|
original_create_and_map = cumem.create_and_map
|
||||||
|
|
||||||
|
def create_and_map_with_poison(handle) -> None:
|
||||||
|
original_create_and_map(handle)
|
||||||
|
_, size, ptr, _ = handle
|
||||||
|
cumem.libcudart.cudaMemset(ptr, byte_value, size)
|
||||||
|
|
||||||
|
cumem.create_and_map = create_and_map_with_poison
|
||||||
|
try:
|
||||||
|
allocator.wake_up()
|
||||||
|
finally:
|
||||||
|
cumem.create_and_map = original_create_and_map
|
||||||
|
|
||||||
|
|
||||||
|
@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn")
|
||||||
|
def test_python_error():
|
||||||
|
"""Test if Python error occurs when there's low-level
|
||||||
|
error happening from the C++ side.
|
||||||
|
"""
|
||||||
|
allocator = get_mem_allocator_instance()
|
||||||
|
total_bytes = torch.accelerator.get_memory_info()[1]
|
||||||
|
alloc_bytes = int(total_bytes * 0.7)
|
||||||
|
tensors = []
|
||||||
|
with allocator.use_memory_pool():
|
||||||
|
# allocate 70% of the total memory
|
||||||
|
x = torch.empty(alloc_bytes, dtype=torch.uint8, device=DEVICE_TYPE)
|
||||||
|
tensors.append(x)
|
||||||
|
# release the memory
|
||||||
|
allocator.sleep(offload_tags=())
|
||||||
|
|
||||||
|
# allocate more memory than the total memory
|
||||||
|
y = torch.empty(alloc_bytes, dtype=torch.uint8, device=DEVICE_TYPE)
|
||||||
|
tensors.append(y)
|
||||||
|
with pytest.raises(RuntimeError):
|
||||||
|
# when the allocator is woken up, it should raise an error
|
||||||
|
# because we don't have enough memory
|
||||||
|
allocator.wake_up()
|
||||||
|
|
||||||
|
|
||||||
|
@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn")
|
||||||
|
def test_basic_cumem():
|
||||||
|
# some tensors from default memory pool
|
||||||
|
shape = (1024, 1024)
|
||||||
|
x = torch.empty(shape, device=DEVICE_TYPE)
|
||||||
|
x.zero_()
|
||||||
|
|
||||||
|
# some tensors from custom memory pool
|
||||||
|
allocator = get_mem_allocator_instance()
|
||||||
|
with allocator.use_memory_pool():
|
||||||
|
# custom memory pool
|
||||||
|
y = torch.empty(shape, device=DEVICE_TYPE)
|
||||||
|
y.zero_()
|
||||||
|
y += 1
|
||||||
|
z = torch.empty(shape, device=DEVICE_TYPE)
|
||||||
|
z.zero_()
|
||||||
|
z += 2
|
||||||
|
|
||||||
|
# they can be used together
|
||||||
|
output = x + y + z
|
||||||
|
assert torch.allclose(output, torch.ones_like(output) * 3)
|
||||||
|
|
||||||
|
# Track the allocator's own mapped bytes, not device-wide free memory.
|
||||||
|
mapped_bytes = mapped_usage(allocator)
|
||||||
|
assert mapped_bytes >= y.nbytes + z.nbytes
|
||||||
|
allocator.sleep()
|
||||||
|
assert mapped_usage(allocator) == 0
|
||||||
|
allocator.wake_up()
|
||||||
|
assert mapped_usage(allocator) == mapped_bytes
|
||||||
|
|
||||||
|
# they can be used together
|
||||||
|
output = x + y + z
|
||||||
|
assert torch.allclose(output, torch.ones_like(output) * 3)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("full_sleep", [0, 1, 2], ids=["kv-only", "sleep-1", "sleep-2"])
|
||||||
|
@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn")
|
||||||
|
def test_release_kv_cache_memory_preserves_generation(full_sleep, monkeypatch):
|
||||||
|
"""Rejected release and KV-only/full sleep cycles preserve greedy tokens."""
|
||||||
|
monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
|
||||||
|
kv_cache_memory_bytes = 256 * 1024 * 1024
|
||||||
|
llm = LLM(
|
||||||
|
"Qwen/Qwen3-0.6B",
|
||||||
|
enable_sleep_mode=True,
|
||||||
|
enforce_eager=True,
|
||||||
|
kv_cache_memory_bytes=kv_cache_memory_bytes,
|
||||||
|
max_model_len=1024,
|
||||||
|
max_num_seqs=4,
|
||||||
|
gpu_memory_utilization=0.05,
|
||||||
|
)
|
||||||
|
prompt = "How are you?"
|
||||||
|
sampling_params = SamplingParams(temperature=0, max_tokens=10)
|
||||||
|
expected = llm.generate(prompt, sampling_params)[0].outputs[0].token_ids
|
||||||
|
|
||||||
|
def get_mapped_bytes_by_tag(worker):
|
||||||
|
allocator = get_mem_allocator_instance()
|
||||||
|
mapped_bytes = mapped_usage(allocator)
|
||||||
|
kv_cache_bytes = sum(
|
||||||
|
data.handle[1]
|
||||||
|
for data in allocator.pointer_to_data.values()
|
||||||
|
if data.tag == "kv_cache" and not data.is_asleep
|
||||||
|
)
|
||||||
|
return mapped_bytes, kv_cache_bytes
|
||||||
|
|
||||||
|
mapped_before, kv_cache_before = llm.collective_rpc(get_mapped_bytes_by_tag)[0]
|
||||||
|
# Utility RPC transports the engine error as a plain Exception.
|
||||||
|
with pytest.raises(Exception, match="requires a completed pause"):
|
||||||
|
llm.release_kv_cache_memory()
|
||||||
|
assert llm.collective_rpc(get_mapped_bytes_by_tag)[0][0] == mapped_before
|
||||||
|
assert not llm.llm_engine.is_sleeping()
|
||||||
|
assert llm.generate(prompt, sampling_params)[0].outputs[0].token_ids == expected
|
||||||
|
|
||||||
|
llm.sleep(level=0)
|
||||||
|
llm.release_kv_cache_memory()
|
||||||
|
mapped_after, kv_cache_after = llm.collective_rpc(get_mapped_bytes_by_tag)[0]
|
||||||
|
assert kv_cache_before > 0
|
||||||
|
assert kv_cache_after == 0
|
||||||
|
assert mapped_before - mapped_after >= kv_cache_before
|
||||||
|
assert mapped_after > 0
|
||||||
|
assert llm.llm_engine.is_sleeping()
|
||||||
|
|
||||||
|
if full_sleep:
|
||||||
|
llm.sleep(level=full_sleep)
|
||||||
|
assert llm.collective_rpc(get_mapped_bytes_by_tag)[0][0] == 0
|
||||||
|
llm.wake_up(tags=None if full_sleep else ["kv_cache"])
|
||||||
|
if full_sleep == 2:
|
||||||
|
llm.collective_rpc("reload_weights")
|
||||||
|
assert llm.collective_rpc(get_mapped_bytes_by_tag)[0][0] == mapped_before
|
||||||
|
assert not llm.llm_engine.is_sleeping()
|
||||||
|
actual = llm.generate(prompt, sampling_params)[0].outputs[0].token_ids
|
||||||
|
assert actual == expected
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("first_level", [1, 2])
|
||||||
|
@pytest.mark.parametrize("second_level", [1, 2])
|
||||||
|
@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn")
|
||||||
|
def test_sleep_with_only_weights_asleep(first_level, second_level, monkeypatch):
|
||||||
|
"""Repeated sleep after KV-only wake preserves mappings and recoverability."""
|
||||||
|
monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
|
||||||
|
llm = LLM(
|
||||||
|
"Qwen/Qwen3-0.6B",
|
||||||
|
enable_sleep_mode=True,
|
||||||
|
enforce_eager=True,
|
||||||
|
kv_cache_memory_bytes=256 * 1024 * 1024,
|
||||||
|
max_model_len=1024,
|
||||||
|
max_num_seqs=4,
|
||||||
|
gpu_memory_utilization=0.05,
|
||||||
|
)
|
||||||
|
prompt = "How are you?"
|
||||||
|
sampling_params = SamplingParams(temperature=0, max_tokens=10)
|
||||||
|
expected = llm.generate(prompt, sampling_params)[0].outputs[0].token_ids
|
||||||
|
|
||||||
|
def get_mapped_bytes(worker):
|
||||||
|
return mapped_usage(get_mem_allocator_instance())
|
||||||
|
|
||||||
|
llm.sleep(level=first_level)
|
||||||
|
llm.wake_up(tags=["kv_cache"])
|
||||||
|
mapped_before = llm.collective_rpc(get_mapped_bytes)[0]
|
||||||
|
assert mapped_before > 0
|
||||||
|
llm.sleep(level=second_level)
|
||||||
|
assert llm.collective_rpc(get_mapped_bytes)[0] == mapped_before
|
||||||
|
llm.wake_up(tags=["weights"])
|
||||||
|
if first_level == 2:
|
||||||
|
llm.collective_rpc("reload_weights")
|
||||||
|
assert not llm.llm_engine.is_sleeping()
|
||||||
|
actual = llm.generate(prompt, sampling_params)[0].outputs[0].token_ids
|
||||||
|
assert actual == expected
|
||||||
|
|
||||||
|
|
||||||
|
@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn")
|
||||||
|
def test_discard_tags():
|
||||||
|
"""Test that discard(tags) selectively frees GPU memory for specific
|
||||||
|
tags while keeping other tags mapped and usable."""
|
||||||
|
allocator = get_mem_allocator_instance()
|
||||||
|
|
||||||
|
with allocator.use_memory_pool("weights"):
|
||||||
|
weights = torch.ones(1024, 1024, device=DEVICE_TYPE)
|
||||||
|
|
||||||
|
with allocator.use_memory_pool("kv_cache"):
|
||||||
|
kv = torch.ones(512, 512, device=DEVICE_TYPE)
|
||||||
|
|
||||||
|
mapped_bytes = mapped_usage(allocator)
|
||||||
|
|
||||||
|
# Discard kv_cache only — weights should remain valid
|
||||||
|
allocator.discard("kv_cache")
|
||||||
|
|
||||||
|
mapped_bytes_after_discard = mapped_usage(allocator)
|
||||||
|
assert mapped_bytes - mapped_bytes_after_discard >= kv.nbytes
|
||||||
|
assert mapped_bytes_after_discard >= weights.nbytes
|
||||||
|
|
||||||
|
# Weights are still usable
|
||||||
|
assert torch.allclose(weights, torch.ones_like(weights))
|
||||||
|
|
||||||
|
# Wake up and verify kv_cache is remapped; discarded contents are undefined.
|
||||||
|
allocator.wake_up()
|
||||||
|
assert kv.shape == (512, 512)
|
||||||
|
|
||||||
|
# Full sleep/wake cycle still works after discard
|
||||||
|
allocator.sleep(offload_tags="weights")
|
||||||
|
allocator.wake_up()
|
||||||
|
assert torch.allclose(weights, torch.ones_like(weights))
|
||||||
|
|
||||||
|
|
||||||
|
@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn")
|
||||||
|
@pytest.mark.skipif(current_platform.is_xpu(), reason="Uses the CuMem allocator")
|
||||||
|
def test_level2_discards_ordinary_tensor_with_weights_tag():
|
||||||
|
"""Reproduce the level-2 variant for an ordinary tensor in weights."""
|
||||||
|
allocator = get_mem_allocator_instance()
|
||||||
|
|
||||||
|
with allocator.use_memory_pool("weights"):
|
||||||
|
fake_weight = torch.full((4096,), 0x44, dtype=torch.uint8, device=DEVICE_TYPE)
|
||||||
|
ordinary_tensor = torch.full(
|
||||||
|
(4096,), 0x55, dtype=torch.uint8, device=DEVICE_TYPE
|
||||||
|
)
|
||||||
|
|
||||||
|
pointers = (fake_weight.data_ptr(), ordinary_tensor.data_ptr())
|
||||||
|
allocator.sleep(offload_tags=())
|
||||||
|
_wake_up_with_poisoned_mappings(allocator)
|
||||||
|
torch.accelerator.synchronize()
|
||||||
|
|
||||||
|
assert (fake_weight.data_ptr(), ordinary_tensor.data_ptr()) == pointers
|
||||||
|
assert torch.all(fake_weight == 0xA5)
|
||||||
|
assert torch.all(ordinary_tensor == 0xA5)
|
||||||
|
|
||||||
|
|
||||||
|
@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn")
|
||||||
|
@pytest.mark.skipif(current_platform.is_xpu(), reason="CUDA graph not supported on XPU")
|
||||||
|
def test_cumem_with_cudagraph():
|
||||||
|
allocator = get_mem_allocator_instance()
|
||||||
|
with allocator.use_memory_pool():
|
||||||
|
weight = torch.eye(1024, device=DEVICE_TYPE)
|
||||||
|
with allocator.use_memory_pool(tag="discard"):
|
||||||
|
cache = torch.empty(1024, 1024, device=DEVICE_TYPE)
|
||||||
|
|
||||||
|
def model(x):
|
||||||
|
out = x @ weight
|
||||||
|
cache[: out.size(0)].copy_(out)
|
||||||
|
return out + 1
|
||||||
|
|
||||||
|
x = torch.empty(128, 1024, device=DEVICE_TYPE)
|
||||||
|
|
||||||
|
# warmup
|
||||||
|
model(x)
|
||||||
|
|
||||||
|
# capture cudagraph
|
||||||
|
model_graph = torch.cuda.CUDAGraph()
|
||||||
|
with torch.cuda.graph(model_graph):
|
||||||
|
y = model(x)
|
||||||
|
|
||||||
|
mapped_bytes = mapped_usage(allocator)
|
||||||
|
assert mapped_bytes >= weight.nbytes + cache.nbytes
|
||||||
|
allocator.sleep()
|
||||||
|
assert mapped_usage(allocator) == 0
|
||||||
|
allocator.wake_up()
|
||||||
|
assert mapped_usage(allocator) == mapped_bytes
|
||||||
|
|
||||||
|
# after waking up, the content in the weight tensor
|
||||||
|
# should be restored, but the content in the cache tensor
|
||||||
|
# should be discarded
|
||||||
|
|
||||||
|
# this operation is also compatible with cudagraph
|
||||||
|
|
||||||
|
x.random_()
|
||||||
|
model_graph.replay()
|
||||||
|
|
||||||
|
# cache content is as expected
|
||||||
|
assert torch.allclose(x, cache[: x.size(0)])
|
||||||
|
|
||||||
|
# output content is as expected
|
||||||
|
assert torch.allclose(y, x + 1)
|
||||||
+3
-297
@@ -9,301 +9,10 @@ import torch
|
|||||||
import vllm.device_allocator.cumem as cumem
|
import vllm.device_allocator.cumem as cumem
|
||||||
import vllm.envs as envs
|
import vllm.envs as envs
|
||||||
from vllm import LLM, AsyncEngineArgs, AsyncLLMEngine, SamplingParams
|
from vllm import LLM, AsyncEngineArgs, AsyncLLMEngine, SamplingParams
|
||||||
from vllm.device_allocator import get_mem_allocator_instance
|
|
||||||
from vllm.platforms import current_platform
|
from vllm.platforms import current_platform
|
||||||
from vllm.utils.mem_constants import GiB_bytes
|
from vllm.utils.mem_constants import GiB_bytes
|
||||||
|
|
||||||
from ..utils import create_new_process_for_each_test, requires_fp8
|
from ....utils import create_new_process_for_each_test, multi_gpu_test, requires_fp8
|
||||||
|
|
||||||
DEVICE_TYPE = current_platform.device_type
|
|
||||||
|
|
||||||
|
|
||||||
def mapped_usage(allocator) -> int:
|
|
||||||
"""Bytes the allocator still has mapped on the device.
|
|
||||||
|
|
||||||
`torch.accelerator.get_memory_info()` is a device-wide `cudaMemGetInfo` /
|
|
||||||
`hipMemGetInfo` query, so its delta also carries whatever else on the GPU
|
|
||||||
allocates or frees while we measure. The allocator's own bookkeeping is
|
|
||||||
process-local and exact.
|
|
||||||
"""
|
|
||||||
return sum(
|
|
||||||
data.handle[1]
|
|
||||||
for data in allocator.pointer_to_data.values()
|
|
||||||
if not data.is_asleep
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def _wake_up_with_poisoned_mappings(allocator, byte_value: int = 0xA5) -> None:
|
|
||||||
"""Wake discarded allocations with deterministic nonzero contents."""
|
|
||||||
original_create_and_map = cumem.create_and_map
|
|
||||||
|
|
||||||
def create_and_map_with_poison(handle) -> None:
|
|
||||||
original_create_and_map(handle)
|
|
||||||
_, size, ptr, _ = handle
|
|
||||||
cumem.libcudart.cudaMemset(ptr, byte_value, size)
|
|
||||||
|
|
||||||
cumem.create_and_map = create_and_map_with_poison
|
|
||||||
try:
|
|
||||||
allocator.wake_up()
|
|
||||||
finally:
|
|
||||||
cumem.create_and_map = original_create_and_map
|
|
||||||
|
|
||||||
|
|
||||||
@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn")
|
|
||||||
def test_python_error():
|
|
||||||
"""Test if Python error occurs when there's low-level
|
|
||||||
error happening from the C++ side.
|
|
||||||
"""
|
|
||||||
allocator = get_mem_allocator_instance()
|
|
||||||
total_bytes = torch.accelerator.get_memory_info()[1]
|
|
||||||
alloc_bytes = int(total_bytes * 0.7)
|
|
||||||
tensors = []
|
|
||||||
with allocator.use_memory_pool():
|
|
||||||
# allocate 70% of the total memory
|
|
||||||
x = torch.empty(alloc_bytes, dtype=torch.uint8, device=DEVICE_TYPE)
|
|
||||||
tensors.append(x)
|
|
||||||
# release the memory
|
|
||||||
allocator.sleep(offload_tags=())
|
|
||||||
|
|
||||||
# allocate more memory than the total memory
|
|
||||||
y = torch.empty(alloc_bytes, dtype=torch.uint8, device=DEVICE_TYPE)
|
|
||||||
tensors.append(y)
|
|
||||||
with pytest.raises(RuntimeError):
|
|
||||||
# when the allocator is woken up, it should raise an error
|
|
||||||
# because we don't have enough memory
|
|
||||||
allocator.wake_up()
|
|
||||||
|
|
||||||
|
|
||||||
@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn")
|
|
||||||
def test_basic_cumem():
|
|
||||||
# some tensors from default memory pool
|
|
||||||
shape = (1024, 1024)
|
|
||||||
x = torch.empty(shape, device=DEVICE_TYPE)
|
|
||||||
x.zero_()
|
|
||||||
|
|
||||||
# some tensors from custom memory pool
|
|
||||||
allocator = get_mem_allocator_instance()
|
|
||||||
with allocator.use_memory_pool():
|
|
||||||
# custom memory pool
|
|
||||||
y = torch.empty(shape, device=DEVICE_TYPE)
|
|
||||||
y.zero_()
|
|
||||||
y += 1
|
|
||||||
z = torch.empty(shape, device=DEVICE_TYPE)
|
|
||||||
z.zero_()
|
|
||||||
z += 2
|
|
||||||
|
|
||||||
# they can be used together
|
|
||||||
output = x + y + z
|
|
||||||
assert torch.allclose(output, torch.ones_like(output) * 3)
|
|
||||||
|
|
||||||
# Track the allocator's own mapped bytes, not device-wide free memory.
|
|
||||||
mapped_bytes = mapped_usage(allocator)
|
|
||||||
assert mapped_bytes >= y.nbytes + z.nbytes
|
|
||||||
allocator.sleep()
|
|
||||||
assert mapped_usage(allocator) == 0
|
|
||||||
allocator.wake_up()
|
|
||||||
assert mapped_usage(allocator) == mapped_bytes
|
|
||||||
|
|
||||||
# they can be used together
|
|
||||||
output = x + y + z
|
|
||||||
assert torch.allclose(output, torch.ones_like(output) * 3)
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize("full_sleep", [0, 1, 2], ids=["kv-only", "sleep-1", "sleep-2"])
|
|
||||||
@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn")
|
|
||||||
def test_release_kv_cache_memory_preserves_generation(full_sleep, monkeypatch):
|
|
||||||
"""Rejected release and KV-only/full sleep cycles preserve greedy tokens."""
|
|
||||||
monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
|
|
||||||
kv_cache_memory_bytes = 256 * 1024 * 1024
|
|
||||||
llm = LLM(
|
|
||||||
"Qwen/Qwen3-0.6B",
|
|
||||||
enable_sleep_mode=True,
|
|
||||||
enforce_eager=True,
|
|
||||||
kv_cache_memory_bytes=kv_cache_memory_bytes,
|
|
||||||
max_model_len=1024,
|
|
||||||
max_num_seqs=4,
|
|
||||||
gpu_memory_utilization=0.05,
|
|
||||||
)
|
|
||||||
prompt = "How are you?"
|
|
||||||
sampling_params = SamplingParams(temperature=0, max_tokens=10)
|
|
||||||
expected = llm.generate(prompt, sampling_params)[0].outputs[0].token_ids
|
|
||||||
|
|
||||||
def get_mapped_bytes_by_tag(worker):
|
|
||||||
allocator = get_mem_allocator_instance()
|
|
||||||
mapped_bytes = mapped_usage(allocator)
|
|
||||||
kv_cache_bytes = sum(
|
|
||||||
data.handle[1]
|
|
||||||
for data in allocator.pointer_to_data.values()
|
|
||||||
if data.tag == "kv_cache" and not data.is_asleep
|
|
||||||
)
|
|
||||||
return mapped_bytes, kv_cache_bytes
|
|
||||||
|
|
||||||
mapped_before, kv_cache_before = llm.collective_rpc(get_mapped_bytes_by_tag)[0]
|
|
||||||
# Utility RPC transports the engine error as a plain Exception.
|
|
||||||
with pytest.raises(Exception, match="requires a completed pause"):
|
|
||||||
llm.release_kv_cache_memory()
|
|
||||||
assert llm.collective_rpc(get_mapped_bytes_by_tag)[0][0] == mapped_before
|
|
||||||
assert not llm.llm_engine.is_sleeping()
|
|
||||||
assert llm.generate(prompt, sampling_params)[0].outputs[0].token_ids == expected
|
|
||||||
|
|
||||||
llm.sleep(level=0)
|
|
||||||
llm.release_kv_cache_memory()
|
|
||||||
mapped_after, kv_cache_after = llm.collective_rpc(get_mapped_bytes_by_tag)[0]
|
|
||||||
assert kv_cache_before > 0
|
|
||||||
assert kv_cache_after == 0
|
|
||||||
assert mapped_before - mapped_after >= kv_cache_before
|
|
||||||
assert mapped_after > 0
|
|
||||||
assert llm.llm_engine.is_sleeping()
|
|
||||||
|
|
||||||
if full_sleep:
|
|
||||||
llm.sleep(level=full_sleep)
|
|
||||||
assert llm.collective_rpc(get_mapped_bytes_by_tag)[0][0] == 0
|
|
||||||
llm.wake_up(tags=None if full_sleep else ["kv_cache"])
|
|
||||||
if full_sleep == 2:
|
|
||||||
llm.collective_rpc("reload_weights")
|
|
||||||
assert llm.collective_rpc(get_mapped_bytes_by_tag)[0][0] == mapped_before
|
|
||||||
assert not llm.llm_engine.is_sleeping()
|
|
||||||
actual = llm.generate(prompt, sampling_params)[0].outputs[0].token_ids
|
|
||||||
assert actual == expected
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize("first_level", [1, 2])
|
|
||||||
@pytest.mark.parametrize("second_level", [1, 2])
|
|
||||||
@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn")
|
|
||||||
def test_sleep_with_only_weights_asleep(first_level, second_level, monkeypatch):
|
|
||||||
"""Repeated sleep after KV-only wake preserves mappings and recoverability."""
|
|
||||||
monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
|
|
||||||
llm = LLM(
|
|
||||||
"Qwen/Qwen3-0.6B",
|
|
||||||
enable_sleep_mode=True,
|
|
||||||
enforce_eager=True,
|
|
||||||
kv_cache_memory_bytes=256 * 1024 * 1024,
|
|
||||||
max_model_len=1024,
|
|
||||||
max_num_seqs=4,
|
|
||||||
gpu_memory_utilization=0.05,
|
|
||||||
)
|
|
||||||
prompt = "How are you?"
|
|
||||||
sampling_params = SamplingParams(temperature=0, max_tokens=10)
|
|
||||||
expected = llm.generate(prompt, sampling_params)[0].outputs[0].token_ids
|
|
||||||
|
|
||||||
def get_mapped_bytes(worker):
|
|
||||||
return mapped_usage(get_mem_allocator_instance())
|
|
||||||
|
|
||||||
llm.sleep(level=first_level)
|
|
||||||
llm.wake_up(tags=["kv_cache"])
|
|
||||||
mapped_before = llm.collective_rpc(get_mapped_bytes)[0]
|
|
||||||
assert mapped_before > 0
|
|
||||||
llm.sleep(level=second_level)
|
|
||||||
assert llm.collective_rpc(get_mapped_bytes)[0] == mapped_before
|
|
||||||
llm.wake_up(tags=["weights"])
|
|
||||||
if first_level == 2:
|
|
||||||
llm.collective_rpc("reload_weights")
|
|
||||||
assert not llm.llm_engine.is_sleeping()
|
|
||||||
actual = llm.generate(prompt, sampling_params)[0].outputs[0].token_ids
|
|
||||||
assert actual == expected
|
|
||||||
|
|
||||||
|
|
||||||
@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn")
|
|
||||||
def test_discard_tags():
|
|
||||||
"""Test that discard(tags) selectively frees GPU memory for specific
|
|
||||||
tags while keeping other tags mapped and usable."""
|
|
||||||
allocator = get_mem_allocator_instance()
|
|
||||||
|
|
||||||
with allocator.use_memory_pool("weights"):
|
|
||||||
weights = torch.ones(1024, 1024, device=DEVICE_TYPE)
|
|
||||||
|
|
||||||
with allocator.use_memory_pool("kv_cache"):
|
|
||||||
kv = torch.ones(512, 512, device=DEVICE_TYPE)
|
|
||||||
|
|
||||||
mapped_bytes = mapped_usage(allocator)
|
|
||||||
|
|
||||||
# Discard kv_cache only — weights should remain valid
|
|
||||||
allocator.discard("kv_cache")
|
|
||||||
|
|
||||||
mapped_bytes_after_discard = mapped_usage(allocator)
|
|
||||||
assert mapped_bytes - mapped_bytes_after_discard >= kv.nbytes
|
|
||||||
assert mapped_bytes_after_discard >= weights.nbytes
|
|
||||||
|
|
||||||
# Weights are still usable
|
|
||||||
assert torch.allclose(weights, torch.ones_like(weights))
|
|
||||||
|
|
||||||
# Wake up and verify kv_cache is remapped; discarded contents are undefined.
|
|
||||||
allocator.wake_up()
|
|
||||||
assert kv.shape == (512, 512)
|
|
||||||
|
|
||||||
# Full sleep/wake cycle still works after discard
|
|
||||||
allocator.sleep(offload_tags="weights")
|
|
||||||
allocator.wake_up()
|
|
||||||
assert torch.allclose(weights, torch.ones_like(weights))
|
|
||||||
|
|
||||||
|
|
||||||
@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn")
|
|
||||||
@pytest.mark.skipif(current_platform.is_xpu(), reason="Uses the CuMem allocator")
|
|
||||||
def test_level2_discards_ordinary_tensor_with_weights_tag():
|
|
||||||
"""Reproduce the level-2 variant for an ordinary tensor in weights."""
|
|
||||||
allocator = get_mem_allocator_instance()
|
|
||||||
|
|
||||||
with allocator.use_memory_pool("weights"):
|
|
||||||
fake_weight = torch.full((4096,), 0x44, dtype=torch.uint8, device=DEVICE_TYPE)
|
|
||||||
ordinary_tensor = torch.full(
|
|
||||||
(4096,), 0x55, dtype=torch.uint8, device=DEVICE_TYPE
|
|
||||||
)
|
|
||||||
|
|
||||||
pointers = (fake_weight.data_ptr(), ordinary_tensor.data_ptr())
|
|
||||||
allocator.sleep(offload_tags=())
|
|
||||||
_wake_up_with_poisoned_mappings(allocator)
|
|
||||||
torch.accelerator.synchronize()
|
|
||||||
|
|
||||||
assert (fake_weight.data_ptr(), ordinary_tensor.data_ptr()) == pointers
|
|
||||||
assert torch.all(fake_weight == 0xA5)
|
|
||||||
assert torch.all(ordinary_tensor == 0xA5)
|
|
||||||
|
|
||||||
|
|
||||||
@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn")
|
|
||||||
@pytest.mark.skipif(current_platform.is_xpu(), reason="CUDA graph not supported on XPU")
|
|
||||||
def test_cumem_with_cudagraph():
|
|
||||||
allocator = get_mem_allocator_instance()
|
|
||||||
with allocator.use_memory_pool():
|
|
||||||
weight = torch.eye(1024, device=DEVICE_TYPE)
|
|
||||||
with allocator.use_memory_pool(tag="discard"):
|
|
||||||
cache = torch.empty(1024, 1024, device=DEVICE_TYPE)
|
|
||||||
|
|
||||||
def model(x):
|
|
||||||
out = x @ weight
|
|
||||||
cache[: out.size(0)].copy_(out)
|
|
||||||
return out + 1
|
|
||||||
|
|
||||||
x = torch.empty(128, 1024, device=DEVICE_TYPE)
|
|
||||||
|
|
||||||
# warmup
|
|
||||||
model(x)
|
|
||||||
|
|
||||||
# capture cudagraph
|
|
||||||
model_graph = torch.cuda.CUDAGraph()
|
|
||||||
with torch.cuda.graph(model_graph):
|
|
||||||
y = model(x)
|
|
||||||
|
|
||||||
mapped_bytes = mapped_usage(allocator)
|
|
||||||
assert mapped_bytes >= weight.nbytes + cache.nbytes
|
|
||||||
allocator.sleep()
|
|
||||||
assert mapped_usage(allocator) == 0
|
|
||||||
allocator.wake_up()
|
|
||||||
assert mapped_usage(allocator) == mapped_bytes
|
|
||||||
|
|
||||||
# after waking up, the content in the weight tensor
|
|
||||||
# should be restored, but the content in the cache tensor
|
|
||||||
# should be discarded
|
|
||||||
|
|
||||||
# this operation is also compatible with cudagraph
|
|
||||||
|
|
||||||
x.random_()
|
|
||||||
model_graph.replay()
|
|
||||||
|
|
||||||
# cache content is as expected
|
|
||||||
assert torch.allclose(x, cache[: x.size(0)])
|
|
||||||
|
|
||||||
# output content is as expected
|
|
||||||
assert torch.allclose(y, x + 1)
|
|
||||||
|
|
||||||
|
|
||||||
@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn")
|
@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn")
|
||||||
@@ -445,8 +154,8 @@ def _lora_logits_mapping_present(model) -> bool:
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@create_new_process_for_each_test()
|
@multi_gpu_test(num_gpus=2)
|
||||||
def test_deep_sleep_lora_tp2(num_gpus_available, monkeypatch):
|
def test_deep_sleep_lora_tp2(monkeypatch):
|
||||||
"""Level-2 sleep/wake/reload with enable_lora=True and TP=2.
|
"""Level-2 sleep/wake/reload with enable_lora=True and TP=2.
|
||||||
|
|
||||||
With TP > 1 the LoRA logits processor carries
|
With TP > 1 the LoRA logits processor carries
|
||||||
@@ -455,9 +164,6 @@ def test_deep_sleep_lora_tp2(num_gpus_available, monkeypatch):
|
|||||||
attribute allocated in the sleep-mode pool, so level-2 sleep destroys
|
attribute allocated in the sleep-mode pool, so level-2 sleep destroys
|
||||||
its contents — it must be restored after reload.
|
its contents — it must be restored after reload.
|
||||||
"""
|
"""
|
||||||
if num_gpus_available < 2:
|
|
||||||
pytest.skip("Requires at least 2 GPUs")
|
|
||||||
|
|
||||||
# Needed for apply_model to reach the multiproc TP workers below.
|
# Needed for apply_model to reach the multiproc TP workers below.
|
||||||
monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
|
monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
|
||||||
|
|
||||||
+4
-4
@@ -2,7 +2,7 @@
|
|||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
"""Compare the short outputs of HF and vLLM when using greedy sampling.
|
"""Compare the short outputs of HF and vLLM when using greedy sampling.
|
||||||
|
|
||||||
Run `pytest tests/basic_correctness/test_basic_correctness.py`.
|
Run `pytest tests/basic_correctness/models/test_basic_correctness.py`.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import os
|
import os
|
||||||
@@ -17,9 +17,9 @@ from vllm import LLM
|
|||||||
from vllm.platforms import current_platform
|
from vllm.platforms import current_platform
|
||||||
from vllm.v1.engine.llm_engine import LLMEngine
|
from vllm.v1.engine.llm_engine import LLMEngine
|
||||||
|
|
||||||
from ..conftest import HfRunner, VllmRunner
|
from ...conftest import HfRunner, VllmRunner
|
||||||
from ..models.utils import check_outputs_equal
|
from ...models.utils import check_outputs_equal
|
||||||
from ..utils import multi_gpu_test
|
from ...utils import multi_gpu_test
|
||||||
|
|
||||||
ATTN_BACKEND = ["ROCM_ATTN"] if current_platform.is_rocm() else ["FLASH_ATTN"]
|
ATTN_BACKEND = ["ROCM_ATTN"] if current_platform.is_rocm() else ["FLASH_ATTN"]
|
||||||
|
|
||||||
+1
-1
@@ -2,7 +2,7 @@
|
|||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
"""Test prefetch offloading correctness with Llama model."""
|
"""Test prefetch offloading correctness with Llama model."""
|
||||||
|
|
||||||
from ..utils import compare_two_settings
|
from ...utils import compare_two_settings
|
||||||
|
|
||||||
|
|
||||||
def test_prefetch_offload_llama():
|
def test_prefetch_offload_llama():
|
||||||
+1
-1
@@ -928,7 +928,7 @@ class HfRunner:
|
|||||||
# shutdown. This is helpful in cases where the HfRunner is
|
# shutdown. This is helpful in cases where the HfRunner is
|
||||||
# initialized after significant GPU memory is already occupied,
|
# initialized after significant GPU memory is already occupied,
|
||||||
# e.g. in
|
# e.g. in
|
||||||
# tests/basic_correctness/test_basic_correctness.py::test_models_distributed
|
# tests/basic_correctness/models/test_basic_correctness.py::test_models_distributed
|
||||||
# where vllm worker processes are still alive and holding GPU
|
# where vllm worker processes are still alive and holding GPU
|
||||||
# memory when hf_runner.__exit__ is called.
|
# memory when hf_runner.__exit__ is called.
|
||||||
from tests.utils import record_gpu_memory_usage_stats
|
from tests.utils import record_gpu_memory_usage_stats
|
||||||
|
|||||||
@@ -2,7 +2,7 @@
|
|||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
|
||||||
# Expanded quantized model tests for CPU offloading
|
# Expanded quantized model tests for CPU offloading
|
||||||
# Base tests: tests/basic_correctness/test_cpu_offload.py
|
# Base tests: tests/basic_correctness/cpu_offload/test_cpu_offload.py
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
|
|||||||
@@ -4,7 +4,7 @@
|
|||||||
|
|
||||||
These cover the registry/factory contract and capability flags. They do not
|
These cover the registry/factory contract and capability flags. They do not
|
||||||
touch CUDA - the ``cumem`` suspend/resume path is exercised end-to-end on GPU
|
touch CUDA - the ``cumem`` suspend/resume path is exercised end-to-end on GPU
|
||||||
in ``tests/basic_correctness/test_cumem.py``.
|
in ``tests/basic_correctness/memory/``.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
|
|||||||
Reference in New Issue
Block a user