[ROCm][CI] Keep startup profiling from aborting when free memory grows (#53591)

Signed-off-by: Stefan Koncarevic <stefan.koncarevic@amd.com>
This commit is contained in:
stefankoncarevic
2026-08-28 00:02:40 -07:00
committed by GitHub
parent c01b50e390
commit 9236159bb6
2 changed files with 130 additions and 14 deletions
+76 -1
View File
@@ -7,7 +7,8 @@ from unittest.mock import patch
import pytest
from vllm.utils.mem_constants import GiB_bytes
from vllm.v1.worker import startup_plan
from vllm.v1.worker import gpu_worker, startup_plan
from vllm.v1.worker.gpu_worker import maybe_rocm_profiling_fallback
from vllm.v1.worker.startup_plan import (
maybe_apply_startup_plan,
maybe_save_startup_plan,
@@ -77,3 +78,77 @@ def test_startup_plan_apply_gate(plan_env):
explicit = _plan_worker(kv_bytes=7 * GiB_bytes)
maybe_apply_startup_plan(explicit)
assert explicit.cache_config.kv_cache_memory_bytes == 7 * GiB_bytes
# Memory accounting of the profiling run (Worker.determine_available_memory).
# The fallback reads only the sign of the measured drop and this process's torch
# reservation; free memory is only logged, so no amount here is a device size.
ANY_FREE_MEMORY = 8 * GiB_bytes
MEASURED_DROP = 4 * GiB_bytes
TORCH_RESERVED = 3 * GiB_bytes
RELEASED_BY_OTHERS = 2 * GiB_bytes
def _snapshot(free_memory, torch_memory=0):
return SimpleNamespace(free_memory=free_memory, torch_memory=torch_memory)
def _profile_result(consumed, reserved_before=0, reserved_after=0):
"""A result whose free-memory readings agree with `consumed`, which
`memory_profiling` derives as the drop in free memory, negative when it grew."""
return SimpleNamespace(
total_consumed=consumed,
transient_peak_headroom=0,
before_create=_snapshot(ANY_FREE_MEMORY, reserved_before),
after_profile=_snapshot(ANY_FREE_MEMORY - consumed, reserved_after),
)
@pytest.fixture
def rocm(request):
with patch.object(
gpu_worker, "current_platform", SimpleNamespace(is_rocm=lambda: request.param)
):
yield request.param
@pytest.mark.parametrize("rocm", [True, False], indirect=True)
def test_profiling_fallback_declines_when_free_memory_dropped(rocm):
"""The profiling measurement is kept as-is whenever free memory dropped."""
result = _profile_result(consumed=MEASURED_DROP)
assert maybe_rocm_profiling_fallback(result) is None
@pytest.mark.parametrize("rocm", [True], indirect=True)
def test_profiling_fallback_replaces_a_released_measurement(rocm):
"""A negative measurement describes the rest of the device, so it is replaced
by this process's reservation, which the rest of the device cannot move."""
result = _profile_result(
consumed=-RELEASED_BY_OTHERS,
reserved_after=TORCH_RESERVED,
)
assert maybe_rocm_profiling_fallback(result) == TORCH_RESERVED
@pytest.mark.parametrize("rocm", [True], indirect=True)
def test_profiling_fallback_never_returns_a_negative_amount(rocm):
"""A reservation that shrank across the run cannot become negative usage."""
result = _profile_result(
consumed=-RELEASED_BY_OTHERS,
reserved_before=TORCH_RESERVED,
reserved_after=0,
)
assert maybe_rocm_profiling_fallback(result) == 0
@pytest.mark.parametrize("rocm", [False], indirect=True)
def test_profiling_fallback_declines_off_rocm(rocm):
"""Platforms that account frees eagerly keep reporting the error, so the
caller's assertion stays reachable there."""
result = _profile_result(consumed=-RELEASED_BY_OTHERS)
assert maybe_rocm_profiling_fallback(result) is None